diff --git a/Cargo.lock b/Cargo.lock index afd7f7e308e..19b9aaabca4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -213,7 +213,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" [[package]] name = "api" -version = "1.1.2" +version = "1.1.3" dependencies = [ "arrow-schema 58.3.0", "common-base", @@ -936,7 +936,7 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "auth" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "async-trait", @@ -1574,7 +1574,7 @@ dependencies = [ [[package]] name = "cache" -version = "1.1.2" +version = "1.1.3" dependencies = [ "catalog", "common-error", @@ -1610,7 +1610,7 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "catalog" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "arrow 58.3.0", @@ -1945,7 +1945,7 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "cli" -version = "1.1.2" +version = "1.1.3" dependencies = [ "async-stream", "async-trait", @@ -2004,7 +2004,7 @@ dependencies = [ [[package]] name = "client" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "arc-swap", @@ -2036,7 +2036,7 @@ dependencies = [ "serde_json", "snafu 0.8.6", "store-api", - "substrait 1.1.2", + "substrait 1.1.3", "tokio", "tokio-stream", "tonic 0.14.2", @@ -2085,7 +2085,7 @@ dependencies = [ [[package]] name = "cmd" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "async-trait", @@ -2235,7 +2235,7 @@ dependencies = [ [[package]] name = "common-base" -version = "1.1.2" +version = "1.1.3" dependencies = [ "ahash 0.8.12", "anymap2", @@ -2255,14 +2255,14 @@ dependencies = [ [[package]] name = "common-catalog" -version = "1.1.2" +version = "1.1.3" dependencies = [ "const_format", ] [[package]] name = "common-config" -version = "1.1.2" +version = "1.1.3" dependencies = [ "common-base", "common-error", @@ -2286,7 +2286,7 @@ dependencies = [ [[package]] name = "common-datasource" -version = "1.1.2" +version = "1.1.3" dependencies = [ "arrow 58.3.0", "arrow-schema 58.3.0", @@ -2322,7 +2322,7 @@ dependencies = [ [[package]] name = "common-decimal" -version = "1.1.2" +version = "1.1.3" dependencies = [ "bigdecimal 0.4.8", "common-error", @@ -2335,7 +2335,7 @@ dependencies = [ [[package]] name = "common-error" -version = "1.1.2" +version = "1.1.3" dependencies = [ "common-macro", "http 1.3.1", @@ -2346,7 +2346,7 @@ dependencies = [ [[package]] name = "common-event-recorder" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "async-trait", @@ -2369,7 +2369,7 @@ dependencies = [ [[package]] name = "common-frontend" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "async-trait", @@ -2390,7 +2390,7 @@ dependencies = [ [[package]] name = "common-function" -version = "1.1.2" +version = "1.1.3" dependencies = [ "ahash 0.8.12", "api", @@ -2453,7 +2453,7 @@ dependencies = [ [[package]] name = "common-greptimedb-telemetry" -version = "1.1.2" +version = "1.1.3" dependencies = [ "async-trait", "common-runtime", @@ -2470,7 +2470,7 @@ dependencies = [ [[package]] name = "common-grpc" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "arrow-flight", @@ -2505,7 +2505,7 @@ dependencies = [ [[package]] name = "common-grpc-expr" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "common-base", @@ -2525,7 +2525,7 @@ dependencies = [ [[package]] name = "common-macro" -version = "1.1.2" +version = "1.1.3" dependencies = [ "greptime-proto", "once_cell", @@ -2536,7 +2536,7 @@ dependencies = [ [[package]] name = "common-mem-prof" -version = "1.1.2" +version = "1.1.3" dependencies = [ "anyhow", "common-error", @@ -2552,7 +2552,7 @@ dependencies = [ [[package]] name = "common-memory-manager" -version = "1.1.2" +version = "1.1.3" dependencies = [ "common-error", "common-macro", @@ -2564,7 +2564,7 @@ dependencies = [ [[package]] name = "common-meta" -version = "1.1.2" +version = "1.1.3" dependencies = [ "anymap2", "api", @@ -2635,7 +2635,7 @@ dependencies = [ [[package]] name = "common-options" -version = "1.1.2" +version = "1.1.3" dependencies = [ "common-grpc", "humantime-serde", @@ -2645,11 +2645,11 @@ dependencies = [ [[package]] name = "common-plugins" -version = "1.1.2" +version = "1.1.3" [[package]] name = "common-pprof" -version = "1.1.2" +version = "1.1.3" dependencies = [ "common-error", "common-macro", @@ -2660,7 +2660,7 @@ dependencies = [ [[package]] name = "common-procedure" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "async-stream", @@ -2689,7 +2689,7 @@ dependencies = [ [[package]] name = "common-procedure-test" -version = "1.1.2" +version = "1.1.3" dependencies = [ "async-trait", "common-procedure", @@ -2699,7 +2699,7 @@ dependencies = [ [[package]] name = "common-query" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "async-trait", @@ -2729,7 +2729,7 @@ dependencies = [ [[package]] name = "common-recordbatch" -version = "1.1.2" +version = "1.1.3" dependencies = [ "arc-swap", "common-base", @@ -2754,7 +2754,7 @@ dependencies = [ [[package]] name = "common-runtime" -version = "1.1.2" +version = "1.1.3" dependencies = [ "async-trait", "clap", @@ -2783,7 +2783,7 @@ dependencies = [ [[package]] name = "common-session" -version = "1.1.2" +version = "1.1.3" dependencies = [ "serde", "strum 0.27.1", @@ -2791,7 +2791,7 @@ dependencies = [ [[package]] name = "common-sql" -version = "1.1.2" +version = "1.1.3" dependencies = [ "arrow-schema 58.3.0", "common-base", @@ -2811,7 +2811,7 @@ dependencies = [ [[package]] name = "common-stat" -version = "1.1.2" +version = "1.1.3" dependencies = [ "common-base", "common-runtime", @@ -2826,7 +2826,7 @@ dependencies = [ [[package]] name = "common-telemetry" -version = "1.1.2" +version = "1.1.3" dependencies = [ "backtrace", "common-base", @@ -2855,7 +2855,7 @@ dependencies = [ [[package]] name = "common-test-util" -version = "1.1.2" +version = "1.1.3" dependencies = [ "client", "common-grpc", @@ -2868,7 +2868,7 @@ dependencies = [ [[package]] name = "common-time" -version = "1.1.2" +version = "1.1.3" dependencies = [ "arrow 58.3.0", "chrono", @@ -2886,7 +2886,7 @@ dependencies = [ [[package]] name = "common-version" -version = "1.1.2" +version = "1.1.3" dependencies = [ "cargo-manifest", "const_format", @@ -2896,7 +2896,7 @@ dependencies = [ [[package]] name = "common-wal" -version = "1.1.2" +version = "1.1.3" dependencies = [ "common-base", "common-error", @@ -2919,7 +2919,7 @@ dependencies = [ [[package]] name = "common-workload" -version = "1.1.2" +version = "1.1.3" dependencies = [ "common-telemetry", "serde", @@ -3642,7 +3642,7 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "datafusion" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "arrow-schema 58.3.0", @@ -3696,7 +3696,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3720,7 +3720,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3742,7 +3742,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -3766,7 +3766,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "futures", "log", @@ -3776,7 +3776,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "async-compression", @@ -3810,7 +3810,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "arrow-ipc 58.3.0", @@ -3833,7 +3833,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3855,7 +3855,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3878,7 +3878,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3907,12 +3907,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" [[package]] name = "datafusion-execution" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "arrow-buffer 58.3.0", @@ -3934,7 +3934,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3956,7 +3956,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -3968,7 +3968,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "arrow-buffer 58.3.0", @@ -3999,7 +3999,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4020,7 +4020,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4032,7 +4032,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "arrow-ord 58.3.0", @@ -4056,7 +4056,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "async-trait", @@ -4071,7 +4071,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4088,7 +4088,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -4097,7 +4097,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "datafusion-doc", "quote", @@ -4107,7 +4107,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "chrono", @@ -4140,9 +4140,9 @@ dependencies = [ [[package]] name = "datafusion-pg-catalog" -version = "0.17.0" +version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c0ecc2378a803326a2dd32df32236144e51fe2629547ae1ad03ef6650024e79" +checksum = "f6b82fb8bd291b718d4226eaba85f78c73f0abdedb256a5dc91d9d9e6b0e0dab" dependencies = [ "arrow-pg", "async-trait", @@ -4156,7 +4156,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4179,7 +4179,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4193,7 +4193,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4209,7 +4209,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4227,7 +4227,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4258,7 +4258,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "chrono", @@ -4285,7 +4285,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4295,7 +4295,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4311,7 +4311,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "async-trait", "datafusion-common", @@ -4324,7 +4324,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "arrow 58.3.0", "bigdecimal 0.4.8", @@ -4342,7 +4342,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7a84c45104d35408f00bbf880268c47193b73b28#7a84c45104d35408f00bbf880268c47193b73b28" dependencies = [ "async-recursion", "async-trait", @@ -4360,7 +4360,7 @@ dependencies = [ [[package]] name = "datanode" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "arrow-flight", @@ -4434,7 +4434,7 @@ checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" [[package]] name = "datatypes" -version = "1.1.2" +version = "1.1.3" dependencies = [ "arrow 58.3.0", "arrow-array 58.3.0", @@ -5171,7 +5171,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file-engine" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "async-trait", @@ -5302,7 +5302,7 @@ checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8" [[package]] name = "flow" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "arrow 58.3.0", @@ -5371,7 +5371,7 @@ dependencies = [ "sql", "store-api", "strum 0.27.1", - "substrait 1.1.2", + "substrait 1.1.3", "table", "tokio", "tokio-stream", @@ -5454,7 +5454,7 @@ checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619" [[package]] name = "frontend" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "arc-swap", @@ -6744,7 +6744,7 @@ dependencies = [ [[package]] name = "index" -version = "1.1.2" +version = "1.1.3" dependencies = [ "async-trait", "asynchronous-codec", @@ -7843,7 +7843,7 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "log-query" -version = "1.1.2" +version = "1.1.3" dependencies = [ "chrono", "common-error", @@ -7855,7 +7855,7 @@ dependencies = [ [[package]] name = "log-store" -version = "1.1.2" +version = "1.1.3" dependencies = [ "async-stream", "async-trait", @@ -8174,7 +8174,7 @@ dependencies = [ [[package]] name = "meta-client" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "async-trait", @@ -8206,7 +8206,7 @@ dependencies = [ [[package]] name = "meta-srv" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "async-trait", @@ -8306,7 +8306,7 @@ dependencies = [ [[package]] name = "metric-engine" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "aquamarine", @@ -8407,7 +8407,7 @@ dependencies = [ [[package]] name = "mito-codec" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "bytes", @@ -8432,7 +8432,7 @@ dependencies = [ [[package]] name = "mito2" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "aquamarine", @@ -9221,7 +9221,7 @@ dependencies = [ [[package]] name = "object-store" -version = "1.1.2" +version = "1.1.3" dependencies = [ "anyhow", "async-trait", @@ -9776,7 +9776,7 @@ dependencies = [ [[package]] name = "operator" -version = "1.1.2" +version = "1.1.3" dependencies = [ "ahash 0.8.12", "api", @@ -9835,7 +9835,7 @@ dependencies = [ "sql", "sqlparser", "store-api", - "substrait 1.1.2", + "substrait 1.1.3", "table", "tokio", "tokio-util", @@ -10120,7 +10120,7 @@ checksum = "e3c406c9e2aa74554e662d2c2ee11cd3e73756988800be7e6f5eddb16fed4699" [[package]] name = "partition" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "async-trait", @@ -10514,7 +10514,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pipeline" -version = "1.1.2" +version = "1.1.3" dependencies = [ "ahash 0.8.12", "api", @@ -10671,7 +10671,7 @@ dependencies = [ [[package]] name = "plugins" -version = "1.1.2" +version = "1.1.3" dependencies = [ "auth", "catalog", @@ -11011,7 +11011,7 @@ dependencies = [ [[package]] name = "promql" -version = "1.1.2" +version = "1.1.3" dependencies = [ "ahash 0.8.12", "async-trait", @@ -11363,7 +11363,7 @@ dependencies = [ [[package]] name = "puffin" -version = "1.1.2" +version = "1.1.3" dependencies = [ "async-compression", "async-trait", @@ -11425,7 +11425,7 @@ dependencies = [ [[package]] name = "query" -version = "1.1.2" +version = "1.1.3" dependencies = [ "ahash 0.8.12", "api", @@ -11495,7 +11495,7 @@ dependencies = [ "sql", "sqlparser", "store-api", - "substrait 1.1.2", + "substrait 1.1.3", "table", "tokio", "tokio-stream", @@ -13067,7 +13067,7 @@ dependencies = [ [[package]] name = "servers" -version = "1.1.2" +version = "1.1.3" dependencies = [ "ahash 0.8.12", "api", @@ -13205,7 +13205,7 @@ dependencies = [ [[package]] name = "session" -version = "1.1.2" +version = "1.1.3" dependencies = [ "ahash 0.8.12", "api", @@ -13558,7 +13558,7 @@ dependencies = [ [[package]] name = "sql" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "arrow-buffer 58.3.0", @@ -13619,7 +13619,7 @@ dependencies = [ [[package]] name = "sqlness-runner" -version = "1.1.2" +version = "1.1.3" dependencies = [ "async-trait", "clap", @@ -13899,7 +13899,7 @@ dependencies = [ [[package]] name = "standalone" -version = "1.1.2" +version = "1.1.3" dependencies = [ "async-trait", "catalog", @@ -13943,7 +13943,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "store-api" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "aquamarine", @@ -14136,7 +14136,7 @@ dependencies = [ [[package]] name = "substrait" -version = "1.1.2" +version = "1.1.3" dependencies = [ "async-trait", "bytes", @@ -14258,7 +14258,7 @@ dependencies = [ [[package]] name = "table" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "arc-swap", @@ -14541,7 +14541,7 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "tests-fuzz" -version = "1.1.2" +version = "1.1.3" dependencies = [ "arbitrary", "async-trait", @@ -14586,7 +14586,7 @@ dependencies = [ [[package]] name = "tests-integration" -version = "1.1.2" +version = "1.1.3" dependencies = [ "api", "arrow-flight", @@ -14666,7 +14666,7 @@ dependencies = [ "sqlx", "standalone", "store-api", - "substrait 1.1.2", + "substrait 1.1.3", "table", "tempfile", "time", diff --git a/Cargo.toml b/Cargo.toml index 29cbc471115..fd6a533ed11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,7 +75,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.1.2" +version = "1.1.3" edition = "2024" license = "Apache-2.0" @@ -137,7 +137,7 @@ datafusion-functions-aggregate-common = "=53.1.0" datafusion-functions-window-common = "=53.1.0" datafusion-optimizer = "=53.1.0" datafusion-orc = { git = "https://github.com/datafusion-contrib/datafusion-orc.git", rev = "6c07fa282dc8d62db2aa4ded06ab55485efc811a" } -datafusion-pg-catalog = "0.17" +datafusion-pg-catalog = "0.17.3" datafusion-physical-expr = "=53.1.0" datafusion-physical-plan = "=53.1.0" datafusion-proto = "=53.1.0" @@ -341,21 +341,21 @@ git = "https://github.com/GreptimeTeam/greptime-meter.git" rev = "5618e779cf2bb4755b499c630fba4c35e91898cb" [patch.crates-io] -datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } -datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } +datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } +datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } +datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } +datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } +datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } +datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } +datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } +datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } +datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } +datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } +datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } +datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } +datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } +datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7a84c45104d35408f00bbf880268c47193b73b28" } sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "2aefa08a8d69c96eec2d6d6703598a009bba6e4c" } # on branch v0.61.x [profile.release] diff --git a/src/catalog/src/information_extension.rs b/src/catalog/src/information_extension.rs index bcb5056bada..08d81ac1f03 100644 --- a/src/catalog/src/information_extension.rs +++ b/src/catalog/src/information_extension.rs @@ -23,7 +23,7 @@ use common_meta::rpc::procedure; use common_procedure::{ProcedureInfo, ProcedureState}; use common_query::request::QueryRequest; use common_recordbatch::SendableRecordBatchStream; -use common_recordbatch::util::ChainedRecordBatchStream; +use common_recordbatch::util::{ChainedRecordBatchStream, LimitedRecordBatchStream}; use meta_client::MetaClientRef; use snafu::ResultExt; use store_api::storage::RegionId; @@ -120,6 +120,7 @@ impl InformationExtension for DistributedInformationExtension { .map_err(BoxedError::new) .context(crate::error::ListNodesSnafu)?; + let limit = request.scan.limit; let plan = request .build_plan() .context(crate::error::DatafusionSnafu)?; @@ -140,6 +141,12 @@ impl InformationExtension for DistributedInformationExtension { let chained = ChainedRecordBatchStream::new(streams).context(crate::error::CreateRecordBatchSnafu)?; - Ok(Box::pin(chained)) + match limit { + Some(limit) => Ok(Box::pin(LimitedRecordBatchStream::new( + Box::pin(chained), + limit, + ))), + None => Ok(Box::pin(chained)), + } } } diff --git a/src/common/recordbatch/Cargo.toml b/src/common/recordbatch/Cargo.toml index efc6b6f60e9..59929ba3477 100644 --- a/src/common/recordbatch/Cargo.toml +++ b/src/common/recordbatch/Cargo.toml @@ -34,3 +34,7 @@ tokio.workspace = true [[bench]] name = "iter_record_batch_rows" harness = false + +[[bench]] +name = "memory_accounting" +harness = false diff --git a/src/common/recordbatch/benches/memory_accounting.rs b/src/common/recordbatch/benches/memory_accounting.rs new file mode 100644 index 00000000000..05fb63555a4 --- /dev/null +++ b/src/common/recordbatch/benches/memory_accounting.rs @@ -0,0 +1,117 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::hint::black_box; +use std::sync::Arc; + +use common_recordbatch::RecordBatch; +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use datatypes::data_type::ConcreteDataType; +use datatypes::schema::{ColumnSchema, Schema}; +use datatypes::vectors::{Int32Vector, StringVector, UInt64Vector, VectorRef}; + +const LARGE_ROWS: usize = 8_192; + +fn batch(columns: Vec<(&str, ConcreteDataType, bool, VectorRef)>) -> RecordBatch { + let schema = Arc::new(Schema::new( + columns + .iter() + .map(|(name, data_type, nullable, _)| { + ColumnSchema::new(*name, data_type.clone(), *nullable) + }) + .collect(), + )); + let vectors: Vec = columns + .into_iter() + .map(|(_, _, _, vector)| vector) + .collect(); + RecordBatch::new(schema, vectors).unwrap() +} + +fn primitive_string_batch(rows: usize) -> RecordBatch { + let strings = (0..rows) + .map(|row| (row % 5 != 0).then(|| format!("value-{row:08}-with-payload"))) + .collect::>(); + batch(vec![ + ( + "i32", + ConcreteDataType::int32_datatype(), + false, + Arc::new(Int32Vector::from_slice( + (0..rows).map(|value| value as i32).collect::>(), + )), + ), + ( + "u64", + ConcreteDataType::uint64_datatype(), + false, + Arc::new(UInt64Vector::from_slice( + (0..rows).map(|value| value as u64).collect::>(), + )), + ), + ( + "string", + ConcreteDataType::string_datatype(), + true, + Arc::new(StringVector::from(strings)), + ), + ]) +} + +fn bench_one_call(c: &mut Criterion) { + let mut group = c.benchmark_group("memory_accounting/one_call"); + for rows in [128, LARGE_ROWS] { + let batch = primitive_string_batch(rows); + group.bench_function(BenchmarkId::new("buffer", rows), |b| { + b.iter(|| black_box(black_box(&batch).buffer_memory_size())) + }); + group.bench_function(BenchmarkId::new("logical", rows), |b| { + b.iter(|| black_box(black_box(&batch).logical_slice_memory_size())) + }); + } + group.finish(); +} + +fn bench_shared_buffer_slices(c: &mut Criterion) { + let backing = primitive_string_batch(LARGE_ROWS); + let mut group = c.benchmark_group("memory_accounting/shared_buffer_slices"); + for slice_len in [1, 16] { + let slices = (0..512) + .map(|index| backing.slice(index * slice_len, slice_len).unwrap()) + .collect::>(); + group.throughput(Throughput::Elements(slices.len() as u64)); + group.bench_function(BenchmarkId::new("buffer", slice_len), |b| { + b.iter(|| { + let total = black_box(&slices) + .iter() + .map(RecordBatch::buffer_memory_size) + .fold(0usize, usize::saturating_add); + black_box(total) + }) + }); + group.bench_function(BenchmarkId::new("logical", slice_len), |b| { + b.iter(|| { + let total = black_box(&slices) + .iter() + .map(RecordBatch::logical_slice_memory_size) + .fold(0usize, usize::saturating_add); + black_box(total) + }) + }); + } + group.finish(); +} + +criterion_group!(benches, bench_one_call, bench_shared_buffer_slices); +criterion_main!(benches); diff --git a/src/common/recordbatch/src/adapter.rs b/src/common/recordbatch/src/adapter.rs index a69eedf7ebf..15c07773146 100644 --- a/src/common/recordbatch/src/adapter.rs +++ b/src/common/recordbatch/src/adapter.rs @@ -366,6 +366,7 @@ impl ExecutionPlanVisitor for MetricCollector { let Some(metric) = plan.metrics() else { self.record_batch_metrics.plan_metrics.push(PlanMetrics { plan: plan.name().to_string(), + plan_name: plan.name().to_string(), level: self.current_level, metrics: vec![], }); @@ -380,6 +381,7 @@ impl ExecutionPlanVisitor for MetricCollector { .timestamps_removed(); let mut plan_metric = PlanMetrics { plan: one_line(plan, self.verbose).to_string(), + plan_name: plan.name().to_string(), level: self.current_level, metrics: Vec::with_capacity(metric.iter().size_hint().0), }; @@ -519,6 +521,9 @@ impl Display for RecordBatchMetrics { pub struct PlanMetrics { /// The plan name pub plan: String, + /// The stable execution plan name. + #[serde(default)] + pub plan_name: String, /// The level of the plan, starts from 0 pub level: usize, /// An ordered key-value list of metrics. @@ -916,6 +921,22 @@ mod test { assert_eq!(metrics.memory_usage, 34); } + #[test] + fn test_plan_metrics_deserializes_without_plan_name() { + let metrics: RecordBatchMetrics = serde_json::from_value(json!({ + "elapsed_compute": 12, + "memory_usage": 34, + "plan_metrics": [{ + "plan": "SeqScan: region=1", + "level": 0, + "metrics": [] + }] + })) + .unwrap(); + + assert_eq!(metrics.plan_metrics[0].plan_name, ""); + } + #[test] fn test_recordbatch_metrics_region_watermarks_serde_roundtrip() { let metrics = RecordBatchMetrics { diff --git a/src/common/recordbatch/src/lib.rs b/src/common/recordbatch/src/lib.rs index 3db3d9c08b6..46375d8fd77 100644 --- a/src/common/recordbatch/src/lib.rs +++ b/src/common/recordbatch/src/lib.rs @@ -769,7 +769,7 @@ impl MemoryTrackedStream { batch: RecordBatch, cx: &mut Context<'_>, ) -> Poll>> { - let additional = batch.buffer_memory_size(); + let additional = batch.logical_slice_memory_size(); let tracker = self.ready_tracker_mut(); if let Err(error) = tracker.try_track(additional) { @@ -860,6 +860,35 @@ mod tests { as usize } + #[tokio::test] + async fn test_memory_tracked_stream_charges_logical_slice_size() { + let schema = Arc::new(Schema::new(vec![ColumnSchema::new( + "payload", + ConcreteDataType::string_datatype(), + false, + )])); + let payloads: Vec<_> = (0..1024) + .map(|value| format!("payload-{value:04}")) + .collect(); + let batch = RecordBatch::new(schema, vec![Arc::new(StringVector::from(payloads)) as _]) + .unwrap() + .slice(512, 1) + .unwrap(); + let expected_bytes = aligned_tracked_bytes(batch.logical_slice_memory_size()); + assert!(expected_bytes < aligned_tracked_bytes(batch.buffer_memory_size())); + let tracker = QueryMemoryTracker::builder(MB, OnExhaustedPolicy::Fail).build(); + let mut stream = MemoryTrackedStream::new( + RecordBatches::try_new(batch.schema.clone(), vec![batch]) + .unwrap() + .as_stream(), + tracker.clone(), + ); + + stream.next().await.unwrap().unwrap(); + + assert_eq!(tracker.current(), expected_bytes); + } + #[test] fn test_recordbatches_try_from_columns() { let schema = Arc::new(Schema::new(vec![ColumnSchema::new( @@ -1052,7 +1081,7 @@ mod tests { }) .build(); let batch = large_string_batch(700 * 1024); - let expected_bytes = aligned_tracked_bytes(batch.buffer_memory_size()); + let expected_bytes = aligned_tracked_bytes(batch.logical_slice_memory_size()); let mut stream1 = MemoryTrackedStream::new( RecordBatches::try_new(batch.schema.clone(), vec![batch.clone()]) diff --git a/src/common/recordbatch/src/recordbatch.rs b/src/common/recordbatch/src/recordbatch.rs index 7280772d05d..77a36836d6a 100644 --- a/src/common/recordbatch/src/recordbatch.rs +++ b/src/common/recordbatch/src/recordbatch.rs @@ -269,6 +269,28 @@ impl RecordBatch { .sum() } + /// Returns the logical memory size of this batch's array slices. + /// + /// This sums Arrow's logical visible slice buffers rather than the capacity of their shared + /// backing buffers. View out-of-line payloads and nested custom payloads are not separately + /// traversed or accounted. It is not an exact measure of live physical memory. If Arrow cannot + /// calculate a slice's size, the full buffer size is used conservatively. + /// + /// Mito's current scan paths do not produce top-level View arrays. If they do in the future, + /// their out-of-line payload accounting must be reassessed here. + pub fn logical_slice_memory_size(&self) -> usize { + self.df_record_batch + .columns() + .iter() + .fold(0, |total, array| { + let array_size = array + .to_data() + .get_slice_memory_size() + .unwrap_or_else(|_| array.get_buffer_memory_size()); + total.saturating_add(array_size) + }) + } + /// Iterate the values as strings in the column at index `i`. /// /// Note that if the underlying array is not a valid GreptimeDB vector, an empty iterator is @@ -382,13 +404,14 @@ fn maybe_align_json_array_with_schema( mod tests { use std::sync::Arc; - use datatypes::arrow::array::{AsArray, BinaryArray, UInt32Array}; + use datatypes::arrow::array::{ + AsArray, BinaryArray, StringArray, StringViewArray, UInt32Array, + }; use datatypes::arrow::datatypes::{DataType, Field, Schema as ArrowSchema, UInt32Type}; - use datatypes::arrow_array::StringArray; use datatypes::data_type::ConcreteDataType; use datatypes::extension::json::{JsonExtensionType, JsonMetadata}; use datatypes::schema::{ColumnSchema, Schema}; - use datatypes::vectors::{StringVector, UInt32Vector}; + use datatypes::vectors::{BinaryVector, StringVector, UInt32Vector}; use super::*; @@ -475,6 +498,97 @@ mod tests { assert!(recordbatch.slice(1, 5).is_err()); } + #[test] + fn test_logical_slice_memory_size_for_visible_primitive_string_binary_slices() { + let schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("numbers", ConcreteDataType::uint32_datatype(), false), + ColumnSchema::new("strings", ConcreteDataType::string_datatype(), true), + ColumnSchema::new("binary", ConcreteDataType::binary_datatype(), true), + ])); + let numbers: Vec<_> = (0..1024).collect(); + let strings = (0..1024) + .map(|value| (value % 3 != 0).then(|| format!("value-{value}"))) + .collect::>(); + let binary = (0_u32..1024) + .map(|value| (value % 3 != 0).then(|| value.to_le_bytes().to_vec())) + .collect::>(); + let columns: Vec = vec![ + Arc::new(UInt32Vector::from_slice(numbers)), + Arc::new(StringVector::from(strings)), + Arc::new(BinaryVector::from(binary)), + ]; + let batch = RecordBatch::new(schema, columns).unwrap(); + let slice = batch.slice(511, 3).unwrap(); + + assert!(slice.columns().iter().any(|column| column.null_count() > 0)); + assert!(slice.logical_slice_memory_size() < slice.buffer_memory_size()); + assert_eq!( + slice.logical_slice_memory_size(), + slice + .columns() + .iter() + .map(|column| column.to_data().get_slice_memory_size().unwrap()) + .sum::() + ); + } + + #[test] + fn test_logical_slice_memory_size_for_many_shared_buffer_slices() { + let schema = Arc::new(Schema::new(vec![ColumnSchema::new( + "strings", + ConcreteDataType::string_datatype(), + false, + )])); + let strings = (0..1024) + .map(|value| format!("shared-value-{value}")) + .collect::>(); + let backing = RecordBatch::new( + schema, + vec![Arc::new(StringVector::from(strings)) as VectorRef], + ) + .unwrap(); + let slices = (0..128) + .map(|index| backing.slice(index * 4, 2).unwrap()) + .collect::>(); + let logical_total = slices + .iter() + .map(RecordBatch::logical_slice_memory_size) + .sum::(); + let buffer_total = slices + .iter() + .map(RecordBatch::buffer_memory_size) + .sum::(); + + assert!(logical_total < buffer_total); + assert!(slices.iter().all(|slice| { + slice.logical_slice_memory_size() + == slice.column(0).to_data().get_slice_memory_size().unwrap() + })); + } + + #[test] + fn test_logical_slice_memory_size_uses_arrow_slice_scope_for_views() { + let schema = Arc::new(Schema::new(vec![ColumnSchema::new( + "strings", + ConcreteDataType::utf8_view_datatype(), + false, + )])); + let columns: Vec = + vec![Arc::new(StringVector::from(StringViewArray::from(vec![ + "unrelated backing payload", + "visible string view payload", + ])))]; + let batch = RecordBatch::new(schema, columns) + .unwrap() + .slice(1, 1) + .unwrap(); + + assert_eq!( + batch.column(0).to_data().get_slice_memory_size().unwrap(), + batch.logical_slice_memory_size() + ); + } + #[test] fn test_merge_record_batch() { let column_schemas = vec![ diff --git a/src/common/recordbatch/src/util.rs b/src/common/recordbatch/src/util.rs index 0a587e303dc..abe84035462 100644 --- a/src/common/recordbatch/src/util.rs +++ b/src/common/recordbatch/src/util.rs @@ -98,6 +98,83 @@ impl ChainedRecordBatchStream { } } +/// A stream that stops after yielding at most `remaining` rows. +pub struct LimitedRecordBatchStream { + input: Option, + remaining: usize, + schema: SchemaRef, + output_ordering: Option>, +} + +impl LimitedRecordBatchStream { + pub fn new(input: SendableRecordBatchStream, limit: usize) -> Self { + let schema = input.schema(); + let output_ordering = input.output_ordering().map(|o| o.to_vec()); + Self { + input: Some(input), + remaining: limit, + schema, + output_ordering, + } + } +} + +impl RecordBatchStream for LimitedRecordBatchStream { + fn name(&self) -> &str { + "LimitedRecordBatchStream" + } + + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + self.output_ordering.as_deref() + } + + fn metrics(&self) -> Option { + self.input.as_ref().and_then(|input| input.metrics()) + } +} + +impl Stream for LimitedRecordBatchStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll> { + if self.remaining == 0 { + self.input.take(); + return Poll::Ready(None); + } + + let Some(input) = self.input.as_mut() else { + return Poll::Ready(None); + }; + + match input.poll_next_unpin(ctx) { + Poll::Ready(Some(Ok(batch))) => { + let num_rows = batch.num_rows(); + if num_rows > self.remaining { + let remaining = self.remaining; + self.remaining = 0; + self.input.take(); + Poll::Ready(Some(batch.slice(0, remaining))) + } else { + self.remaining -= num_rows; + if self.remaining == 0 { + self.input.take(); + } + Poll::Ready(Some(Ok(batch))) + } + } + Poll::Ready(None) => { + self.input.take(); + Poll::Ready(None) + } + other => other, + } + } +} + impl RecordBatchStream for ChainedRecordBatchStream { fn name(&self) -> &str { "ChainedRecordBatchStream" @@ -172,6 +249,75 @@ mod tests { } } + #[tokio::test] + async fn test_limited_chained_stream() { + let column_schemas = vec![ColumnSchema::new( + "number", + ConcreteDataType::uint32_datatype(), + false, + )]; + + let schema = Arc::new(Schema::try_new(column_schemas).unwrap()); + let first = RecordBatch::new( + schema.clone(), + [Arc::new(UInt32Vector::from_vec(vec![0, 1, 2])) as _], + ) + .unwrap(); + let second = RecordBatch::new( + schema.clone(), + [Arc::new(UInt32Vector::from_vec(vec![3, 4, 5])) as _], + ) + .unwrap(); + let chained = ChainedRecordBatchStream::new(vec![ + Box::pin(MockRecordBatchStream { + schema: schema.clone(), + batch: Some(first), + }), + Box::pin(MockRecordBatchStream { + schema: schema.clone(), + batch: Some(second), + }), + ]) + .unwrap(); + + let batches = collect(Box::pin(LimitedRecordBatchStream::new( + Box::pin(chained), + 4, + ))) + .await + .unwrap(); + + assert_eq!(2, batches.len()); + assert_eq!(3, batches[0].num_rows()); + assert_eq!(1, batches[1].num_rows()); + } + + #[tokio::test] + async fn test_limited_stream_with_zero_limit() { + let column_schemas = vec![ColumnSchema::new( + "number", + ConcreteDataType::uint32_datatype(), + false, + )]; + + let schema = Arc::new(Schema::try_new(column_schemas).unwrap()); + let batch = RecordBatch::new( + schema.clone(), + [Arc::new(UInt32Vector::from_vec(vec![0])) as _], + ) + .unwrap(); + let stream = MockRecordBatchStream { + schema, + batch: Some(batch), + }; + + let batches = collect(Box::pin(LimitedRecordBatchStream::new(Box::pin(stream), 0))) + .await + .unwrap(); + + assert!(batches.is_empty()); + } + #[tokio::test] async fn test_collect() { let column_schemas = vec![ColumnSchema::new( diff --git a/src/log-query/src/log_query.rs b/src/log-query/src/log_query.rs index 2955b7bdedf..78e622200d4 100644 --- a/src/log-query/src/log_query.rs +++ b/src/log-query/src/log_query.rs @@ -214,9 +214,9 @@ impl TimeFilter { (Some(start), Some(end), _) => { // Both 'start' and 'end' are provided let (start, _) = Self::parse_datetime(start)?; - let (end, _) = Self::parse_datetime(end)?; + let (end, inferred_end) = Self::parse_datetime(end)?; start_dt = Some(start); - end_dt = Some(end); + end_dt = Some(inferred_end.unwrap_or(end)); } (Some(start), None, Some(span)) => { let (start, _) = Self::parse_datetime(start)?; @@ -276,41 +276,39 @@ impl TimeFilter { if let Ok(dt) = DateTime::parse_from_rfc3339(s) { Ok((dt.with_timezone(&Utc), None)) } else { - let formats = ["%Y-%m-%d", "%Y-%m", "%Y"]; - for format in &formats { - if let Ok(naive_date) = NaiveDate::parse_from_str(s, format) { - let start = Utc.from_utc_datetime( - &naive_date.and_time(NaiveTime::from_hms_opt(0, 0, 0).unwrap()), - ); - let end = match *format { - "%Y-%m-%d" => start + Duration::days(1), - "%Y-%m" => { - let next_month = if naive_date.month() == 12 { - NaiveDate::from_ymd_opt(naive_date.year() + 1, 1, 1).unwrap() - } else { - NaiveDate::from_ymd_opt( - naive_date.year(), - naive_date.month() + 1, - 1, - ) - .unwrap() - }; - Utc.from_utc_datetime(&next_month.and_hms_opt(0, 0, 0).unwrap()) - } - "%Y" => { - let next_year = - NaiveDate::from_ymd_opt(naive_date.year() + 1, 1, 1).unwrap(); - Utc.from_utc_datetime(&next_year.and_hms_opt(0, 0, 0).unwrap()) - } - _ => unreachable!(), - }; - return Ok((start, Some(end))); + let (naive_date, format) = if let Ok(date) = NaiveDate::parse_from_str(s, "%Y-%m-%d") { + (date, "%Y-%m-%d") + } else if let Ok(date) = NaiveDate::parse_from_str(&format!("{s}-01"), "%Y-%m-%d") { + (date, "%Y-%m") + } else if let Ok(date) = NaiveDate::parse_from_str(&format!("{s}-01-01"), "%Y-%m-%d") { + (date, "%Y") + } else { + return Err(InvalidDateFormatSnafu { + input: s.to_string(), } - } - Err(InvalidDateFormatSnafu { - input: s.to_string(), - } - .build()) + .build()); + }; + + let start = Utc + .from_utc_datetime(&naive_date.and_time(NaiveTime::from_hms_opt(0, 0, 0).unwrap())); + let end = match format { + "%Y-%m-%d" => start + Duration::days(1), + "%Y-%m" => { + let next_month = if naive_date.month() == 12 { + NaiveDate::from_ymd_opt(naive_date.year() + 1, 1, 1).unwrap() + } else { + NaiveDate::from_ymd_opt(naive_date.year(), naive_date.month() + 1, 1) + .unwrap() + }; + Utc.from_utc_datetime(&next_month.and_hms_opt(0, 0, 0).unwrap()) + } + "%Y" => { + let next_year = NaiveDate::from_ymd_opt(naive_date.year() + 1, 1, 1).unwrap(); + Utc.from_utc_datetime(&next_year.and_hms_opt(0, 0, 0).unwrap()) + } + _ => unreachable!(), + }; + Ok((start, Some(end))) } } @@ -545,4 +543,44 @@ mod tests { let result = tf.canonicalize(); assert!(matches!(result, Err(Error::EndBeforeStart { .. }))); } + + #[test] + fn test_canonicalize_date_end() { + let mut date_end = TimeFilter { + start: Some("2023-10-01".to_string()), + end: Some("2023-10-02".to_string()), + span: None, + }; + date_end.canonicalize().unwrap(); + assert_eq!(date_end.end.as_deref(), Some("2023-10-03T00:00:00+00:00")); + + let mut timestamp_end = TimeFilter { + start: Some("2023-10-01".to_string()), + end: Some("2023-10-02T12:34:56Z".to_string()), + span: None, + }; + timestamp_end.canonicalize().unwrap(); + assert_eq!( + timestamp_end.end.as_deref(), + Some("2023-10-02T12:34:56+00:00") + ); + + let mut month = TimeFilter { + start: Some("2023-10".to_string()), + end: Some("2023-10".to_string()), + span: None, + }; + month.canonicalize().unwrap(); + assert_eq!(month.start.as_deref(), Some("2023-10-01T00:00:00+00:00")); + assert_eq!(month.end.as_deref(), Some("2023-11-01T00:00:00+00:00")); + + let mut year = TimeFilter { + start: Some("2023".to_string()), + end: Some("2023".to_string()), + span: None, + }; + year.canonicalize().unwrap(); + assert_eq!(year.start.as_deref(), Some("2023-01-01T00:00:00+00:00")); + assert_eq!(year.end.as_deref(), Some("2024-01-01T00:00:00+00:00")); + } } diff --git a/src/log-store/src/kafka/util/record.rs b/src/log-store/src/kafka/util/record.rs index 720f989139d..aefb098e0db 100644 --- a/src/log-store/src/kafka/util/record.rs +++ b/src/log-store/src/kafka/util/record.rs @@ -227,13 +227,17 @@ pub fn remaining_entries( /// - Emits a [RecordType::Full] type record immediately. /// /// For type of [Entry::MultiplePart] Entry: -/// - Emits a complete or incomplete [Entry] while the next same [RegionId] record arrives. +/// - Emits a complete [Entry] immediately when a [RecordType::Last] record arrives with +/// buffered parts. +/// - Emits an incomplete [Entry] when a new [RecordType::First] record replaces +/// buffered incomplete records for the same [RegionId]. /// -/// **Incomplete Entry:** -/// If the records arrive in the following order, it emits **the incomplete [Entry]** when the next record arrives. -/// - **[RecordType::First], [RecordType::Middle]**, [RecordType::First] -/// - **[RecordType::Middle]**, [RecordType::First] -/// - **[RecordType::Last]** +/// **Discarded records:** +/// - A standalone [RecordType::First] record is discarded when another [RecordType::First] +/// record with the same payload for the same [RegionId] arrives. +/// - A [RecordType::Last] record without buffered parts is discarded. +/// +/// A trailing incomplete entry is emitted by [`remaining_entries`]. pub(crate) fn maybe_emit_entry( provider: &Arc, record: Record, @@ -244,7 +248,16 @@ pub(crate) fn maybe_emit_entry( RecordType::Full => entry = Some(convert_to_naive_entry(provider.clone(), record)), RecordType::First => { let region_id = record.meta.ns.region_id.into(); + let duplicate = buffered_records.get(®ion_id).is_some_and(|records| { + records.len() == 1 + && records[0].meta.tp == RecordType::First + && records[0].data == record.data + }); if let Some(records) = buffered_records.insert(region_id, vec![record]) { + if duplicate { + // A duplicate standalone First cannot form a complete entry. + return Ok(None); + } // Incomplete entry entry = Some(convert_to_multiple_entry( provider.clone(), @@ -261,6 +274,11 @@ pub(crate) fn maybe_emit_entry( if !records.is_empty() { // Safety: the records are guaranteed not empty if the key exists. let last_record = records.last().unwrap(); + if matches!(last_record.meta.tp, RecordType::Middle(last_seq) if last_seq == seq) + && last_record.data == record.data + { + return Ok(None); + } let legal = match last_record.meta.tp { // Legal if this record follows a First record. RecordType::First => seq == 1, @@ -290,14 +308,10 @@ pub(crate) fn maybe_emit_entry( provider.clone(), region_id, records, - )) + )); } else { - // Incomplete entry - entry = Some(convert_to_multiple_entry( - provider.clone(), - region_id, - vec![record], - )) + // Intentionally discard a Last record without buffered parts. It is either a + // duplicate Last record or the tail of an incomplete multipart entry. } } } @@ -359,7 +373,16 @@ mod tests { .unwrap() .is_none() ); - let record = new_test_record(RecordType::First, 2, region_id.as_u64(), vec![2; 100]); + let record = new_test_record(RecordType::First, 2, region_id.as_u64(), vec![1; 100]); + // An identical First is a duplicate and does not emit an incomplete entry. + assert!( + maybe_emit_entry(&provider, record, &mut buffer) + .unwrap() + .is_none() + ); + assert_eq!(buffer[®ion_id][0].meta.entry_id, 2); + + let record = new_test_record(RecordType::First, 3, region_id.as_u64(), vec![2; 100]); let incomplete_entry = maybe_emit_entry(&provider, record, &mut buffer) .unwrap() .unwrap(); @@ -369,29 +392,21 @@ mod tests { Entry::MultiplePart(MultiplePartEntry { provider: Provider::Kafka(provider.clone()), region_id, - entry_id: 1, + entry_id: 2, headers: vec![MultiplePartHeader::First], parts: vec![vec![1; 100]], }) ); - // `Last` overwrite `None` + // `Last` after `None` let mut buffer = HashMap::new(); let record = new_test_record(RecordType::Last, 1, region_id.as_u64(), vec![1; 100]); - let incomplete_entry = maybe_emit_entry(&provider, record, &mut buffer) - .unwrap() - .unwrap(); - - assert_eq!( - incomplete_entry, - Entry::MultiplePart(MultiplePartEntry { - provider: Provider::Kafka(provider.clone()), - region_id, - entry_id: 1, - headers: vec![MultiplePartHeader::Last], - parts: vec![vec![1; 100]], - }) + assert!( + maybe_emit_entry(&provider, record, &mut buffer) + .unwrap() + .is_none() ); + assert!(buffer.is_empty()); // `First` overwrite `Middle(0)` let mut buffer = HashMap::new(); @@ -451,6 +466,99 @@ mod tests { assert_matches!(err, error::Error::IllegalSequence { .. }); } + #[test] + fn test_maybe_emit_entry_deduplicates_multipart_records() { + let provider = Arc::new(KafkaProvider::new("my_topic".to_string())); + let region_id = RegionId::new(1, 1); + let mut buffer = HashMap::new(); + + for record in [ + new_test_record(RecordType::First, 1, region_id.as_u64(), vec![1; 100]), + new_test_record(RecordType::Middle(1), 1, region_id.as_u64(), vec![2; 100]), + new_test_record(RecordType::Middle(1), 1, region_id.as_u64(), vec![2; 100]), + ] { + assert!( + maybe_emit_entry(&provider, record, &mut buffer) + .unwrap() + .is_none() + ); + } + + let last = new_test_record(RecordType::Last, 1, region_id.as_u64(), vec![3; 100]); + let entry = maybe_emit_entry(&provider, last, &mut buffer) + .unwrap() + .unwrap(); + + assert_eq!( + entry, + Entry::MultiplePart(MultiplePartEntry { + provider: Provider::Kafka(provider.clone()), + region_id, + entry_id: 1, + headers: vec![ + MultiplePartHeader::First, + MultiplePartHeader::Middle(1), + MultiplePartHeader::Last, + ], + parts: vec![vec![1; 100], vec![2; 100], vec![3; 100]], + }) + ); + let duplicate = new_test_record(RecordType::Last, 1, region_id.as_u64(), vec![3; 100]); + assert!( + maybe_emit_entry(&provider, duplicate, &mut buffer) + .unwrap() + .is_none() + ); + } + + #[test] + fn test_maybe_emit_entry_rejects_conflicting_duplicate_middle_record() { + let provider = Arc::new(KafkaProvider::new("my_topic".to_string())); + let region_id = RegionId::new(1, 1); + let mut buffer = HashMap::new(); + + for record in [ + new_test_record(RecordType::First, 1, region_id.as_u64(), vec![1; 100]), + new_test_record(RecordType::Middle(1), 1, region_id.as_u64(), vec![2; 100]), + ] { + assert!( + maybe_emit_entry(&provider, record, &mut buffer) + .unwrap() + .is_none() + ); + } + + let duplicate = new_test_record(RecordType::Middle(1), 1, region_id.as_u64(), vec![3; 100]); + let err = maybe_emit_entry(&provider, duplicate, &mut buffer).unwrap_err(); + + assert_matches!(err, error::Error::IllegalSequence { .. }); + } + + #[test] + fn test_maybe_emit_entry_preserves_multipart_entry_order_before_full_record() { + let provider = Arc::new(KafkaProvider::new("my_topic".to_string())); + let region_id = RegionId::new(1, 1); + let mut buffer = HashMap::new(); + + let first = new_test_record(RecordType::First, 1, region_id.as_u64(), vec![1; 100]); + assert!( + maybe_emit_entry(&provider, first, &mut buffer) + .unwrap() + .is_none() + ); + let last = new_test_record(RecordType::Last, 2, region_id.as_u64(), vec![2; 100]); + let multipart_entry = maybe_emit_entry(&provider, last, &mut buffer) + .unwrap() + .unwrap(); + let full = new_test_record(RecordType::Full, 3, region_id.as_u64(), vec![3; 100]); + let full_entry = maybe_emit_entry(&provider, full, &mut buffer) + .unwrap() + .unwrap(); + + assert_eq!(multipart_entry.entry_id(), 2); + assert_eq!(full_entry.entry_id(), 3); + } + #[test] fn test_meta_size() { let meta = RecordMeta { diff --git a/src/mito2/src/compaction.rs b/src/mito2/src/compaction.rs index 10900b5540c..9c0c0520458 100644 --- a/src/mito2/src/compaction.rs +++ b/src/mito2/src/compaction.rs @@ -64,7 +64,9 @@ use crate::error::{ RegionClosedSnafu, RegionDroppedSnafu, RegionTruncatedSnafu, RemoteCompactionSnafu, Result, TimeRangePredicateOverflowSnafu, TimeoutSnafu, }; -use crate::metrics::{COMPACTION_STAGE_ELAPSED, INFLIGHT_COMPACTION_COUNT}; +use crate::metrics::{ + COMPACTION_MEMORY_REJECTED, COMPACTION_STAGE_ELAPSED, INFLIGHT_COMPACTION_COUNT, +}; use crate::read::FlatSource; use crate::read::flat_projection::FlatProjectionMapper; use crate::read::read_columns::ReadColumns; @@ -613,8 +615,21 @@ impl CompactionScheduler { waiters }; - // Create a local compaction task. + // Check whether this local compaction can ever fit before submitting it. let estimated_bytes = estimate_compaction_bytes(&picker_output); + if let Some(limit_bytes) = self.exceeds_compaction_memory_limit(estimated_bytes) { + COMPACTION_MEMORY_REJECTED + .with_label_values(&["oversized"]) + .inc(); + warn!( + "Skip compaction for region {} because estimated memory {} bytes exceeds compaction memory limit {} bytes", + region_id, estimated_bytes, limit_bytes, + ); + for waiter in waiters { + waiter.send(Ok(0)); + } + return Ok(None); + } let cancel_handle = Arc::new(CancellationHandle::default()); let state = LocalCompactionState::new(cancel_handle.clone()); @@ -652,6 +667,15 @@ impl CompactionScheduler { ) } + fn exceeds_compaction_memory_limit(&self, estimated_bytes: u64) -> Option { + let limit_bytes = self.memory_manager.limit_bytes(); + if limit_bytes > 0 && estimated_bytes > limit_bytes { + Some(limit_bytes) + } else { + None + } + } + fn remove_region_on_failure(&mut self, region_id: RegionId, err: Arc) { // Remove this region. let Some(status) = self.region_status.remove(®ion_id) else { @@ -1551,6 +1575,65 @@ mod tests { assert!(scheduler.region_status.contains_key(®ion_id)); } + #[tokio::test] + async fn test_schedule_compaction_skips_task_exceeding_memory_limit() { + let job_scheduler = Arc::new(VecScheduler::default()); + let env = SchedulerEnv::new().await.scheduler(job_scheduler.clone()); + let (tx, _rx) = mpsc::channel(4); + let mut scheduler = env.mock_compaction_scheduler(tx); + scheduler.memory_manager = Arc::new(new_compaction_memory_manager(1024 * 1024)); + + let mut builder = VersionControlBuilder::new(); + let region_id = builder.region_id(); + let end = 1000 * 1000; + let version_control = Arc::new( + builder + .push_l0_file_with_max_row_group_size(0, end, 1024 * 1024) + .push_l0_file_with_max_row_group_size(10, end, 1024 * 1024) + .push_l0_file_with_max_row_group_size(50, end, 1024 * 1024) + .push_l0_file_with_max_row_group_size(80, end, 1024 * 1024) + .push_l0_file_with_max_row_group_size(90, end, 1024 * 1024) + .build(), + ); + let manifest_ctx = env + .mock_manifest_context(version_control.current().version.metadata.clone()) + .await; + let (schema_metadata_manager, kv_backend) = mock_schema_metadata_manager(); + schema_metadata_manager + .register_region_table_info( + region_id.table_id(), + "test_table", + "test_catalog", + "test_schema", + None, + kv_backend, + ) + .await; + let (output_tx, output_rx) = oneshot::channel(); + let rejected = COMPACTION_MEMORY_REJECTED.with_label_values(&["oversized"]); + let rejected_before = rejected.get(); + + let scheduled = scheduler + .schedule_compaction( + region_id, + Options::Regular(Default::default()), + &version_control, + &env.access_layer, + OptionOutputTx::from(output_tx), + &manifest_ctx, + schema_metadata_manager, + 1, + ) + .await + .unwrap(); + + assert!(!scheduled); + assert_eq!(output_rx.await.unwrap().unwrap(), 0); + assert_eq!(rejected_before + 1, rejected.get()); + assert_eq!(0, job_scheduler.num_jobs()); + assert!(!scheduler.region_status.contains_key(®ion_id)); + } + #[tokio::test] async fn test_schedule_on_finished() { common_telemetry::init_default_ut_logging(); diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs index ee679cb2250..4daa6d140e2 100644 --- a/src/mito2/src/engine.rs +++ b/src/mito2/src/engine.rs @@ -39,6 +39,8 @@ mod drop_test; #[cfg(test)] mod edit_region_test; #[cfg(test)] +mod file_ref_test; +#[cfg(test)] mod filter_deleted_test; #[cfg(test)] mod flush_test; @@ -356,10 +358,13 @@ impl MitoEngine { ); let mut dst_region_to_src_regions = Vec::with_capacity(dst2src.len()); for (dst_region, srcs) in dst2src { - let Some(dst_region) = self.find_region(dst_region) else { - continue; + let Some(region) = self.find_region(dst_region) else { + return RegionNotFoundSnafu { + region_id: dst_region, + } + .fail(); }; - dst_region_to_src_regions.push((dst_region, srcs)); + dst_region_to_src_regions.push((region, srcs)); } dst_region_to_src_regions }; diff --git a/src/mito2/src/engine/file_ref_test.rs b/src/mito2/src/engine/file_ref_test.rs new file mode 100644 index 00000000000..96f5f6661f4 --- /dev/null +++ b/src/mito2/src/engine/file_ref_test.rs @@ -0,0 +1,55 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::{HashMap, HashSet}; + +use common_error::ext::ErrorExt; +use common_error::status_code::StatusCode; +use store_api::storage::RegionId; + +use crate::config::MitoConfig; +use crate::test_util::TestEnv; + +#[tokio::test] +async fn get_snapshot_of_file_refs_errors_on_missing_related_region() { + let mut env = TestEnv::new().await; + let engine = env.create_engine(MitoConfig::default()).await; + + let source_region = RegionId::new(1, 1); + let missing_target_region = RegionId::new(1, 2); + let related_regions = HashMap::from([(source_region, HashSet::from([missing_target_region]))]); + + let err = engine + .get_snapshot_of_file_refs([], related_regions) + .await + .expect_err("missing related target region should fail file ref snapshot"); + + assert_eq!(StatusCode::RegionNotFound, err.status_code()); +} + +#[tokio::test] +async fn get_snapshot_of_file_refs_skips_missing_file_handle_region() { + let mut env = TestEnv::new().await; + let engine = env.create_engine(MitoConfig::default()).await; + + let missing_region = RegionId::new(1, 1); + let manifest = engine + .get_snapshot_of_file_refs([missing_region], HashMap::new()) + .await + .expect("missing file handle region should be skipped"); + + assert!(manifest.file_refs.is_empty()); + assert!(manifest.manifest_version.is_empty()); + assert!(manifest.cross_region_refs.is_empty()); +} diff --git a/src/mito2/src/test_util/version_util.rs b/src/mito2/src/test_util/version_util.rs index b14eea09581..1c0d7bfebc8 100644 --- a/src/mito2/src/test_util/version_util.rs +++ b/src/mito2/src/test_util/version_util.rs @@ -89,6 +89,15 @@ impl VersionControlBuilder { } pub(crate) fn push_l0_file(&mut self, start_ms: i64, end_ms: i64) -> &mut Self { + self.push_l0_file_with_max_row_group_size(start_ms, end_ms, 0) + } + + pub(crate) fn push_l0_file_with_max_row_group_size( + &mut self, + start_ms: i64, + end_ms: i64, + max_row_group_uncompressed_size: u64, + ) -> &mut Self { let file_id = FileId::random(); self.files.insert( file_id, @@ -101,7 +110,7 @@ impl VersionControlBuilder { ), level: 0, file_size: 0, // We don't care file size. - max_row_group_uncompressed_size: 0, + max_row_group_uncompressed_size, available_indexes: Default::default(), indexes: Default::default(), index_file_size: 0, diff --git a/src/query/src/dist_plan.rs b/src/query/src/dist_plan.rs index 1983e7824c7..969f00e7c9c 100644 --- a/src/query/src/dist_plan.rs +++ b/src/query/src/dist_plan.rs @@ -27,6 +27,7 @@ mod remote_dyn_filter_registry; pub use analyzer::{DistPlannerAnalyzer, DistPlannerOptions}; pub use filter_id::{FilterFingerprint, FilterId, ParseFilterIdError, RemoteDynFilterProducerId}; pub use merge_scan::{MergeScanExec, MergeScanLogicalPlan}; +pub(crate) use merge_sort::MergeSortExec; pub use planner::{DistExtensionPlanner, MergeSortExtensionPlanner}; pub use predicate_extractor::PredicateExtractor; pub use region_pruner::ConstraintPruner; diff --git a/src/query/src/dist_plan/analyzer.rs b/src/query/src/dist_plan/analyzer.rs index f98565e4f08..c41f7b618dc 100644 --- a/src/query/src/dist_plan/analyzer.rs +++ b/src/query/src/dist_plan/analyzer.rs @@ -36,6 +36,7 @@ use datafusion_optimizer::propagate_empty_relation::PropagateEmptyRelation; use datafusion_optimizer::push_down_filter::PushDownFilter; use datafusion_optimizer::rewrite_set_comparison::RewriteSetComparison; use datafusion_optimizer::scalar_subquery_to_join::ScalarSubqueryToJoin; +use datafusion_optimizer::simplify_expressions::SimplifyExpressions; use promql::extension_plan::SeriesDivide; use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan}; use table::metadata::TableType; @@ -219,6 +220,7 @@ fn pre_merge_scan_optimizer() -> Optimizer { Arc::new(PropagateEmptyRelation::new()), Arc::new(FilterNullJoinKeys::default()), Arc::new(PushDownFilter::new()), + Arc::new(SimplifyExpressions::new()), ]) } diff --git a/src/query/src/dist_plan/analyzer/test.rs b/src/query/src/dist_plan/analyzer/test.rs index a2c0ca44fee..200d105fed6 100644 --- a/src/query/src/dist_plan/analyzer/test.rs +++ b/src/query/src/dist_plan/analyzer/test.rs @@ -2263,6 +2263,90 @@ fn test_table_scan_projection() { assert_eq!(expected, result.to_string()); } +#[test] +fn test_table_scan_cast_projection_pushdown() { + init_default_ut_logging(); + let test_table = TestTable::table_with_name(0, "t".to_string()); + let table_provider = Arc::new(DfTableProviderAdapter::new(test_table)); + let table_source = Arc::new(DefaultTableSource::new(table_provider.clone() as _)); + let ctx = SessionContext::new(); + ctx.register_table(TableReference::bare("t"), table_provider.clone() as _) + .unwrap(); + + let scan = LogicalPlanBuilder::scan_with_filters("t", table_source, Some(vec![3]), vec![]) + .unwrap() + .build() + .unwrap(); + let ts_cast_type = DataType::Timestamp(TimeUnit::Millisecond, Some("+00:00".into())); + let ts_cast_expr = col("ts").cast_to(&ts_cast_type, scan.schema()).unwrap(); + let plan = LogicalPlanBuilder::from(scan) + .project(vec![ts_cast_expr.alias("ts")]) + .unwrap() + .build() + .unwrap(); + + let config = ConfigOptions::default(); + let result = DistPlannerAnalyzer {} + .analyze(plan.clone(), &config) + .unwrap(); + let expected = [ + "Projection: ts", + " MergeScan [is_placeholder=false, remote_input=[", + "Projection: CAST(t.ts AS Timestamp(ms, \"+00:00\")) AS ts", + " TableScan: t projection=[ts]", + "]]", + ] + .join("\n"); + assert_eq!(expected, result.to_string()); +} + +#[test] +fn test_cast_filter_simplified_after_pushdown() { + // This test invokes `DistPlannerAnalyzer` directly rather than the full + // query `SessionState`, so the globally-registered `ConstNormalizationRule` + // does not run here. The native timestamp filter below proves the focused + // pre-MergeScan pass can do this by running DataFusion `SimplifyExpressions` + // after `PushDownFilter` has pushed the alias predicate through the cast + // projection into `TableScan.filters`. + init_default_ut_logging(); + let test_table = + TestTable::table_with_filter_pushdown(0, "t".to_string(), FilterPushDownType::Inexact); + let table_provider = Arc::new(DfTableProviderAdapter::new(test_table)); + let table_source = Arc::new(DefaultTableSource::new(table_provider.clone() as _)); + let ctx = SessionContext::new(); + ctx.register_table(TableReference::bare("t"), table_provider.clone() as _) + .unwrap(); + + let scan = LogicalPlanBuilder::scan_with_filters("t", table_source, Some(vec![3]), vec![]) + .unwrap() + .build() + .unwrap(); + let ts_cast_type = DataType::Timestamp(TimeUnit::Second, None); + let ts_cast_expr = col("ts").cast_to(&ts_cast_type, scan.schema()).unwrap(); + let plan = LogicalPlanBuilder::from(scan) + .project(vec![ts_cast_expr.alias("ts")]) + .unwrap() + .filter(col("ts").gt_eq(lit(ScalarValue::TimestampSecond(Some(10), None)))) + .unwrap() + .build() + .unwrap(); + + let config = ConfigOptions::default(); + let result = DistPlannerAnalyzer {} + .analyze(plan.clone(), &config) + .unwrap(); + let expected = [ + "Projection: ts", + " MergeScan [is_placeholder=false, remote_input=[", + "Projection: CAST(t.ts AS Timestamp(s)) AS ts", + " Filter: t.ts >= TimestampMillisecond(10000, None)", + " TableScan: t projection=[ts], partial_filters=[t.ts >= TimestampMillisecond(10000, None)]", + "]]", + ] + .join("\n"); + assert_eq!(expected, result.to_string()); +} + /// When `ScheduledTimeExtension` is injected into `ConfigOptions`, the /// `SimplifyExpressions` pass (driven by `PatchOptimizerContext`) uses the /// scheduled time instead of wall-clock time. The remote sub-plan must contain diff --git a/src/query/src/dist_plan/commutativity.rs b/src/query/src/dist_plan/commutativity.rs index 7366aa0d5b3..b380fbba05e 100644 --- a/src/query/src/dist_plan/commutativity.rs +++ b/src/query/src/dist_plan/commutativity.rs @@ -322,14 +322,14 @@ impl Categorizer { | Expr::SimilarTo(_) | Expr::IsUnknown(_) | Expr::IsNotUnknown(_) - | Expr::Cast(_) - | Expr::TryCast(_) | Expr::WindowFunction(_) | Expr::InSubquery(_) | Expr::ScalarSubquery(_) | Expr::Wildcard { .. } => Commutativity::Unimplemented, Expr::Alias(alias) => Self::check_expr(&alias.expr), + Expr::Cast(cast) => Self::check_expr(&cast.expr), + Expr::TryCast(try_cast) => Self::check_expr(&try_cast.expr), Expr::Unnest(_) | Expr::GroupingSet(_) diff --git a/src/query/src/dist_plan/merge_scan.rs b/src/query/src/dist_plan/merge_scan.rs index 0b03d4f53ba..1d8369c5479 100644 --- a/src/query/src/dist_plan/merge_scan.rs +++ b/src/query/src/dist_plan/merge_scan.rs @@ -47,6 +47,7 @@ use meter_core::data::ReadItem; use meter_macros::read_meter; use session::context::QueryContextRef; use store_api::storage::RegionId; +use table::table::scan::REGION_SCAN_EXEC_NAME; use table::table_name::TableName; use tokio::time::Instant; use tracing::{Instrument, Span}; @@ -496,13 +497,14 @@ impl MergeScanExec { // process metrics after all data is drained. if let Some(metrics) = stream.metrics() { + let output_bytes = scan_output_bytes(&metrics); let (c, s) = parse_catalog_and_schema_from_db_string(&dbname); let value = read_meter!( c, s, ReadItem { cpu_time: metrics.elapsed_compute as u64, - table_scan: metrics.memory_usage as u64 + table_scan: output_bytes as u64, }, current_channel as u8 ); @@ -557,6 +559,24 @@ impl MergeScanExec { return None; } + let hash_expr_col_names: HashSet<_> = hash_exprs + .iter() + .filter_map(|expr| { + expr.as_any() + .downcast_ref::() + .map(|col_expr| col_expr.name()) + }) + .collect(); + + let covers_all_partition_cols = self.partition_cols.values().all(|aliases| { + aliases + .iter() + .any(|col| hash_expr_col_names.contains(col.name())) + }); + if !covers_all_partition_cols { + return None; + } + let all_partition_col_aliases: HashSet<_> = self .partition_cols .values() @@ -916,6 +936,17 @@ impl DisplayAs for MergeScanExec { } } +/// Extract total `output_bytes` from [`RegionScanExec`] plan nodes. +fn scan_output_bytes(metrics: &RecordBatchMetrics) -> usize { + metrics + .plan_metrics + .iter() + .filter(|pm| pm.plan_name == REGION_SCAN_EXEC_NAME) + .flat_map(|pm| &pm.metrics) + .filter_map(|(name, value)| (name == "output_bytes").then_some(*value)) + .sum() +} + #[derive(Debug, Clone)] struct MergeScanMetric { /// Nanosecond elapsed till the scan operator is ready to emit data @@ -969,11 +1000,12 @@ mod tests { use async_trait::async_trait; use common_query::request::INITIAL_REMOTE_DYN_FILTER_REGISTRATIONS_EXTENSION_KEY; + use common_recordbatch::adapter::{PlanMetrics, RecordBatchMetrics}; use datafusion::config::ConfigOptions; use datafusion::execution::SessionStateBuilder; use datafusion::physical_plan::filter_pushdown::ChildFilterPushdownResult; use datafusion_common::TableReference; - use datafusion_expr::{LogicalPlanBuilder, lit}; + use datafusion_expr::{LogicalPlanBuilder, col, lit}; use datafusion_physical_expr::Distribution; use datafusion_physical_expr::expressions::{ Column, DynamicFilterPhysicalExpr, lit as physical_lit, @@ -992,6 +1024,64 @@ mod tests { QueryId::from(Uuid::from_u128(value)) } + fn merge_scan_exec_with_sorted_input( + region_count: u64, + target_partition: usize, + ) -> MergeScanExec { + let session_state = SessionStateBuilder::new().build(); + let plan = LogicalPlanBuilder::empty(true) + .project(vec![lit(1i64).alias("ts")]) + .unwrap() + .sort(vec![col("ts").sort(false, true)]) + .unwrap() + .build() + .unwrap(); + let schema = plan.schema().as_arrow().clone(); + let regions = (0..region_count) + .map(|region_number| RegionId::new(1024, region_number as u32)) + .collect(); + + MergeScanExec::new( + &session_state, + // The table name is not relevant to these ordering metadata tests; + // `MergeScanExec::new` requires one to model the production plan. + TableName::new("catalog", "schema", "table"), + regions, + plan, + &schema, + Arc::new(TestRegionQueryHandler), + QueryContext::arc(), + target_partition, + AliasMapping::new(), + None, + ) + .unwrap() + } + + #[test] + fn merge_scan_does_not_advertise_ordering_when_partition_may_merge_regions() { + let exec = merge_scan_exec_with_sorted_input(3, 2); + + assert!( + exec.properties().output_ordering().is_none(), + "target_partition < region_count means one output partition may concatenate multiple sorted region streams" + ); + } + + #[test] + fn merge_scan_advertises_ordering_when_each_partition_reads_at_most_one_region() { + let exec = merge_scan_exec_with_sorted_input(3, 3); + + assert!(exec.properties().output_ordering().is_some()); + } + + #[test] + fn merge_scan_advertises_ordering_when_partitions_exceed_regions() { + let exec = merge_scan_exec_with_sorted_input(3, 4); + + assert!(exec.properties().output_ordering().is_some()); + } + #[test] fn remote_dyn_filter_region_query_context_registers_before_do_get() { let registry_manager = Arc::new(DynFilterRegistryManager::default()); @@ -1210,4 +1300,57 @@ mod tests { assert_eq!(exec.captured_remote_dyn_filters().len(), 1); assert!(matches!(propagation.filters.as_slice(), [PushedDown::Yes])); } + + #[test] + fn scan_output_bytes_uses_plan_name() { + let metrics = RecordBatchMetrics { + plan_metrics: vec![PlanMetrics { + plan: "SeqScan: region=1".to_string(), + plan_name: REGION_SCAN_EXEC_NAME.to_string(), + level: 0, + metrics: vec![("output_bytes".to_string(), 42)], + }], + ..Default::default() + }; + + assert_eq!(scan_output_bytes(&metrics), 42); + } + + #[test] + fn scan_output_bytes_defaults_to_zero_without_region_scan() { + let metrics = RecordBatchMetrics { + plan_metrics: vec![PlanMetrics { + plan: "ProjectionExec".to_string(), + plan_name: "ProjectionExec".to_string(), + level: 0, + metrics: vec![("output_bytes".to_string(), 42)], + }], + ..Default::default() + }; + + assert_eq!(scan_output_bytes(&metrics), 0); + } + + #[test] + fn scan_output_bytes_sums_multiple_region_scans() { + let metrics = RecordBatchMetrics { + plan_metrics: vec![ + PlanMetrics { + plan: "RegionScanExec: region=1".to_string(), + plan_name: REGION_SCAN_EXEC_NAME.to_string(), + level: 0, + metrics: vec![("output_bytes".to_string(), 42)], + }, + PlanMetrics { + plan: "RegionScanExec: region=2".to_string(), + plan_name: REGION_SCAN_EXEC_NAME.to_string(), + level: 0, + metrics: vec![("output_bytes".to_string(), 18)], + }, + ], + ..Default::default() + }; + + assert_eq!(scan_output_bytes(&metrics), 60); + } } diff --git a/src/query/src/dist_plan/merge_sort.rs b/src/query/src/dist_plan/merge_sort.rs index d9bc6119c08..2c8f4d9fd8a 100644 --- a/src/query/src/dist_plan/merge_sort.rs +++ b/src/query/src/dist_plan/merge_sort.rs @@ -16,11 +16,23 @@ //! `SortPreservingMergeExec` operator in datafusion //! +use std::any::Any; use std::fmt; use std::sync::Arc; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::execution_plan::CardinalityEffect; +use datafusion::physical_plan::metrics::MetricsSet; +use datafusion::physical_plan::projection::{ProjectionExec, make_with_child, update_ordering}; +use datafusion::physical_plan::sorts::sort::SortExec; +use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SendableRecordBatchStream, + Statistics, +}; use datafusion_common::{DataFusionError, Result}; use datafusion_expr::{Extension, LogicalPlan, SortExpr, UserDefinedLogicalNodeCore}; +use datafusion_physical_expr::{Distribution, LexOrdering, OrderingRequirements}; /// MergeSort Logical Plan, have same field as `Sort`, but indicate it is a merge sort, /// which assume each input partition is a sorted stream, and will use `SortPreserveingMergeExec` @@ -53,14 +65,249 @@ impl MergeSortLogicalPlan { node: Arc::new(self), }) } +} - /// Convert self to a [`Sort`] logical plan with same input and expressions - pub fn into_sort(self) -> LogicalPlan { - LogicalPlan::Sort(datafusion::logical_expr::Sort { - input: self.input.clone(), - expr: self.expr, - fetch: self.fetch, - }) +/// An opaque physical execution node for [`MergeSortLogicalPlan`]. +/// +/// It delegates execution and physical properties to DataFusion's +/// [`SortPreservingMergeExec`], but intentionally does not expose itself as a +/// `SortPreservingMergeExec`. `EnforceSorting` is allowed to replace a bare +/// `SortPreservingMergeExec` with `CoalescePartitionsExec` when the parent does +/// not require ordering. `MergeSortExec` represents the distributed TopK merge +/// stage itself, so later physical optimizer rules must not rewrite it into an +/// unordered fetch. +#[derive(Debug, Clone)] +pub(crate) struct MergeSortExec { + inner: SortPreservingMergeExec, +} + +impl MergeSortExec { + pub(crate) fn new( + ordering: LexOrdering, + input: Arc, + fetch: Option, + ) -> Self { + Self { + inner: SortPreservingMergeExec::new(ordering, input).with_fetch(fetch), + } + } + + fn input_with_fetch(&self, fetch: Option) -> Arc { + let input = Arc::clone(self.inner.input()); + if let Some(sort) = input.as_any().downcast_ref::() + && sort.preserve_partitioning() + && sort.expr() == self.inner.expr() + { + // Mirror DataFusion's bare SPM plan quality for distributed TopK: + // keep the parent `MergeSortExec(fetch)` as the global merge, and + // bound the partition-preserving child sort to the same local TopK. + // Local top-K is safe because every global top-K row must be within + // the top-K rows of its own input partition. + Arc::new(sort.with_fetch(fetch)) + } else { + input + } + } +} + +impl DisplayAs for MergeSortExec { + fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result { + match t { + DisplayFormatType::Default | DisplayFormatType::Verbose => { + write!(f, "MergeSortExec: [{}]", self.inner.expr())?; + if let Some(fetch) = self.inner.fetch() { + write!(f, ", fetch={fetch}")?; + } + Ok(()) + } + DisplayFormatType::TreeRender => { + if let Some(fetch) = self.inner.fetch() { + writeln!(f, "limit={fetch}")?; + } + + for (i, expr) in self.inner.expr().iter().enumerate() { + expr.fmt_sql(f)?; + if i != self.inner.expr().len() - 1 { + write!(f, ", ")?; + } + } + + Ok(()) + } + } + } +} + +impl ExecutionPlan for MergeSortExec { + fn name(&self) -> &str { + "MergeSortExec" + } + + /// Keeps this node intentionally opaque to DataFusion's type-specialized + /// optimizer rewrites. + /// + /// `MergeSortExec` delegates most behavior to DataFusion's + /// `SortPreservingMergeExec`, but it must not expose itself as that type. + /// DataFusion's `EnforceSorting` optimizer recognizes a bare + /// `SortPreservingMergeExec` via `as_any().downcast_ref::<...>()` and may + /// replace it with an unordered `CoalescePartitionsExec(fetch)` when the + /// parent does not require sorted output. + /// + /// That rewrite is valid for an ordinary SPM used only to satisfy parent + /// ordering, but not for GreptimeDB's distributed TopK merge stage. In a + /// scalar-subquery shape like `ORDER BY ts DESC LIMIT 1`, this node is the + /// operator that merges region-local TopK streams into the global TopK. + /// Replacing it with unordered coalescing can return a partial/latest row + /// from one region instead of the global latest row. + /// + /// `required_input_ordering()` separately tells DataFusion what ordering this + /// node needs from its child, so `EnforceSorting` can insert a `SortExec` + /// below `MergeSortExec` when `MergeScanExec` cannot preserve per-partition + /// ordering. This opacity is specifically about protecting the merge stage + /// itself from the `EnforceSorting` rewrite above. + fn as_any(&self) -> &dyn Any { + self + } + + fn properties(&self) -> &Arc { + self.inner.properties() + } + + /// Forwards DataFusion's order-preserving scan hint through this wrapper. + /// + /// This mirrors `SortPreservingMergeExec::with_preserve_order()`: if the + /// child can produce an order-preserving variant, rebuild the same merge + /// stage on top of that child. The returned plan must stay a + /// `MergeSortExec`, not a bare SPM, so the distributed TopK merge remains + /// opaque to `EnforceSorting`'s SPM-specific rewrite. + fn with_preserve_order(&self, preserve_order: bool) -> Option> { + self.inner + .input() + .with_preserve_order(preserve_order) + .map(|new_input| { + Arc::new(Self::new( + self.inner.expr().clone(), + new_input, + self.inner.fetch(), + )) as Arc + }) + } + + fn required_input_distribution(&self) -> Vec { + self.inner.required_input_distribution() + } + + fn benefits_from_input_partitioning(&self) -> Vec { + self.inner.benefits_from_input_partitioning() + } + + /// Tells DataFusion that `MergeSortExec` requires each input partition to be + /// ordered. This is the contract that makes `EnforceSorting` insert a + /// `SortExec` below `MergeSortExec` when the input cannot preserve ordering. + /// + /// The opacity of `MergeSortExec::as_any`, not this requirement, is what + /// prevents DataFusion from rewriting the merge stage itself as a bare + /// `SortPreservingMergeExec`. + fn required_input_ordering(&self) -> Vec> { + vec![Some(OrderingRequirements::from(self.inner.expr().clone()))] + } + + fn maintains_input_order(&self) -> Vec { + self.inner.maintains_input_order() + } + + fn children(&self) -> Vec<&Arc> { + self.inner.children() + } + + fn with_new_children( + self: Arc, + mut children: Vec>, + ) -> Result> { + if children.len() != 1 { + return Err(DataFusionError::Internal(format!( + "MergeSortExec expects exactly one child, got {}", + children.len() + ))); + } + + Ok(Arc::new(Self::new( + self.inner.expr().clone(), + children.swap_remove(0), + self.inner.fetch(), + ))) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + self.inner.execute(partition, context) + } + + fn metrics(&self) -> Option { + self.inner.metrics() + } + + fn partition_statistics(&self, partition: Option) -> Result { + self.inner.partition_statistics(partition) + } + + fn cardinality_effect(&self) -> CardinalityEffect { + self.inner.cardinality_effect() + } + + /// Intentionally keeps DataFusion's generic limit pushdown disabled. + /// + /// `MergeSortExec` still supports its own global fetch through + /// `with_fetch()`. What we must not allow is pushing an external limit below + /// this required distributed TopK merge. DataFusion's limit pushdown rules + /// know how to treat a bare `SortPreservingMergeExec` as a + /// partition-combining node, but `MergeSortExec` is intentionally opaque to + /// those SPM-specific downcasts. Enabling generic limit pushdown without also + /// teaching the optimizer about this wrapper could return partition-local + /// rows instead of the global TopK. + fn supports_limit_pushdown(&self) -> bool { + false + } + + fn fetch(&self) -> Option { + self.inner.fetch() + } + + fn with_fetch(&self, limit: Option) -> Option> { + Some(Arc::new(Self::new( + self.inner.expr().clone(), + self.input_with_fetch(limit), + limit, + ))) + } + + /// Lets DataFusion push a projection below this merge when it can rewrite + /// the ordering expressions safely. + /// + /// This mirrors `SortPreservingMergeExec::try_swapping_with_projection()` + /// for plan quality, but re-wraps the result as `MergeSortExec` so the + /// distributed merge stage keeps its type identity and opacity. + fn try_swapping_with_projection( + &self, + projection: &ProjectionExec, + ) -> Result>> { + if projection.expr().len() >= projection.input().schema().fields().len() { + return Ok(None); + } + + let Some(updated_exprs) = update_ordering(self.inner.expr().clone(), projection.expr())? + else { + return Ok(None); + }; + + Ok(Some(Arc::new(Self::new( + updated_exprs, + make_with_child(projection, self.inner.input())?, + self.inner.fetch(), + )))) } } @@ -127,3 +374,420 @@ pub fn merge_sort_transformer(plan: &LogicalPlan) -> Option { None } } + +#[cfg(test)] +mod tests { + use arrow_schema::{DataType, Field, Schema, SortOptions}; + use datafusion::physical_optimizer::enforce_sorting::replace_with_order_preserving_variants::{ + OrderPreservationContext, plan_with_order_breaking_variants, + }; + use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; + use datafusion::physical_plan::displayable; + use datafusion::physical_plan::empty::EmptyExec; + use datafusion_physical_expr::PhysicalSortExpr; + use datafusion_physical_expr::expressions::col as physical_col; + + use super::*; + + /// Test double that records DataFusion's preserve-order signal while + /// otherwise behaving like a transparent wrapper around its child. + #[derive(Debug, Clone)] + struct PreserveOrderProbeExec { + inner: Arc, + preserve_order: bool, + } + + impl PreserveOrderProbeExec { + fn new(inner: Arc) -> Self { + Self { + inner, + preserve_order: false, + } + } + } + + impl DisplayAs for PreserveOrderProbeExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "PreserveOrderProbeExec: preserve_order={}", + self.preserve_order + ) + } + } + + impl ExecutionPlan for PreserveOrderProbeExec { + fn name(&self) -> &str { + "PreserveOrderProbeExec" + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn properties(&self) -> &Arc { + self.inner.properties() + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.inner] + } + + fn with_new_children( + self: Arc, + mut children: Vec>, + ) -> Result> { + if children.len() != 1 { + return Err(DataFusionError::Internal(format!( + "PreserveOrderProbeExec expects exactly one child, got {}", + children.len() + ))); + } + + Ok(Arc::new(Self { + inner: children.swap_remove(0), + preserve_order: self.preserve_order, + })) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + self.inner.execute(partition, context) + } + + fn with_preserve_order(&self, preserve_order: bool) -> Option> { + Some(Arc::new(Self { + inner: Arc::clone(&self.inner), + preserve_order, + })) + } + } + + fn test_ordering(schema: &Schema) -> LexOrdering { + LexOrdering::new([PhysicalSortExpr::new( + physical_col("ts", schema).unwrap(), + SortOptions { + descending: true, + nulls_first: false, + }, + )]) + .unwrap() + } + + #[test] + fn merge_sort_exec_is_opaque_and_preserves_topk_requirements() { + let schema = Arc::new(Schema::new(vec![Field::new("ts", DataType::Int64, false)])); + let input = Arc::new(EmptyExec::new(schema.clone()).with_partitions(2)) as _; + let ordering = test_ordering(schema.as_ref()); + + let merge_sort = + Arc::new(MergeSortExec::new(ordering, input, Some(1))) as Arc; + + assert_eq!(merge_sort.name(), "MergeSortExec"); + assert!( + merge_sort + .as_any() + .downcast_ref::() + .is_none(), + "MergeSortExec must stay opaque to EnforceSorting's bare SortPreservingMerge rewrite" + ); + assert_eq!(merge_sort.fetch(), Some(1)); + assert!(!merge_sort.supports_limit_pushdown()); + assert!(merge_sort.required_input_ordering()[0].is_some()); + + let tree = displayable(merge_sort.as_ref()).tree_render().to_string(); + assert!(tree.contains("MergeSortExec")); + assert!(!tree.contains("SortPreservingMergeExec")); + + let fetched = merge_sort.with_fetch(Some(2)).unwrap(); + assert!(fetched.as_any().downcast_ref::().is_some()); + assert_eq!(fetched.fetch(), Some(2)); + } + + #[test] + fn merge_sort_exec_required_input_ordering_matches_spm() { + let schema = Arc::new(Schema::new(vec![Field::new("ts", DataType::Int64, false)])); + let input = Arc::new(EmptyExec::new(schema.clone()).with_partitions(2)) as _; + let ordering = test_ordering(schema.as_ref()); + + let merge_sort = MergeSortExec::new(ordering.clone(), Arc::clone(&input), Some(1)); + let bare_spm = + SortPreservingMergeExec::new(ordering.clone(), Arc::clone(&input)).with_fetch(Some(1)); + + assert_eq!( + merge_sort.required_input_ordering(), + vec![Some(OrderingRequirements::from(ordering))], + "MergeSortExec must require locally sorted input partitions for the merge key" + ); + assert_eq!( + merge_sort.required_input_ordering(), + bare_spm.required_input_ordering(), + "MergeSortExec's child ordering contract should mirror SortPreservingMergeExec" + ); + assert_eq!( + merge_sort.maintains_input_order(), + bare_spm.maintains_input_order() + ); + } + + #[test] + fn merge_sort_exec_with_fetch_pushes_fetch_to_child_sort() { + let schema = Arc::new(Schema::new(vec![Field::new("ts", DataType::Int64, false)])); + let input = Arc::new(EmptyExec::new(schema.clone()).with_partitions(2)) as _; + let ordering = test_ordering(schema.as_ref()); + let child_sort = + Arc::new(SortExec::new(ordering.clone(), input).with_preserve_partitioning(true)) + as Arc; + let merge_sort = MergeSortExec::new(ordering, child_sort, None); + + let fetched = merge_sort.with_fetch(Some(2)).unwrap(); + + assert!(fetched.as_any().downcast_ref::().is_some()); + assert_eq!(fetched.fetch(), Some(2)); + let child_sort = fetched.children()[0] + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(child_sort.fetch(), Some(2)); + assert!(child_sort.preserve_partitioning()); + } + + #[test] + fn merge_sort_exec_with_preserve_order_matches_spm_but_keeps_wrapper() { + let schema = Arc::new(Schema::new(vec![Field::new("ts", DataType::Int64, false)])); + let input = Arc::new(PreserveOrderProbeExec::new(Arc::new( + EmptyExec::new(schema.clone()).with_partitions(2), + ))) as _; + let ordering = test_ordering(schema.as_ref()); + + let bare_spm = + SortPreservingMergeExec::new(ordering.clone(), Arc::clone(&input)).with_fetch(Some(1)); + let preserved_spm = bare_spm.with_preserve_order(true).unwrap(); + assert!( + preserved_spm + .as_any() + .downcast_ref::() + .is_some(), + "bare SPM should rebuild as bare SPM" + ); + assert!( + preserved_spm.children()[0] + .as_any() + .downcast_ref::() + .unwrap() + .preserve_order + ); + + let merge_sort = MergeSortExec::new(ordering, input, Some(1)); + let preserved_merge_sort = merge_sort.with_preserve_order(true).unwrap(); + assert!( + preserved_merge_sort + .as_any() + .downcast_ref::() + .is_some(), + "MergeSortExec must rewrap the preserve-order child as MergeSortExec" + ); + assert!( + preserved_merge_sort + .as_any() + .downcast_ref::() + .is_none(), + "MergeSortExec must not expose a bare SPM after with_preserve_order" + ); + assert_eq!(preserved_merge_sort.fetch(), Some(1)); + assert_eq!( + preserved_merge_sort.required_input_ordering(), + preserved_spm.required_input_ordering(), + "preserve-order rewrite should keep the same SPM child-ordering contract" + ); + assert!( + preserved_merge_sort.children()[0] + .as_any() + .downcast_ref::() + .unwrap() + .preserve_order + ); + } + + #[test] + fn merge_sort_exec_projection_swap_matches_spm_but_keeps_wrapper() -> Result<()> { + let schema = Arc::new(Schema::new(vec![ + Field::new("value", DataType::Int64, false), + Field::new("ts", DataType::Int64, false), + Field::new("tag", DataType::Utf8, false), + ])); + let input = Arc::new(EmptyExec::new(schema.clone()).with_partitions(2)) as _; + let ordering = test_ordering(schema.as_ref()); + + let bare_spm = Arc::new( + SortPreservingMergeExec::new(ordering.clone(), Arc::clone(&input)).with_fetch(Some(1)), + ) as Arc; + let spm_projection = ProjectionExec::try_new( + vec![ + (physical_col("ts", schema.as_ref())?, "ts".to_string()), + (physical_col("tag", schema.as_ref())?, "tag".to_string()), + ], + Arc::clone(&bare_spm), + )?; + let swapped_spm = bare_spm + .try_swapping_with_projection(&spm_projection)? + .expect("SPM should accept a narrowing projection that preserves the sort key"); + assert!( + swapped_spm + .as_any() + .downcast_ref::() + .is_some(), + "bare SPM should rebuild as bare SPM" + ); + + let merge_sort = + Arc::new(MergeSortExec::new(ordering, input, Some(1))) as Arc; + let merge_projection = ProjectionExec::try_new( + vec![ + (physical_col("ts", schema.as_ref())?, "ts".to_string()), + (physical_col("tag", schema.as_ref())?, "tag".to_string()), + ], + Arc::clone(&merge_sort), + )?; + let swapped_merge_sort = merge_sort + .try_swapping_with_projection(&merge_projection)? + .expect("MergeSortExec should accept the same projection swap as SPM"); + + assert!( + swapped_merge_sort + .as_any() + .downcast_ref::() + .is_some(), + "MergeSortExec must rewrap projection swaps as MergeSortExec" + ); + assert!( + swapped_merge_sort + .as_any() + .downcast_ref::() + .is_none(), + "MergeSortExec must not expose a bare SPM after projection swap" + ); + assert_eq!(swapped_merge_sort.fetch(), Some(1)); + assert!( + swapped_merge_sort.children()[0] + .as_any() + .downcast_ref::() + .is_some(), + "the projection should move below MergeSortExec" + ); + let swapped_schema = swapped_merge_sort.schema(); + assert_eq!( + swapped_schema + .fields() + .iter() + .map(|field| field.name().as_str()) + .collect::>(), + vec!["ts", "tag"], + "swapped MergeSortExec should expose the projected schema" + ); + + let projected_ordering = LexOrdering::new([PhysicalSortExpr::new( + physical_col("ts", swapped_merge_sort.children()[0].schema().as_ref())?, + SortOptions { + descending: true, + nulls_first: false, + }, + )]) + .unwrap(); + assert_eq!( + swapped_merge_sort.required_input_ordering(), + vec![Some(OrderingRequirements::from(projected_ordering))], + "projection swap must rewrite the ordering to the child projection's schema" + ); + assert_eq!( + swapped_merge_sort.required_input_ordering(), + swapped_spm.required_input_ordering(), + "MergeSortExec projection swap should mirror SPM's ordering rewrite" + ); + + let spm_projection_without_sort_key = ProjectionExec::try_new( + vec![(physical_col("tag", schema.as_ref())?, "tag".to_string())], + Arc::clone(&bare_spm), + )?; + let merge_projection_without_sort_key = ProjectionExec::try_new( + vec![(physical_col("tag", schema.as_ref())?, "tag".to_string())], + Arc::clone(&merge_sort), + )?; + assert!( + bare_spm + .try_swapping_with_projection(&spm_projection_without_sort_key)? + .is_none(), + "SPM must reject projection swaps that drop the sort key" + ); + assert!( + merge_sort + .try_swapping_with_projection(&merge_projection_without_sort_key)? + .is_none(), + "MergeSortExec should reject the same projection swap as SPM" + ); + + Ok(()) + } + + #[test] + fn enforce_sorting_rewrite_keeps_merge_sort_exec_opaque() { + let schema = Arc::new(Schema::new(vec![Field::new("ts", DataType::Int64, false)])); + let input = Arc::new(EmptyExec::new(schema.clone()).with_partitions(2)) as _; + let ordering = test_ordering(schema.as_ref()); + + let bare_spm = Arc::new( + SortPreservingMergeExec::new(ordering.clone(), Arc::clone(&input)).with_fetch(Some(1)), + ) as Arc; + let optimized_spm = plan_with_order_breaking_variants(OrderPreservationContext::new( + bare_spm, + false, + vec![OrderPreservationContext::new( + Arc::clone(&input), + false, + vec![], + )], + )) + .unwrap() + .plan; + assert!( + optimized_spm + .as_any() + .downcast_ref::() + .is_some(), + "this regression test must exercise EnforceSorting's bare SPM -> CoalescePartitionsExec rewrite" + ); + + let merge_sort = + Arc::new(MergeSortExec::new(ordering, input, Some(1))) as Arc; + let optimized_merge_sort = + plan_with_order_breaking_variants(OrderPreservationContext::new( + Arc::clone(&merge_sort), + false, + vec![OrderPreservationContext::new( + Arc::clone(merge_sort.children()[0]), + false, + vec![], + )], + )) + .unwrap() + .plan; + assert!( + optimized_merge_sort + .as_any() + .downcast_ref::() + .is_some(), + "MergeSortExec must stay opaque to the bare SPM rewrite" + ); + assert!( + optimized_merge_sort + .as_any() + .downcast_ref::() + .is_none(), + "MergeSortExec(fetch) is the required distributed TopK merge stage, not an unordered coalesce" + ); + assert_eq!(optimized_merge_sort.fetch(), Some(1)); + } +} diff --git a/src/query/src/dist_plan/planner.rs b/src/query/src/dist_plan/planner.rs index a14aefbf5b0..820f6ff2690 100644 --- a/src/query/src/dist_plan/planner.rs +++ b/src/query/src/dist_plan/planner.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use ahash::HashMap; +use arrow_schema::SortOptions; use async_trait::async_trait; use catalog::CatalogManagerRef; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; @@ -29,6 +30,7 @@ use datafusion::physical_planner::{ExtensionPlanner, PhysicalPlanner}; use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion, TreeNodeVisitor}; use datafusion_common::{DataFusionError, TableReference}; use datafusion_expr::{LogicalPlan, UserDefinedLogicalNode}; +use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr}; use partition::manager::{PartitionRuleManagerRef, create_partitions_from_region_routes}; use session::context::QueryContext; use snafu::{OptionExt, ResultExt}; @@ -39,27 +41,54 @@ use table::table_name::TableName; use crate::dist_plan::PredicateExtractor; use crate::dist_plan::merge_scan::{MergeScanExec, MergeScanLogicalPlan}; -use crate::dist_plan::merge_sort::MergeSortLogicalPlan; +use crate::dist_plan::merge_sort::{MergeSortExec, MergeSortLogicalPlan}; use crate::dist_plan::region_pruner::ConstraintPruner; use crate::error::{CatalogSnafu, PartitionRuleManagerSnafu, TableNotFoundSnafu}; use crate::region_query::RegionQueryHandlerRef; -/// Planner for convert merge sort logical plan to physical plan +/// Planner for converting merge sort logical plan to physical plan. /// -/// it is currently a fallback to sort, and doesn't change the execution plan: -/// `MergeSort(MergeScan) -> Sort(MergeScan) - to physical plan -> ...` -/// It should be applied after `DistExtensionPlanner` -/// -/// (Later when actually impl this merge sort) -/// -/// We should ensure the number of partition is not smaller than the number of region at present. Otherwise this would result in incorrect output. +/// `MergeSortExec` always represents the distributed merge stage. It declares +/// the required input ordering to DataFusion, so `EnforceSorting` inserts a +/// `SortExec` below it when the input `MergeScanExec` cannot preserve per-region +/// ordering, for example when one output partition may merge multiple region +/// streams. pub struct MergeSortExtensionPlanner {} +impl MergeSortExtensionPlanner { + fn ordering( + session_state: &SessionState, + merge_sort: &MergeSortLogicalPlan, + ) -> Result { + let ordering = merge_sort + .expr + .iter() + .map(|sort_expr| { + let physical_expr = session_state + .create_physical_expr(sort_expr.expr.clone(), merge_sort.input.schema())?; + Ok(PhysicalSortExpr::new( + physical_expr, + SortOptions { + descending: !sort_expr.asc, + nulls_first: sort_expr.nulls_first, + }, + )) + }) + .collect::>>()?; + + LexOrdering::new(ordering).ok_or_else(|| { + DataFusionError::Internal( + "Expect MergeSort to have non-empty sort expressions".to_string(), + ) + }) + } +} + #[async_trait] impl ExtensionPlanner for MergeSortExtensionPlanner { async fn plan_extension( &self, - planner: &dyn PhysicalPlanner, + _planner: &dyn PhysicalPlanner, node: &dyn UserDefinedLogicalNode, _logical_inputs: &[&LogicalPlan], physical_inputs: &[Arc], @@ -73,28 +102,24 @@ impl ExtensionPlanner for MergeSortExtensionPlanner { .downcast_ref::() .is_some() { - let merge_scan_exec = physical_inputs - .first() - .and_then(|p| p.as_any().downcast_ref::()) - .ok_or(DataFusionError::Internal(format!( + let input = physical_inputs.first().cloned().ok_or_else(|| { + DataFusionError::Internal( + "Expect MergeSort to have one physical input".to_string(), + ) + })?; + if input.as_any().downcast_ref::().is_none() { + return Err(DataFusionError::Internal(format!( "Expect MergeSort's input is a MergeScanExec, found {:?}", physical_inputs - )))?; - - let partition_cnt = merge_scan_exec.partition_count(); - let region_cnt = merge_scan_exec.region_count(); - // if partition >= region, we know that every partition stream of merge scan is ordered - // and we only need to do a merge sort, otherwise fallback to quick sort - let can_merge_sort = partition_cnt >= region_cnt; - if can_merge_sort { - // TODO(discord9): use `SortPreservingMergeExec here` + ))); } - // for now merge sort only exist in logical plan, and have the same effect as `Sort` - // doesn't change the execution plan, this will change in the future - let ret = planner - .create_physical_plan(&merge_sort.clone().into_sort(), session_state) - .await?; - Ok(Some(ret)) + + let ordering = Self::ordering(session_state, merge_sort)?; + Ok(Some(Arc::new(MergeSortExec::new( + ordering, + input, + merge_sort.fetch, + )))) } else { Ok(None) } diff --git a/src/query/src/log_query/planner.rs b/src/query/src/log_query/planner.rs index f9c641b23b5..05aeb0795d2 100644 --- a/src/query/src/log_query/planner.rs +++ b/src/query/src/log_query/planner.rs @@ -34,8 +34,6 @@ use crate::log_query::error::{ UnknownTableSnafu, }; -const DEFAULT_LIMIT: usize = 1000; - pub struct LogQueryPlanner { table_provider: DfTableSourceProvider, session_state: SessionState, @@ -99,19 +97,18 @@ impl LogQueryPlanner { .context(DataFusionPlanningSnafu)?; } - // Apply limit - plan_builder = plan_builder - .limit( - query.limit.skip.unwrap_or(0), - Some(query.limit.fetch.unwrap_or(DEFAULT_LIMIT)), - ) - .context(DataFusionPlanningSnafu)?; - // Apply log expressions for expr in &query.exprs { plan_builder = self.process_log_expr(plan_builder, expr)?; } + // Apply pagination to the final result after all log expressions. + if query.limit.skip.is_some() || query.limit.fetch.is_some() { + plan_builder = plan_builder + .limit(query.limit.skip.unwrap_or(0), query.limit.fetch) + .context(DataFusionPlanningSnafu)?; + } + // Build the final plan let plan = plan_builder.build().context(DataFusionPlanningSnafu)?; @@ -126,15 +123,14 @@ impl LogQueryPlanner { .clone(); let start_time = ScalarValue::Utf8(time_filter.start.clone()); - let end_time = ScalarValue::Utf8( - time_filter - .end - .clone() - .or(Some("9999-12-31T23:59:59Z".to_string())), - ); let expr = col(timestamp_col.clone()) .gt_eq(lit(start_time)) - .and(col(timestamp_col).lt_eq(lit(end_time))); + .and(match &time_filter.end { + Some(end) => col(timestamp_col).lt(lit(ScalarValue::Utf8(Some(end.clone())))), + None => col(timestamp_col).lt_eq(lit(ScalarValue::Utf8(Some( + "9999-12-31T23:59:59Z".to_string(), + )))), + }); Ok(expr) } @@ -730,7 +726,7 @@ mod tests { let plan = planner.query_to_plan(log_query).await.unwrap(); let expected = "Limit: skip=0, fetch=100 [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ -\n Filter: greptime.public.test_table.timestamp >= Utf8(\"2021-01-01T00:00:00Z\") AND greptime.public.test_table.timestamp <= Utf8(\"2021-01-02T00:00:00Z\") AND greptime.public.test_table.message LIKE Utf8(\"%error%\") [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ +\n Filter: greptime.public.test_table.timestamp >= Utf8(\"2021-01-01T00:00:00Z\") AND greptime.public.test_table.timestamp < Utf8(\"2021-01-02T00:00:00Z\") AND greptime.public.test_table.message LIKE Utf8(\"%error%\") [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ \n TableScan: greptime.public.test_table [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]"; assert_eq!(plan.display_indent_schema().to_string(), expected); @@ -757,7 +753,7 @@ mod tests { .gt_eq(lit(ScalarValue::Utf8(Some( "2021-01-01T00:00:00Z".to_string(), )))) - .and(col("timestamp").lt_eq(lit(ScalarValue::Utf8(Some( + .and(col("timestamp").lt(lit(ScalarValue::Utf8(Some( "2021-01-02T00:00:00Z".to_string(), ))))); @@ -850,8 +846,8 @@ mod tests { }; let plan = planner.query_to_plan(log_query).await.unwrap(); - let expected = "Limit: skip=10, fetch=1000 [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ -\n Filter: greptime.public.test_table.timestamp >= Utf8(\"2021-01-01T00:00:00Z\") AND greptime.public.test_table.timestamp <= Utf8(\"2021-01-02T00:00:00Z\") AND greptime.public.test_table.message LIKE Utf8(\"%error%\") [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ + let expected = "Limit: skip=10, fetch=None [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ +\n Filter: greptime.public.test_table.timestamp >= Utf8(\"2021-01-01T00:00:00Z\") AND greptime.public.test_table.timestamp < Utf8(\"2021-01-02T00:00:00Z\") AND greptime.public.test_table.message LIKE Utf8(\"%error%\") [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ \n TableScan: greptime.public.test_table [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]"; assert_eq!(plan.display_indent_schema().to_string(), expected); @@ -885,9 +881,8 @@ mod tests { }; let plan = planner.query_to_plan(log_query).await.unwrap(); - let expected = "Limit: skip=0, fetch=1000 [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ -\n Filter: greptime.public.test_table.timestamp >= Utf8(\"2021-01-01T00:00:00Z\") AND greptime.public.test_table.timestamp <= Utf8(\"2021-01-02T00:00:00Z\") AND greptime.public.test_table.message LIKE Utf8(\"%error%\") [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ -\n TableScan: greptime.public.test_table [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]"; + let expected = "Filter: greptime.public.test_table.timestamp >= Utf8(\"2021-01-01T00:00:00Z\") AND greptime.public.test_table.timestamp < Utf8(\"2021-01-02T00:00:00Z\") AND greptime.public.test_table.message LIKE Utf8(\"%error%\") [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ +\n TableScan: greptime.public.test_table [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]"; assert_eq!(plan.display_indent_schema().to_string(), expected); } @@ -932,9 +927,9 @@ mod tests { }; let plan = planner.query_to_plan(log_query).await.unwrap(); - let expected = "Aggregate: groupBy=[[greptime.public.test_table.host]], aggr=[[count(greptime.public.test_table.message) AS count_result]] [host:Utf8;N, count_result:Int64]\ -\n Limit: skip=0, fetch=100 [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ -\n Filter: greptime.public.test_table.timestamp >= Utf8(\"2021-01-01T00:00:00Z\") AND greptime.public.test_table.timestamp <= Utf8(\"2021-01-02T00:00:00Z\") [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ + let expected = "Limit: skip=0, fetch=100 [host:Utf8;N, count_result:Int64]\ +\n Aggregate: groupBy=[[greptime.public.test_table.host]], aggr=[[count(greptime.public.test_table.message) AS count_result]] [host:Utf8;N, count_result:Int64]\ +\n Filter: greptime.public.test_table.timestamp >= Utf8(\"2021-01-01T00:00:00Z\") AND greptime.public.test_table.timestamp < Utf8(\"2021-01-02T00:00:00Z\") [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ \n TableScan: greptime.public.test_table [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]"; assert_eq!(plan.display_indent_schema().to_string(), expected); @@ -972,9 +967,9 @@ mod tests { }; let plan = planner.query_to_plan(log_query).await.unwrap(); - let expected = "Projection: date_trunc(Utf8(\"day\"), greptime.public.test_table.timestamp) AS time_bucket [time_bucket:Timestamp(ms)]\ - \n Limit: skip=0, fetch=100 [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ - \n Filter: greptime.public.test_table.timestamp >= Utf8(\"2021-01-01T00:00:00Z\") AND greptime.public.test_table.timestamp <= Utf8(\"2021-01-02T00:00:00Z\") [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ + let expected = "Limit: skip=0, fetch=100 [time_bucket:Timestamp(ms)]\ + \n Projection: date_trunc(Utf8(\"day\"), greptime.public.test_table.timestamp) AS time_bucket [time_bucket:Timestamp(ms)]\ + \n Filter: greptime.public.test_table.timestamp >= Utf8(\"2021-01-01T00:00:00Z\") AND greptime.public.test_table.timestamp < Utf8(\"2021-01-02T00:00:00Z\") [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ \n TableScan: greptime.public.test_table [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]"; assert_eq!(plan.display_indent_schema().to_string(), expected); @@ -1055,10 +1050,10 @@ mod tests { }; let plan = planner.query_to_plan(log_query).await.unwrap(); - let expected = "Aggregate: groupBy=[[2__date_histogram__time_bucket]], aggr=[[count(2__date_histogram__time_bucket) AS count_result]] [2__date_histogram__time_bucket:Timestamp(ns);N, count_result:Int64]\ -\n Projection: date_bin(Utf8(\"30 seconds\"), greptime.public.test_table.timestamp) AS 2__date_histogram__time_bucket [2__date_histogram__time_bucket:Timestamp(ns);N]\ -\n Limit: skip=0, fetch=1000 [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ -\n Filter: greptime.public.test_table.timestamp >= Utf8(\"2021-01-01T00:00:00Z\") AND greptime.public.test_table.timestamp <= Utf8(\"2021-01-02T00:00:00Z\") [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ + let expected = "Limit: skip=0, fetch=None [2__date_histogram__time_bucket:Timestamp(ns);N, count_result:Int64]\ +\n Aggregate: groupBy=[[2__date_histogram__time_bucket]], aggr=[[count(2__date_histogram__time_bucket) AS count_result]] [2__date_histogram__time_bucket:Timestamp(ns);N, count_result:Int64]\ +\n Projection: date_bin(Utf8(\"30 seconds\"), greptime.public.test_table.timestamp) AS 2__date_histogram__time_bucket [2__date_histogram__time_bucket:Timestamp(ns);N]\ +\n Filter: greptime.public.test_table.timestamp >= Utf8(\"2021-01-01T00:00:00Z\") AND greptime.public.test_table.timestamp < Utf8(\"2021-01-02T00:00:00Z\") [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]\ \n TableScan: greptime.public.test_table [message:Utf8, timestamp:Timestamp(ms), host:Utf8;N, is_active:Boolean;N]"; assert_eq!(plan.display_indent_schema().to_string(), expected); diff --git a/src/query/src/optimizer.rs b/src/query/src/optimizer.rs index ffbfff5ee26..8827d48ed63 100644 --- a/src/query/src/optimizer.rs +++ b/src/query/src/optimizer.rs @@ -16,6 +16,7 @@ pub mod const_normalization; pub mod constant_term; pub mod count_nest_aggr; pub mod count_wildcard; +pub mod global_limit; pub(crate) mod json_type_concretize; pub mod parallelize_scan; pub mod pass_distribution; diff --git a/src/query/src/optimizer/const_normalization.rs b/src/query/src/optimizer/const_normalization.rs index fac0e90538f..480a541b942 100644 --- a/src/query/src/optimizer/const_normalization.rs +++ b/src/query/src/optimizer/const_normalization.rs @@ -615,6 +615,7 @@ mod tests { use datafusion_optimizer::analyzer::AnalyzerRule; use datafusion_optimizer::optimizer::{Optimizer, OptimizerContext}; use datafusion_optimizer::push_down_filter::PushDownFilter; + use datafusion_optimizer::simplify_expressions::SimplifyExpressions; use table::predicate::build_time_range_predicate; use super::{ @@ -982,6 +983,127 @@ mod tests { ); } + #[test] + fn test_const_normalization_vs_datafusion_cast_preimage_overlap() { + struct Case { + name: &'static str, + fields: Vec, + predicate: Expr, + expected_greptime: &'static str, + expected_datafusion: &'static str, + } + + let cases = [ + Case { + name: "integer widening binary", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: cast(col("v"), DataType::Int64).gt_eq(lit(42_i64)), + expected_greptime: "Filter: t.v >= Int16(42)\n TableScan: t", + expected_datafusion: "Filter: t.v >= Int16(42)\n TableScan: t", + }, + Case { + name: "swapped integer comparison", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: lit(42_i64).lt_eq(cast(col("v"), DataType::Int64)), + expected_greptime: "Filter: t.v >= Int16(42)\n TableScan: t", + expected_datafusion: "Filter: t.v >= Int16(42)\n TableScan: t", + }, + Case { + name: "try_cast integer widening binary", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: try_cast(col("v"), DataType::Int64).gt_eq(lit(42_i64)), + expected_greptime: "Filter: t.v >= Int16(42)\n TableScan: t", + expected_datafusion: "Filter: t.v >= Int16(42)\n TableScan: t", + }, + Case { + name: "exact in-list", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: cast(col("v"), DataType::Int64) + .in_list(vec![lit(1_i64), lit(2_i64)], false), + expected_greptime: "Filter: t.v IN ([Int16(1), Int16(2)])\n TableScan: t", + expected_datafusion: "Filter: t.v = Int16(1) OR t.v = Int16(2)\n TableScan: t", + }, + Case { + name: "integer between", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: cast(col("v"), DataType::Int64).between(lit(3_i64), lit(5_i64)), + expected_greptime: "Filter: t.v BETWEEN Int16(3) AND Int16(5)\n TableScan: t", + expected_datafusion: "Filter: t.v >= Int16(3) AND t.v <= Int16(5)\n TableScan: t", + }, + Case { + name: "not between", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: Expr::Between(Between { + expr: Box::new(cast(col("v"), DataType::Int64)), + negated: true, + low: Box::new(lit(3_i64)), + high: Box::new(lit(5_i64)), + }), + expected_greptime: "Filter: t.v NOT BETWEEN Int16(3) AND Int16(5)\n TableScan: t", + expected_datafusion: "Filter: t.v < Int16(3) OR t.v > Int16(5)\n TableScan: t", + }, + Case { + name: "plain literal", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: col("v").gt_eq(lit(42_i64)), + expected_greptime: "Filter: t.v >= Int16(42)\n TableScan: t", + expected_datafusion: "Filter: t.v >= Int64(42)\n TableScan: t", + }, + Case { + name: "casted constant", + fields: vec![Field::new("v", DataType::Int16, false)], + predicate: col("v").gt_eq(cast(lit(42_i8), DataType::Int64)), + expected_greptime: "Filter: t.v >= Int16(42)\n TableScan: t", + expected_datafusion: "Filter: t.v >= Int64(42)\n TableScan: t", + }, + Case { + name: "timestamp downcast equality", + fields: vec![Field::new( + "ts_ns", + DataType::Timestamp(ArrowTimeUnit::Nanosecond, None), + false, + )], + predicate: cast( + col("ts_ns"), + DataType::Timestamp(ArrowTimeUnit::Millisecond, None), + ) + .eq(ts_ms_literal(5000)), + expected_greptime: "Filter: CAST(t.ts_ns AS Timestamp(ms)) = TimestampMillisecond(5000, None)\n TableScan: t", + expected_datafusion: "Filter: t.ts_ns >= TimestampNanosecond(5000000000, None) AND t.ts_ns < TimestampNanosecond(5001000000, None)\n TableScan: t", + }, + Case { + name: "timestamp widening exact", + fields: vec![Field::new( + "ts_ms", + DataType::Timestamp(ArrowTimeUnit::Millisecond, None), + false, + )], + predicate: cast( + col("ts_ms"), + DataType::Timestamp(ArrowTimeUnit::Nanosecond, None), + ) + .eq(lit(ScalarValue::TimestampNanosecond( + Some(5_000_000_000), + None, + ))), + expected_greptime: "Filter: CAST(t.ts_ms AS Timestamp(ns)) = TimestampNanosecond(5000000000, None)\n TableScan: t", + expected_datafusion: "Filter: t.ts_ms = TimestampMillisecond(5000, None)\n TableScan: t", + }, + ]; + + for case in cases { + let greptime = + greptime_const_normalized_filter(case.fields.clone(), case.predicate.clone()); + let datafusion = datafusion_simplified_filter(case.fields, case.predicate); + assert_eq!(case.expected_greptime, greptime, "{} greptime", case.name); + assert_eq!( + case.expected_datafusion, datafusion, + "{} datafusion", + case.name + ); + } + } + fn assert_pattern_match_plan(kind: PatternMatchKind, pattern: ScalarValue, expected: &str) { let predicate = match kind { PatternMatchKind::Like => Expr::Like(Like::new( @@ -1033,6 +1155,18 @@ mod tests { analyze_plan(build_filter_plan(test_schema(fields), predicate)) } + fn greptime_const_normalized_filter(fields: Vec, predicate: Expr) -> String { + analyze_filter(fields, predicate).to_string() + } + + fn datafusion_simplified_filter(fields: Vec, predicate: Expr) -> String { + let plan = build_filter_plan(test_schema(fields), predicate); + Optimizer::with_rules(vec![Arc::new(SimplifyExpressions::new())]) + .optimize(plan, &OptimizerContext::new(), |_, _| {}) + .unwrap() + .to_string() + } + fn analyze_plan(plan: LogicalPlan) -> LogicalPlan { ConstNormalizationRule .analyze(plan, &ConfigOptions::default()) diff --git a/src/query/src/optimizer/global_limit.rs b/src/query/src/optimizer/global_limit.rs new file mode 100644 index 00000000000..4b2ff721f64 --- /dev/null +++ b/src/query/src/optimizer/global_limit.rs @@ -0,0 +1,629 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use datafusion::config::ConfigOptions; +use datafusion::physical_optimizer::PhysicalOptimizerRule; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; +use datafusion::physical_plan::filter::FilterExec; +use datafusion::physical_plan::limit::GlobalLimitExec; +use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; +use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; +use datafusion_common::Result as DfResult; +use datafusion_physical_expr::{Distribution, OrderingRequirements, Partitioning}; + +use crate::dist_plan::MergeSortExec; + +#[derive(Debug)] +pub struct EnsureGlobalLimitForFetch; + +impl PhysicalOptimizerRule for EnsureGlobalLimitForFetch { + fn optimize( + &self, + plan: Arc, + _config: &ConfigOptions, + ) -> DfResult> { + Self::optimize_plan(plan, ParentContext::default()) + } + + fn name(&self) -> &str { + "EnsureGlobalLimitForFetch" + } + + fn schema_check(&self) -> bool { + true + } +} + +impl EnsureGlobalLimitForFetch { + fn optimize_plan( + plan: Arc, + parent: ParentContext, + ) -> DfResult> { + let children = plan.children(); + let plan = if children.is_empty() { + plan + } else { + let required_input_distribution = plan.required_input_distribution(); + let required_input_ordering = plan.required_input_ordering(); + let maintains_input_order = plan.maintains_input_order(); + let child_parent = ParentContext { + global_fetch: provided_global_fetch(&plan), + required_ordering: None, + required_distribution: Distribution::UnspecifiedDistribution, + partitioning_to_restore: None, + preserve_hash_partitioning: false, + }; + let children = children + .into_iter() + .enumerate() + .map(|(idx, child)| { + let required_distribution = required_input_distribution + .get(idx) + .cloned() + .unwrap_or(Distribution::UnspecifiedDistribution); + let partitioning_to_restore = + partitioning_to_restore_for(child, &required_distribution) + .or_else(|| inherited_partitioning_to_restore(&plan, child, &parent)); + let preserve_hash_partitioning = partitioning_to_restore.is_some(); + let required_ordering = required_input_ordering + .get(idx) + .cloned() + .unwrap_or(None) + .or_else(|| { + maintains_input_order + .get(idx) + .copied() + .unwrap_or(false) + .then(|| parent.required_ordering.clone()) + .flatten() + }); + let parent = ParentContext { + required_ordering, + required_distribution, + partitioning_to_restore, + preserve_hash_partitioning, + ..child_parent.clone() + }; + Self::optimize_plan(Arc::clone(child), parent) + }) + .collect::>>()?; + plan.with_new_children(children)? + }; + + let Some(fetch) = plan.fetch() else { + return Ok(plan); + }; + + if parent + .global_fetch + .is_some_and(|parent_fetch| parent_fetch <= fetch) + || !plan.as_any().is::() + || plan.output_partitioning().partition_count() <= 1 + { + return Ok(plan); + } + + add_global_fetch( + plan, + fetch, + parent.required_ordering, + parent.partitioning_to_restore, + ) + } +} + +#[derive(Clone)] +struct ParentContext { + global_fetch: Option, + required_ordering: Option, + required_distribution: Distribution, + partitioning_to_restore: Option, + preserve_hash_partitioning: bool, +} + +impl Default for ParentContext { + fn default() -> Self { + Self { + global_fetch: None, + required_ordering: None, + required_distribution: Distribution::UnspecifiedDistribution, + partitioning_to_restore: None, + preserve_hash_partitioning: false, + } + } +} + +fn provided_global_fetch(plan: &Arc) -> Option { + let fetch = plan.fetch()?; + (plan.as_any().is::() + || plan.as_any().is::() + || plan.as_any().is::() + || plan.as_any().is::()) + .then_some(fetch) +} + +fn add_global_fetch( + plan: Arc, + fetch: usize, + required_ordering: Option, + partitioning_to_restore: Option, +) -> DfResult> { + let plan = if required_ordering.is_some() + && let Some(ordering) = plan.output_ordering().cloned() + { + Arc::new(SortPreservingMergeExec::new(ordering, plan).with_fetch(Some(fetch))) + as Arc + } else { + Arc::new(CoalescePartitionsExec::new(plan).with_fetch(Some(fetch))) + as Arc + }; + + restore_required_partitioning(plan, partitioning_to_restore) +} + +fn restore_required_partitioning( + plan: Arc, + partitioning_to_restore: Option, +) -> DfResult> { + let Some(partitioning) = partitioning_to_restore else { + return Ok(plan); + }; + + if partitioning.partition_count() <= 1 || !matches!(&partitioning, Partitioning::Hash(_, _)) { + return Ok(plan); + } + + Ok(Arc::new( + RepartitionExec::try_new(plan, partitioning)?.with_preserve_order(), + )) +} + +fn partitioning_to_restore_for( + child: &Arc, + required_distribution: &Distribution, +) -> Option { + if !matches!(required_distribution, Distribution::HashPartitioned(_)) + || child.output_partitioning().partition_count() <= 1 + { + return None; + } + + if child + .output_partitioning() + .satisfaction(required_distribution, child.equivalence_properties(), false) + .is_satisfied() + { + Some(child.output_partitioning().clone()) + } else { + Some( + required_distribution + .clone() + .create_partitioning(child.output_partitioning().partition_count()), + ) + } +} + +fn inherited_partitioning_to_restore( + plan: &Arc, + child: &Arc, + parent: &ParentContext, +) -> Option { + if child.output_partitioning().partition_count() <= 1 + || !matches!(child.output_partitioning(), Partitioning::Hash(_, _)) + || !matches!(plan.output_partitioning(), Partitioning::Hash(_, _)) + || plan.output_partitioning().partition_count() + != child.output_partitioning().partition_count() + { + return None; + } + + let satisfies_parent_distribution = matches!( + parent.required_distribution, + Distribution::HashPartitioned(_) + ) && plan + .output_partitioning() + .satisfaction( + &parent.required_distribution, + plan.equivalence_properties(), + false, + ) + .is_satisfied(); + + (satisfies_parent_distribution || parent.preserve_hash_partitioning) + .then(|| child.output_partitioning().clone()) +} + +#[cfg(test)] +mod tests { + use datafusion::arrow::array::Int32Array; + use datafusion::arrow::compute::SortOptions; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::arrow::record_batch::RecordBatch; + use datafusion::physical_expr::expressions::{col, lit}; + use datafusion::physical_plan::filter::FilterExecBuilder; + use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode}; + use datafusion::physical_plan::limit::GlobalLimitExec; + use datafusion::physical_plan::projection::ProjectionExec; + use datafusion::physical_plan::repartition::RepartitionExec; + use datafusion::physical_plan::test::TestMemoryExec; + use datafusion_common::{JoinType, NullEquality}; + use datafusion_physical_expr::{LexOrdering, Partitioning, PhysicalSortExpr}; + + use super::*; + + #[test] + fn adds_global_limit_for_multi_partition_filter_fetch() { + let filter = filter_fetch(unordered_input(), 1); + + let optimized = + EnsureGlobalLimitForFetch::optimize_plan(filter, ParentContext::default()).unwrap(); + + assert!(optimized.as_any().is::()); + assert_eq!(optimized.fetch(), Some(1)); + assert_eq!(optimized.output_partitioning().partition_count(), 1); + } + + #[test] + fn still_visits_subtree_under_global_limit() { + let filter = filter_fetch(unordered_input(), 5); + let projection = Arc::new( + ProjectionExec::try_new( + vec![(col("a", filter.schema().as_ref()).unwrap(), "a".to_string())], + filter, + ) + .unwrap(), + ); + let limit = + Arc::new(GlobalLimitExec::new(projection, 0, Some(10))) as Arc; + + let optimized = + EnsureGlobalLimitForFetch::optimize_plan(limit, ParentContext::default()).unwrap(); + let projection = optimized.children()[0]; + let coalesce = projection.children()[0]; + + assert!(coalesce.as_any().is::()); + assert_eq!(coalesce.fetch(), Some(5)); + } + + #[test] + fn keeps_filter_under_parent_global_fetch() { + let (input, ordering) = ordered_input(); + let filter = filter_fetch(input, 1); + let merge = Arc::new(SortPreservingMergeExec::new(ordering, filter).with_fetch(Some(1))) + as Arc; + + let optimized = + EnsureGlobalLimitForFetch::optimize_plan(merge, ParentContext::default()).unwrap(); + let child = optimized.children()[0]; + + assert!(optimized.as_any().is::()); + assert!(child.as_any().is::()); + } + + #[test] + fn adds_tighter_global_fetch_under_looser_parent_fetch() { + let (input, ordering) = ordered_input(); + let filter = filter_fetch(input, 5); + let merge = Arc::new(SortPreservingMergeExec::new(ordering, filter).with_fetch(Some(10))) + as Arc; + + let optimized = + EnsureGlobalLimitForFetch::optimize_plan(merge, ParentContext::default()).unwrap(); + let child = optimized.children()[0]; + + assert!(optimized.as_any().is::()); + assert!(child.as_any().is::()); + assert_eq!(child.fetch(), Some(5)); + } + + #[test] + fn keeps_filter_under_parent_merge_sort_fetch() { + let (input, ordering) = ordered_input(); + let filter = filter_fetch(input, 1); + let merge = merge_sort_fetch(ordering, filter, 1); + + let optimized = + EnsureGlobalLimitForFetch::optimize_plan(merge, ParentContext::default()).unwrap(); + let child = optimized.children()[0]; + + assert!(optimized.as_any().is::()); + assert!(child.as_any().is::()); + } + + #[test] + fn adds_tighter_global_fetch_under_looser_merge_sort_fetch() { + let (input, ordering) = ordered_input(); + let filter = filter_fetch(input, 5); + let merge = merge_sort_fetch(ordering, filter, 10); + + let optimized = + EnsureGlobalLimitForFetch::optimize_plan(merge, ParentContext::default()).unwrap(); + let child = optimized.children()[0]; + + assert!(optimized.as_any().is::()); + assert!(child.as_any().is::()); + assert_eq!(child.fetch(), Some(5)); + assert!(child.children()[0].as_any().is::()); + } + + #[test] + fn preserves_parent_ordering_requirement() { + let (input, ordering) = ordered_input(); + let filter = filter_fetch(input, 1); + let merge = + Arc::new(SortPreservingMergeExec::new(ordering, filter)) as Arc; + + let optimized = + EnsureGlobalLimitForFetch::optimize_plan(merge, ParentContext::default()).unwrap(); + let child = optimized.children()[0]; + + assert!(optimized.as_any().is::()); + assert!(child.as_any().is::()); + assert_eq!(child.fetch(), Some(1)); + } + + #[test] + fn uses_child_output_ordering_for_merge() { + let schema = schema(); + let required_ordering = ordering(schema.as_ref(), false); + let actual_ordering = ordering(schema.as_ref(), true); + let batch = batch(schema.clone()); + let partitions = vec![vec![batch.clone()], vec![batch.clone()], vec![batch]]; + let input = TestMemoryExec::try_new(&partitions, schema, None) + .unwrap() + .try_with_sort_information(vec![actual_ordering.clone()]) + .unwrap(); + let filter = filter_fetch(Arc::new(input), 1); + + let optimized = add_global_fetch( + filter, + 1, + Some(OrderingRequirements::from(required_ordering)), + None, + ) + .unwrap(); + let merge = optimized + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(merge.expr(), &actual_ordering); + } + + #[test] + fn preserves_inherited_ordering_requirement_through_projection() { + let (input, ordering) = ordered_input(); + let filter = filter_fetch(input, 1); + let projection = Arc::new( + ProjectionExec::try_new( + vec![(col("a", filter.schema().as_ref()).unwrap(), "a".to_string())], + filter, + ) + .unwrap(), + ); + let merge = + Arc::new(SortPreservingMergeExec::new(ordering, projection)) as Arc; + + let optimized = + EnsureGlobalLimitForFetch::optimize_plan(merge, ParentContext::default()).unwrap(); + let projection = optimized.children()[0]; + let child = projection.children()[0]; + + assert!(optimized.as_any().is::()); + assert!(projection.as_any().is::()); + assert!(child.as_any().is::()); + assert_eq!(child.fetch(), Some(1)); + } + + #[test] + fn restores_parent_hash_distribution_after_global_fetch() { + let left = filter_fetch(hash_repartition(unordered_input()), 1); + let right = hash_repartition(unordered_input()); + let on = vec![( + col("a", left.schema().as_ref()).unwrap(), + col("a", right.schema().as_ref()).unwrap(), + )]; + let join = Arc::new( + HashJoinExec::try_new( + left, + right, + on, + None, + &JoinType::Inner, + None, + PartitionMode::Partitioned, + NullEquality::NullEqualsNothing, + false, + ) + .unwrap(), + ) as Arc; + + let optimized = + EnsureGlobalLimitForFetch::optimize_plan(join, ParentContext::default()).unwrap(); + let left = optimized.children()[0]; + let repartition = left.as_any().downcast_ref::().unwrap(); + + assert!(matches!( + repartition.partitioning(), + Partitioning::Hash(_, 3) + )); + assert!(repartition.input().as_any().is::()); + assert_eq!(repartition.input().fetch(), Some(1)); + } + + #[test] + fn restores_inherited_hash_distribution_through_projection() { + let filter = filter_fetch(hash_repartition(unordered_input()), 1); + let projection = Arc::new( + ProjectionExec::try_new( + vec![(col("a", filter.schema().as_ref()).unwrap(), "a".to_string())], + filter, + ) + .unwrap(), + ) as Arc; + let right = hash_repartition(unordered_input()); + let on = vec![( + col("a", projection.schema().as_ref()).unwrap(), + col("a", right.schema().as_ref()).unwrap(), + )]; + let join = Arc::new( + HashJoinExec::try_new( + projection, + right, + on, + None, + &JoinType::Inner, + None, + PartitionMode::Partitioned, + NullEquality::NullEqualsNothing, + false, + ) + .unwrap(), + ) as Arc; + + let optimized = + EnsureGlobalLimitForFetch::optimize_plan(join, ParentContext::default()).unwrap(); + let projection = optimized.children()[0]; + let repartition = projection.children()[0] + .as_any() + .downcast_ref::() + .unwrap(); + + assert!(projection.as_any().is::()); + assert!(matches!( + repartition.partitioning(), + Partitioning::Hash(_, 3) + )); + assert!(repartition.input().as_any().is::()); + assert_eq!(repartition.input().fetch(), Some(1)); + } + + #[test] + fn restores_inherited_hash_distribution_through_multiple_projections() { + let filter = filter_fetch(hash_repartition(unordered_input()), 1); + let projection = project_a(filter); + let projection = project_a(projection); + let right = hash_repartition(unordered_input()); + let on = vec![( + col("a", projection.schema().as_ref()).unwrap(), + col("a", right.schema().as_ref()).unwrap(), + )]; + let join = Arc::new( + HashJoinExec::try_new( + projection, + right, + on, + None, + &JoinType::Inner, + None, + PartitionMode::Partitioned, + NullEquality::NullEqualsNothing, + false, + ) + .unwrap(), + ) as Arc; + + let optimized = + EnsureGlobalLimitForFetch::optimize_plan(join, ParentContext::default()).unwrap(); + let outer_projection = optimized.children()[0]; + let inner_projection = outer_projection.children()[0]; + let repartition = inner_projection.children()[0] + .as_any() + .downcast_ref::() + .unwrap(); + + assert!(outer_projection.as_any().is::()); + assert!(inner_projection.as_any().is::()); + assert!(matches!( + repartition.partitioning(), + Partitioning::Hash(_, 3) + )); + assert!(repartition.input().as_any().is::()); + assert_eq!(repartition.input().fetch(), Some(1)); + } + + fn unordered_input() -> Arc { + let schema = schema(); + let batch = batch(schema.clone()); + let partitions = vec![vec![batch.clone()], vec![batch.clone()], vec![batch]]; + Arc::new(TestMemoryExec::try_new(&partitions, schema, None).unwrap()) + } + + fn ordered_input() -> (Arc, LexOrdering) { + let schema = schema(); + let ordering = ordering(schema.as_ref(), false); + let batch = batch(schema.clone()); + let partitions = vec![vec![batch.clone()], vec![batch.clone()], vec![batch]]; + let input = TestMemoryExec::try_new(&partitions, schema, None) + .unwrap() + .try_with_sort_information(vec![ordering.clone()]) + .unwrap(); + + (Arc::new(input), ordering) + } + + fn filter_fetch(input: Arc, fetch: usize) -> Arc { + Arc::new( + FilterExecBuilder::new(lit(true), input) + .with_fetch(Some(fetch)) + .build() + .unwrap(), + ) + } + + fn merge_sort_fetch( + ordering: LexOrdering, + input: Arc, + fetch: usize, + ) -> Arc { + Arc::new(MergeSortExec::new(ordering, input, Some(fetch))) + } + + fn hash_repartition(input: Arc) -> Arc { + let partitioning = Partitioning::Hash(vec![col("a", input.schema().as_ref()).unwrap()], 3); + Arc::new(RepartitionExec::try_new(input, partitioning).unwrap()) + } + + fn project_a(input: Arc) -> Arc { + Arc::new( + ProjectionExec::try_new( + vec![(col("a", input.schema().as_ref()).unwrap(), "a".to_string())], + input, + ) + .unwrap(), + ) + } + + fn schema() -> Arc { + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])) + } + + fn batch(schema: Arc) -> RecordBatch { + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]).unwrap() + } + + fn ordering(schema: &Schema, descending: bool) -> LexOrdering { + LexOrdering::new([PhysicalSortExpr::new( + col("a", schema).unwrap(), + SortOptions { + descending, + nulls_first: descending, + }, + )]) + .unwrap() + } +} diff --git a/src/query/src/optimizer/pass_distribution.rs b/src/query/src/optimizer/pass_distribution.rs index e9cd5f028d6..89ac43c5a93 100644 --- a/src/query/src/optimizer/pass_distribution.rs +++ b/src/query/src/optimizer/pass_distribution.rs @@ -16,8 +16,9 @@ use std::sync::Arc; use datafusion::config::ConfigOptions; use datafusion::physical_optimizer::PhysicalOptimizerRule; -use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::projection::ProjectionExec; +use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::physical_plan::{ExecutionPlan, Partitioning}; use datafusion_common::Result as DfResult; use datafusion_physical_expr::Distribution; use datafusion_physical_expr::utils::map_columns_before_projection; @@ -68,11 +69,26 @@ impl PassDistribution { ) -> DfResult> { // If this is a MergeScanExec, try to apply the current requirement. if let Some(merge_scan) = plan.as_any().downcast_ref::() - && let Some(distribution) = current_req.as_ref() - && let Some(new_plan) = merge_scan.try_with_new_distribution(distribution.clone()) + && let Some(Distribution::HashPartitioned(hash_exprs)) = current_req.as_ref() { - // Leaf node; no children to process - return Ok(Arc::new(new_plan) as _); + if let Partitioning::Hash(current_hash_exprs, _) = &merge_scan.properties().partitioning + && *current_hash_exprs == *hash_exprs + { + return Ok(plan); + } + + if let Some(new_plan) = merge_scan + .try_with_new_distribution(Distribution::HashPartitioned(hash_exprs.clone())) + { + // Leaf node; no children to process + return Ok(Arc::new(new_plan) as _); + } + + let partitioning = Partitioning::Hash( + hash_exprs.clone(), + merge_scan.properties().partitioning.partition_count(), + ); + return Ok(Arc::new(RepartitionExec::try_new(plan, partitioning)?) as _); } // Compute per-child requirements from the current node. @@ -266,6 +282,20 @@ mod tests { ); } + #[test] + fn merge_scan_rejects_hash_requirement_on_partition_key_subset() { + let merge_scan = test_merge_scan_exec(test_schema()); + + let new_plan = merge_scan.try_with_new_distribution(Distribution::HashPartitioned(vec![ + partition_column(DATA_SCHEMA_TSID_COLUMN_NAME, 1), + ])); + + assert!( + new_plan.is_none(), + "partitioning by a subset of multi-column partition keys is not sufficient" + ); + } + fn test_merge_scan_exec(schema: SchemaRef) -> MergeScanExec { let session_state = SessionStateBuilder::new().with_default_features().build(); let partition_cols = BTreeMap::from([ diff --git a/src/query/src/query_engine/state.rs b/src/query/src/query_engine/state.rs index 44c75292b19..58ad21c5bf5 100644 --- a/src/query/src/query_engine/state.rs +++ b/src/query/src/query_engine/state.rs @@ -66,6 +66,7 @@ use crate::optimizer::const_normalization::ConstNormalizationRule; use crate::optimizer::constant_term::MatchesConstantTermOptimizer; use crate::optimizer::count_nest_aggr::CountNestAggrRule; use crate::optimizer::count_wildcard::CountWildcardToTimeIndexRule; +use crate::optimizer::global_limit::EnsureGlobalLimitForFetch; use crate::optimizer::json_type_concretize::JsonTypeConcretizeRule; use crate::optimizer::parallelize_scan::ParallelizeScan; use crate::optimizer::pass_distribution::PassDistribution; @@ -214,6 +215,9 @@ impl QueryEngineState { physical_optimizer .rules .push(Arc::new(MatchesConstantTermOptimizer)); + physical_optimizer + .rules + .push(Arc::new(EnsureGlobalLimitForFetch)); // Add rule to remove duplicate nodes generated by other rules. Run this in the last. physical_optimizer.rules.push(Arc::new(RemoveDuplicate)); // Place SanityCheckPlan at the end of the list to ensure that it runs after all other rules. diff --git a/src/servers/dashboard/VERSION b/src/servers/dashboard/VERSION index 5ba40509d6e..9266439916f 100644 --- a/src/servers/dashboard/VERSION +++ b/src/servers/dashboard/VERSION @@ -1 +1 @@ -v0.13.6 +v0.13.7 diff --git a/src/servers/src/http/logs.rs b/src/servers/src/http/logs.rs index b9f23b68089..ca7c061da05 100644 --- a/src/servers/src/http/logs.rs +++ b/src/servers/src/http/logs.rs @@ -19,24 +19,31 @@ use axum::extract::State; use axum::response::{IntoResponse, Response}; use axum::{Extension, Json}; use common_telemetry::tracing; -use log_query::LogQuery; +use log_query::{Limit, LogQuery}; use session::context::{Channel, QueryContext}; use crate::http::result::greptime_result_v1::GreptimedbV1Response; use crate::query_handler::LogQueryHandlerRef; +const DEFAULT_FETCH: usize = 1000; + +fn apply_default_fetch(limit: &mut Limit) { + limit.fetch.get_or_insert(DEFAULT_FETCH); +} + #[axum_macros::debug_handler] #[tracing::instrument(skip_all, fields(protocol = "http", request_type = "logs"))] pub async fn logs( State(handler): State, Extension(mut query_ctx): Extension, - Json(params): Json, + Json(mut params): Json, ) -> Response { let exec_start = Instant::now(); let db = query_ctx.get_db_string(); query_ctx.set_channel(Channel::Log); let query_ctx = Arc::new(query_ctx); + apply_default_fetch(&mut params.limit); let _timer = crate::metrics::METRIC_HTTP_LOGS_ELAPSED .with_label_values(&[db.as_str()]) @@ -48,3 +55,34 @@ pub async fn logs( resp.with_execution_time(exec_start.elapsed().as_millis() as u64) .into_response() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_apply_default_fetch() { + let mut limit = Limit { + skip: None, + fetch: None, + }; + apply_default_fetch(&mut limit); + assert_eq!(limit.fetch, Some(DEFAULT_FETCH)); + + let mut limit = Limit { + skip: Some(10), + fetch: None, + }; + apply_default_fetch(&mut limit); + assert_eq!(limit.skip, Some(10)); + assert_eq!(limit.fetch, Some(DEFAULT_FETCH)); + + let mut limit = Limit { + skip: Some(10), + fetch: Some(42), + }; + apply_default_fetch(&mut limit); + assert_eq!(limit.skip, Some(10)); + assert_eq!(limit.fetch, Some(42)); + } +} diff --git a/src/servers/src/postgres/types.rs b/src/servers/src/postgres/types.rs index 9fc5dccf5f3..8d55fb3a6cc 100644 --- a/src/servers/src/postgres/types.rs +++ b/src/servers/src/postgres/types.rs @@ -582,9 +582,10 @@ pub(super) fn parameters_to_scalar_values( Some(st @ ConcreteDataType::Timestamp(unit)) => { to_timestamp_scalar_value(data.and_then(|n| n.to_i64()), unit, st)? } - Some(ConcreteDataType::UInt64(_)) | None => { - ScalarValue::UInt64(data.and_then(|n| n.to_u64())) - } + Some(ConcreteDataType::UInt64(_)) | None => ScalarValue::UInt64( + data.map(|n| n.to_u64().ok_or_else(|| numeric_out_of_range_error(n))) + .transpose()?, + ), Some(st) => { return Err(invalid_parameter_error( "invalid_parameter_type", @@ -924,21 +925,28 @@ pub(super) fn parameters_to_scalar_values( &Type::NUMERIC_ARRAY => { let data = portal.parameter::>>(idx, &client_type)?; if let Some(data) = data { - let build_u64_list = |data: Vec>| { + let build_u64_list = |data: Vec>| -> PgWireResult { let values = data .into_iter() - .map(|n| ScalarValue::UInt64(n.and_then(|n| n.to_u64()))) - .collect::>(); - ScalarValue::List(ScalarValue::new_list( + .map(|n| { + Ok(ScalarValue::UInt64( + n.map(|n| { + n.to_u64().ok_or_else(|| numeric_out_of_range_error(n)) + }) + .transpose()?, + )) + }) + .collect::>>()?; + Ok(ScalarValue::List(ScalarValue::new_list( &values, &ArrowDataType::UInt64, true, - )) + ))) }; if let Some(server_type) = &server_type { match server_type { ConcreteDataType::List(list_type) => match list_type.item_type() { - ConcreteDataType::UInt64(_) => build_u64_list(data), + ConcreteDataType::UInt64(_) => build_u64_list(data)?, ConcreteDataType::Decimal128(dt) => { let values = data .into_iter() @@ -975,7 +983,7 @@ pub(super) fn parameters_to_scalar_values( } } else { // server type not provided - build_u64_list(data) + build_u64_list(data)? } } else { ScalarValue::Null @@ -2025,4 +2033,114 @@ mod test { let values = parameters_to_scalar_values(&plan, &portal).unwrap(); assert_eq!(values[0], ScalarValue::Int8(None)); } + + fn numeric_uint64_array_plan() -> LogicalPlan { + build_plan_with_params(vec![( + "$1", + DataType::List(Arc::new(Field::new("item", DataType::UInt64, true))), + )]) + } + + fn assert_numeric_out_of_range(result: PgWireResult>) { + match result.unwrap_err() { + PgWireError::UserError(error) => { + assert_eq!("22023", error.code); + assert_eq!("numeric_value_out_of_range", error.message); + } + error => panic!("expected numeric out-of-range error, got {error:?}"), + } + } + + #[test] + fn test_numeric_uint64_scalar_negative_rejected() { + let plan = build_plan_with_params(vec![("$1", DataType::UInt64)]); + let portal = make_portal(vec![Some(Type::NUMERIC)], vec![s("-1")]); + + assert_numeric_out_of_range(parameters_to_scalar_values(&plan, &portal)); + } + + #[test] + fn test_numeric_uint64_scalar_above_u64_max_rejected() { + let plan = build_plan_with_params(vec![("$1", DataType::UInt64)]); + let portal = make_portal(vec![Some(Type::NUMERIC)], vec![s("18446744073709551616")]); + + assert_numeric_out_of_range(parameters_to_scalar_values(&plan, &portal)); + } + + #[test] + fn test_numeric_uint64_scalar_null_preserved() { + let plan = build_plan_with_params(vec![("$1", DataType::UInt64)]); + let portal = make_portal(vec![Some(Type::NUMERIC)], vec![None]); + + let values = parameters_to_scalar_values(&plan, &portal).unwrap(); + assert_eq!(ScalarValue::UInt64(None), values[0]); + } + + #[test] + fn test_numeric_uint64_scalar_u64_max_preserved() { + let plan = build_plan_with_params(vec![("$1", DataType::UInt64)]); + let portal = make_portal(vec![Some(Type::NUMERIC)], vec![s(&u64::MAX.to_string())]); + + let values = parameters_to_scalar_values(&plan, &portal).unwrap(); + assert_eq!(ScalarValue::UInt64(Some(u64::MAX)), values[0]); + } + + #[test] + fn test_numeric_uint64_array_outer_null_preserved() { + let portal = make_portal(vec![Some(Type::NUMERIC_ARRAY)], vec![None]); + + let values = parameters_to_scalar_values(&numeric_uint64_array_plan(), &portal).unwrap(); + assert_eq!(ScalarValue::Null, values[0]); + } + + #[test] + fn test_numeric_uint64_array_preserves_values_and_null_slots() { + let portal = make_portal( + vec![Some(Type::NUMERIC_ARRAY)], + vec![s("{42,NULL,18446744073709551615}")], + ); + + let values = parameters_to_scalar_values(&numeric_uint64_array_plan(), &portal).unwrap(); + let expected = ScalarValue::List(ScalarValue::new_list( + &[ + ScalarValue::UInt64(Some(42)), + ScalarValue::UInt64(None), + ScalarValue::UInt64(Some(u64::MAX)), + ], + &ArrowDataType::UInt64, + true, + )); + assert_eq!(expected, values[0]); + } + + #[test] + fn test_numeric_uint64_array_invalid_after_valid_and_null_prefix_rejected() { + let portal = make_portal(vec![Some(Type::NUMERIC_ARRAY)], vec![s("{42,NULL,-1}")]); + + assert_numeric_out_of_range(parameters_to_scalar_values( + &numeric_uint64_array_plan(), + &portal, + )); + } + + #[test] + fn test_numeric_uint64_array_above_u64_max_rejected() { + let portal = make_portal( + vec![Some(Type::NUMERIC_ARRAY)], + vec![s("{18446744073709551616}")], + ); + + assert_numeric_out_of_range(parameters_to_scalar_values( + &numeric_uint64_array_plan(), + &portal, + )); + } + + #[test] + fn test_numeric_uint64_uninferred_array_invalid_value_rejected() { + let plan = LogicalPlanBuilder::empty(true).build().unwrap(); + let portal = make_portal(vec![Some(Type::NUMERIC_ARRAY)], vec![s("{-1}")]); + + assert_numeric_out_of_range(parameters_to_scalar_values(&plan, &portal)); + } } diff --git a/src/table/src/table/metrics.rs b/src/table/src/table/metrics.rs index 23356a408d8..b76be231f1f 100644 --- a/src/table/src/table/metrics.rs +++ b/src/table/src/table/metrics.rs @@ -15,7 +15,7 @@ use std::time::Duration; use datafusion::physical_plan::metrics::{ - Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, ScopedTimerGuard, Time, Timestamp, + Count, ExecutionPlanMetricsSet, MetricBuilder, ScopedTimerGuard, Time, Timestamp, }; /// This metrics struct is used to record and hold metrics like memory usage @@ -25,10 +25,10 @@ use datafusion::physical_plan::metrics::{ pub struct StreamMetrics { /// Timestamp when the stream finished end_time: Timestamp, - /// Used memory in bytes - mem_used: Gauge, /// Number of rows in output output_rows: Count, + /// Number of bytes in output + output_bytes: Count, /// Elapsed time used to `poll` the stream poll_elapsed: Time, /// Elapsed time used to `.await`ing the stream @@ -36,28 +36,28 @@ pub struct StreamMetrics { } impl StreamMetrics { - /// Create a new MemoryUsageMetrics structure, and set `start_time` to now + /// Create a new [`StreamMetrics`] structure, and set `start_time` to now. pub fn new(metrics: &ExecutionPlanMetricsSet, partition: usize) -> Self { let start_time = MetricBuilder::new(metrics).start_timestamp(partition); start_time.record(); Self { end_time: MetricBuilder::new(metrics).end_timestamp(partition), - mem_used: MetricBuilder::new(metrics).mem_used(partition), output_rows: MetricBuilder::new(metrics).output_rows(partition), + output_bytes: MetricBuilder::new(metrics).output_bytes(partition), poll_elapsed: MetricBuilder::new(metrics).subset_time("elapsed_poll", partition), await_elapsed: MetricBuilder::new(metrics).subset_time("elapsed_await", partition), } } - pub fn record_mem_usage(&self, mem_used: usize) { - self.mem_used.add(mem_used); - } - pub fn record_output(&self, num_rows: usize) { self.output_rows.add(num_rows); } + pub fn record_output_bytes(&self, num_bytes: usize) { + self.output_bytes.add(num_bytes); + } + /// Record the end time of the query pub fn try_done(&self) { if self.end_time.value().is_none() { diff --git a/src/table/src/table/scan.rs b/src/table/src/table/scan.rs index d301202e172..199b4f1d8b7 100644 --- a/src/table/src/table/scan.rs +++ b/src/table/src/table/scan.rs @@ -54,6 +54,9 @@ use store_api::storage::{ScanRequest, TimeSeriesDistribution}; use crate::table::metrics::StreamMetrics; +/// The name of [`RegionScanExec`] plan node. +pub const REGION_SCAN_EXEC_NAME: &str = "RegionScanExec"; + /// A plan to read multiple partitions from a region of a table. #[derive(Clone)] pub struct RegionScanExec { @@ -453,7 +456,7 @@ impl ExecutionPlan for RegionScanExec { } fn name(&self) -> &str { - "RegionScanExec" + REGION_SCAN_EXEC_NAME } fn handle_child_pushdown_result( @@ -519,8 +522,8 @@ impl Stream for StreamWithMetricWrapper { Ok(record_batch) => { // we don't record elapsed time here // since it's calling storage api involving I/O ops - this.metric - .record_mem_usage(record_batch.buffer_memory_size()); + let batch_bytes = record_batch.logical_slice_memory_size(); + this.metric.record_output_bytes(batch_bytes); this.metric.record_output(record_batch.num_rows()); Poll::Ready(Some(Ok(record_batch.into_df_record_batch()))) } diff --git a/tests-integration/src/tests/instance_test.rs b/tests-integration/src/tests/instance_test.rs index 59f6a626c7e..c4a7e6d598a 100644 --- a/tests-integration/src/tests/instance_test.rs +++ b/tests-integration/src/tests/instance_test.rs @@ -28,6 +28,7 @@ use datatypes::arrow::array::{ use frontend::error::Error; use frontend::instance::Instance; use operator::error::Error as OperatorError; +use query::datafusion::QUERY_PARALLELISM_HINT; use rstest::rstest; use rstest_reuse::apply; use servers::error as server_error; @@ -89,6 +90,98 @@ async fn test_create_database_and_insert_query(instance: Arc) } } +#[tokio::test(flavor = "multi_thread")] +async fn test_distributed_scalar_latest_with_query_parallelism_below_regions() { + common_telemetry::init_default_ut_logging(); + + let distributed = crate::tests::create_distributed_instance( + "test_distributed_scalar_latest_with_query_parallelism_below_regions", + ) + .await; + let frontend = distributed.frontend(); + + execute_sql( + &frontend, + r#" +CREATE TABLE cpu ( + rack STRING NULL, + os STRING NULL, + usage_user BIGINT NULL, + greptime_timestamp TIMESTAMP(9) NOT NULL, + TIME INDEX (greptime_timestamp) +) +PARTITION ON COLUMNS (rack) ( + rack < '2', + rack >= '2' AND rack < '4', + rack >= '4' AND rack < '6', + rack >= '6' AND rack < '8', + rack >= '8' +) +ENGINE = mito +WITH (append_mode = 'true', sst_format = 'flat') +"#, + ) + .await; + + execute_sql( + &frontend, + r#" +INSERT INTO cpu VALUES + ('1', 'linux', 10, '2023-06-12 01:04:49'), + ('1', 'linux', 15, '2023-06-12 01:04:50'), + ('3', 'windows', 25, '2023-06-12 01:05:00'), + ('5', 'mac', 30, '2023-06-12 01:03:00'), + ('7', 'linux', 45, '2023-06-12 02:00:00'), + ('2', 'linux', 20, '2023-06-12 01:04:51'), + ('2', 'windows', 22, '2023-06-12 01:06:00'), + ('4', 'mac', 12, '2023-06-12 00:59:00'), + ('6', 'linux', 35, '2023-06-12 01:04:55'), + ('8', 'windows', 50, '2023-06-12 02:10:00') +"#, + ) + .await; + + let latest_sql = r#" +SELECT rack, os, greptime_timestamp +FROM cpu +WHERE greptime_timestamp = ( + SELECT greptime_timestamp + FROM cpu + ORDER BY greptime_timestamp DESC + LIMIT 1 +) +"#; + + let result = execute_sql_with_query_parallelism(&frontend, latest_sql, 1) + .await + .data + .pretty_print() + .await; + assert_eq!( + result, + r#"+------+---------+---------------------+ +| rack | os | greptime_timestamp | ++------+---------+---------------------+ +| 8 | windows | 2023-06-12T02:10:00 | ++------+---------+---------------------+"# + ); + + let explain = + execute_sql_with_query_parallelism(&frontend, &format!("EXPLAIN {latest_sql}"), 1) + .await + .data + .pretty_print() + .await; + assert!( + explain.contains("SortExec"), + "query_parallelism=1 with five regions should insert SortExec below MergeSortExec; explain:\n{explain}" + ); + assert!( + explain.contains("MergeSortExec"), + "query_parallelism=1 with five regions should keep the distributed merge stage stable; explain:\n{explain}" + ); +} + #[apply(both_instances_cases)] async fn test_show_create_table(instance: Arc) { let frontend = instance.frontend(); @@ -2744,6 +2837,16 @@ async fn execute_sql(instance: &Arc, sql: &str) -> Output { execute_sql_with(instance, sql, QueryContext::arc()).await } +async fn execute_sql_with_query_parallelism( + instance: &Arc, + sql: &str, + parallelism: usize, +) -> Output { + let mut query_ctx = QueryContext::with_db_name(None); + query_ctx.set_extension(QUERY_PARALLELISM_HINT, parallelism.to_string()); + execute_sql_with(instance, sql, Arc::new(query_ctx)).await +} + async fn try_execute_sql(instance: &Arc, sql: &str) -> server_error::Result { try_execute_sql_with(instance, sql, QueryContext::arc()).await } diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index bef0ba1530f..c4c61ddb3f6 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -40,7 +40,7 @@ use common_memory_manager::OnExhaustedPolicy; use common_options::plugin_options::StandaloneFlag; use flate2::Compression; use flate2::write::GzEncoder; -use log_query::{Context, Limit, LogQuery, TimeFilter}; +use log_query::{AggFunc, Context, Limit, LogExpr, LogQuery, TimeFilter}; use loki_proto::logproto::{EntryAdapter, LabelPairAdapter, PushRequest, StreamAdapter}; use loki_proto::prost_types::Timestamp; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; @@ -7147,14 +7147,16 @@ pub async fn test_log_query(store_type: StorageType) { .await; assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await); let res = client - .post("/v1/sql?sql=insert into logs values ('2024-11-07 10:53:50', 'hello');") + .post( + "/v1/sql?sql=insert into logs values ('2024-11-06 23:59:59', 'before-date-end'), ('2024-11-07 10:53:50', 'before-explicit-end'), ('2024-11-07 10:53:51', 'at-explicit-end'), ('2024-11-07 10:53:52', 'after-explicit-end');", + ) .header("Content-Type", "application/x-www-form-urlencoded") .send() .await; assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await); // test log query - let log_query = LogQuery { + let mut log_query = LogQuery { table: TableName { catalog_name: "greptime".to_string(), schema_name: "public".to_string(), @@ -7162,12 +7164,12 @@ pub async fn test_log_query(store_type: StorageType) { }, time_filter: TimeFilter { start: Some("2024-11-07".to_string()), - end: None, + end: Some("2024-11-07T10:53:51Z".to_string()), span: None, }, limit: Limit { skip: None, - fetch: Some(1), + fetch: Some(3), }, columns: vec!["ts".to_string(), "message".to_string()], filters: Default::default(), @@ -7184,7 +7186,114 @@ pub async fn test_log_query(store_type: StorageType) { assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await); let resp = res.text().await; let v = get_rows_from_output(&resp); - assert_eq!(v, "[[1730976830000,\"hello\"]]"); + assert_eq!(v, "[[1730976830000,\"before-explicit-end\"]]"); + + log_query.time_filter.start = Some("2024-11-06".to_string()); + log_query.time_filter.end = Some("2024-11-07".to_string()); + log_query.limit.fetch = Some(4); + let res = client + .post("/v1/logs") + .header("Content-Type", "application/json") + .body(serde_json::to_string(&log_query).unwrap()) + .send() + .await; + + assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await); + let resp = res.text().await; + let output = serde_json::from_str::(&resp).unwrap(); + let rows = output["output"][0]["records"]["rows"].as_array().unwrap(); + let mut messages = rows + .iter() + .map(|row| row[1].as_str().unwrap()) + .collect::>(); + messages.sort_unstable(); + assert_eq!( + messages, + vec![ + "after-explicit-end", + "at-explicit-end", + "before-date-end", + "before-explicit-end", + ] + ); + + let res = client + .get("/v1/sql?sql=create table logs_limit (`ts` timestamp time index, `message` string);") + .send() + .await; + assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await); + let res = client + .get("/v1/sql?sql=insert into logs_limit select number, 'hello' from numbers limit 1001;") + .send() + .await; + assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await); + + let aggregate_query = LogQuery { + table: TableName::new("greptime", "public", "logs_limit"), + time_filter: TimeFilter { + start: Some("1970-01-01T00:00:00Z".to_string()), + end: Some("1970-01-01T00:00:02Z".to_string()), + span: None, + }, + limit: Limit { + skip: None, + fetch: None, + }, + columns: vec![], + filters: Default::default(), + context: Context::None, + exprs: vec![LogExpr::AggrFunc { + expr: vec![AggFunc::new( + "count".to_string(), + vec![LogExpr::NamedIdent("message".to_string())], + Some("count_result".to_string()), + )], + by: vec![], + }], + }; + let res = client + .post("/v1/logs") + .header("Content-Type", "application/json") + .body(serde_json::to_string(&aggregate_query).unwrap()) + .send() + .await; + assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await); + assert_eq!(get_rows_from_output(&res.text().await), "[[1001]]"); + + let mut output_query = LogQuery { + table: TableName::new("greptime", "public", "logs_limit"), + time_filter: TimeFilter { + start: Some("1970-01-01T00:00:00Z".to_string()), + end: Some("1970-01-01T00:00:02Z".to_string()), + span: None, + }, + limit: Limit { + skip: None, + fetch: None, + }, + columns: vec!["ts".to_string(), "message".to_string()], + filters: Default::default(), + context: Context::None, + exprs: vec![], + }; + let res = client + .post("/v1/logs") + .header("Content-Type", "application/json") + .body(serde_json::to_string(&output_query).unwrap()) + .send() + .await; + assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await); + assert_eq!(get_output_row_count(&res.text().await), 1000); + + output_query.limit.fetch = Some(17); + let res = client + .post("/v1/logs") + .header("Content-Type", "application/json") + .body(serde_json::to_string(&output_query).unwrap()) + .send() + .await; + assert_eq!(res.status(), StatusCode::OK, "{:?}", res.text().await); + assert_eq!(get_output_row_count(&res.text().await), 17); guard.remove_all().await; } @@ -8968,6 +9077,18 @@ fn get_rows_from_output(output: &str) -> String { .to_string() } +fn get_output_row_count(output: &str) -> usize { + let resp: Value = serde_json::from_str(output).unwrap(); + resp.get("output") + .and_then(Value::as_array) + .and_then(|v| v.first()) + .and_then(|v| v.get("records")) + .and_then(|v| v.get("rows")) + .and_then(Value::as_array) + .unwrap() + .len() +} + fn compress_vec_with_gzip(data: Vec) -> Vec { let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); encoder.write_all(&data).unwrap(); diff --git a/tests-integration/tests/repartition.rs b/tests-integration/tests/repartition.rs index 2adec87edcb..32f80a32cba 100644 --- a/tests-integration/tests/repartition.rs +++ b/tests-integration/tests/repartition.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::BTreeSet; use std::sync::Arc; use std::time::Duration; @@ -33,7 +34,7 @@ use servers::query_handler::sql::SqlQueryHandler; use session::context::{QueryContext, QueryContextRef}; use store_api::codec::PrimaryKeyEncoding; use store_api::storage::RegionId; -use tests_integration::cluster::GreptimeDbClusterBuilder; +use tests_integration::cluster::{GreptimeDbCluster, GreptimeDbClusterBuilder}; use tests_integration::test_util::{StorageType, get_test_store_config}; use tokio::sync::oneshot; @@ -115,6 +116,217 @@ macro_rules! repartition_tests { }; } +#[tokio::test(flavor = "multi_thread")] +async fn test_repartition_multi_hop_fulltext_index_gc_file() { + if StorageType::File.test_on() { + common_telemetry::init_default_ut_logging(); + test_repartition_multi_hop_fulltext_index_gc().await; + } +} + +async fn test_repartition_multi_hop_fulltext_index_gc() { + let store_type = StorageType::File; + let cluster_name = "test_repartition_multi_hop_fulltext_index_gc"; + let table_name = "repartition_multi_hop_index_table"; + let (store_config, _guard) = get_test_store_config(&store_type); + let datanodes = 3u64; + let home_dir = create_temp_dir("test_repartition_multi_hop_fulltext_index_gc_data_home"); + + let cluster = GreptimeDbClusterBuilder::new(cluster_name) + .await + .with_shared_home_dir(Arc::new(home_dir)) + .with_datanodes(datanodes as u32) + .with_store_config(store_config) + .with_datanode_wal_config(DatanodeWalConfig::Noop) + .with_metasrv_gc_config(GcSchedulerOptions { + enable: true, + gc_cooldown_period: Duration::from_nanos(1), + ..Default::default() + }) + .with_datanode_gc_config(GcConfig { + enable: true, + lingering_time: Some(Duration::from_secs(0)), + unknown_file_lingering_time: Duration::from_secs(0), + ..Default::default() + }) + .build(true) + .await; + let metasrv = &cluster.metasrv; + let ticker = metasrv.gc_ticker().unwrap(); + + let query_ctx = QueryContext::arc(); + let instance = cluster.fe_instance(); + + run_sql( + instance, + r#" + CREATE TABLE `repartition_multi_hop_index_table`( + `id` INT, + `msg` STRING FULLTEXT INDEX, + `ts` TIMESTAMP TIME INDEX, + PRIMARY KEY(`id`) + ) PARTITION ON COLUMNS (`id`) ( + `id` < 100, + `id` >= 100 + ) ENGINE = mito + WITH ('sst_format' = 'flat'); + "#, + query_ctx.clone(), + ) + .await + .unwrap(); + + run_sql( + instance, + r#" + INSERT INTO `repartition_multi_hop_index_table` VALUES + (10, 'quick fox in low partition', '2022-01-01 00:00:00'), + (60, 'quick fox in middle partition', '2022-01-01 00:00:01'), + (90, 'quick fox in high partition', '2022-01-01 00:00:02'), + (120, 'quick fox in outer partition', '2022-01-01 00:00:03'); + "#, + query_ctx.clone(), + ) + .await + .unwrap(); + run_sql( + instance, + "ADMIN FLUSH_TABLE('repartition_multi_hop_index_table')", + query_ctx.clone(), + ) + .await + .unwrap(); + + run_sql( + instance, + r#" + ALTER TABLE `repartition_multi_hop_index_table` SPLIT PARTITION ( + `id` < 100 + ) INTO ( + `id` < 50, + `id` >= 50 AND `id` < 100 + ); + "#, + query_ctx.clone(), + ) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(500)).await; + + run_sql( + instance, + r#" + ALTER TABLE `repartition_multi_hop_index_table` SPLIT PARTITION ( + `id` >= 50 AND `id` < 100 + ) INTO ( + `id` >= 50 AND `id` < 75, + `id` >= 75 AND `id` < 100 + ); + "#, + query_ctx.clone(), + ) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(500)).await; + + run_sql( + instance, + r#" + INSERT INTO `repartition_multi_hop_index_table` VALUES + (80, 'fresh fox in soon dropped source', '2022-01-01 00:00:04'); + "#, + query_ctx.clone(), + ) + .await + .unwrap(); + run_sql( + instance, + "ADMIN FLUSH_TABLE('repartition_multi_hop_index_table')", + query_ctx.clone(), + ) + .await + .unwrap(); + + let fox_query = "SELECT id, msg FROM `repartition_multi_hop_index_table` WHERE MATCHES(msg, 'fox') ORDER BY id"; + let expected = "\ ++-----+----------------------------------+ +| id | msg | ++-----+----------------------------------+ +| 10 | quick fox in low partition | +| 60 | quick fox in middle partition | +| 80 | fresh fox in soon dropped source | +| 90 | quick fox in high partition | +| 120 | quick fox in outer partition | ++-----+----------------------------------+"; + let result = run_sql(instance, fox_query, query_ctx.clone()) + .await + .unwrap(); + check_output_stream(result.data, expected).await; + + let manifest_entries = cluster_manifest_index_entries(&cluster).await; + assert!( + manifest_entries + .iter() + .any(|(_, origin_region_id, region_id)| origin_region_id != region_id), + "expected at least one cross-origin indexed SST before GC, got {manifest_entries:?}" + ); + + run_sql( + instance, + r#" + ALTER TABLE `repartition_multi_hop_index_table` MERGE PARTITION ( + `id` >= 50 AND `id` < 75, + `id` >= 75 AND `id` < 100 + ); + "#, + query_ctx.clone(), + ) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(500)).await; + + trigger_full_gc(&ticker).await; + trigger_table_gc(metasrv, table_name).await; + + let result = run_sql(instance, fox_query, query_ctx.clone()) + .await + .unwrap(); + check_output_stream(result.data, expected).await; + + let manifest_index_paths = cluster_manifest_index_paths(&cluster).await; + assert!( + !manifest_index_paths.is_empty(), + "expected manifest index paths after GC" + ); + let storage_paths = cluster.list_sst_files_from_all_datanodes().await; + assert!( + manifest_index_paths.is_subset(&storage_paths), + "manifest index paths should exist in storage after GC, missing: {:?}", + manifest_index_paths + .difference(&storage_paths) + .collect::>() + ); + + let cross_origin_index_paths = cluster_manifest_index_entries(&cluster) + .await + .into_iter() + .filter_map(|(index_file_path, origin_region_id, region_id)| { + (origin_region_id != region_id).then_some(index_file_path) + }) + .collect::>(); + assert!( + !cross_origin_index_paths.is_empty(), + "expected cross-origin indexed SSTs after GC" + ); + assert!( + cross_origin_index_paths.is_subset(&storage_paths), + "cross-origin manifest index paths should exist in storage after GC, missing: {:?}", + cross_origin_index_paths + .difference(&storage_paths) + .collect::>() + ); +} + pub async fn test_partition_unpartitioned_mito(store_type: StorageType) { info!( "test_partition_unpartitioned_mito: store_type: {:?}", @@ -583,6 +795,35 @@ async fn trigger_full_gc(ticker: &GcTickerRef) { let _ = rx.await.unwrap().unwrap(); } +async fn cluster_manifest_index_entries( + cluster: &GreptimeDbCluster, +) -> Vec<(String, RegionId, RegionId)> { + let mut entries = Vec::new(); + for datanode in cluster.datanode_instances.values() { + let region_server = datanode.region_server(); + let mito = region_server.mito_engine().unwrap(); + entries.extend( + mito.all_ssts_from_manifest() + .await + .into_iter() + .filter_map(|entry| { + entry.index_file_path.map(|index_file_path| { + (index_file_path, entry.origin_region_id, entry.region_id) + }) + }), + ); + } + entries +} + +async fn cluster_manifest_index_paths(cluster: &GreptimeDbCluster) -> BTreeSet { + cluster_manifest_index_entries(cluster) + .await + .into_iter() + .map(|(index_file_path, _, _)| index_file_path) + .collect() +} + fn query_partitions_sql(table_name: &str) -> String { // We query information_schema.partitions to assert repartition results across engines, // rather than relying on SHOW CREATE TABLE formatting differences. diff --git a/tests-integration/tests/sql.rs b/tests-integration/tests/sql.rs index 156de7c415c..0e477061bb7 100644 --- a/tests-integration/tests/sql.rs +++ b/tests-integration/tests/sql.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::HashMap; +use std::str::FromStr; use std::time::Duration; use auth::user_provider_from_option; @@ -1353,6 +1354,68 @@ pub async fn test_postgres_uint64_parameter(store_type: StorageType) { let count: i64 = rows[0].get(0); assert_eq!(count, 1); + let scalar_statement = client + .prepare("SELECT arrow_cast($1, 'UInt64')") + .await + .unwrap(); + let scalar_null_statement = client + .prepare("SELECT arrow_cast($1, 'UInt64') IS NULL") + .await + .unwrap(); + let array_statement = client + .prepare("SELECT arrow_cast($1, 'List(UInt64)')") + .await + .unwrap(); + let array_null_statement = client + .prepare("SELECT arrow_cast($1, 'List(UInt64)') IS NULL") + .await + .unwrap(); + + let ordinary = Decimal::from(42); + let max = Decimal::from_str("18446744073709551615").unwrap(); + for value in [&ordinary, &max] { + let row = client.query_one(&scalar_statement, &[value]).await.unwrap(); + assert_eq!(*value, row.get::<_, Decimal>(0)); + } + + let scalar_null: Option = None; + let row = client + .query_one(&scalar_null_statement, &[&scalar_null]) + .await + .unwrap(); + assert!(row.get::<_, bool>(0)); + + let array_values = vec![Some(ordinary), None, Some(max)]; + let row = client + .query_one(&array_statement, &[&array_values]) + .await + .unwrap(); + assert_eq!(array_values, row.get::<_, Vec>>(0)); + + let array_null: Option>> = None; + let row = client + .query_one(&array_null_statement, &[&array_null]) + .await + .unwrap(); + assert!(row.get::<_, bool>(0)); + + for value in [ + Decimal::from(-1), + Decimal::from_str("18446744073709551616").unwrap(), + ] { + let error = client + .query(&scalar_statement, &[&value]) + .await + .unwrap_err(); + assert_pg_numeric_range_error(error); + + let error = client + .query(&array_statement, &[&vec![Some(value)]]) + .await + .unwrap_err(); + assert_pg_numeric_range_error(error); + } + drop(client); rx.await.unwrap(); @@ -1360,6 +1423,12 @@ pub async fn test_postgres_uint64_parameter(store_type: StorageType) { guard.remove_all().await; } +fn assert_pg_numeric_range_error(error: tokio_postgres::Error) { + let error = error.as_db_error().expect("expected PostgreSQL user error"); + assert_eq!("22023", error.code().code()); + assert_eq!("numeric_value_out_of_range", error.message()); +} + pub async fn test_postgres_explain_bind_parameter(store_type: StorageType) { // Regression test for #8029: EXPLAIN / EXPLAIN ANALYZE must accept bind // parameters over the Postgres extended query protocol. diff --git a/tests/cases/distributed/explain/multi_partitions.result b/tests/cases/distributed/explain/multi_partitions.result index 0c3d4fa4b70..47d807542d8 100644 --- a/tests/cases/distributed/explain/multi_partitions.result +++ b/tests/cases/distributed/explain/multi_partitions.result @@ -30,7 +30,7 @@ explain SELECT * FROM multi_partitions_test_table WHERE ts > cast(1000000000 as |_|_Filter: multi_partitions_test_table.ts > TimestampMillisecond(1000000000, None)_| |_|_TableScan: multi_partitions_test_table, partial_filters=[multi_partitions_test_table.ts > TimestampMillisecond(1000000000, None)]_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [host@0 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [host@0 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ diff --git a/tests/cases/distributed/explain/step_aggr_advance.result b/tests/cases/distributed/explain/step_aggr_advance.result index c2304df5eb9..7ea86dbccbf 100644 --- a/tests/cases/distributed/explain/step_aggr_advance.result +++ b/tests/cases/distributed/explain/step_aggr_advance.result @@ -37,7 +37,7 @@ tql explain (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr | | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) | | | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] | | | ]] | -| physical_plan | SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] | +| physical_plan | MergeSortExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] | | | MergeScanExec: REDACTED | | | +---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -54,7 +54,7 @@ tql analyze (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED +| 0_| 0_|_MergeSortExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED @@ -244,7 +244,7 @@ tql explain (1752591864, 1752592164, '30s') count by (a, b, c, d) (max_over_time | | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) | | | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] | | | ]] | -| physical_plan | SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, d@3 ASC NULLS LAST, greptime_timestamp@4 ASC NULLS LAST] | +| physical_plan | MergeSortExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, d@3 ASC NULLS LAST, greptime_timestamp@4 ASC NULLS LAST] | | | MergeScanExec: REDACTED | | | +---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -261,7 +261,7 @@ tql analyze (1752591864, 1752592164, '30s') count by (a, b, c, d) (max_over_time +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, d@3 ASC NULLS LAST, greptime_timestamp@4 ASC NULLS LAST] REDACTED +| 0_| 0_|_MergeSortExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, d@3 ASC NULLS LAST, greptime_timestamp@4 ASC NULLS LAST] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, d@3 ASC NULLS LAST, greptime_timestamp@4 ASC NULLS LAST] REDACTED @@ -485,9 +485,10 @@ tql explain (1752591864, 1752592164, '30s') sum by (a, b, c) (rate(aggr_optimize | | PromSeriesDivideExec: tags=["a", "b", "c", "d"] | | | SortExec: expr=[a@0 ASC, b@1 ASC, c@2 ASC, d@3 ASC, greptime_timestamp@4 ASC], preserve_partitioning=[true] | | | MergeScanExec: REDACTED -| | SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] | -| | CooperativeExec | -| | MergeScanExec: REDACTED +| | RepartitionExec: partitioning=REDACTED +| | MergeSortExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] | +| | SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] | +| | MergeScanExec: REDACTED | | | +---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -514,8 +515,9 @@ tql analyze (1752591864, 1752592164, '30s') sum by (a, b, c) (rate(aggr_optimize |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED |_|_|_SortExec: expr=[a@0 ASC, b@1 ASC, c@2 ASC, d@3 ASC, greptime_timestamp@4 ASC], preserve_partitioning=[true] REDACTED |_|_|_MergeScanExec: REDACTED +|_|_|_RepartitionExec: partitioning=REDACTED +|_|_|_MergeSortExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_CooperativeExec REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_CooperativeExec REDACTED @@ -974,7 +976,7 @@ EXPLAIN SELECT pk_col_1, pk_col_2, sum(val_col_1) FROM step_aggr_extended GROUP |_|_Aggregate: groupBy=[[step_aggr_extended.pk_col_1, step_aggr_extended.pk_col_2]], aggr=[[sum(step_aggr_extended.val_col_1)]] | |_|_TableScan: step_aggr_extended_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [pk_col_1@0 ASC NULLS LAST, pk_col_2@1 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [pk_col_1@0 ASC NULLS LAST, pk_col_2@1 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ diff --git a/tests/cases/distributed/explain/subqueries.result b/tests/cases/distributed/explain/subqueries.result index cabc20f0a6c..e6f7286535e 100644 --- a/tests/cases/distributed/explain/subqueries.result +++ b/tests/cases/distributed/explain/subqueries.result @@ -91,7 +91,7 @@ order by t.i desc; |_|_Cross Join:_| |_|_Projection: integers.i_| |_|_MergeScan [is_placeholder=false, remote_input=[_| -|_| Filter: integers.i IS NOT NULL AND Boolean(true)_| +|_| Filter: integers.i IS NOT NULL_| |_|_TableScan: integers, partial_filters=[integers.i IS NOT NULL, Boolean(true)] | |_| ]]_| |_|_Projection:_| @@ -287,7 +287,7 @@ EXPLAIN SELECT x FROM (SELECT a AS x FROM t) sq ORDER BY x; |_|_Projection: t.a AS x_| |_|_TableScan: t_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [x@0 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [x@0 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -326,7 +326,7 @@ EXPLAIN SELECT x, COUNT(*) AS c FROM (SELECT a AS x FROM t) sq GROUP BY x ORDER |_|_Projection: t.a AS x_| |_|_TableScan: t_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [x@0 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [x@0 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -397,7 +397,7 @@ EXPLAIN SELECT sq.x FROM (SELECT a AS x FROM t) sq ORDER BY sq.x; |_|_Projection: t.a AS x_| |_|_TableScan: t_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [x@0 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [x@0 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -433,7 +433,7 @@ EXPLAIN SELECT y FROM (SELECT x AS y FROM (SELECT a AS x FROM t) sq1) sq2 ORDER |_|_Projection: t.a AS x_| |_|_TableScan: t_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [y@0 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [y@0 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -471,7 +471,7 @@ EXPLAIN SELECT x, x + 1 AS y FROM (SELECT a AS x FROM t) sq ORDER BY x; |_|_Projection: t.a AS x_| |_|_TableScan: t_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [x@0 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [x@0 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -625,7 +625,7 @@ EXPLAIN SELECT x FROM (SELECT a AS x FROM t) sq ORDER BY x LIMIT 2; |_|_Projection: t.a AS x_| |_|_TableScan: t_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [x@0 ASC NULLS LAST], fetch=2_| +| physical_plan | MergeSortExec: [x@0 ASC NULLS LAST], fetch=2_| |_|_SortExec: TopK(fetch=2), expr=[x@0 ASC NULLS LAST], preserve_partitioning=[true]_| |_|_MergeScanExec: REDACTED |_|_| diff --git a/tests/cases/distributed/information_schema/ssts_limit.result b/tests/cases/distributed/information_schema/ssts_limit.result new file mode 100644 index 00000000000..98a19842995 --- /dev/null +++ b/tests/cases/distributed/information_schema/ssts_limit.result @@ -0,0 +1,69 @@ +CREATE TABLE ssts_limit_case ( + a INT PRIMARY KEY INVERTED INDEX, + b STRING SKIPPING INDEX, + c STRING FULLTEXT INDEX, + ts TIMESTAMP TIME INDEX, +) +PARTITION ON COLUMNS (a) ( + a < 1000, + a >= 1000 AND a < 2000, + a >= 2000 +); + +Affected Rows: 0 + +INSERT INTO ssts_limit_case VALUES + (500, 'a', 'a', 1), + (1500, 'b', 'b', 2), + (2500, 'c', 'c', 3); + +Affected Rows: 3 + +ADMIN FLUSH_TABLE('ssts_limit_case'); + ++--------------------------------------+ +| ADMIN FLUSH_TABLE('ssts_limit_case') | ++--------------------------------------+ +| 0 | ++--------------------------------------+ + +SELECT COUNT(DISTINCT node_id) > 1 AS has_multi_datanodes +FROM information_schema.ssts_manifest; + ++---------------------+ +| has_multi_datanodes | ++---------------------+ +| true | ++---------------------+ + +SELECT COUNT(*) AS limited_rows +FROM ( + SELECT region_id + FROM information_schema.ssts_manifest + LIMIT 1 +); + ++--------------+ +| limited_rows | ++--------------+ +| 1 | ++--------------+ + +SELECT COUNT(*) AS filtered_limited_rows +FROM ( + SELECT region_id + FROM information_schema.ssts_manifest + WHERE region_id > 0 + LIMIT 1 +); + ++-----------------------+ +| filtered_limited_rows | ++-----------------------+ +| 1 | ++-----------------------+ + +DROP TABLE ssts_limit_case; + +Affected Rows: 0 + diff --git a/tests/cases/distributed/information_schema/ssts_limit.sql b/tests/cases/distributed/information_schema/ssts_limit.sql new file mode 100644 index 00000000000..c02f31d63d7 --- /dev/null +++ b/tests/cases/distributed/information_schema/ssts_limit.sql @@ -0,0 +1,38 @@ +CREATE TABLE ssts_limit_case ( + a INT PRIMARY KEY INVERTED INDEX, + b STRING SKIPPING INDEX, + c STRING FULLTEXT INDEX, + ts TIMESTAMP TIME INDEX, +) +PARTITION ON COLUMNS (a) ( + a < 1000, + a >= 1000 AND a < 2000, + a >= 2000 +); + +INSERT INTO ssts_limit_case VALUES + (500, 'a', 'a', 1), + (1500, 'b', 'b', 2), + (2500, 'c', 'c', 3); + +ADMIN FLUSH_TABLE('ssts_limit_case'); + +SELECT COUNT(DISTINCT node_id) > 1 AS has_multi_datanodes +FROM information_schema.ssts_manifest; + +SELECT COUNT(*) AS limited_rows +FROM ( + SELECT region_id + FROM information_schema.ssts_manifest + LIMIT 1 +); + +SELECT COUNT(*) AS filtered_limited_rows +FROM ( + SELECT region_id + FROM information_schema.ssts_manifest + WHERE region_id > 0 + LIMIT 1 +); + +DROP TABLE ssts_limit_case; diff --git a/tests/cases/distributed/optimizer/first_value_advance.result b/tests/cases/distributed/optimizer/first_value_advance.result index ac977b6fdfb..08e1b8a4112 100644 --- a/tests/cases/distributed/optimizer/first_value_advance.result +++ b/tests/cases/distributed/optimizer/first_value_advance.result @@ -240,7 +240,7 @@ order by ordered_host; |_|_Aggregate: groupBy=[[t.host]], aggr=[[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]]] | |_|_TableScan: t_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [ordered_host@0 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -267,7 +267,7 @@ order by ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED +| 0_| 0_|_MergeSortExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_ProjectionExec: expr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@0 as ordered_host, first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST]@1 as first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED @@ -606,7 +606,7 @@ order by ordered_host; |_|_Aggregate: groupBy=[[t1.host]], aggr=[[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]]] | |_|_TableScan: t1_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [ordered_host@0 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -632,7 +632,7 @@ order by ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED +| 0_| 0_|_MergeSortExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_ProjectionExec: expr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@0 as ordered_host, first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@1 as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED diff --git a/tests/cases/distributed/optimizer/last_value_advance.result b/tests/cases/distributed/optimizer/last_value_advance.result index d01ea184300..7b131d44f37 100644 --- a/tests/cases/distributed/optimizer/last_value_advance.result +++ b/tests/cases/distributed/optimizer/last_value_advance.result @@ -240,7 +240,7 @@ order by ordered_host; |_|_Aggregate: groupBy=[[t.host]], aggr=[[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]]] | |_|_TableScan: t_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [ordered_host@0 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -267,7 +267,7 @@ order by ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED +| 0_| 0_|_MergeSortExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_ProjectionExec: expr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@0 as ordered_host, last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST]@1 as last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED @@ -606,7 +606,7 @@ order by ordered_host; |_|_Aggregate: groupBy=[[t1.host]], aggr=[[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]]] | |_|_TableScan: t1_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [ordered_host@0 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -632,7 +632,7 @@ order by ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED +| 0_| 0_|_MergeSortExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_ProjectionExec: expr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@0 as ordered_host, last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@1 as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED diff --git a/tests/cases/distributed/optimizer/pass_distribution_partition_subset_join.result b/tests/cases/distributed/optimizer/pass_distribution_partition_subset_join.result new file mode 100644 index 00000000000..91c0e3e851d --- /dev/null +++ b/tests/cases/distributed/optimizer/pass_distribution_partition_subset_join.result @@ -0,0 +1,118 @@ +CREATE TABLE pd_sub_l( + ts TIMESTAMP TIME INDEX, + device_id INTEGER, + area STRING, + lv INTEGER, + PRIMARY KEY(device_id, area) +) PARTITION ON COLUMNS (device_id, area) ( + device_id < 2 AND area < 'm', + device_id < 2 AND area >= 'm', + device_id >= 2 +) ENGINE=mito; + +Affected Rows: 0 + +CREATE TABLE pd_sub_r( + ts TIMESTAMP TIME INDEX, + device_id INTEGER, + area STRING, + rv INTEGER, + PRIMARY KEY(device_id, area) +) PARTITION ON COLUMNS (device_id, area) ( + device_id < 2 AND area < 'm', + device_id < 2 AND area >= 'm', + device_id >= 2 +) ENGINE=mito; + +Affected Rows: 0 + +INSERT INTO pd_sub_l VALUES + (1, 1, 'a', 10), + (2, 1, 'z', 20); + +Affected Rows: 2 + +INSERT INTO pd_sub_r VALUES + (1, 1, 'a', 100), + (2, 1, 'z', 200); + +Affected Rows: 2 + +ADMIN FLUSH_TABLE('pd_sub_l'); + ++-------------------------------+ +| ADMIN FLUSH_TABLE('pd_sub_l') | ++-------------------------------+ +| 0 | ++-------------------------------+ + +ADMIN FLUSH_TABLE('pd_sub_r'); + ++-------------------------------+ +| ADMIN FLUSH_TABLE('pd_sub_r') | ++-------------------------------+ +| 0 | ++-------------------------------+ + +-- Joining on only a subset of a multi-column partition key must still compare +-- rows across all matching partitions for that subset key. +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \d+\(\d+,\s+\d+\) REDACTED_REGION +-- SQLNESS REPLACE Hash\(\[device_id@0\],\s+\d+\) Hash([device_id@0],REDACTED_PARTITIONS) +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE input_partitions=REDACTED\s+\| input_partitions=REDACTED| +EXPLAIN SELECT l.area, r.area +FROM pd_sub_l l JOIN pd_sub_r r ON l.device_id = r.device_id +ORDER BY l.area, r.area; + ++---------------+-------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-------------------------------------------------------------------------------------------------------------------+ +| logical_plan | Sort: l.area ASC NULLS LAST, r.area ASC NULLS LAST | +| | Projection: l.area, r.area | +| | Inner Join: l.device_id = r.device_id | +| | Projection: l.device_id, l.area | +| | MergeScan [is_placeholder=false, remote_input=[ | +| | SubqueryAlias: l | +| | Filter: pd_sub_l.device_id IS NOT NULL | +| | TableScan: pd_sub_l, partial_filters=[pd_sub_l.device_id IS NOT NULL] | +| | ]] | +| | Projection: r.device_id, r.area | +| | MergeScan [is_placeholder=false, remote_input=[ | +| | SubqueryAlias: r | +| | Filter: pd_sub_r.device_id IS NOT NULL | +| | TableScan: pd_sub_r, partial_filters=[pd_sub_r.device_id IS NOT NULL] | +| | ]] | +| physical_plan | SortPreservingMergeExec: [area@0 ASC NULLS LAST, area@1 ASC NULLS LAST] | +| | SortExec: expr=[area@0 ASC NULLS LAST, area@1 ASC NULLS LAST], preserve_partitioning=[true] | +| | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(device_id@0, device_id@0)], projection=[area@1, area@3] | +| | RepartitionExec: partitioning=Hash([device_id@0],REDACTED_PARTITIONS), input_partitions=REDACTED| +| | ProjectionExec: expr=[device_id@1 as device_id, area@2 as area] | +| | MergeScanExec: peers=[REDACTED_REGION, REDACTED_REGION, REDACTED_REGION, ] | +| | RepartitionExec: partitioning=Hash([device_id@0],REDACTED_PARTITIONS), input_partitions=REDACTED| +| | ProjectionExec: expr=[device_id@1 as device_id, area@2 as area] | +| | MergeScanExec: peers=[REDACTED_REGION, REDACTED_REGION, REDACTED_REGION, ] | +| | | ++---------------+-------------------------------------------------------------------------------------------------------------------+ + +SELECT l.area, r.area +FROM pd_sub_l l JOIN pd_sub_r r ON l.device_id = r.device_id +ORDER BY l.area, r.area; + ++------+------+ +| area | area | ++------+------+ +| a | a | +| a | z | +| z | a | +| z | z | ++------+------+ + +DROP TABLE pd_sub_l; + +Affected Rows: 0 + +DROP TABLE pd_sub_r; + +Affected Rows: 0 + diff --git a/tests/cases/distributed/optimizer/pass_distribution_partition_subset_join.sql b/tests/cases/distributed/optimizer/pass_distribution_partition_subset_join.sql new file mode 100644 index 00000000000..41337e74a0f --- /dev/null +++ b/tests/cases/distributed/optimizer/pass_distribution_partition_subset_join.sql @@ -0,0 +1,52 @@ +CREATE TABLE pd_sub_l( + ts TIMESTAMP TIME INDEX, + device_id INTEGER, + area STRING, + lv INTEGER, + PRIMARY KEY(device_id, area) +) PARTITION ON COLUMNS (device_id, area) ( + device_id < 2 AND area < 'm', + device_id < 2 AND area >= 'm', + device_id >= 2 +) ENGINE=mito; + +CREATE TABLE pd_sub_r( + ts TIMESTAMP TIME INDEX, + device_id INTEGER, + area STRING, + rv INTEGER, + PRIMARY KEY(device_id, area) +) PARTITION ON COLUMNS (device_id, area) ( + device_id < 2 AND area < 'm', + device_id < 2 AND area >= 'm', + device_id >= 2 +) ENGINE=mito; + +INSERT INTO pd_sub_l VALUES + (1, 1, 'a', 10), + (2, 1, 'z', 20); + +INSERT INTO pd_sub_r VALUES + (1, 1, 'a', 100), + (2, 1, 'z', 200); + +ADMIN FLUSH_TABLE('pd_sub_l'); +ADMIN FLUSH_TABLE('pd_sub_r'); + +-- Joining on only a subset of a multi-column partition key must still compare +-- rows across all matching partitions for that subset key. +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \d+\(\d+,\s+\d+\) REDACTED_REGION +-- SQLNESS REPLACE Hash\(\[device_id@0\],\s+\d+\) Hash([device_id@0],REDACTED_PARTITIONS) +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE input_partitions=REDACTED\s+\| input_partitions=REDACTED| +EXPLAIN SELECT l.area, r.area +FROM pd_sub_l l JOIN pd_sub_r r ON l.device_id = r.device_id +ORDER BY l.area, r.area; + +SELECT l.area, r.area +FROM pd_sub_l l JOIN pd_sub_r r ON l.device_id = r.device_id +ORDER BY l.area, r.area; + +DROP TABLE pd_sub_l; +DROP TABLE pd_sub_r; diff --git a/tests/cases/distributed/optimizer/time_index_filter_pushdown.result b/tests/cases/distributed/optimizer/time_index_filter_pushdown.result index 22e96793747..016c49deea0 100644 --- a/tests/cases/distributed/optimizer/time_index_filter_pushdown.result +++ b/tests/cases/distributed/optimizer/time_index_filter_pushdown.result @@ -84,6 +84,82 @@ group by | windows | 2 | +---------+----------+ +-- Query the row at the latest timestamp through a scalar subquery. +-- Keep the distributed merge sort visible, but redact the optional local SortExec +-- that appears when query parallelism is lower than the region count. +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (?m)^\|_\|_SortExec:.*\n +EXPLAIN SELECT + rack, + os, + greptime_timestamp +FROM + cpu +WHERE + greptime_timestamp = ( + SELECT + greptime_timestamp + FROM + cpu + ORDER BY + greptime_timestamp DESC + LIMIT + 1 + ); + ++-+-+ +| plan_type_| plan_| ++-+-+ +| logical_plan_| Projection: cpu.rack, cpu.os, cpu.greptime_timestamp_| +|_|_Inner Join: cpu.greptime_timestamp = __scalar_sq_1.greptime_timestamp_| +|_|_Projection: cpu.rack, cpu.os, cpu.greptime_timestamp_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| TableScan: cpu_| +|_| ]]_| +|_|_SubqueryAlias: __scalar_sq_1_| +|_|_Limit: skip=0, fetch=1_| +|_|_MergeSort: cpu.greptime_timestamp DESC NULLS FIRST_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| Limit: skip=0, fetch=1_| +|_|_Sort: cpu.greptime_timestamp DESC NULLS FIRST_| +|_|_Projection: cpu.greptime_timestamp_| +|_|_TableScan: cpu_| +|_| ]]_| +| physical_plan | HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(greptime_timestamp@0, greptime_timestamp@2)], projection=[rack@1, os@2, greptime_timestamp@3]_| +|_|_MergeSortExec: [greptime_timestamp@0 DESC], fetch=1_| +|_|_MergeScanExec: REDACTED +|_|_ProjectionExec: expr=[rack@0 as rack, os@1 as os, greptime_timestamp@3 as greptime_timestamp]_| +|_|_CooperativeExec_| +|_|_MergeScanExec: REDACTED +|_|_| ++-+-+ + +SELECT + rack, + os, + greptime_timestamp +FROM + cpu +WHERE + greptime_timestamp = ( + SELECT + greptime_timestamp + FROM + cpu + ORDER BY + greptime_timestamp DESC + LIMIT + 1 + ); + ++------+---------+---------------------+ +| rack | os | greptime_timestamp | ++------+---------+---------------------+ +| 8 | windows | 2023-06-12T02:10:00 | ++------+---------+---------------------+ + drop table cpu; Affected Rows: 0 diff --git a/tests/cases/distributed/optimizer/time_index_filter_pushdown.sql b/tests/cases/distributed/optimizer/time_index_filter_pushdown.sql index f5fa24efe77..c9c80c24a61 100644 --- a/tests/cases/distributed/optimizer/time_index_filter_pushdown.sql +++ b/tests/cases/distributed/optimizer/time_index_filter_pushdown.sql @@ -59,4 +59,47 @@ where group by os; -drop table cpu; \ No newline at end of file +-- Query the row at the latest timestamp through a scalar subquery. +-- Keep the distributed merge sort visible, but redact the optional local SortExec +-- that appears when query parallelism is lower than the region count. +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (?m)^\|_\|_SortExec:.*\n +EXPLAIN SELECT + rack, + os, + greptime_timestamp +FROM + cpu +WHERE + greptime_timestamp = ( + SELECT + greptime_timestamp + FROM + cpu + ORDER BY + greptime_timestamp DESC + LIMIT + 1 + ); + +SELECT + rack, + os, + greptime_timestamp +FROM + cpu +WHERE + greptime_timestamp = ( + SELECT + greptime_timestamp + FROM + cpu + ORDER BY + greptime_timestamp DESC + LIMIT + 1 + ); + +drop table cpu; diff --git a/tests/cases/standalone/common/filter/cast_preimage.result b/tests/cases/standalone/common/filter/cast_preimage.result new file mode 100644 index 00000000000..3209a842e04 --- /dev/null +++ b/tests/cases/standalone/common/filter/cast_preimage.result @@ -0,0 +1,394 @@ +-- SQL cast predicate preimage / const-normalization filter pushdown. +-- This is deliberately plain SQL, not PromQL/TQL, so it covers the generic +-- planner path where cast predicates should become scan-local filters. +CREATE TABLE cast_preimage_ts ( + host STRING PRIMARY KEY, + ts TIMESTAMP(9) TIME INDEX, + v INTEGER, +); + +Affected Rows: 0 + +INSERT INTO cast_preimage_ts VALUES + ('host1', 0, 1), + ('host1', 5000000000, 2), + ('host1', 10000000000, 3), + ('host2', 15000000000, 4); + +Affected Rows: 4 + +-- Timestamp downcast comparison: +-- CAST(ts_ns AS TIMESTAMP(3)) >= TIMESTAMP(3) '1970-01-01 00:00:05' +-- should be pushed as a native nanosecond bound on the scan. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE CAST(ts AS TIMESTAMP(3)) >= '1970-01-01 00:00:05'::TIMESTAMP(3) +ORDER BY host, v; + ++---------------+-------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_ts.host ASC NULLS LAST, cast_preimage_ts.v ASC NULLS LAST | +| | Projection: cast_preimage_ts.host, cast_preimage_ts.v | +| | Filter: cast_preimage_ts.ts >= TimestampNanosecond(5000000000, None) | +| | TableScan: cast_preimage_ts, partial_filters=[cast_preimage_ts.ts >= TimestampNanosecond(5000000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-------------------------------------------------------------------------------------------------------------------+ + +-- Timestamp downcast BETWEEN should be lowered through DataFusion's BETWEEN +-- simplification and cast preimage, not left as a CAST predicate in scan filters. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE CAST(ts AS TIMESTAMP(3)) BETWEEN '1970-01-01 00:00:00'::TIMESTAMP(3) + AND '1970-01-01 00:00:10'::TIMESTAMP(3) +ORDER BY host, v; + ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_ts.host ASC NULLS LAST, cast_preimage_ts.v ASC NULLS LAST | +| | Projection: cast_preimage_ts.host, cast_preimage_ts.v | +| | Filter: cast_preimage_ts.ts >= TimestampNanosecond(-999999, None) AND cast_preimage_ts.ts < TimestampNanosecond(10001000000, None) | +| | TableScan: cast_preimage_ts, partial_filters=[cast_preimage_ts.ts >= TimestampNanosecond(-999999, None), cast_preimage_ts.ts < TimestampNanosecond(10001000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- Timestamp downcast equality should become a native half-open nanosecond range. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE CAST(ts AS TIMESTAMP(3)) = '1970-01-01 00:00:05'::TIMESTAMP(3) +ORDER BY host, v; + ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_ts.host ASC NULLS LAST, cast_preimage_ts.v ASC NULLS LAST | +| | Projection: cast_preimage_ts.host, cast_preimage_ts.v | +| | Filter: cast_preimage_ts.ts >= TimestampNanosecond(5000000000, None) AND cast_preimage_ts.ts < TimestampNanosecond(5001000000, None) | +| | TableScan: cast_preimage_ts, partial_filters=[cast_preimage_ts.ts >= TimestampNanosecond(5000000000, None), cast_preimage_ts.ts < TimestampNanosecond(5001000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- Literal-left timestamp inequality should normalize before cast preimage. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE '1970-01-01 00:00:05'::TIMESTAMP(3) < CAST(ts AS TIMESTAMP(3)) +ORDER BY host, v; + ++---------------+-------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_ts.host ASC NULLS LAST, cast_preimage_ts.v ASC NULLS LAST | +| | Projection: cast_preimage_ts.host, cast_preimage_ts.v | +| | Filter: cast_preimage_ts.ts >= TimestampNanosecond(5001000000, None) | +| | TableScan: cast_preimage_ts, partial_filters=[cast_preimage_ts.ts >= TimestampNanosecond(5001000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-------------------------------------------------------------------------------------------------------------------+ + +-- Integer widening comparison and IN-list cover the overlap with the old +-- Greptime ConstNormalizationRule's lossless-cast cases. +CREATE TABLE cast_preimage_int ( + ts TIMESTAMP TIME INDEX, + host STRING PRIMARY KEY, + v SMALLINT, +); + +Affected Rows: 0 + +INSERT INTO cast_preimage_int VALUES + (0, 'host1', 1), + (1000, 'host2', 2), + (2000, 'host3', 3); + +Affected Rows: 3 + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) >= 2 +ORDER BY host; + ++---------------+---------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v >= Int16(2) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v >= Int16(2)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------+ + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) IN (1, 3) +ORDER BY host; + ++---------------+------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v = Int16(1) OR cast_preimage_int.v = Int16(3) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v = Int16(1) OR cast_preimage_int.v = Int16(3)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+------------------------------------------------------------------------------------------------------------------------+ + +-- Swapped comparison and BETWEEN/NOT BETWEEN cover const-normalization overlap. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE 2 <= CAST(v AS BIGINT) +ORDER BY host; + ++---------------+---------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v >= Int16(2) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v >= Int16(2)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------+ + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) BETWEEN 1 AND 2 +ORDER BY host; + ++---------------+------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v >= Int16(1) AND cast_preimage_int.v <= Int16(2) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v >= Int16(1), cast_preimage_int.v <= Int16(2)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+------------------------------------------------------------------------------------------------------------------------+ + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) NOT BETWEEN 1 AND 2 +ORDER BY host; + ++---------------+------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v < Int16(1) OR cast_preimage_int.v > Int16(2) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v < Int16(1) OR cast_preimage_int.v > Int16(2)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+------------------------------------------------------------------------------------------------------------------------+ + +-- Plain and casted constants should become scan-local typed literals. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE v >= 2 +ORDER BY host; + ++---------------+---------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v >= Int16(2) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v >= Int16(2)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------+ + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE v >= CAST(2 AS BIGINT) +ORDER BY host; + ++---------------+---------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v >= Int16(2) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v >= Int16(2)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------+ + +-- TRY_CAST widening should also simplify to a scan-local typed literal. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE TRY_CAST(v AS BIGINT) >= 2 +ORDER BY host; + ++---------------+---------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_int.host ASC NULLS LAST | +| | Projection: cast_preimage_int.host, cast_preimage_int.v | +| | Filter: cast_preimage_int.v >= Int16(2) | +| | TableScan: cast_preimage_int, partial_filters=[cast_preimage_int.v >= Int16(2)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------+ + +-- Unsafe narrowing and int-to-string roundtrip stay result-correct. +SELECT host, v FROM cast_preimage_int +WHERE CAST(CAST(v AS TINYINT) AS SMALLINT) = v +ORDER BY host; + ++-------+---+ +| host | v | ++-------+---+ +| host1 | 1 | +| host2 | 2 | +| host3 | 3 | ++-------+---+ + +SELECT host, v FROM cast_preimage_int +WHERE CAST(CAST(v AS STRING) AS SMALLINT) = v +ORDER BY host; + ++-------+---+ +| host | v | ++-------+---+ +| host1 | 1 | +| host2 | 2 | +| host3 | 3 | ++-------+---+ + +CREATE TABLE cast_preimage_ts_ms ( + host STRING PRIMARY KEY, + ts TIMESTAMP(3) TIME INDEX, + v INTEGER, +); + +Affected Rows: 0 + +INSERT INTO cast_preimage_ts_ms VALUES + ('host1', 0, 1), + ('host2', 5000, 2), + ('host3', 5001, 3); + +Affected Rows: 3 + +-- Timestamp widening equality is exact at millisecond precision. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts_ms +WHERE CAST(ts AS TIMESTAMP(9)) = '1970-01-01 00:00:05'::TIMESTAMP(9) +ORDER BY host; + ++---------------+-------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_ts_ms.host ASC NULLS LAST | +| | Projection: cast_preimage_ts_ms.host, cast_preimage_ts_ms.v | +| | Filter: cast_preimage_ts_ms.ts = TimestampMillisecond(5000, None) | +| | TableScan: cast_preimage_ts_ms, partial_filters=[cast_preimage_ts_ms.ts = TimestampMillisecond(5000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-------------------------------------------------------------------------------------------------------------------+ + +-- Non-exact nanosecond literal should remain semantically correct. +SELECT host, v FROM cast_preimage_ts_ms +WHERE CAST(ts AS TIMESTAMP(9)) = '1970-01-01 00:00:05.000000001'::TIMESTAMP(9) +ORDER BY host; + +++ +++ + +DROP TABLE cast_preimage_ts; + +Affected Rows: 0 + +DROP TABLE cast_preimage_int; + +Affected Rows: 0 + +DROP TABLE cast_preimage_ts_ms; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/filter/cast_preimage.sql b/tests/cases/standalone/common/filter/cast_preimage.sql new file mode 100644 index 00000000000..7ee7e8198f3 --- /dev/null +++ b/tests/cases/standalone/common/filter/cast_preimage.sql @@ -0,0 +1,175 @@ +-- SQL cast predicate preimage / const-normalization filter pushdown. +-- This is deliberately plain SQL, not PromQL/TQL, so it covers the generic +-- planner path where cast predicates should become scan-local filters. + +CREATE TABLE cast_preimage_ts ( + host STRING PRIMARY KEY, + ts TIMESTAMP(9) TIME INDEX, + v INTEGER, +); + +INSERT INTO cast_preimage_ts VALUES + ('host1', 0, 1), + ('host1', 5000000000, 2), + ('host1', 10000000000, 3), + ('host2', 15000000000, 4); + +-- Timestamp downcast comparison: +-- CAST(ts_ns AS TIMESTAMP(3)) >= TIMESTAMP(3) '1970-01-01 00:00:05' +-- should be pushed as a native nanosecond bound on the scan. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE CAST(ts AS TIMESTAMP(3)) >= '1970-01-01 00:00:05'::TIMESTAMP(3) +ORDER BY host, v; + +-- Timestamp downcast BETWEEN should be lowered through DataFusion's BETWEEN +-- simplification and cast preimage, not left as a CAST predicate in scan filters. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE CAST(ts AS TIMESTAMP(3)) BETWEEN '1970-01-01 00:00:00'::TIMESTAMP(3) + AND '1970-01-01 00:00:10'::TIMESTAMP(3) +ORDER BY host, v; + +-- Timestamp downcast equality should become a native half-open nanosecond range. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE CAST(ts AS TIMESTAMP(3)) = '1970-01-01 00:00:05'::TIMESTAMP(3) +ORDER BY host, v; + +-- Literal-left timestamp inequality should normalize before cast preimage. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts +WHERE '1970-01-01 00:00:05'::TIMESTAMP(3) < CAST(ts AS TIMESTAMP(3)) +ORDER BY host, v; + +-- Integer widening comparison and IN-list cover the overlap with the old +-- Greptime ConstNormalizationRule's lossless-cast cases. +CREATE TABLE cast_preimage_int ( + ts TIMESTAMP TIME INDEX, + host STRING PRIMARY KEY, + v SMALLINT, +); + +INSERT INTO cast_preimage_int VALUES + (0, 'host1', 1), + (1000, 'host2', 2), + (2000, 'host3', 3); + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) >= 2 +ORDER BY host; + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) IN (1, 3) +ORDER BY host; + +-- Swapped comparison and BETWEEN/NOT BETWEEN cover const-normalization overlap. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE 2 <= CAST(v AS BIGINT) +ORDER BY host; + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) BETWEEN 1 AND 2 +ORDER BY host; + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE CAST(v AS BIGINT) NOT BETWEEN 1 AND 2 +ORDER BY host; + +-- Plain and casted constants should become scan-local typed literals. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE v >= 2 +ORDER BY host; + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE v >= CAST(2 AS BIGINT) +ORDER BY host; + +-- TRY_CAST widening should also simplify to a scan-local typed literal. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_int +WHERE TRY_CAST(v AS BIGINT) >= 2 +ORDER BY host; + +-- Unsafe narrowing and int-to-string roundtrip stay result-correct. +SELECT host, v FROM cast_preimage_int +WHERE CAST(CAST(v AS TINYINT) AS SMALLINT) = v +ORDER BY host; + +SELECT host, v FROM cast_preimage_int +WHERE CAST(CAST(v AS STRING) AS SMALLINT) = v +ORDER BY host; + +CREATE TABLE cast_preimage_ts_ms ( + host STRING PRIMARY KEY, + ts TIMESTAMP(3) TIME INDEX, + v INTEGER, +); + +INSERT INTO cast_preimage_ts_ms VALUES + ('host1', 0, 1), + ('host2', 5000, 2), + ('host3', 5001, 3); + +-- Timestamp widening equality is exact at millisecond precision. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts_ms +WHERE CAST(ts AS TIMESTAMP(9)) = '1970-01-01 00:00:05'::TIMESTAMP(9) +ORDER BY host; + +-- Non-exact nanosecond literal should remain semantically correct. +SELECT host, v FROM cast_preimage_ts_ms +WHERE CAST(ts AS TIMESTAMP(9)) = '1970-01-01 00:00:05.000000001'::TIMESTAMP(9) +ORDER BY host; + +DROP TABLE cast_preimage_ts; + +DROP TABLE cast_preimage_int; + +DROP TABLE cast_preimage_ts_ms; diff --git a/tests/cases/standalone/common/order/order_by.result b/tests/cases/standalone/common/order/order_by.result index ceebb94ff5d..aed1a491c27 100644 --- a/tests/cases/standalone/common/order/order_by.result +++ b/tests/cases/standalone/common/order/order_by.result @@ -296,7 +296,7 @@ explain analyze select tag from t where num > 6 order by ts desc limit 2; | stage | node | plan_| +-+-+-+ | 0_| 0_|_ProjectionExec: expr=[tag@0 as tag] REDACTED -|_|_|_SortPreservingMergeExec: [ts@1 DESC], fetch=2 REDACTED +|_|_|_MergeSortExec: [ts@1 DESC], fetch=2 REDACTED |_|_|_SortExec: TopK(fetch=2), expr=[ts@1 DESC], preserve_partitioning=[true] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| diff --git a/tests/cases/standalone/common/promql/precisions.result b/tests/cases/standalone/common/promql/precisions.result index ef67f1aec78..e57f5b04ee4 100644 --- a/tests/cases/standalone/common/promql/precisions.result +++ b/tests/cases/standalone/common/promql/precisions.result @@ -38,6 +38,26 @@ INSERT INTO TABLE host_micro VALUES Affected Rows: 8 +CREATE TABLE host_nano ( + ts timestamp(9) time index, + host STRING PRIMARY KEY, + val DOUBLE, +); + +Affected Rows: 0 + +INSERT INTO TABLE host_nano VALUES + (0, 'host1', 1), + (0, 'host2', 2), + (5000000000, 'host1', 3), + (5000000000, 'host2', 4), + (10000000000, 'host1', 5), + (10000000000, 'host2', 6), + (15000000000, 'host1', 7), + (15000000000, 'host2', 8); + +Affected Rows: 8 + -- Test on Timestamps of different precisions -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') host_sec{host="host1"}; @@ -111,6 +131,118 @@ TQL EVAL (0, 15, '5s') avg_over_time(host_sec{host="host1"}[5s]) + avg_over_time | host1 | 1970-01-01T00:00:15 | 14.0 | +-------+---------------------+-----------------------------------------------------------------------------------------+ +-- Verify that PromQL time predicates on non-millisecond time indexes are +-- pushed into the scan as native timestamp range filters. +-- Original instant selector filter is built on the millisecond alias: +-- host = "host1" AND ts_ms >= -299999ms AND ts_ms <= 10000ms +-- After pushing through `CAST(raw_ts AS Timestamp(ms)) AS ts` and applying +-- DataFusion cast preimage, it becomes a native half-open range on raw_ts. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_micro.__table_id\s*=\s*UInt32\(\d+\) host_micro.__table_id=UInt32(REDACTED) +-- SQLNESS REPLACE host_nano.__table_id\s*=\s*UInt32\(\d+\) host_nano.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') host_micro{host="host1"}; + ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts] | +| | PromSeriesDivide: tags=["host"] | +| | Sort: host_micro.host ASC NULLS FIRST, host_micro.ts ASC NULLS FIRST | +| | Projection: host_micro.val, host_micro.host, CAST(host_micro.ts AS Timestamp(ms)) AS ts | +| | Filter: host_micro.host = Utf8("host1") AND host_micro.ts >= TimestampMicrosecond(-299999999, None) AND host_micro.ts < TimestampMicrosecond(10001000, None) | +| | TableScan: host_micro, partial_filters=[host_micro.host = Utf8("host1"), host_micro.ts >= TimestampMicrosecond(-299999999, None), host_micro.ts < TimestampMicrosecond(10001000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- The same instant-selector cast-preimage path should work for nanosecond indexes. +-- Expected native bounds: ts_ns >= -299999999999ns AND ts_ns < 10001000000ns. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_nano.__table_id\s*=\s*UInt32\(\d+\) host_nano.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') host_nano{host="host1"}; + ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts] | +| | PromSeriesDivide: tags=["host"] | +| | Sort: host_nano.host ASC NULLS FIRST, host_nano.ts ASC NULLS FIRST | +| | Projection: host_nano.val, host_nano.host, CAST(host_nano.ts AS Timestamp(ms)) AS ts | +| | Filter: host_nano.host = Utf8("host1") AND host_nano.ts >= TimestampNanosecond(-299999999999, None) AND host_nano.ts < TimestampNanosecond(10001000000, None) | +| | TableScan: host_nano, partial_filters=[host_nano.host = Utf8("host1"), host_nano.ts >= TimestampNanosecond(-299999999999, None), host_nano.ts < TimestampNanosecond(10001000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- Range selectors use their range window instead of the default lookback. +-- Original range selector filter for [5s]: +-- host = "host1" AND ts_ms >= -4999ms AND ts_ms <= 10000ms +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_micro.__table_id\s*=\s*UInt32\(\d+\) host_micro.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') avg_over_time(host_micro{host="host1"}[5s]); + ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Filter: prom_avg_over_time(ts_range,val) IS NOT NULL | +| | Projection: host_micro.ts, prom_avg_over_time(ts_range, val) AS prom_avg_over_time(ts_range,val), host_micro.host | +| | PromRangeManipulate: req range=[0..10000], interval=[5000], eval range=[5000], time index=[ts], values=["val"] | +| | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] | +| | PromSeriesDivide: tags=["host"] | +| | Sort: host_micro.host ASC NULLS FIRST, host_micro.ts ASC NULLS FIRST | +| | Projection: host_micro.val, host_micro.host, CAST(host_micro.ts AS Timestamp(ms)) AS ts | +| | Filter: host_micro.host = Utf8("host1") AND host_micro.ts >= TimestampMicrosecond(-4999999, None) AND host_micro.ts < TimestampMicrosecond(10001000, None) | +| | TableScan: host_micro, partial_filters=[host_micro.host = Utf8("host1"), host_micro.ts >= TimestampMicrosecond(-4999999, None), host_micro.ts < TimestampMicrosecond(10001000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- The same range-selector cast-preimage path should work for nanosecond indexes. +-- Expected native bounds: ts_ns >= -4999999999ns AND ts_ns < 10001000000ns. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_nano.__table_id\s*=\s*UInt32\(\d+\) host_nano.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') avg_over_time(host_nano{host="host1"}[5s]); + ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Filter: prom_avg_over_time(ts_range,val) IS NOT NULL | +| | Projection: host_nano.ts, prom_avg_over_time(ts_range, val) AS prom_avg_over_time(ts_range,val), host_nano.host | +| | PromRangeManipulate: req range=[0..10000], interval=[5000], eval range=[5000], time index=[ts], values=["val"] | +| | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] | +| | PromSeriesDivide: tags=["host"] | +| | Sort: host_nano.host ASC NULLS FIRST, host_nano.ts ASC NULLS FIRST | +| | Projection: host_nano.val, host_nano.host, CAST(host_nano.ts AS Timestamp(ms)) AS ts | +| | Filter: host_nano.host = Utf8("host1") AND host_nano.ts >= TimestampNanosecond(-4999999999, None) AND host_nano.ts < TimestampNanosecond(10001000000, None) | +| | TableScan: host_nano, partial_filters=[host_nano.host = Utf8("host1"), host_nano.ts >= TimestampNanosecond(-4999999999, None), host_nano.ts < TimestampNanosecond(10001000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + DROP TABLE host_sec; Affected Rows: 0 @@ -119,3 +251,7 @@ DROP TABLE host_micro; Affected Rows: 0 +DROP TABLE host_nano; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/promql/precisions.sql b/tests/cases/standalone/common/promql/precisions.sql index 6966e5bca20..01e6cae9fec 100644 --- a/tests/cases/standalone/common/promql/precisions.sql +++ b/tests/cases/standalone/common/promql/precisions.sql @@ -30,6 +30,22 @@ INSERT INTO TABLE host_micro VALUES (15000000, 'host1', 7), (15000000, 'host2', 8); +CREATE TABLE host_nano ( + ts timestamp(9) time index, + host STRING PRIMARY KEY, + val DOUBLE, +); + +INSERT INTO TABLE host_nano VALUES + (0, 'host1', 1), + (0, 'host2', 2), + (5000000000, 'host1', 3), + (5000000000, 'host2', 4), + (10000000000, 'host1', 5), + (10000000000, 'host2', 6), + (15000000000, 'host1', 7), + (15000000000, 'host2', 8); + -- Test on Timestamps of different precisions -- SQLNESS SORT_RESULT 3 1 @@ -50,6 +66,50 @@ TQL EVAL (0, 15, '5s') host_sec{host="host1"} + host_micro{host="host1"}; -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') avg_over_time(host_sec{host="host1"}[5s]) + avg_over_time(host_micro{host="host1"}[5s]); +-- Verify that PromQL time predicates on non-millisecond time indexes are +-- pushed into the scan as native timestamp range filters. +-- Original instant selector filter is built on the millisecond alias: +-- host = "host1" AND ts_ms >= -299999ms AND ts_ms <= 10000ms +-- After pushing through `CAST(raw_ts AS Timestamp(ms)) AS ts` and applying +-- DataFusion cast preimage, it becomes a native half-open range on raw_ts. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_micro.__table_id\s*=\s*UInt32\(\d+\) host_micro.__table_id=UInt32(REDACTED) +-- SQLNESS REPLACE host_nano.__table_id\s*=\s*UInt32\(\d+\) host_nano.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') host_micro{host="host1"}; + +-- The same instant-selector cast-preimage path should work for nanosecond indexes. +-- Expected native bounds: ts_ns >= -299999999999ns AND ts_ns < 10001000000ns. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_nano.__table_id\s*=\s*UInt32\(\d+\) host_nano.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') host_nano{host="host1"}; + +-- Range selectors use their range window instead of the default lookback. +-- Original range selector filter for [5s]: +-- host = "host1" AND ts_ms >= -4999ms AND ts_ms <= 10000ms +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_micro.__table_id\s*=\s*UInt32\(\d+\) host_micro.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') avg_over_time(host_micro{host="host1"}[5s]); + +-- The same range-selector cast-preimage path should work for nanosecond indexes. +-- Expected native bounds: ts_ns >= -4999999999ns AND ts_ns < 10001000000ns. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE host_nano.__table_id\s*=\s*UInt32\(\d+\) host_nano.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 10, '5s') avg_over_time(host_nano{host="host1"}[5s]); + DROP TABLE host_sec; DROP TABLE host_micro; + +DROP TABLE host_nano; diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.result b/tests/cases/standalone/common/tql-explain-analyze/explain.result index 810dd276442..a13e004bd12 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/explain.result +++ b/tests/cases/standalone/common/tql-explain-analyze/explain.result @@ -202,6 +202,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; | physical_plan after FilterPushdown(Post)_| SAME TEXT AS ABOVE_| | physical_plan after WindowedSortRule_| SAME TEXT AS ABOVE_| | physical_plan after MatchesConstantTerm_| SAME TEXT AS ABOVE_| +| physical_plan after EnsureGlobalLimitForFetch_| SAME TEXT AS ABOVE_| | physical_plan after RemoveDuplicateRule_| SAME TEXT AS ABOVE_| | physical_plan after SanityCheckPlan_| SAME TEXT AS ABOVE_| | physical_plan_| CooperativeExec_| @@ -353,6 +354,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test AS series; | physical_plan after FilterPushdown(Post)_| SAME TEXT AS ABOVE_| | physical_plan after WindowedSortRule_| SAME TEXT AS ABOVE_| | physical_plan after MatchesConstantTerm_| SAME TEXT AS ABOVE_| +| physical_plan after EnsureGlobalLimitForFetch_| SAME TEXT AS ABOVE_| | physical_plan after RemoveDuplicateRule_| SAME TEXT AS ABOVE_| | physical_plan after SanityCheckPlan_| SAME TEXT AS ABOVE_| | physical_plan_| CooperativeExec_| @@ -380,25 +382,21 @@ Affected Rows: 3 -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED TQL EXPLAIN (0, 10, '5s') test_nano; -+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| plan_type | plan | -+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| logical_plan | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] | -| | PromSeriesDivide: tags=["k"] | -| | Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST | -| | Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j | -| | MergeScan [is_placeholder=false, remote_input=[ | -| | Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None) | -| | TableScan: test_nano, partial_filters=[CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None), CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)] | -| | ]] | -| physical_plan | PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] | -| | PromSeriesDivideExec: tags=["k"] | -| | SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true] | -| | RepartitionExec: REDACTED -| | ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j] | -| | MergeScanExec: REDACTED -| | | -+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] | +| | PromSeriesDivide: tags=["k"] | +| | Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST | +| | Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j | +| | Filter: test_nano.j >= TimestampNanosecond(-299999999999, None) AND test_nano.j < TimestampNanosecond(10001000000, None) | +| | TableScan: test_nano, partial_filters=[test_nano.j >= TimestampNanosecond(-299999999999, None), test_nano.j < TimestampNanosecond(10001000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+ -- explain verbose at 0s, 5s and 10s for a nanosecond time index. -- SQLNESS REPLACE (-+) - @@ -425,14 +423,14 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; | logical_plan after resolve_grouping_function_| SAME TEXT AS ABOVE_| | logical_plan after type_coercion_| SAME TEXT AS ABOVE_| | logical_plan after ConstNormalizationRule_| SAME TEXT AS ABOVE_| -| logical_plan after DistPlannerAnalyzer_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +| logical_plan after DistPlannerAnalyzer_| Projection: test_nano.i, test_nano.k, test_nano.j_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| |_|_PromSeriesDivide: tags=["k"]_| |_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| |_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| -|_|_Projection: test_nano.i, test_nano.j, test_nano.k_| -|_|_MergeScan [is_placeholder=false, remote_input=[_| -|_| Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_| -|_|_TableScan: test_nano, partial_filters=[CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None), CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)] | +|_|_Filter: test_nano.j >= TimestampNanosecond(-299999999999, None) AND test_nano.j < TimestampNanosecond(10001000000, None)_| +|_|_TableScan: test_nano, partial_filters=[test_nano.j >= TimestampNanosecond(-299999999999, None), test_nano.j < TimestampNanosecond(10001000000, None)] | |_| ]]_| | logical_plan after FixStateUdafOrderingAnalyzer_| SAME TEXT AS ABOVE_| | analyzed_logical_plan_| SAME TEXT AS ABOVE_| @@ -459,13 +457,13 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; | logical_plan after common_sub_expression_eliminate_| SAME TEXT AS ABOVE_| | logical_plan after extract_leaf_expressions_| SAME TEXT AS ABOVE_| | logical_plan after push_down_leaf_projections_| SAME TEXT AS ABOVE_| -| logical_plan after optimize_projections_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +| logical_plan after optimize_projections_| MergeScan [is_placeholder=false, remote_input=[_| +|_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| |_|_PromSeriesDivide: tags=["k"]_| |_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| |_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| -|_|_MergeScan [is_placeholder=false, remote_input=[_| -|_| Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_| -|_|_TableScan: test_nano, partial_filters=[CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None), CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)] | +|_|_Filter: test_nano.j >= TimestampNanosecond(-299999999999, None) AND test_nano.j < TimestampNanosecond(10001000000, None)_| +|_|_TableScan: test_nano, partial_filters=[test_nano.j >= TimestampNanosecond(-299999999999, None), test_nano.j < TimestampNanosecond(10001000000, None)] | |_| ]]_| | logical_plan after ScanHintRule_| SAME TEXT AS ABOVE_| | logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_| @@ -495,37 +493,21 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; | logical_plan after optimize_projections_| SAME TEXT AS ABOVE_| | logical_plan after ScanHintRule_| SAME TEXT AS ABOVE_| | logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_| -| logical_plan_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| +| logical_plan_| MergeScan [is_placeholder=false, remote_input=[_| +|_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| |_|_PromSeriesDivide: tags=["k"]_| |_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| |_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| -|_|_MergeScan [is_placeholder=false, remote_input=[_| -|_| Filter: CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None) AND CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)_| -|_|_TableScan: test_nano, partial_filters=[CAST(test_nano.j AS Timestamp(ms)) >= TimestampMillisecond(-299999, None), CAST(test_nano.j AS Timestamp(ms)) <= TimestampMillisecond(10000, None)] | +|_|_Filter: test_nano.j >= TimestampNanosecond(-299999999999, None) AND test_nano.j < TimestampNanosecond(10001000000, None)_| +|_|_TableScan: test_nano, partial_filters=[test_nano.j >= TimestampNanosecond(-299999999999, None), test_nano.j < TimestampNanosecond(10001000000, None)] | |_| ]]_| -| initial_physical_plan_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false]_| -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| -|_|_MergeScanExec: REDACTED +| initial_physical_plan_| MergeScanExec: REDACTED |_|_| -| initial_physical_plan_with_stats_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], statistics=[Rows=Inexact(2), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| -|_|_PromSeriesDivideExec: tags=["k"], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| -|_|_MergeScanExec: REDACTED +| initial_physical_plan_with_stats_| MergeScanExec: REDACTED |_|_| -| initial_physical_plan_with_schema_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| -|_|_PromSeriesDivideExec: tags=["k"], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| -|_|_MergeScanExec: REDACTED +| initial_physical_plan_with_schema_| MergeScanExec: REDACTED |_|_| | physical_plan after OutputRequirements_| OutputRequirementExec: order_by=[], dist_by=Unspecified_| -|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[false]_| -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| |_|_MergeScanExec: REDACTED |_|_| | physical_plan after aggregate_statistics_| SAME TEXT AS ABOVE_| @@ -535,70 +517,35 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; | physical_plan after parallelize_scan_| SAME TEXT AS ABOVE_| | physical_plan after PassDistributionRule_| SAME TEXT AS ABOVE_| | physical_plan after PromqlTsidNarrowJoin_| SAME TEXT AS ABOVE_| -| physical_plan after EnforceSorting_| OutputRequirementExec: order_by=[], dist_by=Unspecified_| -|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| -|_|_MergeScanExec: REDACTED -|_|_| -| physical_plan after EnforceDistribution_| OutputRequirementExec: order_by=[], dist_by=Unspecified_| -|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| -|_|_RepartitionExec: REDACTED -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| -|_|_MergeScanExec: REDACTED -|_|_| +| physical_plan after EnforceSorting_| SAME TEXT AS ABOVE_| +| physical_plan after EnforceDistribution_| SAME TEXT AS ABOVE_| | physical_plan after CombinePartialFinalAggregate_| SAME TEXT AS ABOVE_| -| physical_plan after EnforceSorting_| OutputRequirementExec: order_by=[], dist_by=Unspecified_| -|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| -|_|_RepartitionExec: REDACTED -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| -|_|_MergeScanExec: REDACTED -|_|_| +| physical_plan after EnforceSorting_| SAME TEXT AS ABOVE_| | physical_plan after OptimizeAggregateOrder_| SAME TEXT AS ABOVE_| | physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_| -| physical_plan after OutputRequirements_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| -|_|_RepartitionExec: REDACTED -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| -|_|_MergeScanExec: REDACTED +| physical_plan after OutputRequirements_| MergeScanExec: REDACTED |_|_| | physical_plan after LimitAggregation_| SAME TEXT AS ABOVE_| | physical_plan after LimitPushPastWindows_| SAME TEXT AS ABOVE_| | physical_plan after LimitPushdown_| SAME TEXT AS ABOVE_| | physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_| | physical_plan after PushdownSort_| SAME TEXT AS ABOVE_| -| physical_plan after EnsureCooperative_| SAME TEXT AS ABOVE_| +| physical_plan after EnsureCooperative_| CooperativeExec_| +|_|_MergeScanExec: REDACTED +|_|_| | physical_plan after FilterPushdown(Post)_| SAME TEXT AS ABOVE_| | physical_plan after WindowedSortRule_| SAME TEXT AS ABOVE_| | physical_plan after MatchesConstantTerm_| SAME TEXT AS ABOVE_| +| physical_plan after EnsureGlobalLimitForFetch_| SAME TEXT AS ABOVE_| | physical_plan after RemoveDuplicateRule_| SAME TEXT AS ABOVE_| | physical_plan after SanityCheckPlan_| SAME TEXT AS ABOVE_| -| physical_plan_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true]_| -|_|_RepartitionExec: REDACTED -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j]_| +| physical_plan_| CooperativeExec_| |_|_MergeScanExec: REDACTED |_|_| -| physical_plan_with_stats_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], statistics=[Rows=Inexact(2), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| -|_|_PromSeriesDivideExec: tags=["k"], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| -|_|_RepartitionExec: REDACTED -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +| physical_plan_with_stats_| CooperativeExec, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| |_|_MergeScanExec: REDACTED |_|_| -| physical_plan_with_schema_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| -|_|_PromSeriesDivideExec: tags=["k"], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| -|_|_SortExec: expr=[k@1 ASC, j@2 ASC], preserve_partitioning=[true], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| -|_|_RepartitionExec: REDACTED -|_|_ProjectionExec: expr=[i@0 as i, k@2 as k, CAST(j@1 AS Timestamp(ms)) as j], schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| +| physical_plan_with_schema_| CooperativeExec, schema=[i:Float64;N, k:Utf8;N, j:Timestamp(ms)]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ diff --git a/tests/cases/standalone/common/tql/general_table.result b/tests/cases/standalone/common/tql/general_table.result index 949c1f8d716..6172f40aaad 100644 --- a/tests/cases/standalone/common/tql/general_table.result +++ b/tests/cases/standalone/common/tql/general_table.result @@ -18,6 +18,10 @@ Affected Rows: 0 -- SQLNESS REPLACE (Hash.*) REDACTED -- SQLNESS REPLACE (-+) - -- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (cpu_usage\.ts_range) ts_range +-- SQLNESS REPLACE (cpu_usage\.ts) ts +-- SQLNESS REPLACE (?m)^\|_\|_\|_SortExec:.*\n\|_\|_\|_RepartitionExec:.*\n +-- SQLNESS REPLACE (SeriesScan:.*|SeqScan:.*) ScanExec: REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED TQL analyze (0, 10, '1s') sum by(job) (irate(cpu_usage{job="fire"}[5s])) / 1e9; @@ -25,7 +29,10 @@ TQL analyze (0, 10, '1s') sum by(job) (irate(cpu_usage{job="fire"}[5s])) / 1e9; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[job@0 as job, ts@1 as ts, sum(prom_irate(ts_range,value))@2 / 1000000000 as sum(prom_irate(ts_range,value)) / Float64(1000000000)] REDACTED +| 0_| 0_|_CooperativeExec REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_ProjectionExec: expr=[job@0 as job, ts@1 as ts, sum(prom_irate(ts_range,value))@2 / 1000000000 as sum(prom_irate(ts_range,value)) / Float64(1000000000)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_SortPreservingMergeExec: [job@0 ASC NULLS LAST, ts@1 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[job@0 ASC NULLS LAST, ts@1 ASC NULLS LAST], preserve_partitioning=[true] REDACTED @@ -35,13 +42,8 @@ TQL analyze (0, 10, '1s') sum by(job) (irate(cpu_usage{job="fire"}[5s])) / 1e9; |_|_|_PromRangeManipulateExec: req range=[0..10000], interval=[1000], eval range=[5000], time index=[ts] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[ts], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["job"] REDACTED -|_|_|_SortExec: expr=[job@1 ASC, ts@2 ASC], preserve_partitioning=[true] REDACTED -|_|_|_RepartitionExec: partitioning=REDACTED |_|_|_ProjectionExec: expr=[value@1 as value, job@0 as job, CAST(ts@2 AS Timestamp(ms)) as ts] REDACTED -|_|_|_MergeScanExec: REDACTED -|_|_|_| -| 1_| 0_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_ScanExec: REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ diff --git a/tests/cases/standalone/common/tql/general_table.sql b/tests/cases/standalone/common/tql/general_table.sql index 573d0cbdff9..6bf9549773c 100644 --- a/tests/cases/standalone/common/tql/general_table.sql +++ b/tests/cases/standalone/common/tql/general_table.sql @@ -16,6 +16,10 @@ WITH( -- SQLNESS REPLACE (Hash.*) REDACTED -- SQLNESS REPLACE (-+) - -- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (cpu_usage\.ts_range) ts_range +-- SQLNESS REPLACE (cpu_usage\.ts) ts +-- SQLNESS REPLACE (?m)^\|_\|_\|_SortExec:.*\n\|_\|_\|_RepartitionExec:.*\n +-- SQLNESS REPLACE (SeriesScan:.*|SeqScan:.*) ScanExec: REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED TQL analyze (0, 10, '1s') sum by(job) (irate(cpu_usage{job="fire"}[5s])) / 1e9; diff --git a/tests/cases/standalone/common/tql/partition.result b/tests/cases/standalone/common/tql/partition.result index 5d0fd981238..45927f8abbd 100644 --- a/tests/cases/standalone/common/tql/partition.result +++ b/tests/cases/standalone/common/tql/partition.result @@ -133,8 +133,8 @@ tql analyze (0, 10, '1s') 100 - (avg by (k) (irate(t[1m])) * 100); |_|_|_PromRangeManipulateExec: req range=[0..10000], interval=[1000], eval range=[60000], time index=[j] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["k", "l"] REDACTED -|_|_|_SortExec: expr=[k@2 ASC, l@3 ASC, j@1 ASC], preserve_partitioning=[true] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED +|_|_|_MergeSortExec: [k@2 ASC, l@3 ASC, j@1 ASC] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [k@2 ASC, l@3 ASC, j@1 ASC] REDACTED diff --git a/tests/cases/standalone/common/window/latest_per_series.result b/tests/cases/standalone/common/window/latest_per_series.result new file mode 100644 index 00000000000..c389c8aeb8a --- /dev/null +++ b/tests/cases/standalone/common/window/latest_per_series.result @@ -0,0 +1,187 @@ +CREATE TABLE netflow_raw ( + greptime_timestamp TIMESTAMP TIME INDEX, + src STRING, + dst_port INT, + dst_addr STRING, + byte_count DOUBLE, + flow_id INT, + PRIMARY KEY (src, dst_port) +); + +Affected Rows: 0 + +INSERT INTO netflow_raw VALUES + (1000, 'source-a', 80, 'target-80-old', 100.0, 1), + (5000, 'source-a', 80, 'target-80-new', 500.0, 2), + (2000, 'source-a', 443, 'target-443-old', 200.0, 3), + (7000, 'source-a', 443, 'target-443-new', 700.0, 4), + (3000, 'source-a', 22, 'target-22-only', 300.0, 5), + (4000, 'source-a', 8080, 'target-8080-only', 400.0, 6), + (8000, 'source-b', 9999, 'target-other-src', 999.0, 7); + +Affected Rows: 7 + +-- Latest row per dst_port using a window subquery. The window result must be +-- filtered outside the subquery, not directly in the WHERE clause. +SELECT greptime_timestamp, dst_port, dst_addr, byte_count, flow_id +FROM ( + SELECT + greptime_timestamp, + dst_port, + dst_addr, + byte_count, + flow_id, + ROW_NUMBER() OVER ( + PARTITION BY dst_port + ORDER BY greptime_timestamp DESC + ) AS rn + FROM netflow_raw + WHERE src = 'source-a' + AND greptime_timestamp >= '1970-01-01T00:00:02'::timestamp +) latest_per_port +WHERE rn = 1 +ORDER BY greptime_timestamp DESC, dst_port +LIMIT 3; + ++---------------------+----------+------------------+------------+---------+ +| greptime_timestamp | dst_port | dst_addr | byte_count | flow_id | ++---------------------+----------+------------------+------------+---------+ +| 1970-01-01T00:00:07 | 443 | target-443-new | 700.0 | 4 | +| 1970-01-01T00:00:05 | 80 | target-80-new | 500.0 | 2 | +| 1970-01-01T00:00:04 | 8080 | target-8080-only | 400.0 | 6 | ++---------------------+----------+------------------+------------+---------+ + +-- DISTINCT ON keeps the first row per dst_port according to its own ORDER BY. +-- Wrap it to apply the final "latest groups first" ordering and LIMIT. +SELECT greptime_timestamp, dst_port, dst_addr, byte_count, flow_id +FROM ( + SELECT DISTINCT ON (dst_port) + greptime_timestamp, + dst_port, + dst_addr, + byte_count, + flow_id + FROM netflow_raw + WHERE src = 'source-a' + AND greptime_timestamp >= '1970-01-01T00:00:02'::timestamp + ORDER BY dst_port, greptime_timestamp DESC +) latest_per_port +ORDER BY greptime_timestamp DESC, dst_port +LIMIT 3; + ++---------------------+----------+------------------+------------+---------+ +| greptime_timestamp | dst_port | dst_addr | byte_count | flow_id | ++---------------------+----------+------------------+------------+---------+ +| 1970-01-01T00:00:07 | 443 | target-443-new | 700.0 | 4 | +| 1970-01-01T00:00:05 | 80 | target-80-new | 500.0 | 2 | +| 1970-01-01T00:00:04 | 8080 | target-8080-only | 400.0 | 6 | ++---------------------+----------+------------------+------------+---------+ + +-- RANK returns all rows tied at the latest timestamp for each dst_port. +SELECT greptime_timestamp, dst_port, dst_addr, byte_count, flow_id, r +FROM ( + SELECT + greptime_timestamp, + dst_port, + dst_addr, + byte_count, + flow_id, + RANK() OVER ( + PARTITION BY dst_port + ORDER BY greptime_timestamp DESC + ) AS r + FROM netflow_raw + WHERE src = 'source-a' + AND greptime_timestamp >= '1970-01-01T00:00:02'::timestamp +) latest_per_port +WHERE r = 1 +ORDER BY greptime_timestamp DESC, dst_port +LIMIT 3; + ++---------------------+----------+------------------+------------+---------+---+ +| greptime_timestamp | dst_port | dst_addr | byte_count | flow_id | r | ++---------------------+----------+------------------+------------+---------+---+ +| 1970-01-01T00:00:07 | 443 | target-443-new | 700.0 | 4 | 1 | +| 1970-01-01T00:00:05 | 80 | target-80-new | 500.0 | 2 | 1 | +| 1970-01-01T00:00:04 | 8080 | target-8080-only | 400.0 | 6 | 1 | ++---------------------+----------+------------------+------------+---------+---+ + +-- Ordered aggregate variant when the caller can enumerate the columns to fetch. +SELECT + dst_port, + last_value(greptime_timestamp ORDER BY greptime_timestamp) AS latest_ts, + last_value(dst_addr ORDER BY greptime_timestamp) AS latest_dst_addr, + last_value(byte_count ORDER BY greptime_timestamp) AS latest_byte_count, + last_value(flow_id ORDER BY greptime_timestamp) AS latest_flow_id +FROM netflow_raw +WHERE src = 'source-a' + AND greptime_timestamp >= '1970-01-01T00:00:02'::timestamp +GROUP BY dst_port +ORDER BY latest_ts DESC, dst_port +LIMIT 3; + ++----------+---------------------+------------------+-------------------+----------------+ +| dst_port | latest_ts | latest_dst_addr | latest_byte_count | latest_flow_id | ++----------+---------------------+------------------+-------------------+----------------+ +| 443 | 1970-01-01T00:00:07 | target-443-new | 700.0 | 4 | +| 80 | 1970-01-01T00:00:05 | target-80-new | 500.0 | 2 | +| 8080 | 1970-01-01T00:00:04 | target-8080-only | 400.0 | 6 | ++----------+---------------------+------------------+-------------------+----------------+ + +CREATE TABLE netflow_raw_ties ( + greptime_timestamp TIMESTAMP TIME INDEX, + src STRING, + dst_port INT, + dst_addr STRING, + byte_count DOUBLE, + flow_id INT, + PRIMARY KEY (src, dst_port, flow_id) +); + +Affected Rows: 0 + +INSERT INTO netflow_raw_ties VALUES + (1000, 'source-a', 443, 'target-443-old', 100.0, 1), + (9000, 'source-a', 443, 'target-443-tie-a', 900.0, 2), + (9000, 'source-a', 443, 'target-443-tie-b', 901.0, 3), + (8000, 'source-a', 80, 'target-80-new', 800.0, 4), + (9500, 'source-b', 443, 'target-other-src', 950.0, 5); + +Affected Rows: 5 + +-- RANK with r = 1 keeps all rows tied at the latest timestamp per dst_port. +SELECT greptime_timestamp, dst_port, dst_addr, byte_count, flow_id, r +FROM ( + SELECT + greptime_timestamp, + dst_port, + dst_addr, + byte_count, + flow_id, + RANK() OVER ( + PARTITION BY dst_port + ORDER BY greptime_timestamp DESC + ) AS r + FROM netflow_raw_ties + WHERE src = 'source-a' + AND greptime_timestamp >= '1970-01-01T00:00:02'::timestamp +) latest_per_port +WHERE r = 1 +ORDER BY greptime_timestamp DESC, dst_port, flow_id; + ++---------------------+----------+------------------+------------+---------+---+ +| greptime_timestamp | dst_port | dst_addr | byte_count | flow_id | r | ++---------------------+----------+------------------+------------+---------+---+ +| 1970-01-01T00:00:09 | 443 | target-443-tie-a | 900.0 | 2 | 1 | +| 1970-01-01T00:00:09 | 443 | target-443-tie-b | 901.0 | 3 | 1 | +| 1970-01-01T00:00:08 | 80 | target-80-new | 800.0 | 4 | 1 | ++---------------------+----------+------------------+------------+---------+---+ + +DROP TABLE netflow_raw_ties; + +Affected Rows: 0 + +DROP TABLE netflow_raw; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/window/latest_per_series.sql b/tests/cases/standalone/common/window/latest_per_series.sql new file mode 100644 index 00000000000..f809c8777ee --- /dev/null +++ b/tests/cases/standalone/common/window/latest_per_series.sql @@ -0,0 +1,134 @@ +CREATE TABLE netflow_raw ( + greptime_timestamp TIMESTAMP TIME INDEX, + src STRING, + dst_port INT, + dst_addr STRING, + byte_count DOUBLE, + flow_id INT, + PRIMARY KEY (src, dst_port) +); + +INSERT INTO netflow_raw VALUES + (1000, 'source-a', 80, 'target-80-old', 100.0, 1), + (5000, 'source-a', 80, 'target-80-new', 500.0, 2), + (2000, 'source-a', 443, 'target-443-old', 200.0, 3), + (7000, 'source-a', 443, 'target-443-new', 700.0, 4), + (3000, 'source-a', 22, 'target-22-only', 300.0, 5), + (4000, 'source-a', 8080, 'target-8080-only', 400.0, 6), + (8000, 'source-b', 9999, 'target-other-src', 999.0, 7); + +-- Latest row per dst_port using a window subquery. The window result must be +-- filtered outside the subquery, not directly in the WHERE clause. +SELECT greptime_timestamp, dst_port, dst_addr, byte_count, flow_id +FROM ( + SELECT + greptime_timestamp, + dst_port, + dst_addr, + byte_count, + flow_id, + ROW_NUMBER() OVER ( + PARTITION BY dst_port + ORDER BY greptime_timestamp DESC + ) AS rn + FROM netflow_raw + WHERE src = 'source-a' + AND greptime_timestamp >= '1970-01-01T00:00:02'::timestamp +) latest_per_port +WHERE rn = 1 +ORDER BY greptime_timestamp DESC, dst_port +LIMIT 3; + +-- DISTINCT ON keeps the first row per dst_port according to its own ORDER BY. +-- Wrap it to apply the final "latest groups first" ordering and LIMIT. +SELECT greptime_timestamp, dst_port, dst_addr, byte_count, flow_id +FROM ( + SELECT DISTINCT ON (dst_port) + greptime_timestamp, + dst_port, + dst_addr, + byte_count, + flow_id + FROM netflow_raw + WHERE src = 'source-a' + AND greptime_timestamp >= '1970-01-01T00:00:02'::timestamp + ORDER BY dst_port, greptime_timestamp DESC +) latest_per_port +ORDER BY greptime_timestamp DESC, dst_port +LIMIT 3; + +-- RANK returns all rows tied at the latest timestamp for each dst_port. +SELECT greptime_timestamp, dst_port, dst_addr, byte_count, flow_id, r +FROM ( + SELECT + greptime_timestamp, + dst_port, + dst_addr, + byte_count, + flow_id, + RANK() OVER ( + PARTITION BY dst_port + ORDER BY greptime_timestamp DESC + ) AS r + FROM netflow_raw + WHERE src = 'source-a' + AND greptime_timestamp >= '1970-01-01T00:00:02'::timestamp +) latest_per_port +WHERE r = 1 +ORDER BY greptime_timestamp DESC, dst_port +LIMIT 3; + +-- Ordered aggregate variant when the caller can enumerate the columns to fetch. +SELECT + dst_port, + last_value(greptime_timestamp ORDER BY greptime_timestamp) AS latest_ts, + last_value(dst_addr ORDER BY greptime_timestamp) AS latest_dst_addr, + last_value(byte_count ORDER BY greptime_timestamp) AS latest_byte_count, + last_value(flow_id ORDER BY greptime_timestamp) AS latest_flow_id +FROM netflow_raw +WHERE src = 'source-a' + AND greptime_timestamp >= '1970-01-01T00:00:02'::timestamp +GROUP BY dst_port +ORDER BY latest_ts DESC, dst_port +LIMIT 3; + +CREATE TABLE netflow_raw_ties ( + greptime_timestamp TIMESTAMP TIME INDEX, + src STRING, + dst_port INT, + dst_addr STRING, + byte_count DOUBLE, + flow_id INT, + PRIMARY KEY (src, dst_port, flow_id) +); + +INSERT INTO netflow_raw_ties VALUES + (1000, 'source-a', 443, 'target-443-old', 100.0, 1), + (9000, 'source-a', 443, 'target-443-tie-a', 900.0, 2), + (9000, 'source-a', 443, 'target-443-tie-b', 901.0, 3), + (8000, 'source-a', 80, 'target-80-new', 800.0, 4), + (9500, 'source-b', 443, 'target-other-src', 950.0, 5); + +-- RANK with r = 1 keeps all rows tied at the latest timestamp per dst_port. +SELECT greptime_timestamp, dst_port, dst_addr, byte_count, flow_id, r +FROM ( + SELECT + greptime_timestamp, + dst_port, + dst_addr, + byte_count, + flow_id, + RANK() OVER ( + PARTITION BY dst_port + ORDER BY greptime_timestamp DESC + ) AS r + FROM netflow_raw_ties + WHERE src = 'source-a' + AND greptime_timestamp >= '1970-01-01T00:00:02'::timestamp +) latest_per_port +WHERE r = 1 +ORDER BY greptime_timestamp DESC, dst_port, flow_id; + +DROP TABLE netflow_raw_ties; + +DROP TABLE netflow_raw; diff --git a/tests/cases/standalone/optimizer/first_value_advance.result b/tests/cases/standalone/optimizer/first_value_advance.result index ffc0b341f0a..afcefae4cff 100644 --- a/tests/cases/standalone/optimizer/first_value_advance.result +++ b/tests/cases/standalone/optimizer/first_value_advance.result @@ -240,7 +240,7 @@ order by ordered_host; |_|_Aggregate: groupBy=[[t.host]], aggr=[[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]]] | |_|_TableScan: t_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [ordered_host@0 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -267,7 +267,7 @@ order by ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED +| 0_| 0_|_MergeSortExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED @@ -603,7 +603,7 @@ order by ordered_host; |_|_Aggregate: groupBy=[[t1.host]], aggr=[[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]]] | |_|_TableScan: t1_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [ordered_host@0 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -629,7 +629,7 @@ order by ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED +| 0_| 0_|_MergeSortExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED diff --git a/tests/cases/standalone/optimizer/last_value_advance.result b/tests/cases/standalone/optimizer/last_value_advance.result index 0af561e6185..a1882a75887 100644 --- a/tests/cases/standalone/optimizer/last_value_advance.result +++ b/tests/cases/standalone/optimizer/last_value_advance.result @@ -240,7 +240,7 @@ order by ordered_host; |_|_Aggregate: groupBy=[[t.host]], aggr=[[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]]] | |_|_TableScan: t_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [ordered_host@0 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -267,7 +267,7 @@ order by ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED +| 0_| 0_|_MergeSortExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED @@ -603,7 +603,7 @@ order by ordered_host; |_|_Aggregate: groupBy=[[t1.host]], aggr=[[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]]] | |_|_TableScan: t1_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST]_| +| physical_plan | MergeSortExec: [ordered_host@0 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -629,7 +629,7 @@ order by ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED +| 0_| 0_|_MergeSortExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED diff --git a/tests/cases/standalone/optimizer/time_index_filter_pushdown.result b/tests/cases/standalone/optimizer/time_index_filter_pushdown.result index de6222ae28e..fd0d869a33f 100644 --- a/tests/cases/standalone/optimizer/time_index_filter_pushdown.result +++ b/tests/cases/standalone/optimizer/time_index_filter_pushdown.result @@ -197,6 +197,31 @@ group by | windows | 2 | +---------+----------+ +-- Query the row at the latest timestamp through a scalar subquery. +SELECT + rack, + os, + greptime_timestamp +FROM + cpu +WHERE + greptime_timestamp = ( + SELECT + greptime_timestamp + FROM + cpu + ORDER BY + greptime_timestamp DESC + LIMIT + 1 + ); + ++------+---------+---------------------+ +| rack | os | greptime_timestamp | ++------+---------+---------------------+ +| 8 | windows | 2023-06-12T02:10:00 | ++------+---------+---------------------+ + drop table cpu; Affected Rows: 0 diff --git a/tests/cases/standalone/optimizer/time_index_filter_pushdown.sql b/tests/cases/standalone/optimizer/time_index_filter_pushdown.sql index c0636e26a4b..7a7030d0925 100644 --- a/tests/cases/standalone/optimizer/time_index_filter_pushdown.sql +++ b/tests/cases/standalone/optimizer/time_index_filter_pushdown.sql @@ -120,4 +120,23 @@ where group by os; +-- Query the row at the latest timestamp through a scalar subquery. +SELECT + rack, + os, + greptime_timestamp +FROM + cpu +WHERE + greptime_timestamp = ( + SELECT + greptime_timestamp + FROM + cpu + ORDER BY + greptime_timestamp DESC + LIMIT + 1 + ); + drop table cpu;