diff --git a/Cargo.lock b/Cargo.lock index a87635711b..ae1bc0147d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3626,7 +3626,7 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "datafusion" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "arrow-schema 58.3.0", @@ -3680,7 +3680,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3704,7 +3704,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3726,7 +3726,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -3750,7 +3750,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "futures", "log", @@ -3760,7 +3760,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "async-compression", @@ -3794,7 +3794,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "arrow-ipc 58.3.0", @@ -3817,7 +3817,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3839,7 +3839,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3862,7 +3862,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3891,12 +3891,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" [[package]] name = "datafusion-execution" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "arrow-buffer 58.3.0", @@ -3918,7 +3918,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3940,7 +3940,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -3952,7 +3952,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "arrow-buffer 58.3.0", @@ -3983,7 +3983,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4004,7 +4004,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4016,7 +4016,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "arrow-ord 58.3.0", @@ -4040,7 +4040,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "async-trait", @@ -4055,7 +4055,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4072,7 +4072,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -4081,7 +4081,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "datafusion-doc", "quote", @@ -4091,7 +4091,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "chrono", @@ -4140,7 +4140,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4163,7 +4163,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4177,7 +4177,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4193,7 +4193,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4211,7 +4211,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4242,7 +4242,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "chrono", @@ -4269,7 +4269,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4279,7 +4279,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4295,7 +4295,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "async-trait", "datafusion-common", @@ -4308,7 +4308,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "arrow 58.3.0", "bigdecimal 0.4.8", @@ -4326,7 +4326,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=9c1ed8d9242408aad0a6d444c7c339bcb62f9be4#9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e8a127c28e8839964bb6aefdd909810dc11cd2c9#e8a127c28e8839964bb6aefdd909810dc11cd2c9" dependencies = [ "async-recursion", "async-trait", @@ -6457,7 +6457,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "tokio", "tower-service", "tracing", diff --git a/Cargo.toml b/Cargo.toml index a4e661f972..f0c8280dda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -338,21 +338,21 @@ git = "https://github.com/GreptimeTeam/greptime-meter.git" rev = "5618e779cf2bb4755b499c630fba4c35e91898cb" [patch.crates-io] -datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } -datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } -datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } -datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } -datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } -datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } -datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } -datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } -datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } -datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } -datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } -datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } -datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } -datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } -datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "9c1ed8d9242408aad0a6d444c7c339bcb62f9be4" } +datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } +datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e8a127c28e8839964bb6aefdd909810dc11cd2c9" } sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "2aefa08a8d69c96eec2d6d6703598a009bba6e4c" } # on branch v0.61.x [profile.release] diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs index 90aedae645..dfba397a14 100644 --- a/src/datanode/src/datanode.rs +++ b/src/datanode/src/datanode.rs @@ -459,6 +459,7 @@ impl DatanodeBuilder { opts.concurrent_query_limiter_timeout, opts.grpc.flight_compression, ); + region_server.install_remote_dyn_filter_receiver_injector(&self.plugins); let object_store_manager = Self::build_object_store_manager(&opts.storage).await?; let engines = self diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs index a5e724ab85..615b3f996b 100644 --- a/src/datanode/src/region_server.rs +++ b/src/datanode/src/region_server.rs @@ -13,6 +13,8 @@ // limitations under the License. mod catalog; +mod registrations; +mod remote_dyn_filter; use std::collections::HashMap; use std::fmt::Debug; @@ -25,11 +27,9 @@ use std::time::Duration; use api::region::RegionResponse; use api::v1::meta::TopicStat; -use api::v1::region::remote_dyn_filter_request::Action; use api::v1::region::sync_request::ManifestInfo; use api::v1::region::{ - ListMetadataRequest, RegionResponse as RegionResponseV1, RemoteDynFilterRequest, SyncRequest, - region_request, + ListMetadataRequest, RegionResponse as RegionResponseV1, SyncRequest, region_request, }; use api::v1::{ResponseHeader, Status}; use arrow_flight::{FlightData, Ticket}; @@ -92,13 +92,16 @@ use crate::error::{ ConcurrentQueryLimiterTimeoutSnafu, DataFusionSnafu, DecodeLogicalPlanSnafu, ExecuteLogicalPlanSnafu, FindLogicalRegionsSnafu, GetRegionMetadataSnafu, HandleBatchDdlRequestSnafu, HandleBatchOpenRequestSnafu, HandleRegionRequestSnafu, - NewPlanDecoderSnafu, NotYetImplementedSnafu, RegionEngineNotFoundSnafu, RegionNotFoundSnafu, - RegionNotReadySnafu, Result, RuntimeJoinSnafu, SerializeJsonSnafu, StopRegionEngineSnafu, - UnexpectedSnafu, UnsupportedOutputSnafu, + NewPlanDecoderSnafu, RegionEngineNotFoundSnafu, RegionNotFoundSnafu, RegionNotReadySnafu, + Result, RuntimeJoinSnafu, SerializeJsonSnafu, StopRegionEngineSnafu, UnexpectedSnafu, + UnsupportedOutputSnafu, }; use crate::event_listener::RegionServerEventListenerRef; use crate::query_stream::QueryRuntimeStream; use crate::region_server::catalog::{NameAwareCatalogList, NameAwareDataSourceInjectorBuilder}; +use crate::region_server::registrations::RemoteDynFilterRegistry; +use crate::region_server::remote_dyn_filter::wrap_remote_dyn_filter_guarded_stream; + const QUERY_RUNTIME_STREAM_BUFFER_SIZE: usize = 8; #[derive(Clone)] @@ -257,11 +260,14 @@ impl RegionServer { RegionNotReadySnafu { region_id } ); - self.inner + let provider = self + .inner .table_provider_factory .create(region_id, status.into_engine(), ctx) .await - .context(ExecuteLogicalPlanSnafu) + .context(ExecuteLogicalPlanSnafu)?; + + Ok(provider) } /// Handle reads from remote. They're often query requests received by our Arrow Flight service. @@ -307,6 +313,8 @@ impl RegionServer { .await .context(DecodeLogicalPlanSnafu)?; + let cleanup = self.register_initial_remote_dyn_filter_cleanup(&query_ctx, region_id); + let stream = self .inner .handle_read( @@ -318,7 +326,13 @@ impl RegionServer { query_ctx.clone(), ) .await?; + let stream = wrap_flow_region_watermark_stream(stream, region_id, &query_ctx); + let stream = if let Some(cleanup) = cleanup { + wrap_remote_dyn_filter_guarded_stream(stream, cleanup) + } else { + stream + }; Ok(maybe_guard_stream(stream, permit)) } @@ -350,12 +364,19 @@ impl RegionServer { .context(DataFusionSnafu)? .data; + let cleanup = self.register_initial_remote_dyn_filter_cleanup(&query_ctx, region_id); + let stream = self .inner .handle_read(QueryRequest { plan, ..request }, query_ctx.clone()) .await?; let stream = wrap_flow_region_watermark_stream(stream, region_id, &query_ctx); + let stream = if let Some(cleanup) = cleanup { + wrap_remote_dyn_filter_guarded_stream(stream, cleanup) + } else { + stream + }; Ok(maybe_guard_stream(stream, permit)) } @@ -742,70 +763,6 @@ impl RegionServer { Ok(response) } - async fn handle_remote_dyn_filter_request( - &self, - request: &RemoteDynFilterRequest, - ) -> Result { - if request.query_id.is_empty() { - return error::MissingRequiredFieldSnafu { name: "query_id" }.fail(); - } - - match request - .action - .as_ref() - .context(error::MissingRequiredFieldSnafu { name: "action" })? - { - Action::Update(update) => { - self.handle_remote_dyn_filter_update(&request.query_id, update) - .await - } - Action::Unregister(unregister) => { - self.handle_remote_dyn_filter_unregister(&request.query_id, unregister) - .await - } - } - } - - async fn handle_remote_dyn_filter_update( - &self, - query_id: &str, - request: &api::v1::region::RemoteDynFilterUpdate, - ) -> Result { - if request.filter_id.is_empty() { - return error::MissingRequiredFieldSnafu { name: "filter_id" }.fail(); - } - - if request.payload.is_empty() { - return error::MissingRequiredFieldSnafu { name: "payload" }.fail(); - } - - NotYetImplementedSnafu { - what: format!( - "remote dyn filter update unary RPC placeholder for query_id {query_id}, filter_id {}", - request.filter_id - ), - } - .fail() - } - - async fn handle_remote_dyn_filter_unregister( - &self, - query_id: &str, - request: &api::v1::region::RemoteDynFilterUnregister, - ) -> Result { - if request.filter_id.is_empty() { - return error::MissingRequiredFieldSnafu { name: "filter_id" }.fail(); - } - - NotYetImplementedSnafu { - what: format!( - "remote dyn filter unregister unary RPC placeholder for query_id {query_id}, filter_id {}", - request.filter_id - ), - } - .fail() - } - /// Sync region manifest and registers new opened logical regions. pub async fn sync_region( &self, @@ -1081,6 +1038,9 @@ struct RegionServerInner { /// server with a concrete engine; acceptable for now to fetch Mito-specific /// info (e.g., list SSTs). Consider a diagnostics trait later. mito_engine: RwLock>, + /// TODO(remote-dyn-filter): Reap this query-scoped placeholder registry on query finish/cancel + /// and later fold it into the real remote dyn filter runtime state lifecycle. + initial_remote_dyn_filter_registrations: RemoteDynFilterRegistry, } struct RegionServerParallelism { @@ -1197,6 +1157,7 @@ impl RegionServerInner { parallelism, topic_stats_reporter: RwLock::new(None), mito_engine: RwLock::new(None), + initial_remote_dyn_filter_registrations: RemoteDynFilterRegistry::new(), } } @@ -1964,10 +1925,6 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; - use api::v1::region::{ - RemoteDynFilterRequest, RemoteDynFilterUnregister, RemoteDynFilterUpdate, - remote_dyn_filter_request, - }; use api::v1::{Rows, SemanticType}; use common_error::ext::ErrorExt; use common_recordbatch::RecordBatches; @@ -1987,7 +1944,6 @@ mod tests { use store_api::storage::RegionId; use super::*; - use crate::error::Result; use crate::tests::{MockRegionEngine, mock_region_server}; #[test] @@ -2779,135 +2735,4 @@ mod tests { .await .unwrap_err(); } - - #[tokio::test] - async fn test_handle_remote_dyn_filter_request_requires_query_id() { - let mock_region_server = mock_region_server(); - - let err = mock_region_server - .handle_remote_dyn_filter_request(&RemoteDynFilterRequest { - query_id: String::new(), - action: Some(remote_dyn_filter_request::Action::Unregister( - RemoteDynFilterUnregister { - filter_id: "filter-1".to_string(), - }, - )), - }) - .await - .unwrap_err(); - - assert_matches!( - err, - crate::error::Error::MissingRequiredField { ref name, .. } if name == "query_id" - ); - } - - #[tokio::test] - async fn test_handle_remote_dyn_filter_request_requires_action() { - let mock_region_server = mock_region_server(); - - let err = mock_region_server - .handle_remote_dyn_filter_request(&RemoteDynFilterRequest { - query_id: "query-1".to_string(), - action: None, - }) - .await - .unwrap_err(); - - assert_matches!( - err, - crate::error::Error::MissingRequiredField { ref name, .. } if name == "action" - ); - } - - #[tokio::test] - async fn test_handle_remote_dyn_filter_update_requires_filter_id() { - let mock_region_server = mock_region_server(); - - let err = mock_region_server - .handle_remote_dyn_filter_request(&RemoteDynFilterRequest { - query_id: "query-1".to_string(), - action: Some(remote_dyn_filter_request::Action::Update( - RemoteDynFilterUpdate { - filter_id: String::new(), - payload: vec![1], - generation: 1, - is_complete: false, - }, - )), - }) - .await - .unwrap_err(); - - assert_matches!( - err, - crate::error::Error::MissingRequiredField { ref name, .. } if name == "filter_id" - ); - } - - #[tokio::test] - async fn test_handle_remote_dyn_filter_update_requires_payload() { - let mock_region_server = mock_region_server(); - - let err = mock_region_server - .handle_remote_dyn_filter_request(&RemoteDynFilterRequest { - query_id: "query-1".to_string(), - action: Some(remote_dyn_filter_request::Action::Update( - RemoteDynFilterUpdate { - filter_id: "filter-1".to_string(), - payload: Vec::new(), - generation: 1, - is_complete: false, - }, - )), - }) - .await - .unwrap_err(); - - assert_matches!( - err, - crate::error::Error::MissingRequiredField { ref name, .. } if name == "payload" - ); - } - - #[tokio::test] - async fn test_handle_remote_dyn_filter_update_placeholder() { - let mock_region_server = mock_region_server(); - - let err = mock_region_server - .handle_remote_dyn_filter_request(&RemoteDynFilterRequest { - query_id: "query-1".to_string(), - action: Some(remote_dyn_filter_request::Action::Update( - RemoteDynFilterUpdate { - filter_id: "filter-1".to_string(), - payload: vec![1], - generation: 1, - is_complete: false, - }, - )), - }) - .await - .unwrap_err(); - - assert_matches!(err, crate::error::Error::NotYetImplemented { .. }); - } - - #[tokio::test] - async fn test_handle_remote_dyn_filter_unregister_placeholder() { - let mock_region_server = mock_region_server(); - - let err = mock_region_server - .handle_remote_dyn_filter_request(&RemoteDynFilterRequest { - query_id: "query-1".to_string(), - action: Some(remote_dyn_filter_request::Action::Unregister( - RemoteDynFilterUnregister { - filter_id: "filter-1".to_string(), - }, - )), - }) - .await - .unwrap_err(); - - assert_matches!(err, crate::error::Error::NotYetImplemented { .. }); - } } diff --git a/src/datanode/src/region_server/registrations.rs b/src/datanode/src/region_server/registrations.rs new file mode 100644 index 0000000000..92a1bb813c --- /dev/null +++ b/src/datanode/src/region_server/registrations.rs @@ -0,0 +1,686 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet}; +use std::fmt::{Display, Formatter}; +use std::sync::{Arc, Mutex, OnceLock}; + +use common_query::request::{ + DynFilterPayload, INITIAL_REMOTE_DYN_FILTER_REGISTRATIONS_EXTENSION_KEY, InitialDynFilterReg, + InitialDynFilterRegs, +}; +use common_telemetry::warn; +use dashmap::DashMap; +use datafusion::arrow::datatypes::{Schema, SchemaRef}; +use datafusion::execution::{SessionStateBuilder, TaskContext}; +use datafusion::physical_plan::PhysicalExpr; +use datafusion::physical_plan::expressions::{DynamicFilterPhysicalExpr, lit}; +use datafusion_common::Result as DataFusionResult; +use session::context::QueryContextRef; +use session::query_id::QueryId; +use store_api::storage::RegionId; + +pub(super) const REMOTE_DYN_FILTER_PAYLOAD_MAX_BYTES: usize = 64 * 1024; + +type QueryRemoteDynFilterRegs = HashMap; + +#[derive(Debug, Default)] +pub(super) struct RemoteDynFilterRegistry { + // Keep cross-query concurrency while making each query's RDF state machine a + // single critical section. RDF count per query is small + queries: DashMap>>, +} + +impl RemoteDynFilterRegistry { + pub(super) fn new() -> Self { + Self::default() + } + + fn get_or_insert_query(&self, query_id: QueryId) -> Arc> { + self.queries + .entry(query_id) + .or_insert_with(|| Arc::new(Mutex::new(HashMap::new()))) + .clone() + } + + fn get_query(&self, query_id: &QueryId) -> Option>> { + self.queries + .get(query_id) + .map(|query_regs| query_regs.clone()) + } + + fn remove_query_if_empty( + &self, + query_id: &QueryId, + expected: &Arc>, + ) { + // Protect against stale cleanup of an old per-query state: remove the + // outer entry only if it still points to the same inner mutex and that + // inner map is still empty. + self.queries.remove_if(query_id, |_, query_regs| { + Arc::ptr_eq(query_regs, expected) && query_regs.lock().unwrap().is_empty() + }); + } + + #[cfg(test)] + pub(super) fn inspect_query( + &self, + query_id: &QueryId, + inspect: impl FnOnce(&QueryRemoteDynFilterRegs) -> R, + ) -> Option { + self.get_query(query_id) + .map(|query_regs| inspect(&query_regs.lock().unwrap())) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(super) struct RemoteDynFilterId(String); + +impl RemoteDynFilterId { + pub(super) fn new(value: impl Into) -> Self { + Self(value.into()) + } +} + +impl Display for RemoteDynFilterId { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum RemoteDynFilterUpdateOutcome { + MissingRegistration, + Buffered, + Applied, + Idempotent, + Stale, + AlreadyComplete, + PayloadTooLarge, + DecodeFailed, +} + +#[derive(Debug, Clone)] +struct PendingDynFilterUpdate { + payload: Vec, + generation: u64, + is_complete: bool, +} + +impl PendingDynFilterUpdate { + fn from_initial_reg(reg: &InitialDynFilterReg) -> Option { + let snapshot = reg.initial_snapshot.as_ref()?; + match &snapshot.payload { + DynFilterPayload::Datafusion(payload) => Some(Self { + payload: payload.clone(), + generation: snapshot.generation, + is_complete: snapshot.is_complete, + }), + _ => None, + } + } +} + +#[derive(Debug)] +struct RemoteDynFilterEpochState { + generation: Option, + is_complete: bool, +} + +#[derive(Debug)] +struct RemoteDynFilterState { + filter: Arc, + input_schema: SchemaRef, + epoch: Mutex, +} + +impl RemoteDynFilterState { + fn new(filter: Arc, input_schema: SchemaRef) -> Self { + Self { + filter, + input_schema, + epoch: Mutex::new(RemoteDynFilterEpochState { + generation: None, + is_complete: false, + }), + } + } + + fn filter(&self) -> Arc { + self.filter.clone() + } + + fn apply_update( + &self, + payload: &[u8], + generation: u64, + is_complete: bool, + ) -> RemoteDynFilterUpdateOutcome { + if !validate_update_payload_size(payload) { + return RemoteDynFilterUpdateOutcome::PayloadTooLarge; + } + + let mut epoch = self.epoch.lock().unwrap(); + if let Some(current_generation) = epoch.generation { + if generation < current_generation { + return RemoteDynFilterUpdateOutcome::Stale; + } + + if generation == current_generation { + if is_complete && !epoch.is_complete { + self.filter.mark_complete(); + epoch.is_complete = true; + return RemoteDynFilterUpdateOutcome::Applied; + } + return RemoteDynFilterUpdateOutcome::Idempotent; + } + } + + if epoch.is_complete { + return RemoteDynFilterUpdateOutcome::AlreadyComplete; + } + + let expr = match decode_update_payload(payload, self.input_schema.as_ref()) { + Ok(expr) => expr, + Err(error) => { + warn!(error; "Failed to decode remote dynamic filter update payload"); + return RemoteDynFilterUpdateOutcome::DecodeFailed; + } + }; + + if let Err(error) = self.filter.update(expr) { + warn!(error; "Failed to apply remote dynamic filter update"); + return RemoteDynFilterUpdateOutcome::DecodeFailed; + } + + epoch.generation = Some(generation); + if is_complete { + self.filter.mark_complete(); + epoch.is_complete = true; + } + + RemoteDynFilterUpdateOutcome::Applied + } +} + +#[derive(Debug)] +pub(super) struct RegisteredDynFilter { + pub(super) filter_id: RemoteDynFilterId, + pub(super) child_exprs_datafusion_proto: Vec>, + pub(super) subscriber_regions: HashSet, + runtime: Option>, + pending_update: Option, +} + +impl RegisteredDynFilter { + fn new( + filter_id: RemoteDynFilterId, + child_exprs_datafusion_proto: Vec>, + pending_update: Option, + region_id: RegionId, + ) -> Self { + let mut subscriber_regions = HashSet::new(); + subscriber_regions.insert(region_id); + + Self { + filter_id, + child_exprs_datafusion_proto, + subscriber_regions, + runtime: None, + pending_update, + } + } + + fn apply_initial_snapshot( + &mut self, + reg: &InitialDynFilterReg, + ) -> RemoteDynFilterUpdateOutcome { + let Some(snapshot) = PendingDynFilterUpdate::from_initial_reg(reg) else { + return RemoteDynFilterUpdateOutcome::Idempotent; + }; + + self.apply_or_buffer_update(&snapshot.payload, snapshot.generation, snapshot.is_complete) + } + + fn register_subscriber(&mut self, region_id: RegionId) -> bool { + if !self.subscriber_regions.insert(region_id) { + warn!( + "Duplicate remote dynamic filter subscriber region, filter_id: {}, region_id: {}", + self.filter_id, region_id + ); + return false; + } + + true + } + + fn has_subscribers(&self) -> bool { + !self.subscriber_regions.is_empty() + } + + fn should_drop_after_remove(&mut self, region_id: RegionId) -> bool { + self.subscriber_regions.remove(®ion_id); + !self.has_subscribers() + } + + fn buffer_update( + &mut self, + payload: &[u8], + generation: u64, + is_complete: bool, + ) -> RemoteDynFilterUpdateOutcome { + if !validate_update_payload_size(payload) { + return RemoteDynFilterUpdateOutcome::PayloadTooLarge; + } + + if let Some(pending) = self.pending_update.as_mut() { + if generation < pending.generation { + return RemoteDynFilterUpdateOutcome::Stale; + } + + if generation == pending.generation { + pending.is_complete |= is_complete; + return RemoteDynFilterUpdateOutcome::Idempotent; + } + + if pending.is_complete { + return RemoteDynFilterUpdateOutcome::AlreadyComplete; + } + } + + self.pending_update = Some(PendingDynFilterUpdate { + payload: payload.to_vec(), + generation, + is_complete, + }); + RemoteDynFilterUpdateOutcome::Buffered + } + + fn apply_or_buffer_update( + &mut self, + payload: &[u8], + generation: u64, + is_complete: bool, + ) -> RemoteDynFilterUpdateOutcome { + if let Some(runtime) = &self.runtime { + return runtime.apply_update(payload, generation, is_complete); + } + + self.buffer_update(payload, generation, is_complete) + } + + fn decode_children( + &self, + input_schema: &Schema, + ) -> DataFusionResult>> { + InitialDynFilterReg::new( + self.filter_id.to_string(), + self.child_exprs_datafusion_proto.clone(), + ) + .decode_children( + remote_dyn_filter_task_context(), + input_schema, + REMOTE_DYN_FILTER_PAYLOAD_MAX_BYTES, + ) + } + + fn dyn_filter(&mut self, input_schema: &Schema) -> Option> { + let children = match self.decode_children(input_schema) { + Ok(children) => children, + Err(error) => { + warn!(error; "Failed to decode remote dynamic filter initial children"); + return None; + } + }; + + let runtime = match &self.runtime { + Some(runtime) => runtime.clone(), + None => { + let filter = Arc::new(DynamicFilterPhysicalExpr::new(children.clone(), lit(true))); + let runtime = Arc::new(RemoteDynFilterState::new( + filter, + Arc::new(input_schema.clone()), + )); + if let Some(pending) = self.pending_update.take() { + let outcome = runtime.apply_update( + &pending.payload, + pending.generation, + pending.is_complete, + ); + if matches!(outcome, RemoteDynFilterUpdateOutcome::DecodeFailed) { + warn!( + "Dropped buffered remote dynamic filter update after decode failure, filter_id: {}, generation: {}", + self.filter_id, pending.generation + ); + } + } + self.runtime = Some(runtime.clone()); + runtime + } + }; + + match runtime.filter().with_new_children(children) { + Ok(expr) => Some(expr), + Err(error) => { + warn!(error; "Failed to remap remote dynamic filter children for scan"); + None + } + } + } + + fn deactivate(&self) { + if let Some(runtime) = &self.runtime { + runtime.filter.mark_complete(); + } + } +} + +impl Drop for RegisteredDynFilter { + fn drop(&mut self) { + self.deactivate(); + } +} + +pub(super) fn initial_dyn_filter_regs_from_query_ctx( + query_ctx: &QueryContextRef, +) -> Option { + let registrations = + query_ctx.extension(INITIAL_REMOTE_DYN_FILTER_REGISTRATIONS_EXTENSION_KEY)?; + match InitialDynFilterRegs::from_extension_value(registrations) { + Ok(registrations) => match registrations.validate_default_bounds() { + Ok(()) => Some(registrations), + Err(error) => { + warn!(error; "Initial remote dyn filter registrations exceeded Task 03 bounds"); + None + } + }, + Err(error) => { + warn!(error; "Failed to decode initial remote dyn filter registrations from query context"); + None + } + } +} + +pub(super) fn register_initial_dyn_filter_regs( + regs_by_query: &RemoteDynFilterRegistry, + query_id: &QueryId, + region_id: RegionId, + regs: &InitialDynFilterRegs, +) -> Vec { + if regs.is_empty() { + return Vec::new(); + } + + if let Err(error) = regs.validate_default_bounds() { + warn!(error; "Ignored invalid initial dyn filter registrations for query_id {}", query_id); + return Vec::new(); + } + + let query_regs = regs_by_query.get_or_insert_query(*query_id); + let mut query_regs = query_regs.lock().unwrap(); + let mut registered_filter_ids = Vec::with_capacity(regs.regs.len()); + + for reg in ®s.regs { + let filter_id = RemoteDynFilterId::new(reg.filter_id.clone()); + match query_regs.entry(filter_id.clone()) { + Entry::Occupied(mut entry) => { + let registered = entry.get_mut(); + if registered.child_exprs_datafusion_proto != reg.child_exprs_datafusion_proto { + warn!( + "Remote dynamic filter registration reused filter_id with different children, query_id: {}, filter_id: {}, region_id: {}", + query_id, filter_id, region_id + ); + } + if registered.register_subscriber(region_id) { + registered_filter_ids.push(filter_id); + } + let _ = registered.apply_initial_snapshot(reg); + } + Entry::Vacant(entry) => { + entry.insert(RegisteredDynFilter::new( + filter_id.clone(), + reg.child_exprs_datafusion_proto.clone(), + PendingDynFilterUpdate::from_initial_reg(reg), + region_id, + )); + registered_filter_ids.push(filter_id); + } + } + } + + registered_filter_ids +} + +pub(super) fn remote_dyn_filter_exprs_for_initial_regs( + regs_by_query: &RemoteDynFilterRegistry, + query_id: &QueryId, + initial_regs: &InitialDynFilterRegs, + input_schema: &Schema, +) -> Vec> { + let Some(query_regs) = regs_by_query.get_query(query_id) else { + return Vec::new(); + }; + + let mut query_regs = query_regs.lock().unwrap(); + initial_regs + .regs + .iter() + .filter_map(|reg| { + let filter_id = RemoteDynFilterId::new(reg.filter_id.clone()); + let registered = query_regs.get_mut(&filter_id)?; + registered.dyn_filter(input_schema) + }) + .collect() +} + +pub(super) fn apply_remote_dyn_filter_update( + regs_by_query: &RemoteDynFilterRegistry, + query_id: &QueryId, + filter_id: &RemoteDynFilterId, + payload: &[u8], + generation: u64, + is_complete: bool, +) -> RemoteDynFilterUpdateOutcome { + if !validate_update_payload_size(payload) { + warn!( + "Ignored oversized remote dynamic filter update, query_id: {}, filter_id: {}, payload_size: {}, max_payload_size: {}", + query_id, + filter_id, + payload.len(), + REMOTE_DYN_FILTER_PAYLOAD_MAX_BYTES + ); + return RemoteDynFilterUpdateOutcome::PayloadTooLarge; + } + + let Some(query_regs) = regs_by_query.get_query(query_id) else { + warn!( + "Ignored remote dynamic filter update without query registration, query_id: {}, filter_id: {}", + query_id, filter_id + ); + return RemoteDynFilterUpdateOutcome::MissingRegistration; + }; + + let mut query_regs = query_regs.lock().unwrap(); + let Some(registered) = query_regs.get_mut(filter_id) else { + warn!( + "Ignored remote dynamic filter update without filter registration, query_id: {}, filter_id: {}", + query_id, filter_id + ); + return RemoteDynFilterUpdateOutcome::MissingRegistration; + }; + + registered.apply_or_buffer_update(payload, generation, is_complete) +} + +pub(super) fn unregister_remote_dyn_filter( + regs_by_query: &RemoteDynFilterRegistry, + query_id: &QueryId, + filter_id: &RemoteDynFilterId, +) -> RemoteDynFilterUpdateOutcome { + let Some(query_regs) = regs_by_query.get_query(query_id) else { + warn!( + "Ignored remote dynamic filter unregister without query registration, query_id: {}, filter_id: {}", + query_id, filter_id + ); + return RemoteDynFilterUpdateOutcome::MissingRegistration; + }; + + let (registered, should_remove_query) = { + let mut locked = query_regs.lock().unwrap(); + let Some(registered) = locked.remove(filter_id) else { + warn!( + "Ignored remote dynamic filter unregister without filter registration, query_id: {}, filter_id: {}", + query_id, filter_id + ); + return RemoteDynFilterUpdateOutcome::MissingRegistration; + }; + let should_remove_query = locked.is_empty(); + (registered, should_remove_query) + }; + + drop(registered); + if should_remove_query { + regs_by_query.remove_query_if_empty(query_id, &query_regs); + } + + RemoteDynFilterUpdateOutcome::Applied +} + +pub(super) fn remove_initial_dyn_filter_regs( + regs_by_query: &RemoteDynFilterRegistry, + query_id: &QueryId, + region_id: RegionId, + filter_ids: &[RemoteDynFilterId], +) { + if filter_ids.is_empty() { + return; + } + + let Some(query_regs) = regs_by_query.get_query(query_id) else { + return; + }; + + let (removed_filters, should_remove_query) = { + let mut locked = query_regs.lock().unwrap(); + let mut removed_filters = Vec::new(); + + for filter_id in filter_ids { + let should_remove_filter = locked + .get_mut(filter_id) + .map(|registered| registered.should_drop_after_remove(region_id)) + .unwrap_or(false); + + if should_remove_filter && let Some(registered) = locked.remove(filter_id) { + removed_filters.push(registered); + } + } + + let should_remove_query = locked.is_empty(); + (removed_filters, should_remove_query) + }; + + drop(removed_filters); + if should_remove_query { + regs_by_query.remove_query_if_empty(query_id, &query_regs); + } +} + +fn decode_update_payload( + payload: &[u8], + input_schema: &Schema, +) -> DataFusionResult> { + DynFilterPayload::Datafusion(payload.to_vec()).decode_datafusion_expr( + remote_dyn_filter_task_context(), + input_schema, + REMOTE_DYN_FILTER_PAYLOAD_MAX_BYTES, + ) +} + +fn remote_dyn_filter_task_context() -> &'static TaskContext { + static TASK_CONTEXT: OnceLock = OnceLock::new(); + + TASK_CONTEXT.get_or_init(|| { + // RDF payloads can contain DataFusion built-in scalar functions. For + // example, multi-column join dynamic filters use `struct(...) IN (...)`. + // `TaskContext::default()` has an empty function registry and cannot + // decode those expressions. + let session_state = SessionStateBuilder::new().with_default_features().build(); + TaskContext::from(&session_state) + }) +} + +fn validate_update_payload_size(payload: &[u8]) -> bool { + payload.len() <= REMOTE_DYN_FILTER_PAYLOAD_MAX_BYTES +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn remote_dyn_filter_same_query_reuses_one_inner_lock() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let query_id = QueryId::new(); + + let first = regs_by_query.get_or_insert_query(query_id); + let second = regs_by_query.get_or_insert_query(query_id); + + assert!(Arc::ptr_eq(&first, &second)); + } + + #[test] + fn remote_dyn_filter_stale_query_remove_does_not_remove_new_query_state() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let query_id = QueryId::new(); + let stale_query_regs = regs_by_query.get_or_insert_query(query_id); + + regs_by_query.remove_query_if_empty(&query_id, &stale_query_regs); + assert!(regs_by_query.get_query(&query_id).is_none()); + + let new_query_regs = regs_by_query.get_or_insert_query(query_id); + let filter_id = RemoteDynFilterId::new("filter-1"); + let region_id = RegionId::new(1024, 7); + new_query_regs.lock().unwrap().insert( + filter_id.clone(), + RegisteredDynFilter::new(filter_id.clone(), vec![], None, region_id), + ); + + regs_by_query.remove_query_if_empty(&query_id, &stale_query_regs); + + let current_query_regs = regs_by_query.get_query(&query_id).unwrap(); + assert!(Arc::ptr_eq(¤t_query_regs, &new_query_regs)); + assert_eq!(current_query_regs.lock().unwrap().len(), 1); + } + + #[test] + fn remote_dyn_filter_register_uses_entry_to_merge_same_filter_subscribers() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let query_id = QueryId::new(); + let first_region_id = RegionId::new(1024, 7); + let second_region_id = RegionId::new(1024, 8); + let regs = InitialDynFilterRegs::new(vec![InitialDynFilterReg::new("filter-1", vec![])]); + + register_initial_dyn_filter_regs(®s_by_query, &query_id, first_region_id, ®s); + register_initial_dyn_filter_regs(®s_by_query, &query_id, second_region_id, ®s); + + let query_regs = regs_by_query.get_query(&query_id).unwrap(); + let query_regs = query_regs.lock().unwrap(); + assert_eq!(query_regs.len(), 1); + let registered = query_regs.get(&RemoteDynFilterId::new("filter-1")).unwrap(); + assert_eq!(registered.subscriber_regions.len(), 2); + assert!(registered.subscriber_regions.contains(&first_region_id)); + assert!(registered.subscriber_regions.contains(&second_region_id)); + } +} diff --git a/src/datanode/src/region_server/remote_dyn_filter.rs b/src/datanode/src/region_server/remote_dyn_filter.rs new file mode 100644 index 0000000000..f750cb896c --- /dev/null +++ b/src/datanode/src/region_server/remote_dyn_filter.rs @@ -0,0 +1,1170 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::pin::Pin; +use std::sync::{Arc, Weak}; +use std::task::{Context, Poll}; + +use api::region::RegionResponse; +use api::v1::region::RemoteDynFilterRequest; +use api::v1::region::remote_dyn_filter_request::Action; +use common_base::Plugins; +use common_recordbatch::adapter::RecordBatchMetrics; +use common_recordbatch::{OrderOption, RecordBatch, RecordBatchStream, SendableRecordBatchStream}; +use common_telemetry::{debug, warn}; +use datafusion_expr::LogicalPlan; +use futures_util::Stream; +use query::dist_plan::{ + RemoteDynFilterReceiverInjector, RemoteDynFilterReceiverInjectorRef, + RemoteDynFilterReceiverLogicalPlan, +}; +use query::options::remote_dyn_filter_pushdown_enabled_from_extensions; +use query::promql::plan_contains_promql_extension; +use session::context::QueryContextRef; +use session::query_id::QueryId; +use snafu::OptionExt; +use store_api::storage::RegionId; + +use crate::error::{self, Result, UnexpectedSnafu}; +use crate::region_server::registrations::{ + RemoteDynFilterId, RemoteDynFilterUpdateOutcome, apply_remote_dyn_filter_update, + initial_dyn_filter_regs_from_query_ctx, register_initial_dyn_filter_regs, + remote_dyn_filter_exprs_for_initial_regs, remove_initial_dyn_filter_regs, + unregister_remote_dyn_filter, +}; +use crate::region_server::{RegionServer, RegionServerInner}; + +fn remote_dyn_filter_receiver_plan( + server: &Weak, + origin: LogicalPlan, + ctx: QueryContextRef, +) -> LogicalPlan { + if !remote_dyn_filter_pushdown_enabled(&ctx) { + return origin; + } + + if plan_contains_promql_extension(&origin) { + return origin; + } + + let Some(query_id) = ctx.remote_query_id_value() else { + return origin; + }; + + let Some(initial_regs) = initial_dyn_filter_regs_from_query_ctx(&ctx) else { + return origin; + }; + + let Some(server) = server.upgrade() else { + return origin; + }; + + let dyn_filters = remote_dyn_filter_exprs_for_initial_regs( + &server.initial_remote_dyn_filter_registrations, + &query_id, + &initial_regs, + origin.schema().as_arrow(), + ); + + if dyn_filters.is_empty() { + return origin; + } + + RemoteDynFilterReceiverLogicalPlan::new(origin, dyn_filters).into_logical_plan() +} + +fn remote_dyn_filter_pushdown_enabled(query_ctx: &QueryContextRef) -> bool { + match remote_dyn_filter_pushdown_enabled_from_extensions(&query_ctx.extensions()) { + Ok(enabled) => enabled, + Err(error) => { + warn!(error; "Remote dynamic filter pushdown is disabled because the query option is invalid"); + false + } + } +} + +impl RegionServer { + pub fn install_remote_dyn_filter_receiver_injector(&self, plugins: &Plugins) { + let server = Arc::downgrade(&self.inner); + plugins.insert::(Arc::new( + RemoteDynFilterReceiverInjector::new(move |origin, ctx| { + remote_dyn_filter_receiver_plan(&server, origin, ctx) + }), + )); + } + + pub(super) fn register_initial_remote_dyn_filter_cleanup( + &self, + query_ctx: &QueryContextRef, + region_id: RegionId, + ) -> Option { + if !remote_dyn_filter_pushdown_enabled(query_ctx) { + return None; + } + + let initial_dyn_filter_regs = initial_dyn_filter_regs_from_query_ctx(query_ctx); + let query_id = query_ctx.remote_query_id_value(); + let registered_filter_ids = if let (Some(query_id), Some(regs)) = + (query_id.as_ref(), initial_dyn_filter_regs.as_ref()) + { + register_initial_dyn_filter_regs( + &self.inner.initial_remote_dyn_filter_registrations, + query_id, + region_id, + regs, + ) + } else { + Vec::new() + }; + + match (query_id, registered_filter_ids.is_empty()) { + (Some(query_id), false) => Some(RemoteDynFilterRegistrationGuard::new( + self.clone(), + query_id, + region_id, + registered_filter_ids, + )), + _ => None, + } + } + + pub(super) async fn handle_remote_dyn_filter_request( + &self, + request: &RemoteDynFilterRequest, + ) -> Result { + if request.query_id.is_empty() { + return error::MissingRequiredFieldSnafu { name: "query_id" }.fail(); + } + + let query_id = request.query_id.parse::().map_err(|_| { + UnexpectedSnafu { + violated: "remote dynamic filter query_id must be a valid QueryId", + } + .build() + })?; + + match request + .action + .as_ref() + .context(error::MissingRequiredFieldSnafu { name: "action" })? + { + Action::Update(update) => { + self.handle_remote_dyn_filter_update(&query_id, update) + .await + } + Action::Unregister(unregister) => { + self.handle_remote_dyn_filter_unregister(&query_id, unregister) + .await + } + } + } + + async fn handle_remote_dyn_filter_update( + &self, + query_id: &QueryId, + request: &api::v1::region::RemoteDynFilterUpdate, + ) -> Result { + if request.filter_id.is_empty() { + return error::MissingRequiredFieldSnafu { name: "filter_id" }.fail(); + } + + if request.payload.is_empty() { + return error::MissingRequiredFieldSnafu { name: "payload" }.fail(); + } + + let filter_id = RemoteDynFilterId::new(request.filter_id.clone()); + let outcome = apply_remote_dyn_filter_update( + &self.inner.initial_remote_dyn_filter_registrations, + query_id, + &filter_id, + &request.payload, + request.generation, + request.is_complete, + ); + self.log_remote_dyn_filter_update_outcome(query_id, &filter_id, outcome); + + Ok(RegionResponse::new(0)) + } + + async fn handle_remote_dyn_filter_unregister( + &self, + query_id: &QueryId, + request: &api::v1::region::RemoteDynFilterUnregister, + ) -> Result { + if request.filter_id.is_empty() { + return error::MissingRequiredFieldSnafu { name: "filter_id" }.fail(); + } + + let filter_id = RemoteDynFilterId::new(request.filter_id.clone()); + let outcome = unregister_remote_dyn_filter( + &self.inner.initial_remote_dyn_filter_registrations, + query_id, + &filter_id, + ); + self.log_remote_dyn_filter_update_outcome(query_id, &filter_id, outcome); + + Ok(RegionResponse::new(0)) + } + + fn log_remote_dyn_filter_update_outcome( + &self, + query_id: &QueryId, + filter_id: &RemoteDynFilterId, + outcome: RemoteDynFilterUpdateOutcome, + ) { + if matches!( + outcome, + RemoteDynFilterUpdateOutcome::MissingRegistration + | RemoteDynFilterUpdateOutcome::AlreadyComplete + | RemoteDynFilterUpdateOutcome::PayloadTooLarge + | RemoteDynFilterUpdateOutcome::DecodeFailed + ) { + warn!( + "Remote dynamic filter update outcome, query_id: {}, filter_id: {}, outcome: {:?}", + query_id, filter_id, outcome + ); + } else { + debug!( + "Remote dynamic filter update outcome, query_id: {}, filter_id: {}, outcome: {:?}", + query_id, filter_id, outcome + ); + } + } +} + +pub(super) fn wrap_remote_dyn_filter_guarded_stream( + stream: SendableRecordBatchStream, + cleanup: RemoteDynFilterRegistrationGuard, +) -> SendableRecordBatchStream { + Box::pin(RemoteDynFilterGuardedStream { stream, cleanup }) +} + +/// Removes query-scoped remote dynamic filter subscriptions unless ownership is moved elsewhere. +pub(super) struct RemoteDynFilterRegistrationGuard { + server: RegionServer, + query_id: QueryId, + region_id: RegionId, + filter_ids: Vec, + cleaned: bool, +} + +impl RemoteDynFilterRegistrationGuard { + fn new( + server: RegionServer, + query_id: QueryId, + region_id: RegionId, + filter_ids: Vec, + ) -> Self { + Self { + server, + query_id, + region_id, + filter_ids, + cleaned: false, + } + } + + fn cleanup_once(&mut self) { + if self.cleaned { + return; + } + + remove_initial_dyn_filter_regs( + &self.server.inner.initial_remote_dyn_filter_registrations, + &self.query_id, + self.region_id, + &self.filter_ids, + ); + self.cleaned = true; + } +} + +impl Drop for RemoteDynFilterRegistrationGuard { + fn drop(&mut self) { + self.cleanup_once(); + } +} + +/// Removes query-scoped remote dynamic filter subscriptions when a remote read stream is done. +struct RemoteDynFilterGuardedStream { + stream: SendableRecordBatchStream, + cleanup: RemoteDynFilterRegistrationGuard, +} + +impl RecordBatchStream for RemoteDynFilterGuardedStream { + fn name(&self) -> &str { + self.stream.name() + } + + fn schema(&self) -> datatypes::schema::SchemaRef { + self.stream.schema() + } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + self.stream.output_ordering() + } + + fn metrics(&self) -> Option { + self.stream.metrics() + } +} + +impl Stream for RemoteDynFilterGuardedStream { + type Item = common_recordbatch::error::Result; + + fn size_hint(&self) -> (usize, Option) { + self.stream.size_hint() + } + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match Pin::new(&mut self.stream).poll_next(cx) { + Poll::Ready(None) => { + self.cleanup.cleanup_once(); + Poll::Ready(None) + } + other => other, + } + } +} + +#[cfg(test)] +mod tests { + use std::assert_matches; + use std::collections::{HashMap, HashSet}; + use std::sync::Arc; + use std::time::Duration; + + use api::v1::region::{ + RemoteDynFilterRequest, RemoteDynFilterUnregister, RemoteDynFilterUpdate, + remote_dyn_filter_request, + }; + use common_query::request::{ + DynFilterPayload, INITIAL_REMOTE_DYN_FILTER_REGISTRATIONS_EXTENSION_KEY, + InitialDynFilterReg, InitialDynFilterRegs, InitialDynFilterSnapshot, + }; + use common_recordbatch::RecordBatches; + use datafusion::arrow::datatypes::Schema as ArrowSchema; + use datafusion::physical_plan::PhysicalExpr; + use datafusion::physical_plan::expressions::{DynamicFilterPhysicalExpr, lit as physical_lit}; + use datafusion_common::DFSchema; + use datafusion_expr::EmptyRelation; + use datatypes::prelude::{ConcreteDataType, VectorRef}; + use datatypes::schema::{ColumnSchema, Schema}; + use datatypes::vectors::Int32Vector; + use futures_util::StreamExt; + use query::options::QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN; + use session::context::QueryContext; + use session::hints::REMOTE_QUERY_ID_EXTENSION_KEY; + use session::query_id::QueryId; + use store_api::storage::RegionId; + + use super::*; + use crate::region_server::registrations::{ + REMOTE_DYN_FILTER_PAYLOAD_MAX_BYTES, RemoteDynFilterRegistry, + }; + use crate::tests::mock_region_server; + + #[derive(Debug, Clone)] + struct RegisteredDynFilterSnapshot { + filter_id: RemoteDynFilterId, + child_exprs_datafusion_proto: Vec>, + subscriber_regions: HashSet, + } + + fn query_regs( + regs_by_query: &RemoteDynFilterRegistry, + query_id: &QueryId, + ) -> Option> { + regs_by_query.inspect_query(query_id, |query_regs| { + query_regs + .iter() + .map(|(filter_id, registered)| { + ( + filter_id.clone(), + RegisteredDynFilterSnapshot { + filter_id: registered.filter_id.clone(), + child_exprs_datafusion_proto: registered + .child_exprs_datafusion_proto + .clone(), + subscriber_regions: registered.subscriber_regions.clone(), + }, + ) + }) + .collect() + }) + } + + fn test_remote_query_id() -> QueryId { + QueryId::new() + } + + fn test_remote_dyn_filter_region_id() -> RegionId { + RegionId::new(1024, 7) + } + + fn single_value_stream() -> common_recordbatch::SendableRecordBatchStream { + let schema = Arc::new(Schema::new(vec![ColumnSchema::new( + "v", + ConcreteDataType::int32_datatype(), + false, + )])); + let values: VectorRef = Arc::new(Int32Vector::from_slice([1])); + let batch = common_recordbatch::RecordBatch::new(schema.clone(), vec![values]).unwrap(); + RecordBatches::try_new(schema, vec![batch]) + .unwrap() + .as_stream() + } + + fn empty_logical_plan() -> LogicalPlan { + LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: Arc::new(DFSchema::empty()), + }) + } + + fn query_context_with_initial_regs(query_id: QueryId) -> QueryContext { + let mut query_ctx = QueryContext::with("greptime", "public"); + query_ctx.set_extension(REMOTE_QUERY_ID_EXTENSION_KEY, query_id.to_string()); + query_ctx.set_extension( + INITIAL_REMOTE_DYN_FILTER_REGISTRATIONS_EXTENSION_KEY, + InitialDynFilterRegs::new(vec![InitialDynFilterReg::new("filter-1", vec![])]) + .to_extension_value() + .unwrap(), + ); + query_ctx + } + + #[test] + fn remote_dyn_filter_receiver_plan_respects_disabled_query_option() { + let mock_region_server = mock_region_server(); + let query_id = test_remote_query_id(); + let mut query_ctx = query_context_with_initial_regs(query_id); + query_ctx.set_extension(QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN, "false"); + + let plan = remote_dyn_filter_receiver_plan( + &Arc::downgrade(&mock_region_server.inner), + empty_logical_plan(), + Arc::new(query_ctx), + ); + + assert!(matches!(plan, LogicalPlan::EmptyRelation(_))); + } + + #[test] + fn remote_dyn_filter_cleanup_registration_respects_disabled_query_option() { + let mock_region_server = mock_region_server(); + let query_id = test_remote_query_id(); + let mut query_ctx = query_context_with_initial_regs(query_id); + query_ctx.set_extension(QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN, "false"); + let query_ctx = Arc::new(query_ctx); + + let cleanup = mock_region_server.register_initial_remote_dyn_filter_cleanup( + &query_ctx, + test_remote_dyn_filter_region_id(), + ); + + assert!(cleanup.is_none()); + assert!( + mock_region_server + .inner + .initial_remote_dyn_filter_registrations + .inspect_query(&query_id, |_| ()) + .is_none() + ); + } + + #[test] + fn initial_dyn_filter_regs_can_be_read_from_query_context() { + let mut query_ctx = QueryContext::with("greptime", "public"); + query_ctx.set_extension( + INITIAL_REMOTE_DYN_FILTER_REGISTRATIONS_EXTENSION_KEY, + InitialDynFilterRegs::new(vec![InitialDynFilterReg::new( + "filter-1", + vec![vec![1, 2, 3]], + )]) + .to_extension_value() + .unwrap(), + ); + + let regs = initial_dyn_filter_regs_from_query_ctx(&Arc::new(query_ctx)).unwrap(); + + assert_eq!(regs.regs.len(), 1); + assert_eq!(regs.regs[0].filter_id, "filter-1"); + } + + #[test] + fn initial_dyn_filter_regs_from_query_context_rejects_duplicate_filter_ids() { + let mut query_ctx = QueryContext::with("greptime", "public"); + query_ctx.set_extension( + INITIAL_REMOTE_DYN_FILTER_REGISTRATIONS_EXTENSION_KEY, + InitialDynFilterRegs::new(vec![ + InitialDynFilterReg::new("filter-1", vec![vec![1, 2, 3]]), + InitialDynFilterReg::new("filter-1", vec![vec![4, 5, 6]]), + ]) + .to_extension_value() + .unwrap(), + ); + + let regs = initial_dyn_filter_regs_from_query_ctx(&Arc::new(query_ctx)); + + assert!(regs.is_none()); + } + + #[test] + fn register_initial_dyn_filter_regs_creates_query_scoped_entries() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let regs = InitialDynFilterRegs::new(vec![ + InitialDynFilterReg::new("filter-1", vec![vec![1, 2, 3]]), + InitialDynFilterReg::new("filter-2", vec![vec![4, 5, 6]]), + ]); + let query_id = test_remote_query_id(); + let region_id = test_remote_dyn_filter_region_id(); + + let registered_filter_ids = + register_initial_dyn_filter_regs(®s_by_query, &query_id, region_id, ®s); + + let query_regs = query_regs(®s_by_query, &query_id).unwrap(); + assert_eq!(query_regs.len(), 2); + assert_eq!( + registered_filter_ids, + vec![ + RemoteDynFilterId::new("filter-1"), + RemoteDynFilterId::new("filter-2") + ] + ); + let registered = query_regs.get(&RemoteDynFilterId::new("filter-1")).unwrap(); + assert_eq!(registered.filter_id, RemoteDynFilterId::new("filter-1")); + assert_eq!(registered.child_exprs_datafusion_proto, vec![vec![1, 2, 3]]); + assert_eq!(registered.subscriber_regions.len(), 1); + assert!(registered.subscriber_regions.contains(®ion_id)); + } + + #[test] + fn register_initial_dyn_filter_regs_same_region_duplicate_is_idempotent() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let regs = InitialDynFilterRegs::new(vec![InitialDynFilterReg::new( + "filter-1", + vec![vec![1, 2, 3]], + )]); + let query_id = test_remote_query_id(); + let region_id = test_remote_dyn_filter_region_id(); + + let first = register_initial_dyn_filter_regs(®s_by_query, &query_id, region_id, ®s); + let duplicate = + register_initial_dyn_filter_regs(®s_by_query, &query_id, region_id, ®s); + + let query_regs = query_regs(®s_by_query, &query_id).unwrap(); + assert_eq!(query_regs.len(), 1); + assert_eq!(first, vec![RemoteDynFilterId::new("filter-1")]); + assert!(duplicate.is_empty()); + let registered = query_regs.get(&RemoteDynFilterId::new("filter-1")).unwrap(); + assert_eq!(registered.subscriber_regions.len(), 1); + assert!(registered.subscriber_regions.contains(®ion_id)); + } + + #[test] + fn register_initial_dyn_filter_regs_ignores_invalid_duplicate_payload_set() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let regs = InitialDynFilterRegs::new(vec![ + InitialDynFilterReg::new("filter-1", vec![vec![1, 2, 3]]), + InitialDynFilterReg::new("filter-1", vec![vec![4, 5, 6]]), + ]); + let query_id = test_remote_query_id(); + let region_id = test_remote_dyn_filter_region_id(); + + register_initial_dyn_filter_regs(®s_by_query, &query_id, region_id, ®s); + + assert!(query_regs(®s_by_query, &query_id).is_none()); + } + + #[test] + fn register_initial_dyn_filter_regs_tracks_different_region_subscribers_for_same_filter() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let regs = InitialDynFilterRegs::new(vec![InitialDynFilterReg::new( + "filter-1", + vec![vec![1, 2, 3]], + )]); + let query_id = test_remote_query_id(); + let first_region_id = RegionId::new(1024, 7); + let second_region_id = RegionId::new(1024, 8); + + register_initial_dyn_filter_regs(®s_by_query, &query_id, first_region_id, ®s); + register_initial_dyn_filter_regs(®s_by_query, &query_id, second_region_id, ®s); + + let query_regs = query_regs(®s_by_query, &query_id).unwrap(); + assert_eq!(query_regs.len(), 1); + let registered = query_regs.get(&RemoteDynFilterId::new("filter-1")).unwrap(); + assert_eq!(registered.subscriber_regions.len(), 2); + assert!(registered.subscriber_regions.contains(&first_region_id)); + assert!(registered.subscriber_regions.contains(&second_region_id)); + } + + #[test] + fn remove_initial_dyn_filter_regs_removes_registered_filter_entries() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let query_id = test_remote_query_id(); + let other_query_id = test_remote_query_id(); + let region_id = test_remote_dyn_filter_region_id(); + + let registered_filter_ids = register_initial_dyn_filter_regs( + ®s_by_query, + &query_id, + region_id, + &InitialDynFilterRegs::new(vec![InitialDynFilterReg::new( + "filter-1", + vec![vec![1, 2, 3]], + )]), + ); + register_initial_dyn_filter_regs( + ®s_by_query, + &other_query_id, + region_id, + &InitialDynFilterRegs::new(vec![InitialDynFilterReg::new( + "filter-2", + vec![vec![4, 5, 6]], + )]), + ); + + remove_initial_dyn_filter_regs( + ®s_by_query, + &query_id, + region_id, + ®istered_filter_ids, + ); + + assert!(query_regs(®s_by_query, &query_id).is_none()); + let other_query_regs = query_regs(®s_by_query, &other_query_id).unwrap(); + assert_eq!(other_query_regs.len(), 1); + } + + #[test] + fn remove_initial_dyn_filter_regs_keeps_other_subscribers() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let query_id = test_remote_query_id(); + let regs = InitialDynFilterRegs::new(vec![InitialDynFilterReg::new( + "filter-1", + vec![vec![1, 2, 3]], + )]); + let first_region_id = RegionId::new(1024, 7); + let second_region_id = RegionId::new(1024, 8); + + let first_subscription = + register_initial_dyn_filter_regs(®s_by_query, &query_id, first_region_id, ®s); + register_initial_dyn_filter_regs(®s_by_query, &query_id, second_region_id, ®s); + + remove_initial_dyn_filter_regs( + ®s_by_query, + &query_id, + first_region_id, + &first_subscription, + ); + + let query_regs = query_regs(®s_by_query, &query_id).unwrap(); + assert_eq!(query_regs.len(), 1); + let registered = query_regs.get(&RemoteDynFilterId::new("filter-1")).unwrap(); + assert_eq!(registered.subscriber_regions.len(), 1); + assert!(registered.subscriber_regions.contains(&second_region_id)); + } + + #[tokio::test] + async fn test_handle_remote_dyn_filter_request_requires_query_id() { + let mock_region_server = mock_region_server(); + + let err = mock_region_server + .handle_remote_dyn_filter_request(&RemoteDynFilterRequest { + query_id: String::new(), + action: Some(remote_dyn_filter_request::Action::Unregister( + RemoteDynFilterUnregister { + filter_id: "filter-1".to_string(), + }, + )), + }) + .await + .unwrap_err(); + + assert_matches!( + err, + crate::error::Error::MissingRequiredField { ref name, .. } if name == "query_id" + ); + } + + #[tokio::test] + async fn test_handle_remote_dyn_filter_request_requires_action() { + let mock_region_server = mock_region_server(); + + let err = mock_region_server + .handle_remote_dyn_filter_request(&RemoteDynFilterRequest { + query_id: test_remote_query_id().to_string(), + action: None, + }) + .await + .unwrap_err(); + + assert_matches!( + err, + crate::error::Error::MissingRequiredField { ref name, .. } if name == "action" + ); + } + + #[tokio::test] + async fn test_handle_remote_dyn_filter_update_requires_filter_id() { + let mock_region_server = mock_region_server(); + + let err = mock_region_server + .handle_remote_dyn_filter_request(&RemoteDynFilterRequest { + query_id: test_remote_query_id().to_string(), + action: Some(remote_dyn_filter_request::Action::Update( + RemoteDynFilterUpdate { + filter_id: String::new(), + payload: vec![1], + generation: 1, + is_complete: false, + }, + )), + }) + .await + .unwrap_err(); + + assert_matches!( + err, + crate::error::Error::MissingRequiredField { ref name, .. } if name == "filter_id" + ); + } + + #[tokio::test] + async fn test_handle_remote_dyn_filter_update_requires_payload() { + let mock_region_server = mock_region_server(); + + let err = mock_region_server + .handle_remote_dyn_filter_request(&RemoteDynFilterRequest { + query_id: test_remote_query_id().to_string(), + action: Some(remote_dyn_filter_request::Action::Update( + RemoteDynFilterUpdate { + filter_id: "filter-1".to_string(), + payload: Vec::new(), + generation: 1, + is_complete: false, + }, + )), + }) + .await + .unwrap_err(); + + assert_matches!( + err, + crate::error::Error::MissingRequiredField { ref name, .. } if name == "payload" + ); + } + + #[test] + fn test_apply_remote_dyn_filter_update_missing_registration() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let query_id = test_remote_query_id(); + let filter_id = RemoteDynFilterId::new("filter-1"); + + let outcome = + apply_remote_dyn_filter_update(®s_by_query, &query_id, &filter_id, &[1], 1, false); + assert_eq!(outcome, RemoteDynFilterUpdateOutcome::MissingRegistration); + + let outcome = unregister_remote_dyn_filter(®s_by_query, &query_id, &filter_id); + assert_eq!(outcome, RemoteDynFilterUpdateOutcome::MissingRegistration); + } + + #[test] + fn test_apply_remote_dyn_filter_update_buffering_before_scan() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let regs = InitialDynFilterRegs::new(vec![InitialDynFilterReg::new( + "filter-1", + vec![vec![1, 2, 3]], + )]); + let query_id = test_remote_query_id(); + let filter_id = RemoteDynFilterId::new("filter-1"); + + register_initial_dyn_filter_regs( + ®s_by_query, + &query_id, + test_remote_dyn_filter_region_id(), + ®s, + ); + + // First update with generation 1: should be Buffered (no runtime installed yet) + let outcome = + apply_remote_dyn_filter_update(®s_by_query, &query_id, &filter_id, &[1], 1, false); + assert_eq!(outcome, RemoteDynFilterUpdateOutcome::Buffered); + + // Update with generation 0: should be Stale (older than pending generation 1) + let outcome = + apply_remote_dyn_filter_update(®s_by_query, &query_id, &filter_id, &[2], 0, false); + assert_eq!(outcome, RemoteDynFilterUpdateOutcome::Stale); + + // Update with generation 1 again: should be Idempotent (same generation) + let outcome = + apply_remote_dyn_filter_update(®s_by_query, &query_id, &filter_id, &[3], 1, false); + assert_eq!(outcome, RemoteDynFilterUpdateOutcome::Idempotent); + } + + fn datafusion_payload_bytes(expr: Arc) -> Vec { + match DynFilterPayload::from_datafusion_expr(&expr, REMOTE_DYN_FILTER_PAYLOAD_MAX_BYTES) + .unwrap() + { + DynFilterPayload::Datafusion(bytes) => bytes, + _ => unreachable!( + "DynFilterPayload::from_datafusion_expr only returns datafusion payloads" + ), + } + } + + fn empty_arrow_schema() -> ArrowSchema { + ArrowSchema::empty() + } + + fn register_empty_remote_dyn_filter( + regs_by_query: &RemoteDynFilterRegistry, + query_id: &QueryId, + ) -> Vec { + register_empty_remote_dyn_filter_for_region( + regs_by_query, + query_id, + test_remote_dyn_filter_region_id(), + ) + } + + fn register_empty_remote_dyn_filter_for_region( + regs_by_query: &RemoteDynFilterRegistry, + query_id: &QueryId, + region_id: RegionId, + ) -> Vec { + register_initial_dyn_filter_regs( + regs_by_query, + query_id, + region_id, + &InitialDynFilterRegs::new(vec![InitialDynFilterReg::new("filter-1", vec![])]), + ) + } + + #[test] + fn initial_remote_dyn_filter_snapshot_initializes_runtime_filter() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let query_id = test_remote_query_id(); + let payload = DynFilterPayload::Datafusion(datafusion_payload_bytes(physical_lit(false))); + let regs = InitialDynFilterRegs::new(vec![ + InitialDynFilterReg::new("filter-1", vec![]) + .with_initial_snapshot(InitialDynFilterSnapshot::new(payload, 7, false)), + ]); + + register_initial_dyn_filter_regs( + ®s_by_query, + &query_id, + test_remote_dyn_filter_region_id(), + ®s, + ); + let exprs = remote_dyn_filter_exprs_for_initial_regs( + ®s_by_query, + &query_id, + ®s, + &empty_arrow_schema(), + ); + + assert_eq!(exprs.len(), 1); + assert_eq!(format!("{}", exprs[0]), "DynamicFilter [ false ]"); + } + + fn only_remote_dyn_filter( + regs_by_query: &RemoteDynFilterRegistry, + query_id: &QueryId, + ) -> Arc { + let initial_regs = + InitialDynFilterRegs::new(vec![InitialDynFilterReg::new("filter-1", vec![])]); + let exprs = remote_dyn_filter_exprs_for_initial_regs( + regs_by_query, + query_id, + &initial_regs, + &empty_arrow_schema(), + ); + assert_eq!(1, exprs.len()); + let expr = exprs.into_iter().next().unwrap(); + let expr = expr as Arc; + expr.downcast::().unwrap() + } + + #[test] + fn test_remote_dyn_filter_rejects_oversized_payload_before_buffering() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let query_id = test_remote_query_id(); + let filter_id = RemoteDynFilterId::new("filter-1"); + register_empty_remote_dyn_filter(®s_by_query, &query_id); + + let oversized = vec![0; REMOTE_DYN_FILTER_PAYLOAD_MAX_BYTES + 1]; + let outcome = apply_remote_dyn_filter_update( + ®s_by_query, + &query_id, + &filter_id, + &oversized, + 1, + false, + ); + assert_eq!(outcome, RemoteDynFilterUpdateOutcome::PayloadTooLarge); + + // The rejected generation must not become the pending generation. + let outcome = + apply_remote_dyn_filter_update(®s_by_query, &query_id, &filter_id, &[1], 0, false); + assert_eq!(outcome, RemoteDynFilterUpdateOutcome::Buffered); + } + + #[test] + fn test_remote_dyn_filter_generation_zero_applies_as_first_update() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let query_id = test_remote_query_id(); + let filter_id = RemoteDynFilterId::new("filter-1"); + register_empty_remote_dyn_filter(®s_by_query, &query_id); + + // Installing the wrapper before the update exercises the runtime apply path directly. + let dyn_filter = only_remote_dyn_filter(®s_by_query, &query_id); + let payload = datafusion_payload_bytes(physical_lit(false)); + let outcome = apply_remote_dyn_filter_update( + ®s_by_query, + &query_id, + &filter_id, + &payload, + 0, + false, + ); + + assert_eq!(outcome, RemoteDynFilterUpdateOutcome::Applied); + assert!(dyn_filter.snapshot_generation() > 1); + assert_eq!(format!("{}", dyn_filter.current().unwrap()), "false"); + } + + #[test] + fn test_buffered_update_applies_when_scan_installs_wrapper() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let query_id = test_remote_query_id(); + let filter_id = RemoteDynFilterId::new("filter-1"); + register_empty_remote_dyn_filter(®s_by_query, &query_id); + + let payload = datafusion_payload_bytes(physical_lit(false)); + let outcome = apply_remote_dyn_filter_update( + ®s_by_query, + &query_id, + &filter_id, + &payload, + 1, + false, + ); + assert_eq!(outcome, RemoteDynFilterUpdateOutcome::Buffered); + + let dyn_filter = only_remote_dyn_filter(®s_by_query, &query_id); + assert!(dyn_filter.snapshot_generation() > 1); + assert_eq!(format!("{}", dyn_filter.current().unwrap()), "false"); + } + + #[tokio::test] + async fn test_unregister_completes_installed_remote_dyn_filter_without_relaxing() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let query_id = test_remote_query_id(); + let filter_id = RemoteDynFilterId::new("filter-1"); + register_empty_remote_dyn_filter(®s_by_query, &query_id); + + let dyn_filter = only_remote_dyn_filter(®s_by_query, &query_id); + let payload = datafusion_payload_bytes(physical_lit(false)); + let outcome = apply_remote_dyn_filter_update( + ®s_by_query, + &query_id, + &filter_id, + &payload, + 1, + false, + ); + assert_eq!(outcome, RemoteDynFilterUpdateOutcome::Applied); + assert_eq!(format!("{}", dyn_filter.current().unwrap()), "false"); + + let outcome = unregister_remote_dyn_filter(®s_by_query, &query_id, &filter_id); + assert_eq!(outcome, RemoteDynFilterUpdateOutcome::Applied); + assert!(query_regs(®s_by_query, &query_id).is_none()); + tokio::time::timeout(Duration::from_secs(1), dyn_filter.wait_complete()) + .await + .unwrap(); + assert_eq!(format!("{}", dyn_filter.current().unwrap()), "false"); + } + + #[tokio::test] + async fn test_remote_dyn_filter_unregister_removes_all_region_subscribers_for_filter() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let query_id = test_remote_query_id(); + let filter_id = RemoteDynFilterId::new("filter-1"); + let first_region_id = RegionId::new(1024, 7); + let second_region_id = RegionId::new(1024, 8); + let regs = InitialDynFilterRegs::new(vec![InitialDynFilterReg::new("filter-1", vec![])]); + + let first_subscription = + register_initial_dyn_filter_regs(®s_by_query, &query_id, first_region_id, ®s); + let second_subscription = + register_initial_dyn_filter_regs(®s_by_query, &query_id, second_region_id, ®s); + let dyn_filter = only_remote_dyn_filter(®s_by_query, &query_id); + + let payload = datafusion_payload_bytes(physical_lit(false)); + let outcome = apply_remote_dyn_filter_update( + ®s_by_query, + &query_id, + &filter_id, + &payload, + 1, + false, + ); + assert_eq!(outcome, RemoteDynFilterUpdateOutcome::Applied); + + let outcome = unregister_remote_dyn_filter(®s_by_query, &query_id, &filter_id); + assert_eq!(outcome, RemoteDynFilterUpdateOutcome::Applied); + assert!(query_regs(®s_by_query, &query_id).is_none()); + tokio::time::timeout(Duration::from_secs(1), dyn_filter.wait_complete()) + .await + .unwrap(); + assert_eq!(format!("{}", dyn_filter.current().unwrap()), "false"); + + // Stream-local cleanup may run after FE's peer-deduplicated unregister. It should be benign. + remove_initial_dyn_filter_regs( + ®s_by_query, + &query_id, + first_region_id, + &first_subscription, + ); + remove_initial_dyn_filter_regs( + ®s_by_query, + &query_id, + second_region_id, + &second_subscription, + ); + assert!(query_regs(®s_by_query, &query_id).is_none()); + } + + #[test] + fn test_remote_dyn_filter_unregister_keeps_other_filters_for_same_query() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let query_id = test_remote_query_id(); + let region_id = test_remote_dyn_filter_region_id(); + + register_initial_dyn_filter_regs( + ®s_by_query, + &query_id, + region_id, + &InitialDynFilterRegs::new(vec![InitialDynFilterReg::new("filter-1", vec![])]), + ); + register_initial_dyn_filter_regs( + ®s_by_query, + &query_id, + region_id, + &InitialDynFilterRegs::new(vec![InitialDynFilterReg::new("filter-2", vec![])]), + ); + + let outcome = unregister_remote_dyn_filter( + ®s_by_query, + &query_id, + &RemoteDynFilterId::new("filter-1"), + ); + assert_eq!(outcome, RemoteDynFilterUpdateOutcome::Applied); + + let query_regs = query_regs(®s_by_query, &query_id).unwrap(); + assert!(!query_regs.contains_key(&RemoteDynFilterId::new("filter-1"))); + assert!(query_regs.contains_key(&RemoteDynFilterId::new("filter-2"))); + } + + #[tokio::test] + async fn test_remote_dyn_filter_guarded_stream_removes_on_eof() { + let mock_region_server = mock_region_server(); + let query_id = test_remote_query_id(); + let region_id = test_remote_dyn_filter_region_id(); + let registered_filter_ids = register_empty_remote_dyn_filter( + &mock_region_server + .inner + .initial_remote_dyn_filter_registrations, + &query_id, + ); + let cleanup = RemoteDynFilterRegistrationGuard::new( + mock_region_server.clone(), + query_id, + region_id, + registered_filter_ids, + ); + + let stream = wrap_remote_dyn_filter_guarded_stream(single_value_stream(), cleanup); + let mut pinned = Box::pin(stream); + while pinned.next().await.is_some() {} + + assert!( + mock_region_server + .inner + .initial_remote_dyn_filter_registrations + .inspect_query(&query_id, |_| ()) + .is_none() + ); + } + + #[tokio::test] + async fn test_remote_dyn_filter_multi_region_subscription_cleanup() { + let regs_by_query = RemoteDynFilterRegistry::new(); + let regs = InitialDynFilterRegs::new(vec![InitialDynFilterReg::new("filter-1", vec![])]); + let query_id = test_remote_query_id(); + let first_region_id = RegionId::new(1024, 7); + let second_region_id = RegionId::new(1024, 8); + + // Register the same logical filter for two regions on the same datanode. + let first_subscription = + register_initial_dyn_filter_regs(®s_by_query, &query_id, first_region_id, ®s); + let second_subscription = + register_initial_dyn_filter_regs(®s_by_query, &query_id, second_region_id, ®s); + let dyn_filter = only_remote_dyn_filter(®s_by_query, &query_id); + + // Verify only one filter entry exists and both region subscribers are tracked explicitly. + { + let query_regs = query_regs(®s_by_query, &query_id).unwrap(); + assert_eq!(query_regs.len(), 1); + let registered = query_regs.get(&RemoteDynFilterId::new("filter-1")).unwrap(); + assert_eq!(registered.subscriber_regions.len(), 2); + assert!(registered.subscriber_regions.contains(&first_region_id)); + assert!(registered.subscriber_regions.contains(&second_region_id)); + } + + // One region cleanup should not drop the entry while another region is subscribed. + remove_initial_dyn_filter_regs( + ®s_by_query, + &query_id, + first_region_id, + &first_subscription, + ); + assert!(query_regs(®s_by_query, &query_id).is_some()); + assert!( + tokio::time::timeout(Duration::from_millis(50), dyn_filter.wait_complete()) + .await + .is_err() + ); + { + let query_regs = query_regs(®s_by_query, &query_id).unwrap(); + let registered = query_regs.get(&RemoteDynFilterId::new("filter-1")).unwrap(); + assert_eq!(registered.subscriber_regions.len(), 1); + assert!(registered.subscriber_regions.contains(&second_region_id)); + } + + // Last region cleanup should drop the entry. + remove_initial_dyn_filter_regs( + ®s_by_query, + &query_id, + second_region_id, + &second_subscription, + ); + assert!(query_regs(®s_by_query, &query_id).is_none()); + tokio::time::timeout(Duration::from_secs(1), dyn_filter.wait_complete()) + .await + .unwrap(); + } +} diff --git a/src/query/src/datafusion.rs b/src/query/src/datafusion.rs index a8d60d42f4..7be6656c6f 100644 --- a/src/query/src/datafusion.rs +++ b/src/query/src/datafusion.rs @@ -53,7 +53,9 @@ use tracing::Span; use crate::analyze::DistAnalyzeExec; pub use crate::datafusion::planner::DfContextProviderAdapter; -use crate::dist_plan::{DistPlannerOptions, MergeScanLogicalPlan}; +use crate::dist_plan::{ + DistPlannerOptions, MergeScanLogicalPlan, RemoteDynFilterReceiverInjectorRef, +}; use crate::error::{ CatalogSnafu, CreateRecordBatchSnafu, MissingTableMutationHandlerSnafu, MissingTimestampColumnSnafu, QueryExecutionSnafu, Result, TableMutationSnafu, @@ -93,15 +95,21 @@ impl DatafusionQueryEngine { query_ctx: QueryContextRef, ) -> Result { let mut ctx = self.engine_context(query_ctx.clone()); + let plan = if let Some(receiver_injector) = + self.plugins.get::() + { + receiver_injector.maybe_inject(plan, query_ctx.clone()) + } else { + plan + }; // `create_physical_plan` will optimize logical plan internally let physical_plan = self.create_physical_plan(&mut ctx, &plan).await?; - let optimized_physical_plan = self.optimize_physical_plan(&mut ctx, physical_plan)?; - + let physical_plan = self.optimize_physical_plan(&mut ctx, physical_plan)?; let physical_plan = if let Some(wrapper) = self.plugins.get::() { - wrapper.wrap(optimized_physical_plan, query_ctx) + wrapper.wrap(physical_plan, query_ctx) } else { - optimized_physical_plan + physical_plan }; let stream = self.execute_stream(&ctx, &physical_plan)?; diff --git a/src/query/src/dist_plan.rs b/src/query/src/dist_plan.rs index 084ca7d1e5..1983e7824c 100644 --- a/src/query/src/dist_plan.rs +++ b/src/query/src/dist_plan.rs @@ -21,6 +21,7 @@ mod merge_sort; mod planner; mod predicate_extractor; mod region_pruner; +mod remote_dyn_filter_receiver; mod remote_dyn_filter_registry; pub use analyzer::{DistPlannerAnalyzer, DistPlannerOptions}; @@ -29,6 +30,10 @@ pub use merge_scan::{MergeScanExec, MergeScanLogicalPlan}; pub use planner::{DistExtensionPlanner, MergeSortExtensionPlanner}; pub use predicate_extractor::PredicateExtractor; pub use region_pruner::ConstraintPruner; +pub use remote_dyn_filter_receiver::{ + RemoteDynFilterReceiverExtensionPlanner, RemoteDynFilterReceiverInjector, + RemoteDynFilterReceiverInjectorRef, RemoteDynFilterReceiverLogicalPlan, +}; pub use remote_dyn_filter_registry::{ DynFilterEntry, DynFilterRegistryManager, EntryRegistration, QueryDynFilterRegistry, RemoteDynFilterRegistryLease, Subscriber, SubscriberRegistration, diff --git a/src/query/src/dist_plan/merge_scan.rs b/src/query/src/dist_plan/merge_scan.rs index 951bc7319a..daf9624374 100644 --- a/src/query/src/dist_plan/merge_scan.rs +++ b/src/query/src/dist_plan/merge_scan.rs @@ -59,7 +59,7 @@ use crate::dist_plan::dyn_filter_bridge::{ }; use crate::dist_plan::{RemoteDynFilterProducerId, RemoteDynFilterRegistryLease}; use crate::metrics::{MERGE_SCAN_ERRORS_TOTAL, MERGE_SCAN_POLL_ELAPSED, MERGE_SCAN_REGIONS}; -use crate::options::FlowQueryExtensions; +use crate::options::{FlowQueryExtensions, remote_dyn_filter_pushdown_enabled_from_extensions}; use crate::query_engine::QueryEngineState; use crate::region_query::RegionQueryHandlerRef; @@ -67,6 +67,11 @@ fn query_engine_state_from_task_context(context: &TaskContext) -> Option Result { + remote_dyn_filter_pushdown_enabled_from_extensions(&query_ctx.extensions()) + .map_err(|err| DataFusionError::External(Box::new(err))) +} + fn acquire_remote_dyn_filter_registry_lease( context: &TaskContext, query_ctx: &QueryContextRef, @@ -334,7 +339,12 @@ impl MergeScanExec { let partition_metrics_moved = self.partition_metrics.clone(); let plan = self.plan.clone(); let target_partition = self.target_partition; - let captured_remote_dyn_filters = self.captured_remote_dyn_filters(); + let remote_dyn_filter_enabled = remote_dyn_filter_enabled(&self.query_ctx)?; + let captured_remote_dyn_filters = if remote_dyn_filter_enabled { + self.captured_remote_dyn_filters() + } else { + Vec::new() + }; let dbname = context.task_id().unwrap_or_default(); let tracing_context = TracingContext::from_json(context.session_id().as_str()); let current_channel = self.query_ctx.channel(); @@ -738,6 +748,20 @@ impl ExecutionPlan for MergeScanExec { .into_iter() .map(|filter| filter.filter) .collect::>(); + + if !remote_dyn_filter_enabled(&self.query_ctx)? { + // Reject remote pushdown instead of pretending success: this keeps + // DataFusion/local dynamic filter semantics intact while disabling + // only FE -> DN remote dynamic filter propagation. + self.captured_remote_dyn_filters.lock().unwrap().clear(); + let new_self = Arc::new(self.clone()); + + return Ok(FilterPushdownPropagation { + filters: parent_filters.into_iter().map(|_| PushedDown::No).collect(), + updated_node: Some(new_self), + }); + } + let Some(remote_dyn_filter_producer_id) = self.remote_dyn_filter_producer_id else { // Missing RDF identity disables only RDF, not normal execution. common_telemetry::warn!( @@ -761,10 +785,12 @@ impl ExecutionPlan for MergeScanExec { filters: remote_dyn_filter_pushdown .pushed_down .into_iter() - .map(|_pushdown_ready| { - // TODO(discord9): Return `PushedDown::Yes` after datanodes consume RDF - // registrations and pending updates. Until then, keep the parent-side filter. - PushedDown::No + .map(|pushdown_ready| { + if pushdown_ready { + PushedDown::Yes + } else { + PushedDown::No + } }) .collect(), updated_node: Some(new_self), @@ -1077,7 +1103,7 @@ mod tests { } #[test] - fn remote_dyn_filter_preflight_keeps_parent_filter_until_dn_runtime_is_ready() { + fn remote_dyn_filter_preflight_removes_parent_filter_after_dn_runtime_is_ready() { let remote_dyn_filter_producer_id = RemoteDynFilterProducerId::new(42); let plan = LogicalPlanBuilder::empty(true) .project(vec![lit(1i32).alias("col1")]) @@ -1124,6 +1150,6 @@ mod tests { .unwrap(); assert_eq!(exec.captured_remote_dyn_filters().len(), 1); - assert!(matches!(propagation.filters.as_slice(), [PushedDown::No])); + assert!(matches!(propagation.filters.as_slice(), [PushedDown::Yes])); } } diff --git a/src/query/src/dist_plan/remote_dyn_filter_receiver.rs b/src/query/src/dist_plan/remote_dyn_filter_receiver.rs new file mode 100644 index 0000000000..b36e725bed --- /dev/null +++ b/src/query/src/dist_plan/remote_dyn_filter_receiver.rs @@ -0,0 +1,319 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::cmp::Ordering; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +use async_trait::async_trait; +use datafusion::common::Result; +use datafusion::execution::context::SessionState; +use datafusion::physical_expr::utils::conjunction; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::expressions::Column; +use datafusion::physical_plan::filter::FilterExec; +use datafusion::physical_planner::{ExtensionPlanner, PhysicalPlanner}; +use datafusion_common::{DFSchemaRef, DataFusionError}; +use datafusion_expr::{ + Expr, Extension, LogicalPlan, UserDefinedLogicalNode, UserDefinedLogicalNodeCore, +}; +use datafusion_physical_expr::PhysicalExpr; +use datafusion_physical_expr::utils::collect_columns; +use session::context::QueryContextRef; + +type InjectRemoteDynFilterReceiver = + dyn Fn(LogicalPlan, QueryContextRef) -> LogicalPlan + Send + Sync + 'static; + +/// Injects a logical remote dynamic filter receiver into a query plan. +pub struct RemoteDynFilterReceiverInjector { + inject: Box, +} + +impl RemoteDynFilterReceiverInjector { + pub fn new( + inject: impl Fn(LogicalPlan, QueryContextRef) -> LogicalPlan + Send + Sync + 'static, + ) -> Self { + Self { + inject: Box::new(inject), + } + } + + pub fn maybe_inject(&self, plan: LogicalPlan, query_ctx: QueryContextRef) -> LogicalPlan { + (self.inject)(plan, query_ctx) + } +} + +pub type RemoteDynFilterReceiverInjectorRef = Arc; + +/// A logical marker that is converted to a [`FilterExec`] carrying remote dynamic filters. +#[derive(Clone)] +pub struct RemoteDynFilterReceiverLogicalPlan { + input: Arc, + dyn_filters: Vec>, +} + +impl RemoteDynFilterReceiverLogicalPlan { + pub fn new(input: LogicalPlan, dyn_filters: Vec>) -> Self { + Self { + input: Arc::new(input), + dyn_filters, + } + } + + pub fn name() -> &'static str { + "RemoteDynFilterReceiver" + } + + pub fn into_logical_plan(self) -> LogicalPlan { + LogicalPlan::Extension(Extension { + node: Arc::new(self), + }) + } + + fn dyn_filters(&self) -> &[Arc] { + &self.dyn_filters + } + + fn ord_key(&self) -> String { + format!("input={:?}, dyn_filters={:?}", self.input, self.dyn_filters) + } +} + +impl fmt::Debug for RemoteDynFilterReceiverLogicalPlan { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + UserDefinedLogicalNodeCore::fmt_for_explain(self, f) + } +} + +impl Hash for RemoteDynFilterReceiverLogicalPlan { + fn hash(&self, state: &mut H) { + self.input.hash(state); + self.dyn_filters.hash(state); + } +} + +impl PartialEq for RemoteDynFilterReceiverLogicalPlan { + fn eq(&self, other: &Self) -> bool { + self.input == other.input && self.dyn_filters == other.dyn_filters + } +} + +impl Eq for RemoteDynFilterReceiverLogicalPlan {} + +impl PartialOrd for RemoteDynFilterReceiverLogicalPlan { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.ord_key().cmp(&other.ord_key())) + } +} + +impl UserDefinedLogicalNodeCore for RemoteDynFilterReceiverLogicalPlan { + fn name(&self) -> &str { + Self::name() + } + + fn inputs(&self) -> Vec<&LogicalPlan> { + vec![self.input.as_ref()] + } + + fn schema(&self) -> &DFSchemaRef { + self.input.schema() + } + + fn expressions(&self) -> Vec { + Vec::new() + } + + fn necessary_children_exprs(&self, output_columns: &[usize]) -> Option>> { + let mut required = output_columns.to_vec(); + + for filter in &self.dyn_filters { + required.extend( + collect_columns(filter) + .into_iter() + .map(|column| column.index()), + ); + } + + required.sort_unstable(); + required.dedup(); + Some(vec![required]) + } + + fn fmt_for_explain(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}: filters={}", Self::name(), self.dyn_filters.len()) + } + + fn with_exprs_and_inputs( + &self, + _exprs: Vec, + mut inputs: Vec, + ) -> Result { + let input = inputs.pop().ok_or_else(|| { + DataFusionError::Internal(format!("Expected exactly one input with {}", Self::name())) + })?; + let dyn_filters = self + .dyn_filters + .iter() + .map(|filter| remap_physical_expr_columns(filter.clone(), input.schema().as_arrow())) + .collect::>>()?; + Ok(Self::new(input, dyn_filters)) + } +} + +fn remap_physical_expr_columns( + expr: Arc, + input_schema: &datafusion::arrow::datatypes::Schema, +) -> Result> { + if let Some(column) = expr.as_any().downcast_ref::() { + return Ok(Arc::new(Column::new_with_schema( + column.name(), + input_schema, + )?)); + } + + let children = expr.children(); + if children.is_empty() { + return Ok(expr); + } + + let new_children = children + .into_iter() + .map(|child| remap_physical_expr_columns(child.clone(), input_schema)) + .collect::>>()?; + expr.with_new_children(new_children) +} + +pub struct RemoteDynFilterReceiverExtensionPlanner; + +#[async_trait] +impl ExtensionPlanner for RemoteDynFilterReceiverExtensionPlanner { + async fn plan_extension( + &self, + _planner: &dyn PhysicalPlanner, + node: &dyn UserDefinedLogicalNode, + _logical_inputs: &[&LogicalPlan], + physical_inputs: &[Arc], + _session_state: &SessionState, + ) -> Result>> { + let Some(receiver) = node + .as_any() + .downcast_ref::() + else { + return Ok(None); + }; + + let input = physical_inputs.first().cloned().ok_or_else(|| { + DataFusionError::Internal(format!("Expected exactly one input with {}", Self::name())) + })?; + if receiver.dyn_filters().is_empty() { + return Ok(Some(input)); + } + + let predicate = conjunction(receiver.dyn_filters().to_vec()); + Ok(Some(Arc::new(FilterExec::try_new(predicate, input)?) as _)) + } +} + +impl RemoteDynFilterReceiverExtensionPlanner { + fn name() -> &'static str { + RemoteDynFilterReceiverLogicalPlan::name() + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::physical_plan::expressions::{Column, DynamicFilterPhysicalExpr, lit}; + use datafusion_common::DFSchema; + use datafusion_expr::{EmptyRelation, UserDefinedLogicalNodeCore}; + + use super::*; + + fn empty_input() -> LogicalPlan { + let schema = Arc::new(Schema::new(vec![ + Field::new( + "ts", + DataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Millisecond, None), + false, + ), + Field::new("value", DataType::Float64, true), + Field::new("instance", DataType::Utf8, true), + Field::new("job", DataType::Utf8, true), + ])); + LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: Arc::new(DFSchema::try_from(schema).unwrap()), + }) + } + + fn pruned_input() -> LogicalPlan { + let schema = Arc::new(Schema::new(vec![ + Field::new( + "ts", + DataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Millisecond, None), + false, + ), + Field::new("instance", DataType::Utf8, true), + ])); + LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: Arc::new(DFSchema::try_from(schema).unwrap()), + }) + } + + #[test] + fn necessary_children_exprs_keeps_parent_and_filter_columns() { + let dyn_filter = Arc::new(DynamicFilterPhysicalExpr::new( + vec![Arc::new(Column::new("ts", 0)) as Arc<_>], + lit(true) as _, + )); + let plan = RemoteDynFilterReceiverLogicalPlan::new(empty_input(), vec![dyn_filter]); + + // Parent only needs `value`, but the receiver must still keep `ts` for + // evaluating its dynamic filter after logical projection pruning. + let required = UserDefinedLogicalNodeCore::necessary_children_exprs(&plan, &[1]).unwrap(); + assert_eq!(required, vec![vec![0, 1]]); + } + + #[test] + fn necessary_children_exprs_is_transparent_without_filters() { + let plan = RemoteDynFilterReceiverLogicalPlan::new(empty_input(), vec![]); + + let required = + UserDefinedLogicalNodeCore::necessary_children_exprs(&plan, &[1, 3]).unwrap(); + assert_eq!(required, vec![vec![1, 3]]); + } + + #[test] + fn with_exprs_and_inputs_remaps_dyn_filter_columns_to_pruned_input() { + let dyn_filter = Arc::new(DynamicFilterPhysicalExpr::new( + vec![Arc::new(Column::new("instance", 2)) as Arc<_>], + lit(true) as _, + )); + let plan = RemoteDynFilterReceiverLogicalPlan::new(empty_input(), vec![dyn_filter]); + + let remapped = + UserDefinedLogicalNodeCore::with_exprs_and_inputs(&plan, vec![], vec![pruned_input()]) + .unwrap(); + + let columns = collect_columns(&remapped.dyn_filters()[0]); + assert_eq!(columns.len(), 1); + let column = columns.iter().next().unwrap(); + assert_eq!(column.name(), "instance"); + assert_eq!(column.index(), 1); + } +} diff --git a/src/query/src/options.rs b/src/query/src/options.rs index 688d6315b0..721d79ea8d 100644 --- a/src/query/src/options.rs +++ b/src/query/src/options.rs @@ -25,6 +25,9 @@ pub const FLOW_INCREMENTAL_AFTER_SEQS: &str = "flow.incremental_after_seqs"; pub const FLOW_INCREMENTAL_MODE: &str = "flow.incremental_mode"; pub const FLOW_RETURN_REGION_SEQ: &str = "flow.return_region_seq"; pub const FLOW_SINK_TABLE_ID: &str = "flow.sink_table_id"; +/// Enable by default, set to false to explicitly disable. +pub const QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN: &str = + "query.enable_remote_dynamic_filter_pushdown"; pub const FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY: &str = "memtable_only"; @@ -106,7 +109,7 @@ impl FlowQueryExtensions { let return_region_seq = extensions .get(FLOW_RETURN_REGION_SEQ) - .map(|value| parse_bool(value.as_str())) + .map(|value| parse_bool(FLOW_RETURN_REGION_SEQ, value.as_str())) .transpose()? .unwrap_or(false); @@ -184,6 +187,22 @@ impl FlowQueryExtensions { } } +/// Returns whether query-level remote dynamic filter propagation is enabled. +/// +/// The option defaults to enabled to preserve existing behavior. Callers may set +/// `query.enable_remote_dynamic_filter_pushdown=false` in query context +/// extensions to disable FE->DN remote dynamic filter propagation for a single +/// query. +pub fn remote_dyn_filter_pushdown_enabled_from_extensions( + extensions: &HashMap, +) -> Result { + extensions + .get(QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN) + .map(|value| parse_bool(QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN, value.as_str())) + .transpose() + .map(|value| value.unwrap_or(true)) +} + /// Returns whether raw Flow query extensions request terminal region watermark collection. /// /// This is only an intent/presence check for transport/scan plumbing; callers that need @@ -249,13 +268,13 @@ fn parse_incremental_after_seqs(value: &str) -> Result> { .collect() } -fn parse_bool(value: &str) -> Result { +fn parse_bool(option_name: &str, value: &str) -> Result { match value { v if v.eq_ignore_ascii_case("true") => Ok(true), v if v.eq_ignore_ascii_case("false") => Ok(false), _ => Err(invalid_query_context_extension(format!( "Invalid value for {}: {}", - FLOW_RETURN_REGION_SEQ, value + option_name, value ))), } } @@ -276,6 +295,37 @@ mod flow_extension_tests { assert_eq!(parsed, None); } + #[test] + fn test_remote_dyn_filter_pushdown_enabled_from_extensions_defaults_true() { + assert!(remote_dyn_filter_pushdown_enabled_from_extensions(&HashMap::new()).unwrap()); + } + + #[test] + fn test_remote_dyn_filter_pushdown_enabled_from_extensions_parses_bool() { + let exts = HashMap::from([( + QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN.to_string(), + "false".to_string(), + )]); + assert!(!remote_dyn_filter_pushdown_enabled_from_extensions(&exts).unwrap()); + + let exts = HashMap::from([( + QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN.to_string(), + "true".to_string(), + )]); + assert!(remote_dyn_filter_pushdown_enabled_from_extensions(&exts).unwrap()); + } + + #[test] + fn test_remote_dyn_filter_pushdown_enabled_from_extensions_rejects_invalid_bool() { + let exts = HashMap::from([( + QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN.to_string(), + "invalid".to_string(), + )]); + + let err = remote_dyn_filter_pushdown_enabled_from_extensions(&exts).unwrap_err(); + assert!(format!("{err}").contains(QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN)); + } + #[test] fn test_parse_flow_extensions_memtable_only_success() { let exts = HashMap::from([ diff --git a/src/query/src/promql.rs b/src/query/src/promql.rs index 27b446e6e4..db682ca6a8 100644 --- a/src/query/src/promql.rs +++ b/src/query/src/promql.rs @@ -15,3 +15,94 @@ pub mod error; pub mod label_values; pub mod planner; + +use datafusion_common::tree_node::{TreeNode as _, TreeNodeRecursion}; +use datafusion_expr::{Extension, LogicalPlan}; +use promql::extension_plan::{ + Absent, EmptyMetric, HistogramFold, InstantManipulate, RangeManipulate, ScalarCalculate, + SeriesDivide, SeriesNormalize, UnionDistinctOn, +}; + +/// Returns true if the plan contains PromQL-specific extension plan nodes. +pub fn plan_contains_promql_extension(plan: &LogicalPlan) -> bool { + let mut found = false; + let _ = plan.apply(|node| { + if is_promql_extension_plan(node) { + found = true; + Ok(TreeNodeRecursion::Stop) + } else { + Ok(TreeNodeRecursion::Continue) + } + }); + found +} + +fn is_promql_extension_plan(plan: &LogicalPlan) -> bool { + let LogicalPlan::Extension(Extension { node }) = plan else { + return false; + }; + + node.as_any().is::() + || node.as_any().is::() + || node.as_any().is::() + || node.as_any().is::() + || node.as_any().is::() + || node.as_any().is::() + || node.as_any().is::() + || node.as_any().is::() + || node.as_any().is::() +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use datafusion_common::DFSchema; + use datafusion_expr::{EmptyRelation, Extension, LogicalPlanBuilder, col}; + + use super::*; + + #[test] + fn plan_contains_promql_extension_returns_true_for_promql_extension() { + let plan = empty_metric_plan(); + + assert!(plan_contains_promql_extension(&plan)); + } + + #[test] + fn plan_contains_promql_extension_returns_true_for_nested_promql_extension() { + let plan = LogicalPlanBuilder::from(empty_metric_plan()) + .project(vec![col("ts")]) + .unwrap() + .build() + .unwrap(); + + assert!(plan_contains_promql_extension(&plan)); + } + + #[test] + fn plan_contains_promql_extension_returns_false_for_non_promql_plan() { + let plan = LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: Arc::new(DFSchema::empty()), + }); + + assert!(!plan_contains_promql_extension(&plan)); + } + + fn empty_metric_plan() -> LogicalPlan { + let empty_metric = EmptyMetric::new( + 0, + 10_000, + 5_000, + "ts".to_string(), + "greptime_value".to_string(), + None, + ) + .unwrap(); + + LogicalPlan::Extension(Extension { + node: Arc::new(empty_metric), + }) + } +} diff --git a/src/query/src/query_engine/state.rs b/src/query/src/query_engine/state.rs index ad2ef92bb9..44c75292b1 100644 --- a/src/query/src/query_engine/state.rs +++ b/src/query/src/query_engine/state.rs @@ -57,7 +57,8 @@ use table::table::adapter::DfTableProviderAdapter; use crate::QueryEngineContext; use crate::dist_plan::{ DistExtensionPlanner, DistPlannerAnalyzer, DistPlannerOptions, DynFilterRegistryManager, - MergeSortExtensionPlanner, RemoteDynFilterRegistryLease, + MergeSortExtensionPlanner, RemoteDynFilterReceiverExtensionPlanner, + RemoteDynFilterRegistryLease, }; use crate::metrics::{QUERY_MEMORY_POOL_REJECTED_TOTAL, QUERY_MEMORY_POOL_USAGE_BYTES}; use crate::optimizer::ExtensionAnalyzerRule; @@ -511,8 +512,11 @@ impl DfQueryPlanner { partition_rule_manager: Option, region_query_handler: Option, ) -> Self { - let mut planners: Vec> = - vec![Arc::new(PromExtensionPlanner), Arc::new(RangeSelectPlanner)]; + let mut planners: Vec> = vec![ + Arc::new(PromExtensionPlanner), + Arc::new(RangeSelectPlanner), + Arc::new(RemoteDynFilterReceiverExtensionPlanner), + ]; if let (Some(region_query_handler), Some(partition_rule_manager)) = (region_query_handler, partition_rule_manager) { diff --git a/src/table/src/table/scan.rs b/src/table/src/table/scan.rs index 02511456ae..d301202e17 100644 --- a/src/table/src/table/scan.rs +++ b/src/table/src/table/scan.rs @@ -40,7 +40,9 @@ use datafusion::physical_plan::{ use datafusion_common::stats::Precision; use datafusion_common::{ColumnStatistics, DataFusionError, Statistics}; use datafusion_physical_expr::expressions::Column; -use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalSortExpr}; +use datafusion_physical_expr::{ + EquivalenceProperties, Partitioning, PhysicalExpr, PhysicalSortExpr, +}; use datatypes::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datatypes::compute::SortOptions; use futures::{Stream, StreamExt}; @@ -339,6 +341,21 @@ impl RegionScanExec { pub fn set_explain_verbose(&mut self, explain_verbose: bool) { self.explain_verbose = explain_verbose; } + + /// Adds dynamic filters directly to the underlying region scanner predicate. + /// + /// This is the same mutation path used by DataFusion child-filter pushdown. Remote dynamic + /// filter updates install their shared runtime wrapper through this method at scan build time + /// and later update the wrapper state out-of-band. + pub fn add_dyn_filters_to_predicate( + &self, + filter_exprs: Vec>, + ) -> Vec { + self.scanner + .lock() + .unwrap() + .add_dyn_filter_to_predicate(filter_exprs) + } } impl ExecutionPlan for RegionScanExec { @@ -451,11 +468,7 @@ impl ExecutionPlan for RegionScanExec { .map(|f| f.filter) .collect::>(); - let supported = self - .scanner - .lock() - .unwrap() - .add_dyn_filter_to_predicate(parent_filters); + let supported = self.add_dyn_filters_to_predicate(parent_filters); // datafusion api require to clone self after mutate, even though we are only mutate inside mutex let new_self = Arc::new(self.clone()); diff --git a/tests-integration/src/tests.rs b/tests-integration/src/tests.rs index ddb9b1c386..d70af02165 100644 --- a/tests-integration/src/tests.rs +++ b/tests-integration/src/tests.rs @@ -18,6 +18,7 @@ mod instance_noop_wal_test; mod instance_test; mod promql_test; mod reconcile_table; +mod remote_dyn_filter_test; pub mod test_util; use std::collections::HashMap; diff --git a/tests-integration/src/tests/remote_dyn_filter_test.rs b/tests-integration/src/tests/remote_dyn_filter_test.rs new file mode 100644 index 0000000000..83c0c7d713 --- /dev/null +++ b/tests-integration/src/tests/remote_dyn_filter_test.rs @@ -0,0 +1,509 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use common_query::Output; +use frontend::instance::Instance; +use query::datafusion::QUERY_PARALLELISM_HINT; +use query::options::QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN; +use servers::query_handler::sql::SqlQueryHandler; +use session::context::QueryContext; + +use crate::test_util::execute_sql; +use crate::tests; + +#[tokio::test(flavor = "multi_thread")] +async fn test_remote_dyn_filter_join_e2e() { + common_telemetry::init_default_ut_logging(); + + let distributed = tests::create_distributed_instance("test_remote_dyn_filter_join_e2e").await; + let frontend = distributed.frontend(); + + prepare_remote_dyn_filter_tables(&frontend).await; + + let join_sql = remote_dyn_filter_join_sql(); + let result = output_to_pretty_string( + execute_sql_with_query_parallelism_one(&frontend, join_sql, true).await, + ) + .await; + assert_eq!( + result, + r#"+---+------+ +| k | v | ++---+------+ +| 2 | 20.0 | +| 4 | 40.0 | ++---+------+"# + ); + + let explain_sql = format!("EXPLAIN ANALYZE VERBOSE {join_sql}"); + let explain = output_to_pretty_string( + execute_sql_with_query_parallelism_one(&frontend, &explain_sql, true).await, + ) + .await; + + assert_contains(&explain, "HashJoinExec: mode=CollectLeft"); + assert_contains(&explain, "MergeScanExec"); + assert_seq_scan_has_dyn_filter(&explain); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_remote_dyn_filter_left_join_e2e() { + common_telemetry::init_default_ut_logging(); + + let distributed = + tests::create_distributed_instance("test_remote_dyn_filter_left_join_e2e").await; + let frontend = distributed.frontend(); + + prepare_remote_dyn_filter_tables(&frontend).await; + execute_sql( + &frontend, + r#" + INSERT INTO rdf_build(k, ts) VALUES (9, 9000) + "#, + ) + .await; + + let join_sql = remote_dyn_filter_left_join_sql(); + let result = output_to_pretty_string( + execute_sql_with_query_parallelism_one(&frontend, join_sql, true).await, + ) + .await; + assert_eq!( + result, + r#"+---+------+ +| k | v | ++---+------+ +| 2 | 20.0 | +| 4 | 40.0 | +| 9 | -1.0 | ++---+------+"# + ); + + let explain_sql = format!("EXPLAIN ANALYZE VERBOSE {join_sql}"); + let explain = output_to_pretty_string( + execute_sql_with_query_parallelism_one(&frontend, &explain_sql, true).await, + ) + .await; + + assert_contains(&explain, "HashJoinExec: mode=CollectLeft, join_type=Left"); + assert_contains(&explain, "MergeScanExec"); + assert_seq_scan_dyn_filter_contains( + &explain, + &[ + "DynamicFilter [ k@0 >= 2 AND k@0 <= 9", + "k@0 IN (SET) ([2, 4, 9])", + ], + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_remote_dyn_filter_multi_column_join_e2e() { + common_telemetry::init_default_ut_logging(); + + let distributed = + tests::create_distributed_instance("test_remote_dyn_filter_multi_column_join_e2e").await; + let frontend = distributed.frontend(); + + prepare_remote_dyn_filter_multi_column_tables(&frontend).await; + + let join_sql = remote_dyn_filter_multi_column_join_sql(); + let result = output_to_pretty_string( + execute_sql_with_query_parallelism_one(&frontend, join_sql, true).await, + ) + .await; + assert_eq!( + result, + r#"+---+----+-------+ +| a | k | v | ++---+----+-------+ +| 1 | 10 | 110.0 | +| 2 | 20 | 220.0 | ++---+----+-------+"# + ); + + let explain_sql = format!("EXPLAIN ANALYZE VERBOSE {join_sql}"); + let explain = output_to_pretty_string( + execute_sql_with_query_parallelism_one(&frontend, &explain_sql, true).await, + ) + .await; + + assert_contains(&explain, "HashJoinExec: mode=CollectLeft"); + assert_contains(&explain, "MergeScanExec"); + assert_seq_scan_has_dyn_filter(&explain); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_remote_dyn_filter_large_join_e2e() { + common_telemetry::init_default_ut_logging(); + + let distributed = + tests::create_distributed_instance("test_remote_dyn_filter_large_join_e2e").await; + let frontend = distributed.frontend(); + + prepare_remote_dyn_filter_large_tables(&frontend).await; + + let join_sql = remote_dyn_filter_large_join_sql(); + let result = output_to_pretty_string( + execute_sql_with_query_parallelism_one(&frontend, join_sql, true).await, + ) + .await; + assert_eq!( + result, + r#"+------+---------+ +| k | v | ++------+---------+ +| 3 | 30.0 | +| 129 | 1290.0 | +| 511 | 5110.0 | +| 900 | 9000.0 | +| 8195 | 81950.0 | ++------+---------+"# + ); + + let result_without_rdf = output_to_pretty_string( + execute_sql_with_query_parallelism_one(&frontend, join_sql, false).await, + ) + .await; + assert_eq!(result_without_rdf, result); + + let explain_sql = format!("EXPLAIN ANALYZE VERBOSE {join_sql}"); + let explain = output_to_pretty_string( + execute_sql_with_query_parallelism_one(&frontend, &explain_sql, true).await, + ) + .await; + + assert_contains(&explain, "HashJoinExec: mode=CollectLeft"); + assert_contains(&explain, "MergeScanExec"); + assert_seq_scan_dyn_filter_contains( + &explain, + &[ + "DynamicFilter [ k@0 >= 3 AND k@0 <= 8195", + "k@0 IN (SET) ([3, 129, 511, 900, 8195])", + ], + ); + + let explain_without_rdf = output_to_pretty_string( + execute_sql_with_query_parallelism_one(&frontend, &explain_sql, false).await, + ) + .await; + assert_no_seq_scan_dyn_filter(&explain_without_rdf); +} + +async fn prepare_remote_dyn_filter_tables(frontend: &Arc) { + execute_sql( + frontend, + r#" + CREATE TABLE rdf_probe( + k INT, + ts TIMESTAMP, + v DOUBLE, + TIME INDEX (ts), + PRIMARY KEY(k) + ) + PARTITION ON COLUMNS (k) ( + k < 2, + k >= 2 AND k < 4, + k >= 4 AND k < 6, + k >= 6 + ) + engine=mito + "#, + ) + .await; + + execute_sql( + frontend, + r#" + CREATE TABLE rdf_build( + k INT, + ts TIMESTAMP, + TIME INDEX (ts), + PRIMARY KEY(k) + ) engine=mito + "#, + ) + .await; + + execute_sql( + frontend, + r#" + INSERT INTO rdf_probe(k, ts, v) VALUES + (1, 1000, 10.0), + (2, 2000, 20.0), + (3, 3000, 30.0), + (4, 4000, 40.0), + (7, 5000, 50.0) + "#, + ) + .await; + + execute_sql( + frontend, + r#" + INSERT INTO rdf_build(k, ts) VALUES + (2, 1000), + (4, 2000) + "#, + ) + .await; +} + +async fn prepare_remote_dyn_filter_multi_column_tables(frontend: &Arc) { + execute_sql( + frontend, + r#" + CREATE TABLE rdf_multi_probe( + a INT, + k INT, + ts TIMESTAMP, + v DOUBLE, + TIME INDEX (ts), + PRIMARY KEY(a, k) + ) + PARTITION ON COLUMNS (a) ( + a < 2, + a >= 2 AND a < 3, + a >= 3 AND a < 4, + a >= 4 + ) + engine=mito + "#, + ) + .await; + + execute_sql( + frontend, + r#" + CREATE TABLE rdf_multi_build( + a INT, + k INT, + ts TIMESTAMP, + TIME INDEX (ts), + PRIMARY KEY(a, k) + ) engine=mito + "#, + ) + .await; + + execute_sql( + frontend, + r#" + INSERT INTO rdf_multi_probe(a, k, ts, v) VALUES + (1, 10, 1000, 110.0), + (1, 11, 1100, 111.0), + (2, 10, 2000, 210.0), + (2, 20, 2200, 220.0), + (3, 30, 3000, 330.0), + (4, 40, 4000, 440.0) + "#, + ) + .await; + + execute_sql( + frontend, + r#" + INSERT INTO rdf_multi_build(a, k, ts) VALUES + (1, 10, 1000), + (2, 20, 2000) + "#, + ) + .await; +} + +async fn prepare_remote_dyn_filter_large_tables(frontend: &Arc) { + execute_sql( + frontend, + r#" + CREATE TABLE rdf_large_probe( + k INT, + ts TIMESTAMP, + v DOUBLE, + TIME INDEX (ts), + PRIMARY KEY(k) + ) + PARTITION ON COLUMNS (k) ( + k < 1024, + k >= 1024 AND k < 2048, + k >= 2048 AND k < 3072, + k >= 3072 AND k < 4096, + k >= 4096 AND k < 5120, + k >= 5120 AND k < 6144, + k >= 6144 AND k < 7168, + k >= 7168 + ) + engine=mito + "#, + ) + .await; + + execute_sql( + frontend, + r#" + CREATE TABLE rdf_large_build( + k INT, + ts TIMESTAMP, + TIME INDEX (ts), + PRIMARY KEY(k) + ) engine=mito + "#, + ) + .await; + + for start in (0..8192).step_by(1024) { + insert_remote_dyn_filter_large_probe_range(frontend, start, start + 1024).await; + } + execute_sql(frontend, "ADMIN FLUSH_TABLE('rdf_large_probe')").await; + + // Keep a few rows in memtable after flush so the same query covers both + // flushed SST/file data and newly written memtable data. + insert_remote_dyn_filter_large_probe_range(frontend, 8192, 8200).await; + + execute_sql( + frontend, + r#" + INSERT INTO rdf_large_build(k, ts) VALUES + (3, 3000), + (129, 129000), + (511, 511000), + (900, 900000), + (8195, 8195000) + "#, + ) + .await; +} + +async fn insert_remote_dyn_filter_large_probe_range( + frontend: &Arc, + start: usize, + end: usize, +) { + let values = (start..end) + .map(|k| format!("({k}, {k}, {}.0)", k * 10)) + .collect::>() + .join(","); + let insert_probe_sql = format!("INSERT INTO rdf_large_probe(k, ts, v) VALUES {values}"); + execute_sql(frontend, &insert_probe_sql).await; +} + +fn remote_dyn_filter_join_sql() -> &'static str { + r#" + SELECT p.k, p.v + FROM rdf_build b + JOIN rdf_probe p ON p.k = b.k + ORDER BY p.k + "# +} + +fn remote_dyn_filter_left_join_sql() -> &'static str { + r#" + SELECT b.k, + CASE WHEN p.v IS NULL THEN -1.0 ELSE p.v END AS v + FROM rdf_build b + LEFT JOIN rdf_probe p ON p.k = b.k + ORDER BY b.k + "# +} + +fn remote_dyn_filter_multi_column_join_sql() -> &'static str { + r#" + SELECT p.a, p.k, p.v + FROM rdf_multi_build b + JOIN rdf_multi_probe p ON p.a = b.a AND p.k = b.k + ORDER BY p.a, p.k + "# +} + +fn remote_dyn_filter_large_join_sql() -> &'static str { + r#" + SELECT p.k, p.v + FROM rdf_large_build b + JOIN rdf_large_probe p ON p.k = b.k + ORDER BY p.k + "# +} + +async fn output_to_pretty_string(output: Output) -> String { + output.data.pretty_print().await +} + +async fn execute_sql_with_query_parallelism_one( + instance: &Arc, + sql: &str, + remote_dyn_filter_enabled: bool, +) -> Output { + let mut query_ctx = QueryContext::with_db_name(None); + query_ctx.set_extension(QUERY_PARALLELISM_HINT, "1"); + if !remote_dyn_filter_enabled { + query_ctx.set_extension(QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN, "false"); + } + SqlQueryHandler::do_query(instance.as_ref(), sql, Arc::new(query_ctx)) + .await + .remove(0) + .unwrap() +} + +fn assert_no_seq_scan_dyn_filter(explain: &str) { + let seq_scan_dyn_filter_lines = explain + .lines() + .filter(|line| line.contains("SeqScan: region=") && line.contains("\"dyn_filters\"")) + .collect::>(); + + assert!( + seq_scan_dyn_filter_lines.is_empty(), + "expected no region SeqScan line with dyn_filters; actual SeqScan dyn_filters lines:\n{}\n\nfull explain:\n{explain}", + seq_scan_dyn_filter_lines.join("\n") + ); +} + +fn assert_contains(haystack: &str, needle: &str) { + assert!( + haystack.contains(needle), + "expected to find {needle:?} in:\n{haystack}" + ); +} + +fn assert_seq_scan_dyn_filter_contains(explain: &str, needles: &[&str]) { + let seq_scan_dyn_filter_lines = explain + .lines() + .filter(|line| line.contains("SeqScan: region=") && line.contains("\"dyn_filters\"")) + .collect::>(); + + assert!( + !seq_scan_dyn_filter_lines.is_empty(), + "expected at least one region SeqScan line with dyn_filters in:\n{explain}" + ); + + let matched_line = seq_scan_dyn_filter_lines + .iter() + .find(|line| needles.iter().all(|needle| line.contains(needle))); + + assert!( + matched_line.is_some(), + "expected one region SeqScan dyn_filters line containing all of {needles:?}; actual SeqScan dyn_filters lines:\n{}\n\nfull explain:\n{explain}", + seq_scan_dyn_filter_lines.join("\n") + ); +} + +fn assert_seq_scan_has_dyn_filter(explain: &str) { + let has_dyn_filter = explain + .lines() + .any(|line| line.contains("SeqScan: region=") && line.contains("\"dyn_filters\"")); + + assert!( + has_dyn_filter, + "expected at least one region SeqScan line with dyn_filters in:\n{explain}" + ); +} diff --git a/tests/cases/standalone/common/filter/hash_join_dyn_filter.result b/tests/cases/standalone/common/filter/hash_join_dyn_filter.result index 3ec77d457e..bdcf00af68 100644 --- a/tests/cases/standalone/common/filter/hash_join_dyn_filter.result +++ b/tests/cases/standalone/common/filter/hash_join_dyn_filter.result @@ -76,6 +76,8 @@ WHERE c.tier = 'gold'; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] +-- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| EXPLAIN ANALYZE VERBOSE SELECT o."id", o.amount, c."name", c.tier FROM orders o JOIN customers c ON o.customer_id = c.customer_id @@ -94,10 +96,10 @@ WHERE c.tier = 'gold'; |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED | +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| | 1_| 0_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ @@ -201,6 +203,8 @@ WHERE c.tier IN ('gold', 'silver'); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] +-- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| EXPLAIN ANALYZE VERBOSE SELECT o."id", o.amount, c."name", c.tier FROM (SELECT "id", customer_id as cid, amount, ts FROM orders) o JOIN customers c ON o.cid = c.customer_id @@ -220,10 +224,10 @@ WHERE c.tier IN ('gold', 'silver'); |_|_|_| | 1_| 0_|_ProjectionExec: expr=[id@0 as id, customer_id@1 as cid, amount@2 as amount, ts@3 as ts] metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED | +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| | 1_| 0_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ diff --git a/tests/cases/standalone/common/filter/hash_join_dyn_filter.sql b/tests/cases/standalone/common/filter/hash_join_dyn_filter.sql index 29bc86b315..d6079bc496 100644 --- a/tests/cases/standalone/common/filter/hash_join_dyn_filter.sql +++ b/tests/cases/standalone/common/filter/hash_join_dyn_filter.sql @@ -41,6 +41,8 @@ WHERE c.tier = 'gold'; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] +-- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| EXPLAIN ANALYZE VERBOSE SELECT o."id", o.amount, c."name", c.tier FROM orders o JOIN customers c ON o.customer_id = c.customer_id @@ -96,6 +98,8 @@ WHERE c.tier IN ('gold', 'silver'); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] +-- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| EXPLAIN ANALYZE VERBOSE SELECT o."id", o.amount, c."name", c.tier FROM (SELECT "id", customer_id as cid, amount, ts FROM orders) o JOIN customers c ON o.cid = c.customer_id diff --git a/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.result b/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.result index c3455f84d8..54c72f4644 100644 --- a/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.result +++ b/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.result @@ -92,6 +92,8 @@ WHERE c.tier IN ('gold', 'bronze'); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] +-- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| EXPLAIN ANALYZE VERBOSE SELECT top_orders."id", top_orders.amount, c."name", c.tier FROM ( SELECT "id", customer_id, amount, ts @@ -116,10 +118,10 @@ WHERE c.tier IN ('gold', 'bronze'); |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [amount@2 DESC], fetch=5 metrics=REDACTED_| |_|_|_SortExec: TopK(fetch=5), expr=[amount@2 DESC], preserve_partitioning=[true] metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "dyn_filters": ["DynamicFilter [ empty ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED | +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| | 1_| 0_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ @@ -246,6 +248,8 @@ LIMIT 4; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] +-- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| EXPLAIN ANALYZE VERBOSE SELECT "id", customer_id, "name", tier, amount FROM ( SELECT o."id", o.customer_id, c."name", c.tier, o.amount @@ -272,10 +276,10 @@ LIMIT 4; |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED | +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| | 1_| 0_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ @@ -420,6 +424,8 @@ WHERE c.tier IN ('gold', 'silver') -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] +-- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| EXPLAIN ANALYZE VERBOSE SELECT o."id", o.amount, c."name", c.tier, p."name" as product_name, p."category" FROM orders o JOIN customers c ON o.customer_id = c.customer_id @@ -447,13 +453,13 @@ WHERE c.tier IN ('gold', 'silver') |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "product_id", "amount", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED | +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "product_id", "amount", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| | 1_| 0_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| | 1_| 0_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["product_id", "name", "category", "ts"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["product_id", "name", "category", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ diff --git a/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.sql b/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.sql index 287d1ca064..97f0c1231d 100644 --- a/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.sql +++ b/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.sql @@ -54,6 +54,8 @@ WHERE c.tier IN ('gold', 'bronze'); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] +-- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| EXPLAIN ANALYZE VERBOSE SELECT top_orders."id", top_orders.amount, c."name", c.tier FROM ( SELECT "id", customer_id, amount, ts @@ -132,6 +134,8 @@ LIMIT 4; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] +-- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| EXPLAIN ANALYZE VERBOSE SELECT "id", customer_id, "name", tier, amount FROM ( SELECT o."id", o.customer_id, c."name", c.tier, o.amount @@ -212,6 +216,8 @@ WHERE c.tier IN ('gold', 'silver') -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] +-- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| EXPLAIN ANALYZE VERBOSE SELECT o."id", o.amount, c."name", c.tier, p."name" as product_name, p."category" FROM orders o JOIN customers c ON o.customer_id = c.customer_id diff --git a/tests/cases/standalone/common/join/rdf_having_filter_regression.result b/tests/cases/standalone/common/join/rdf_having_filter_regression.result new file mode 100644 index 0000000000..d38ecbfae2 --- /dev/null +++ b/tests/cases/standalone/common/join/rdf_having_filter_regression.result @@ -0,0 +1,46 @@ +-- Regression test for remote dynamic filter pushdown preserving a HAVING filter. +CREATE TABLE rdf_having_dim(k INTEGER, label STRING, ts TIMESTAMP TIME INDEX); + +Affected Rows: 0 + +CREATE TABLE rdf_having_fact(k INTEGER, v DOUBLE, ts TIMESTAMP TIME INDEX); + +Affected Rows: 0 + +INSERT INTO rdf_having_dim VALUES + (1, 'keep', 1000), + (2, 'drop', 2000); + +Affected Rows: 2 + +INSERT INTO rdf_having_fact VALUES + (1, 200.0, 1000), + (1, 150.0, 2000), + (2, 100.0, 3000); + +Affected Rows: 3 + +SELECT d.label, s.total +FROM rdf_having_dim d +INNER JOIN ( + SELECT k, SUM(v) AS total + FROM rdf_having_fact + GROUP BY k + HAVING SUM(v) > 300 +) s ON d.k = s.k +ORDER BY s.total DESC, d.label; + ++-------+-------+ +| label | total | ++-------+-------+ +| keep | 350.0 | ++-------+-------+ + +DROP TABLE rdf_having_dim; + +Affected Rows: 0 + +DROP TABLE rdf_having_fact; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/join/rdf_having_filter_regression.sql b/tests/cases/standalone/common/join/rdf_having_filter_regression.sql new file mode 100644 index 0000000000..22d84d0e45 --- /dev/null +++ b/tests/cases/standalone/common/join/rdf_having_filter_regression.sql @@ -0,0 +1,26 @@ +-- Regression test for remote dynamic filter pushdown preserving a HAVING filter. + +CREATE TABLE rdf_having_dim(k INTEGER, label STRING, ts TIMESTAMP TIME INDEX); +CREATE TABLE rdf_having_fact(k INTEGER, v DOUBLE, ts TIMESTAMP TIME INDEX); + +INSERT INTO rdf_having_dim VALUES + (1, 'keep', 1000), + (2, 'drop', 2000); + +INSERT INTO rdf_having_fact VALUES + (1, 200.0, 1000), + (1, 150.0, 2000), + (2, 100.0, 3000); + +SELECT d.label, s.total +FROM rdf_having_dim d +INNER JOIN ( + SELECT k, SUM(v) AS total + FROM rdf_having_fact + GROUP BY k + HAVING SUM(v) > 300 +) s ON d.k = s.k +ORDER BY s.total DESC, d.label; + +DROP TABLE rdf_having_dim; +DROP TABLE rdf_having_fact; diff --git a/tests/cases/standalone/common/order/order_by.result b/tests/cases/standalone/common/order/order_by.result index 6a2807e4b6..91113f82b5 100644 --- a/tests/cases/standalone/common/order/order_by.result +++ b/tests/cases/standalone/common/order/order_by.result @@ -299,12 +299,16 @@ explain analyze select tag from t where num > 6 order by ts desc limit 2; |_|_|_SortExec: TopK(fetch=2), expr=[ts@1 DESC], preserve_partitioning=[true] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| -| 1_| 0_|_SortPreservingMergeExec: [ts@1 DESC], fetch=2 REDACTED +| 1_| 0_|_FilterExec: DynamicFilter [ true ] REDACTED +|_|_|_RepartitionExec: partitioning=REDACTED +|_|_|_SortPreservingMergeExec: [ts@1 DESC], fetch=2 REDACTED |_|_|_SortExec: TopK(fetch=2), expr=[ts@1 DESC], preserve_partitioning=[true] REDACTED |_|_|_FilterExec: num@2 > 6, projection=[tag@0, ts@1] REDACTED |_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED |_|_|_| -| 1_| 1_|_SortPreservingMergeExec: [ts@1 DESC], fetch=2 REDACTED +| 1_| 1_|_FilterExec: DynamicFilter [ true ] REDACTED +|_|_|_RepartitionExec: partitioning=REDACTED +|_|_|_SortPreservingMergeExec: [ts@1 DESC], fetch=2 REDACTED |_|_|_SortExec: TopK(fetch=2), expr=[ts@1 DESC], preserve_partitioning=[true] REDACTED |_|_|_FilterExec: num@2 > 6, projection=[tag@0, ts@1] REDACTED |_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED diff --git a/tests/cases/standalone/common/types/timestamp/timestamp_types.result b/tests/cases/standalone/common/types/timestamp/timestamp_types.result index 25218d9333..525f03504b 100644 --- a/tests/cases/standalone/common/types/timestamp/timestamp_types.result +++ b/tests/cases/standalone/common/types/timestamp/timestamp_types.result @@ -319,7 +319,7 @@ select '2008-01-01 00:00:11.1'::TIMESTAMP_US = '2008-01-01 00:00:11.1'::TIMESTAM +----------------------------------------------------------------------------------------------------------------------------------+ | arrow_cast(Utf8("2008-01-01 00:00:11.1"),Utf8("Timestamp(µs)")) = arrow_cast(Utf8("2008-01-01 00:00:11.1"),Utf8("Timestamp(s)")) | +----------------------------------------------------------------------------------------------------------------------------------+ -| true | +| false | +----------------------------------------------------------------------------------------------------------------------------------+ select '2008-01-01 00:00:11.1'::TIMESTAMP_MS = '2008-01-01 00:00:11'::TIMESTAMP_NS; @@ -335,7 +335,7 @@ select '2008-01-01 00:00:11.1'::TIMESTAMP_MS = '2008-01-01 00:00:11'::TIMESTAMP_ +--------------------------------------------------------------------------------------------------------------------------------+ | arrow_cast(Utf8("2008-01-01 00:00:11.1"),Utf8("Timestamp(ms)")) = arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(s)")) | +--------------------------------------------------------------------------------------------------------------------------------+ -| true | +| false | +--------------------------------------------------------------------------------------------------------------------------------+ select '2008-01-01 00:00:11.1'::TIMESTAMP_NS = '2008-01-01 00:00:11'::TIMESTAMP_S; @@ -343,7 +343,7 @@ select '2008-01-01 00:00:11.1'::TIMESTAMP_NS = '2008-01-01 00:00:11'::TIMESTAMP_ +--------------------------------------------------------------------------------------------------------------------------------+ | arrow_cast(Utf8("2008-01-01 00:00:11.1"),Utf8("Timestamp(ns)")) = arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(s)")) | +--------------------------------------------------------------------------------------------------------------------------------+ -| true | +| false | +--------------------------------------------------------------------------------------------------------------------------------+ select '2008-01-01 00:00:11'::TIMESTAMP_MS = '2008-01-01 00:00:11'::TIMESTAMP_NS;