diff --git a/Cargo.lock b/Cargo.lock index 2cae7d03322..597ad270305 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3672,7 +3672,7 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "datafusion" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -3724,7 +3724,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3748,7 +3748,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3771,7 +3771,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "arrow-ipc 59.2.0", @@ -3797,7 +3797,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "futures", "log", @@ -3807,7 +3807,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "async-compression", @@ -3843,7 +3843,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "arrow-ipc 59.2.0", @@ -3867,7 +3867,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3890,7 +3890,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3913,7 +3913,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -3945,12 +3945,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" [[package]] name = "datafusion-execution" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "arrow-buffer 59.2.0", @@ -3975,7 +3975,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -3999,7 +3999,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4010,7 +4010,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "arrow-buffer 59.2.0", @@ -4041,7 +4041,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4061,7 +4061,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4072,7 +4072,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "arrow-ord 59.2.0", @@ -4096,7 +4096,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "async-trait", @@ -4111,7 +4111,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4127,7 +4127,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -4136,7 +4136,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "datafusion-doc", "quote", @@ -4146,7 +4146,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "chrono", @@ -4194,7 +4194,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -4217,7 +4217,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4231,7 +4231,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "chrono", @@ -4248,7 +4248,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4267,7 +4267,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "arrow-data 59.2.0", @@ -4304,7 +4304,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "datafusion-catalog", @@ -4330,7 +4330,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4340,7 +4340,7 @@ dependencies = [ [[package]] name = "datafusion-proto-models" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "datafusion-common", "datafusion-proto-common", @@ -4350,7 +4350,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4365,7 +4365,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow-schema 59.2.0", "async-trait", @@ -4379,7 +4379,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "arrow 59.2.0", "bigdecimal 0.4.8", @@ -4398,7 +4398,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" dependencies = [ "async-recursion", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 5dc37afceba..e057db65826 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -363,21 +363,21 @@ rev = "f743c3a5fe8c1f57363eac5866c356b7d91f9b39" [patch.crates-io] substrait = { git = "https://github.com/GreptimeTeam/substrait-rs.git", rev = "91ec978b0649417ad3da8390e7a515baec723b1b" } -datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } -datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } -datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } -datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } -datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } -datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } -datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } -datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } -datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } -datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } -datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } -datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } -datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } -datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } -datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" } +datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } datafusion-pg-catalog = { git = "https://github.com/datafusion-contrib/datafusion-postgres.git", rev = "ffb14a52b3d7c7489559812e1ec38096f225ce93" } arrow-pg = { git = "https://github.com/datafusion-contrib/datafusion-postgres.git", rev = "ffb14a52b3d7c7489559812e1ec38096f225ce93" } sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "9e9019bb1c7040ed956f654e39378dd42ab17884" } diff --git a/src/query/src/optimizer/global_limit.rs b/src/query/src/optimizer/global_limit.rs index 5abb9953b7b..b29907f01ab 100644 --- a/src/query/src/optimizer/global_limit.rs +++ b/src/query/src/optimizer/global_limit.rs @@ -254,22 +254,215 @@ fn inherited_partitioning_to_restore( #[cfg(test)] mod tests { - use datafusion::arrow::array::Int32Array; + use datafusion::arrow::array::{Array, Int32Array}; use datafusion::arrow::compute::SortOptions; use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::arrow::record_batch::RecordBatch; + use datafusion::execution::TaskContext; use datafusion::physical_expr::expressions::{col, lit}; + use datafusion::physical_optimizer::optimizer::PhysicalOptimizer; + use datafusion::physical_plan::aggregates::{ + AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy, + }; use datafusion::physical_plan::filter::FilterExecBuilder; use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode}; - use datafusion::physical_plan::limit::GlobalLimitExec; + use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec}; use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::repartition::RepartitionExec; + use datafusion::physical_plan::sorts::sort::SortExec; use datafusion::physical_plan::test::TestMemoryExec; use datafusion_common::{JoinType, NullEquality}; use datafusion_physical_expr::{LexOrdering, Partitioning, PhysicalSortExpr}; use super::*; + async fn optimize_and_collect_twice( + mut plan: Arc, + config: &ConfigOptions, + ) -> Vec> { + let mut results = Vec::with_capacity(2); + for _ in 0..2 { + for rule in PhysicalOptimizer::new().rules { + plan = rule.optimize(plan, config).unwrap(); + } + let batches = datafusion::physical_plan::collect( + Arc::clone(&plan), + Arc::new(TaskContext::default()), + ) + .await + .unwrap(); + results.push( + batches + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(), + ); + } + results + } + + #[tokio::test] + async fn physical_optimizer_keeps_global_distinct_limit_across_two_passes() { + for soft_limit in [false, true] { + let mut config = ConfigOptions::new(); + config.execution.target_partitions = 3; + config.optimizer.enable_distinct_aggregation_soft_limit = soft_limit; + + for mode in [ + AggregateMode::FinalPartitioned, + AggregateMode::SinglePartitioned, + ] { + // The query limit is the only initial limit. + let input = input_with_all_hash_partitions(); + let aggregate = match mode { + AggregateMode::FinalPartitioned => { + agg(agg(input, AggregateMode::Partial), mode) + } + AggregateMode::SinglePartitioned => agg(hash_repartition(input), mode), + _ => unreachable!(), + }; + let plan = + Arc::new(GlobalLimitExec::new(aggregate, 0, Some(1))) as Arc; + + let results = optimize_and_collect_twice(plan, &config).await; + assert_eq!( + results.iter().map(Vec::len).collect::>(), + vec![1, 1], + "soft limit enabled: {soft_limit}, mode: {mode:?}", + ); + } + } + } + + #[tokio::test] + async fn physical_optimizer_keeps_count_over_distinct_limit_across_two_passes() { + use datafusion::datasource::MemTable; + use datafusion::execution::context::{SessionConfig, SessionContext as DFSessionContext}; + use datafusion_common::ScalarValue; + + for soft_limit in [false, true] { + let mut config = SessionConfig::new().with_target_partitions(3); + config + .options_mut() + .optimizer + .enable_distinct_aggregation_soft_limit = soft_limit; + let ctx = DFSessionContext::new_with_config(config.clone()); + let schema = schema(); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from((0..100).collect::>()))], + ) + .unwrap(); + let table = MemTable::try_new(schema, vec![vec![batch]; 3]).unwrap(); + ctx.register_table("t", Arc::new(table)).unwrap(); + let mut plan = ctx + .sql("SELECT COUNT(*) FROM (SELECT DISTINCT a FROM t LIMIT 1) AS limited") + .await + .unwrap() + .create_physical_plan() + .await + .unwrap(); + + for pass in 0..=2 { + if pass > 0 { + for rule in PhysicalOptimizer::new().rules { + plan = rule.optimize(plan, config.options()).unwrap(); + } + } + let batches = datafusion::physical_plan::collect(Arc::clone(&plan), ctx.task_ctx()) + .await + .unwrap(); + let values = batches + .iter() + .flat_map(|batch| { + (0..batch.num_rows()).map(|row| { + ScalarValue::try_from_array(batch.column(0).as_ref(), row).unwrap() + }) + }) + .collect::>(); + assert_eq!( + values, + vec![ScalarValue::Int64(Some(1))], + "soft limit enabled: {soft_limit}, additional optimizer passes: {pass}", + ); + } + } + } + + #[tokio::test] + async fn physical_optimizer_keeps_local_limit_per_partition_across_two_passes() { + let mut config = ConfigOptions::new(); + config.execution.target_partitions = 3; + let input = input_with_all_hash_partitions(); + let local_limit = Arc::new(LocalLimitExec::new( + agg(hash_repartition(input), AggregateMode::SinglePartitioned), + 1, + )) as Arc; + + let results = optimize_and_collect_twice(local_limit, &config).await; + assert_eq!(results.iter().map(Vec::len).collect::>(), vec![3, 3]); + } + + #[tokio::test] + async fn physical_optimizer_keeps_ordered_topk_across_two_passes() { + let schema = schema(); + let losing = + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![100]))]) + .unwrap(); + let winning = + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1]))]) + .unwrap(); + let input = Arc::new( + TestMemoryExec::try_new(&[vec![losing], vec![winning]], schema.clone(), None).unwrap(), + ); + let ordering = ordering(schema.as_ref(), false); + let topk = Arc::new( + SortExec::new( + ordering, + agg_with_limit_options( + input, + AggregateMode::FinalPartitioned, + Some(LimitOptions::new_with_order(1, false)), + ), + ) + .with_fetch(Some(1)), + ) as Arc; + + let mut config = ConfigOptions::new(); + config.execution.target_partitions = 3; + assert_eq!( + optimize_and_collect_twice(topk, &config).await, + vec![vec![1], vec![1]], + ); + } + + #[tokio::test] + async fn physical_optimizer_keeps_ordered_offset_limit_across_two_passes() { + let mut config = ConfigOptions::new(); + config.execution.target_partitions = 3; + let offset = Arc::new(GlobalLimitExec::new( + Arc::new(SortExec::new( + ordering(schema().as_ref(), false), + unordered_input(), + )), + 3, + Some(1), + )) as Arc; + + assert_eq!( + optimize_and_collect_twice(offset, &config).await, + vec![vec![2], vec![2]], + ); + } + #[test] fn adds_global_limit_for_multi_partition_filter_fetch() { let filter = filter_fetch(unordered_input(), 1); @@ -563,6 +756,38 @@ mod tests { Arc::new(TestMemoryExec::try_new(&partitions, schema, None).unwrap()) } + fn input_with_all_hash_partitions() -> Arc { + let schema = schema(); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from((0..100).collect::>()))], + ) + .unwrap(); + let partitions = vec![vec![batch.clone()], vec![batch.clone()], vec![batch]]; + Arc::new(TestMemoryExec::try_new(&partitions, schema, None).unwrap()) + } + + fn agg(input: Arc, mode: AggregateMode) -> Arc { + agg_with_limit_options(input, mode, None) + } + + fn agg_with_limit_options( + input: Arc, + mode: AggregateMode, + limit_options: Option, + ) -> Arc { + let schema = input.schema(); + let group_by = PhysicalGroupBy::new_single(vec![( + col("a", schema.as_ref()).unwrap(), + "a".to_string(), + )]); + Arc::new( + AggregateExec::try_new(mode, group_by, vec![], vec![], input, schema) + .unwrap() + .with_limit_options(limit_options), + ) + } + fn ordered_input() -> (Arc, LexOrdering) { let schema = schema(); let ordering = ordering(schema.as_ref(), false); diff --git a/tests/cases/distributed/information_schema/ssts_limit.result b/tests/cases/distributed/information_schema/ssts_limit.result index 248593410fc..1b21a84c7af 100644 --- a/tests/cases/distributed/information_schema/ssts_limit.result +++ b/tests/cases/distributed/information_schema/ssts_limit.result @@ -72,27 +72,28 @@ FROM ( LIMIT 1 ); -+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| plan_type | plan | -+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | -| | Projection: count(Int64(1)) AS count(*) AS filtered_limited_rows | -| | Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]] | -| | Limit: skip=0, fetch=1 | -| | Projection: information_schema.ssts_manifest.region_id | -| | Filter: information_schema.ssts_manifest.table_id > UInt32(0) | -| | TableScan: information_schema.ssts_manifest, partial_filters=[information_schema.ssts_manifest.table_id > UInt32(0)] | -| | ]] | -| physical_plan | ProjectionExec: expr=[count(Int64(1))@0 as filtered_limited_rows] | -| | AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] | -| | CoalescePartitionsExec | -| | AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] | -| | CoalescePartitionsExec: fetch=1 | -| | FilterExec: table_id@0 > 0, projection=[], fetch=1 | -| | RepartitionExec: REDACTED -| | DistributedInspectExec: kind=SstManifest, scan=ScanRequest { projection: [2], filters: [table_id > UInt32(0)] }, schema=Schema { fields: [Field { name: "table_id", data_type: UInt32 }], metadata: {"greptime:version": "0"} } | -| | | -+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Projection: count(Int64(1)) AS count(*) AS filtered_limited_rows | +| | Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]] | +| | Limit: skip=0, fetch=1 | +| | Projection: information_schema.ssts_manifest.region_id | +| | Filter: information_schema.ssts_manifest.table_id > UInt32(0) | +| | TableScan: information_schema.ssts_manifest, partial_filters=[information_schema.ssts_manifest.table_id > UInt32(0)] | +| | ]] | +| physical_plan | ProjectionExec: expr=[count(Int64(1))@0 as filtered_limited_rows] | +| | AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] | +| | CoalescePartitionsExec | +| | AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] | +| | RepartitionExec: REDACTED +| | CoalescePartitionsExec: fetch=1 | +| | FilterExec: table_id@0 > 0, projection=[], fetch=1 | +| | RepartitionExec: REDACTED +| | DistributedInspectExec: kind=SstManifest, scan=ScanRequest { projection: [2], filters: [table_id > UInt32(0)] }, schema=Schema { fields: [Field { name: "table_id", data_type: UInt32 }], metadata: {"greptime:version": "0"} } | +| | | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ DROP TABLE ssts_limit_case; diff --git a/tests/cases/standalone/common/information_schema/ssts.result b/tests/cases/standalone/common/information_schema/ssts.result index 9777e3b12a9..289b4599387 100644 --- a/tests/cases/standalone/common/information_schema/ssts.result +++ b/tests/cases/standalone/common/information_schema/ssts.result @@ -229,6 +229,20 @@ SELECT * FROM information_schema.ssts_storage order by file_path; | data/greptime/public//_/index/.puffin || || +---------------------------------------------------------------------------------------------+-----------+-------------------------+---------+ +-- Regression: repeated physical optimization must preserve the global DISTINCT limit. +SELECT COUNT(*) AS distinct_limited_rows +FROM ( + SELECT DISTINCT region_id + FROM information_schema.ssts_manifest + LIMIT 1 +); + ++-----------------------+ +| distinct_limited_rows | ++-----------------------+ +| 1 | ++-----------------------+ + DROP TABLE sst_case; Affected Rows: 0 diff --git a/tests/cases/standalone/common/information_schema/ssts.sql b/tests/cases/standalone/common/information_schema/ssts.sql index 43512de6808..7b40b0056bb 100644 --- a/tests/cases/standalone/common/information_schema/ssts.sql +++ b/tests/cases/standalone/common/information_schema/ssts.sql @@ -72,4 +72,12 @@ SELECT * FROM information_schema.ssts_index_meta ORDER BY meta_json; -- SQLNESS REPLACE (/public/\d+/\d+_\d+) /public//_ SELECT * FROM information_schema.ssts_storage order by file_path; +-- Regression: repeated physical optimization must preserve the global DISTINCT limit. +SELECT COUNT(*) AS distinct_limited_rows +FROM ( + SELECT DISTINCT region_id + FROM information_schema.ssts_manifest + LIMIT 1 +); + DROP TABLE sst_case;