From 8826f690a2f6f74fa3caa39233fd6b05d022472c Mon Sep 17 00:00:00 2001 From: luofucong Date: Thu, 15 Jan 2026 19:28:28 +0800 Subject: [PATCH] chore: upgrade DataFusion family Signed-off-by: luofucong --- Cargo.lock | 878 ++++++++---------- Cargo.toml | 71 +- src/common/datasource/Cargo.toml | 2 +- src/common/datasource/src/error.rs | 7 +- src/common/datasource/src/file_format.rs | 9 +- .../datasource/src/file_format/tests.rs | 5 +- src/common/datasource/src/test_util.rs | 12 +- src/common/function/src/aggrs/count_hash.rs | 2 +- src/common/query/src/logical_plan.rs | 2 +- src/file-engine/src/error.rs | 9 +- src/file-engine/src/query.rs | 8 +- src/file-engine/src/query/file_stream.rs | 6 +- .../src/memtable/bulk/row_group_reader.rs | 10 +- src/mito2/src/sst/parquet/row_group.rs | 39 +- src/operator/src/expr_helper.rs | 13 +- src/promql/src/extension_plan/empty_metric.rs | 2 +- .../src/extension_plan/range_manipulate.rs | 2 +- .../src/extension_plan/scalar_calculate.rs | 2 +- src/promql/src/range_array.rs | 49 - src/query/src/dist_plan/analyzer/utils.rs | 2 +- src/query/src/dist_plan/commutativity.rs | 3 +- src/query/src/optimizer/remove_duplicate.rs | 53 +- src/query/src/optimizer/windowed_sort.rs | 45 +- src/query/src/range_select/plan.rs | 2 +- src/query/src/range_select/plan_rewrite.rs | 5 +- src/sql/src/parsers/create_parser.rs | 27 +- src/sql/src/parsers/delete_parser.rs | 7 +- src/sql/src/parsers/insert_parser.rs | 14 +- src/sql/src/parsers/utils.rs | 13 +- src/sql/src/parsers/with_tql_parser.rs | 1 + src/sql/src/statements.rs | 40 +- 31 files changed, 574 insertions(+), 766 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c25d495a97..7404c9696a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -214,7 +214,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" name = "api" version = "1.0.0-beta.4" dependencies = [ - "arrow-schema 57.0.0", + "arrow-schema 57.2.0", "common-base", "common-decimal", "common-error", @@ -316,23 +316,23 @@ dependencies = [ [[package]] name = "arrow" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae" +checksum = "2a2b10dcb159faf30d3f81f6d56c1211a5bea2ca424eabe477648a44b993320e" dependencies = [ - "arrow-arith 57.0.0", - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-cast 57.0.0", - "arrow-csv 57.0.0", - "arrow-data 57.0.0", - "arrow-ipc 57.0.0", - "arrow-json 57.0.0", - "arrow-ord 57.0.0", - "arrow-row 57.0.0", - "arrow-schema 57.0.0", - "arrow-select 57.0.0", - "arrow-string 57.0.0", + "arrow-arith 57.2.0", + "arrow-array 57.2.0", + "arrow-buffer 57.2.0", + "arrow-cast 57.2.0", + "arrow-csv 57.2.0", + "arrow-data 57.2.0", + "arrow-ipc 57.2.0", + "arrow-json 57.2.0", + "arrow-ord 57.2.0", + "arrow-row 57.2.0", + "arrow-schema 57.2.0", + "arrow-select 57.2.0", + "arrow-string 57.2.0", ] [[package]] @@ -351,14 +351,14 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491" +checksum = "288015089e7931843c80ed4032c5274f02b37bcb720c4a42096d50b390e70372" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", + "arrow-array 57.2.0", + "arrow-buffer 57.2.0", + "arrow-data 57.2.0", + "arrow-schema 57.2.0", "chrono", "num-traits", ] @@ -375,24 +375,24 @@ dependencies = [ "arrow-schema 56.2.0", "chrono", "half", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "num", ] [[package]] name = "arrow-array" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31" +checksum = "65ca404ea6191e06bf30956394173337fa9c35f445bd447fe6c21ab944e1a23c" dependencies = [ "ahash 0.8.12", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", + "arrow-buffer 57.2.0", + "arrow-data 57.2.0", + "arrow-schema 57.2.0", "chrono", "chrono-tz", "half", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "num-complex", "num-integer", "num-traits", @@ -411,9 +411,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76885a2697a7edf6b59577f568b456afc94ce0e2edc15b784ce3685b6c3c5c27" +checksum = "36356383099be0151dacc4245309895f16ba7917d79bdb71a7148659c9206c56" dependencies = [ "bytes", "half", @@ -443,15 +443,16 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168" +checksum = "9c8e372ed52bd4ee88cc1e6c3859aa7ecea204158ac640b10e187936e7e87074" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", - "arrow-select 57.0.0", + "arrow-array 57.2.0", + "arrow-buffer 57.2.0", + "arrow-data 57.2.0", + "arrow-ord 57.2.0", + "arrow-schema 57.2.0", + "arrow-select 57.2.0", "atoi", "base64 0.22.1", "chrono", @@ -479,13 +480,13 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c" +checksum = "8e4100b729fe656f2e4fb32bc5884f14acf9118d4ad532b7b33c1132e4dce896" dependencies = [ - "arrow-array 57.0.0", - "arrow-cast 57.0.0", - "arrow-schema 57.0.0", + "arrow-array 57.2.0", + "arrow-cast 57.2.0", + "arrow-schema 57.2.0", "chrono", "csv", "csv-core", @@ -506,12 +507,12 @@ dependencies = [ [[package]] name = "arrow-data" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b" +checksum = "bf87f4ff5fc13290aa47e499a8b669a82c5977c6a1fedce22c7f542c1fd5a597" dependencies = [ - "arrow-buffer 57.0.0", - "arrow-schema 57.0.0", + "arrow-buffer 57.2.0", + "arrow-schema 57.2.0", "half", "num-integer", "num-traits", @@ -519,15 +520,15 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f70bb56412a007b0cfc116d15f24dda6adeed9611a213852a004cda20085a3b9" +checksum = "f63654f21676be802d446c6c4bc54f6a47e18d55f9ae6f7195a6f6faf2ecdbeb" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-cast 57.0.0", - "arrow-ipc 57.0.0", - "arrow-schema 57.0.0", + "arrow-array 57.2.0", + "arrow-buffer 57.2.0", + "arrow-cast 57.2.0", + "arrow-ipc 57.2.0", + "arrow-schema 57.2.0", "base64 0.22.1", "bytes", "futures", @@ -549,23 +550,23 @@ dependencies = [ "arrow-schema 56.2.0", "arrow-select 56.2.0", "flatbuffers", - "zstd 0.13.3", + "zstd", ] [[package]] name = "arrow-ipc" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27" +checksum = "eb3ca63edd2073fcb42ba112f8ae165df1de935627ead6e203d07c99445f2081" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", - "arrow-select 57.0.0", + "arrow-array 57.2.0", + "arrow-buffer 57.2.0", + "arrow-data 57.2.0", + "arrow-schema 57.2.0", + "arrow-select 57.2.0", "flatbuffers", - "lz4_flex", - "zstd 0.13.3", + "lz4_flex 0.12.0", + "zstd", ] [[package]] @@ -581,7 +582,7 @@ dependencies = [ "arrow-schema 56.2.0", "chrono", "half", - "indexmap 2.12.0", + "indexmap 2.13.0", "lexical-core", "memchr", "num", @@ -592,18 +593,18 @@ dependencies = [ [[package]] name = "arrow-json" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8" +checksum = "a36b2332559d3310ebe3e173f75b29989b4412df4029a26a30cc3f7da0869297" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-cast 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", + "arrow-array 57.2.0", + "arrow-buffer 57.2.0", + "arrow-cast 57.2.0", + "arrow-data 57.2.0", + "arrow-schema 57.2.0", "chrono", "half", - "indexmap 2.12.0", + "indexmap 2.13.0", "itoa", "lexical-core", "memchr", @@ -629,15 +630,15 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b" +checksum = "13c4e0530272ca755d6814218dffd04425c5b7854b87fa741d5ff848bf50aa39" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", - "arrow-select 57.0.0", + "arrow-array 57.2.0", + "arrow-buffer 57.2.0", + "arrow-data 57.2.0", + "arrow-schema 57.2.0", + "arrow-select 57.2.0", ] [[package]] @@ -655,14 +656,14 @@ dependencies = [ [[package]] name = "arrow-row" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2" +checksum = "b07f52788744cc71c4628567ad834cadbaeb9f09026ff1d7a4120f69edf7abd3" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", + "arrow-array 57.2.0", + "arrow-buffer 57.2.0", + "arrow-data 57.2.0", + "arrow-schema 57.2.0", "half", ] @@ -674,9 +675,9 @@ checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" [[package]] name = "arrow-schema" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5" +checksum = "6bb63203e8e0e54b288d0d8043ca8fa1013820822a27692ef1b78a977d879f2c" dependencies = [ "serde", "serde_core", @@ -699,15 +700,15 @@ dependencies = [ [[package]] name = "arrow-select" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47" +checksum = "c96d8a1c180b44ecf2e66c9a2f2bbcb8b1b6f14e165ce46ac8bde211a363411b" dependencies = [ "ahash 0.8.12", - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", + "arrow-array 57.2.0", + "arrow-buffer 57.2.0", + "arrow-data 57.2.0", + "arrow-schema 57.2.0", "num-traits", ] @@ -730,15 +731,15 @@ dependencies = [ [[package]] name = "arrow-string" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2" +checksum = "a8ad6a81add9d3ea30bf8374ee8329992c7fd246ffd8b7e2f48a3cea5aa0cc9a" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", - "arrow-select 57.0.0", + "arrow-array 57.2.0", + "arrow-buffer 57.2.0", + "arrow-data 57.2.0", + "arrow-schema 57.2.0", + "arrow-select 57.2.0", "memchr", "num-traits", "regex", @@ -791,39 +792,15 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.3.15" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" +checksum = "d10e4f991a553474232bc0a31799f6d24b034a84c0971d80d2e2f78b2e576e40" dependencies = [ - "bzip2 0.4.4", - "flate2", - "futures-core", + "compression-codecs", + "compression-core", "futures-io", - "memchr", "pin-project-lite", "tokio", - "xz2", - "zstd 0.11.2+zstd.1.5.2", - "zstd-safe 5.0.2+zstd.1.5.2", -] - -[[package]] -name = "async-compression" -version = "0.4.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" -dependencies = [ - "brotli 7.0.0", - "bzip2 0.5.2", - "flate2", - "futures-core", - "futures-io", - "memchr", - "pin-project-lite", - "tokio", - "xz2", - "zstd 0.13.3", - "zstd-safe 7.2.4", ] [[package]] @@ -1433,17 +1410,6 @@ dependencies = [ "syn 2.0.114", ] -[[package]] -name = "brotli" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor 4.0.3", -] - [[package]] name = "brotli" version = "8.0.1" @@ -1452,17 +1418,7 @@ checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor 5.0.0", -] - -[[package]] -name = "brotli-decompressor" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", + "brotli-decompressor", ] [[package]] @@ -1553,32 +1509,13 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" dependencies = [ "serde", ] -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - [[package]] name = "bzip2" version = "0.6.1" @@ -1588,16 +1525,6 @@ dependencies = [ "libbz2-rs-sys", ] -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "cache" version = "1.0.0-beta.4" @@ -1638,8 +1565,8 @@ name = "catalog" version = "1.0.0-beta.4" dependencies = [ "api", - "arrow 57.0.0", - "arrow-schema 57.0.0", + "arrow 57.2.0", + "arrow-schema 57.2.0", "async-stream", "async-trait", "bytes", @@ -1772,7 +1699,7 @@ version = "0.13.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fe45e18904af7af10e4312df7c97251e98af98c70f42f1f2587aecfcbee56bf" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.13.0", "lazy_static", "num-traits", "regex", @@ -2301,9 +2228,9 @@ dependencies = [ name = "common-datasource" version = "1.0.0-beta.4" dependencies = [ - "arrow 57.0.0", - "arrow-schema 57.0.0", - "async-compression 0.3.15", + "arrow 57.2.0", + "arrow-schema 57.2.0", + "async-compression", "async-trait", "bytes", "common-base", @@ -2409,9 +2336,9 @@ dependencies = [ "api", "approx 0.5.1", "arc-swap", - "arrow 57.0.0", - "arrow-cast 57.0.0", - "arrow-schema 57.0.0", + "arrow 57.2.0", + "arrow-cast 57.2.0", + "arrow-schema 57.2.0", "async-trait", "bincode", "catalog", @@ -2801,7 +2728,7 @@ dependencies = [ name = "common-sql" version = "1.0.0-beta.4" dependencies = [ - "arrow-schema 57.0.0", + "arrow-schema 57.2.0", "common-base", "common-decimal", "common-error", @@ -2878,7 +2805,7 @@ dependencies = [ name = "common-time" version = "1.0.0-beta.4" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "chrono", "chrono-tz", "common-error", @@ -2948,6 +2875,28 @@ dependencies = [ "sha1", ] +[[package]] +name = "compression-codecs" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a" +dependencies = [ + "brotli", + "bzip2", + "compression-core", + "flate2", + "liblzma", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -3439,7 +3388,7 @@ checksum = "7a9bc1a22964ff6a355fbec24cf68266a0ed28f8b84c0864c386474ea3d0e479" dependencies = [ "cc", "codespan-reporting 0.13.1", - "indexmap 2.12.0", + "indexmap 2.13.0", "proc-macro2", "quote", "scratch", @@ -3454,7 +3403,7 @@ checksum = "b1f29a879d35f7906e3c9b77d7a1005a6a0787d330c09dfe4ffb5f617728cb44" dependencies = [ "clap 4.5.40", "codespan-reporting 0.13.1", - "indexmap 2.12.0", + "indexmap 2.13.0", "proc-macro2", "quote", "syn 2.0.114", @@ -3472,7 +3421,7 @@ version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d187e019e7b05a1f3e69a8396b70800ee867aa9fc2ab972761173ccee03742df" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.13.0", "proc-macro2", "quote", "syn 2.0.114", @@ -3605,14 +3554,14 @@ dependencies = [ [[package]] name = "datafusion" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", - "arrow-schema 57.0.0", + "arrow 57.2.0", + "arrow-schema 57.2.0", "async-trait", "bytes", - "bzip2 0.6.1", + "bzip2", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -3642,28 +3591,27 @@ dependencies = [ "flate2", "futures", "itertools 0.14.0", + "liblzma", "log", "object_store", "parking_lot 0.12.4", "parquet", "rand 0.9.1", "regex", - "rstest 0.26.1", "sqlparser", "tempfile", "tokio", "url", "uuid", - "xz2", - "zstd 0.13.3", + "zstd", ] [[package]] name = "datafusion-catalog" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "async-trait", "dashmap", "datafusion-common", @@ -3684,10 +3632,10 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -3702,21 +3650,20 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "tokio", ] [[package]] name = "datafusion-common" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ "ahash 0.8.12", - "arrow 57.0.0", - "arrow-ipc 57.0.0", + "arrow 57.2.0", + "arrow-ipc 57.2.0", "chrono", "half", - "hashbrown 0.14.5", - "indexmap 2.12.0", + "hashbrown 0.16.1", + "indexmap 2.13.0", "libc", "log", "object_store", @@ -3730,8 +3677,8 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ "futures", "log", @@ -3740,14 +3687,14 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", - "async-compression 0.4.19", + "arrow 57.2.0", + "async-compression", "async-trait", "bytes", - "bzip2 0.6.1", + "bzip2", "chrono", "datafusion-common", "datafusion-common-runtime", @@ -3762,23 +3709,23 @@ dependencies = [ "futures", "glob", "itertools 0.14.0", + "liblzma", "log", "object_store", "rand 0.9.1", "tokio", "tokio-util", "url", - "xz2", - "zstd 0.13.3", + "zstd", ] [[package]] name = "datafusion-datasource-arrow" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", - "arrow-ipc 57.0.0", + "arrow 57.2.0", + "arrow-ipc 57.2.0", "async-trait", "bytes", "datafusion-common", @@ -3797,10 +3744,10 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "async-trait", "bytes", "datafusion-common", @@ -3819,10 +3766,10 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "async-trait", "bytes", "datafusion-common", @@ -3840,10 +3787,10 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "async-trait", "bytes", "datafusion-common", @@ -3869,19 +3816,21 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" [[package]] name = "datafusion-execution" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "async-trait", + "chrono", "dashmap", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr-common", "futures", "log", "object_store", @@ -3893,10 +3842,10 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "async-trait", "chrono", "datafusion-common", @@ -3905,7 +3854,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.12.0", + "indexmap 2.13.0", "itertools 0.14.0", "paste", "recursive", @@ -3915,27 +3864,28 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "datafusion-common", - "indexmap 2.12.0", + "indexmap 2.13.0", "itertools 0.14.0", "paste", ] [[package]] name = "datafusion-functions" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", - "arrow-buffer 57.0.0", + "arrow 57.2.0", + "arrow-buffer 57.2.0", "base64 0.22.1", "blake2", "blake3", "chrono", + "chrono-tz", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -3956,11 +3906,11 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ "ahash 0.8.12", - "arrow 57.0.0", + "arrow 57.2.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -3976,11 +3926,11 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ "ahash 0.8.12", - "arrow 57.0.0", + "arrow 57.2.0", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", @@ -3988,11 +3938,11 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", - "arrow-ord 57.0.0", + "arrow 57.2.0", + "arrow-ord 57.2.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -4010,10 +3960,10 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -4025,10 +3975,10 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "datafusion-common", "datafusion-doc", "datafusion-expr", @@ -4042,8 +3992,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -4051,8 +4001,8 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ "datafusion-doc", "quote", @@ -4061,16 +4011,16 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "chrono", "datafusion-common", "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", - "indexmap 2.12.0", + "indexmap 2.13.0", "itertools 0.14.0", "log", "recursive", @@ -4080,13 +4030,13 @@ dependencies = [ [[package]] name = "datafusion-orc" -version = "0.6.0" -source = "git+https://github.com/GreptimeTeam/datafusion-orc.git?rev=35f2e04bf81f2ab7b6f86c0450d6a77b7098d43e#35f2e04bf81f2ab7b6f86c0450d6a77b7098d43e" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3afe28e177f33d54b7841add7679176c4354934ff3db350f43c013b4f9b0bad8" dependencies = [ "async-trait", "bytes", "datafusion", - "datafusion-datasource", "futures", "futures-util", "object_store", @@ -4096,8 +4046,9 @@ dependencies = [ [[package]] name = "datafusion-pg-catalog" -version = "0.13.1" -source = "git+https://github.com/GreptimeTeam/datafusion-postgres.git?rev=74ac8e2806be6de91ff192b97f64735392539d16#74ac8e2806be6de91ff192b97f64735392539d16" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daafc06d0478b70b13e8f3d906f2d47c49027efd3718263851137cf6d1d3e0a4" dependencies = [ "async-trait", "datafusion", @@ -4109,31 +4060,33 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ "ahash 0.8.12", - "arrow 57.0.0", + "arrow 57.2.0", "datafusion-common", "datafusion-expr", "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.14.5", - "indexmap 2.12.0", + "hashbrown 0.16.1", + "indexmap 2.13.0", "itertools 0.14.0", "parking_lot 0.12.4", "paste", "petgraph 0.8.3", + "recursive", + "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "datafusion-common", "datafusion-expr", "datafusion-functions", @@ -4144,23 +4097,26 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ "ahash 0.8.12", - "arrow 57.0.0", + "arrow 57.2.0", + "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.14.5", + "hashbrown 0.16.1", + "indexmap 2.13.0", "itertools 0.14.0", + "parking_lot 0.12.4", ] [[package]] name = "datafusion-physical-optimizer" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -4175,29 +4131,30 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ "ahash 0.8.12", - "arrow 57.0.0", - "arrow-ord 57.0.0", - "arrow-schema 57.0.0", + "arrow 57.2.0", + "arrow-ord 57.2.0", + "arrow-schema 57.2.0", "async-trait", - "chrono", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.14.5", - "indexmap 2.12.0", + "hashbrown 0.16.1", + "indexmap 2.13.0", "itertools 0.14.0", "log", + "num-traits", "parking_lot 0.12.4", "pin-project-lite", "tokio", @@ -4205,10 +4162,10 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "datafusion-common", "datafusion-datasource", "datafusion-expr-common", @@ -4221,8 +4178,8 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ "async-trait", "datafusion-common", @@ -4234,15 +4191,16 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "bigdecimal 0.4.8", "chrono", "datafusion-common", "datafusion-expr", - "indexmap 2.12.0", + "datafusion-functions-nested", + "indexmap 2.13.0", "log", "recursive", "regex", @@ -4251,8 +4209,8 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "51.0.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +version = "52.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=dc4129282abc8a74eb1674690115ec899d4e5741#dc4129282abc8a74eb1674690115ec899d4e5741" dependencies = [ "async-recursion", "async-trait", @@ -4339,9 +4297,9 @@ dependencies = [ name = "datatypes" version = "1.0.0-beta.4" dependencies = [ - "arrow 57.0.0", - "arrow-array 57.0.0", - "arrow-schema 57.0.0", + "arrow 57.2.0", + "arrow-array 57.2.0", + "arrow-schema 57.2.0", "common-base", "common-decimal", "common-error", @@ -5091,14 +5049,14 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.4" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9" +checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369" dependencies = [ "crc32fast", - "libz-rs-sys", "libz-sys", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -5149,8 +5107,8 @@ name = "flow" version = "1.0.0-beta.4" dependencies = [ "api", - "arrow 57.0.0", - "arrow-schema 57.0.0", + "arrow 57.2.0", + "arrow-schema 57.2.0", "async-recursion", "async-trait", "auth", @@ -5759,7 +5717,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.12.0", + "indexmap 2.13.0", "slab", "tokio", "tokio-util", @@ -5778,7 +5736,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.3.1", - "indexmap 2.12.0", + "indexmap 2.13.0", "slab", "tokio", "tokio-util", @@ -5867,9 +5825,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", @@ -6556,12 +6514,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.0" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", - "hashbrown 0.16.0", + "hashbrown 0.16.1", ] [[package]] @@ -6577,7 +6535,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash 0.8.12", - "indexmap 2.12.0", + "indexmap 2.13.0", "is-terminal", "itoa", "log", @@ -6600,7 +6558,7 @@ dependencies = [ "crossbeam-utils", "dashmap", "env_logger", - "indexmap 2.12.0", + "indexmap 2.13.0", "itoa", "log", "num-format", @@ -6976,7 +6934,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.13.0", ] [[package]] @@ -7276,9 +7234,9 @@ checksum = "775bf80d5878ab7c2b1080b5351a48b2f737d9f6f8b383574eebcc22be0dfccb" [[package]] name = "libc" -version = "0.2.177" +version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] name = "libflate" @@ -7324,6 +7282,26 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "liblzma" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73c36d08cad03a3fbe2c4e7bb3a9e84c57e4ee4135ed0b065cade3d98480c648" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.15" @@ -7351,15 +7329,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "libz-rs-sys" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" -dependencies = [ - "zlib-rs", -] - [[package]] name = "libz-sys" version = "1.1.22" @@ -7544,7 +7513,7 @@ dependencies = [ "cactus", "cfgrammar", "filetime", - "indexmap 2.12.0", + "indexmap 2.13.0", "lazy_static", "lrtable", "num-traits", @@ -7614,14 +7583,12 @@ dependencies = [ ] [[package]] -name = "lzma-sys" -version = "0.1.20" +name = "lz4_flex" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" dependencies = [ - "cc", - "libc", - "pkg-config", + "twox-hash", ] [[package]] @@ -8065,7 +8032,7 @@ dependencies = [ "regex", "roaring", "rskafka", - "rstest 0.25.0", + "rstest", "rstest_reuse", "scopeguard", "serde", @@ -8302,7 +8269,7 @@ dependencies = [ "thiserror 1.0.69", "time", "uuid", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -8335,7 +8302,7 @@ dependencies = [ "thiserror 1.0.69", "time", "uuid", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -9078,8 +9045,8 @@ version = "1.0.0-beta.4" dependencies = [ "ahash 0.8.12", "api", - "arrow 57.0.0", - "arrow-ipc 57.0.0", + "arrow 57.2.0", + "arrow-ipc 57.2.0", "async-stream", "async-trait", "bytes", @@ -9148,7 +9115,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8f1a357fb58dd9aab45ca21166d852f7894320d67ebeaa6d6b3fe557a9356c8" dependencies = [ - "arrow 57.0.0", + "arrow 57.2.0", "async-trait", "bytemuck", "bytes", @@ -9158,14 +9125,14 @@ dependencies = [ "flate2", "futures", "futures-util", - "lz4_flex", + "lz4_flex 0.11.5", "lzokay-native", "num", "prost 0.13.5", "snafu 0.8.6", "snap", "tokio", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -9373,27 +9340,27 @@ dependencies = [ [[package]] name = "parquet" -version = "57.0.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0f31027ef1af7549f7cec603a9a21dce706d3f8d7c2060a68f43c1773be95a" +checksum = "5f6a2926a30477c0b95fea6c28c3072712b139337a242c2cc64817bdc20a8854" dependencies = [ "ahash 0.8.12", - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-cast 57.0.0", - "arrow-data 57.0.0", - "arrow-ipc 57.0.0", - "arrow-schema 57.0.0", - "arrow-select 57.0.0", + "arrow-array 57.2.0", + "arrow-buffer 57.2.0", + "arrow-cast 57.2.0", + "arrow-data 57.2.0", + "arrow-ipc 57.2.0", + "arrow-schema 57.2.0", + "arrow-select 57.2.0", "base64 0.22.1", - "brotli 8.0.1", + "brotli", "bytes", "chrono", "flate2", "futures", "half", - "hashbrown 0.16.0", - "lz4_flex", + "hashbrown 0.16.1", + "lz4_flex 0.12.0", "num-bigint", "num-integer", "num-traits", @@ -9405,7 +9372,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -9600,7 +9567,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset 0.4.2", - "indexmap 2.12.0", + "indexmap 2.13.0", ] [[package]] @@ -9610,7 +9577,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset 0.5.7", - "indexmap 2.12.0", + "indexmap 2.13.0", ] [[package]] @@ -9621,7 +9588,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset 0.5.7", "hashbrown 0.15.4", - "indexmap 2.12.0", + "indexmap 2.13.0", "serde", ] @@ -9779,8 +9746,8 @@ version = "1.0.0-beta.4" dependencies = [ "ahash 0.8.12", "api", - "arrow 57.0.0", - "arrow-schema 57.0.0", + "arrow 57.2.0", + "arrow-schema 57.2.0", "async-trait", "catalog", "chrono", @@ -9896,7 +9863,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d77244ce2d584cd84f6a15f86195b8c9b2a0dfbfd817c09e0464244091a58ed" dependencies = [ "base64 0.22.1", - "indexmap 2.12.0", + "indexmap 2.13.0", "quick-xml 0.37.5", "serde", "time", @@ -10585,7 +10552,7 @@ dependencies = [ name = "puffin" version = "1.0.0-beta.4" dependencies = [ - "async-compression 0.4.19", + "async-compression", "async-trait", "async-walkdir", "auto_impl", @@ -10600,7 +10567,7 @@ dependencies = [ "common-test-util", "derive_builder 0.20.2", "futures", - "lz4_flex", + "lz4_flex 0.11.5", "moka", "pin-project", "prometheus", @@ -10650,8 +10617,8 @@ dependencies = [ "ahash 0.8.12", "api", "arc-swap", - "arrow 57.0.0", - "arrow-schema 57.0.0", + "arrow 57.2.0", + "arrow-schema 57.2.0", "async-recursion", "async-stream", "async-trait", @@ -11074,7 +11041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c11639076bf147be211b90e47790db89f4c22b6c8a9ca6e960833869da67166" dependencies = [ "aho-corasick", - "indexmap 2.12.0", + "indexmap 2.13.0", "itertools 0.13.0", "nohash", "regex", @@ -11115,7 +11082,7 @@ version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2057b2325e68a893284d1538021ab90279adac1139957ca2a74426c6f118fb48" dependencies = [ - "hashbrown 0.16.0", + "hashbrown 0.16.1", "memchr", ] @@ -11381,7 +11348,7 @@ dependencies = [ "tokio", "tokio-rustls", "tracing", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -11403,21 +11370,10 @@ checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d" dependencies = [ "futures-timer", "futures-util", - "rstest_macros 0.25.0", + "rstest_macros", "rustc_version", ] -[[package]] -name = "rstest" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" -dependencies = [ - "futures-timer", - "futures-util", - "rstest_macros 0.26.1", -] - [[package]] name = "rstest_macros" version = "0.25.0" @@ -11436,24 +11392,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "rstest_macros" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" -dependencies = [ - "cfg-if", - "glob", - "proc-macro-crate 3.3.0", - "proc-macro2", - "quote", - "regex", - "relative-path", - "rustc_version", - "syn 2.0.114", - "unicode-ident", -] - [[package]] name = "rstest_reuse" version = "0.7.0" @@ -12094,7 +12032,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.13.0", "itoa", "ryu", "serde", @@ -12107,10 +12045,10 @@ version = "1.0.0-beta.4" dependencies = [ "ahash 0.8.12", "api", - "arrow 57.0.0", + "arrow 57.2.0", "arrow-flight", - "arrow-ipc 57.0.0", - "arrow-schema 57.0.0", + "arrow-ipc 57.2.0", + "arrow-schema 57.2.0", "async-trait", "auth", "axum 0.8.4", @@ -12160,7 +12098,7 @@ dependencies = [ "humantime", "humantime-serde", "hyper 1.6.0", - "indexmap 2.12.0", + "indexmap 2.13.0", "influxdb_line_protocol", "itertools 0.14.0", "json5", @@ -12227,7 +12165,7 @@ dependencies = [ "urlencoding", "uuid", "vrl", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -12579,7 +12517,7 @@ name = "sql" version = "1.0.0-beta.4" dependencies = [ "api", - "arrow-buffer 57.0.0", + "arrow-buffer 57.2.0", "chrono", "common-base", "common-catalog", @@ -12666,15 +12604,15 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.59.0" -source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=d7d95a44889e099e32d78e9bad9bc00598faef28#d7d95a44889e099e32d78e9bad9bc00598faef28" +version = "0.60.0" +source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=e8a3ac44693768965abf7030cd38ebcff53a4d29#e8a3ac44693768965abf7030cd38ebcff53a4d29" dependencies = [ "lazy_static", "log", "recursive", "regex", "serde", - "sqlparser_derive 0.3.0", + "sqlparser_derive 0.4.0", ] [[package]] @@ -12690,8 +12628,8 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.3.0" -source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=d7d95a44889e099e32d78e9bad9bc00598faef28#d7d95a44889e099e32d78e9bad9bc00598faef28" +version = "0.4.0" +source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=e8a3ac44693768965abf7030cd38ebcff53a4d29#e8a3ac44693768965abf7030cd38ebcff53a4d29" dependencies = [ "proc-macro2", "quote", @@ -12730,7 +12668,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.4", "hashlink", - "indexmap 2.12.0", + "indexmap 2.13.0", "log", "memchr", "once_cell", @@ -13363,7 +13301,7 @@ dependencies = [ "levenshtein_automata", "log", "lru", - "lz4_flex", + "lz4_flex 0.11.5", "measure_time", "memmap2", "once_cell", @@ -13388,7 +13326,7 @@ dependencies = [ "time", "uuid", "winapi", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -13473,7 +13411,7 @@ dependencies = [ "tantivy-bitpacker", "tantivy-common", "tantivy-fst", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -13670,7 +13608,7 @@ dependencies = [ "prost 0.14.1", "query", "rand 0.9.1", - "rstest 0.25.0", + "rstest", "rstest_reuse", "sea-query", "serde_json", @@ -13695,7 +13633,7 @@ dependencies = [ "urlencoding", "uuid", "yaml-rust", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -13884,9 +13822,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.48.0" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" dependencies = [ "bytes", "libc", @@ -14023,9 +13961,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.16" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", @@ -14051,7 +13989,7 @@ version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.13.0", "serde", "serde_spanned", "toml_datetime", @@ -14073,7 +14011,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.13.0", "toml_datetime", "winnow 0.5.40", ] @@ -14084,7 +14022,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.13.0", "serde", "serde_spanned", "toml_datetime", @@ -14182,7 +14120,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -14267,7 +14205,7 @@ dependencies = [ "futures-core", "futures-util", "hdrhistogram", - "indexmap 2.12.0", + "indexmap 2.13.0", "pin-project-lite", "slab", "sync_wrapper 1.0.2", @@ -14303,7 +14241,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ - "async-compression 0.4.19", + "async-compression", "base64 0.22.1", "bitflags 2.9.1", "bytes", @@ -14792,14 +14730,14 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.18.1" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" dependencies = [ "getrandom 0.3.3", "js-sys", "rand 0.9.1", - "serde", + "serde_core", "wasm-bindgen", ] @@ -14925,13 +14863,13 @@ dependencies = [ "hostname 0.4.1", "iana-time-zone", "idna", - "indexmap 2.12.0", + "indexmap 2.13.0", "indoc", "influxdb-line-protocol", "itertools 0.14.0", "lalrpop", "lalrpop-util", - "lz4_flex", + "lz4_flex 0.11.5", "md-5", "nom 7.1.3", "ofb", @@ -14974,7 +14912,7 @@ dependencies = [ "utf8-width", "uuid", "woothee", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -15664,15 +15602,6 @@ dependencies = [ "rustix 1.0.7", ] -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yaml-rust" version = "0.4.5" @@ -15808,9 +15737,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.1" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" +checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" [[package]] name = "zmij" @@ -15818,32 +15747,13 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fc5a66a20078bf1251bde995aa2fdcc4b800c70b5d92dd2c62abc5c60f679f8" -[[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe 5.0.2+zstd.1.5.2", -] - [[package]] name = "zstd" version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe 7.2.4", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", + "zstd-safe", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 077e22abe0..72858a0636 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -100,13 +100,13 @@ rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] } # See for more detaiils: https://github.com/rust-lang/cargo/issues/11329 ahash = { version = "0.8", features = ["compile-time-rng"] } aquamarine = "0.6" -arrow = { version = "57.0", features = ["prettyprint"] } -arrow-array = { version = "57.0", default-features = false, features = ["chrono-tz"] } -arrow-buffer = "57.0" -arrow-cast = "57.0" -arrow-flight = "57.0" -arrow-ipc = { version = "57.0", default-features = false, features = ["lz4", "zstd"] } -arrow-schema = { version = "57.0", features = ["serde"] } +arrow = { version = "57.2", features = ["prettyprint"] } +arrow-array = { version = "57.2", default-features = false, features = ["chrono-tz"] } +arrow-buffer = "57.2" +arrow-cast = "57.2" +arrow-flight = "57.2" +arrow-ipc = { version = "57.2", default-features = false, features = ["lz4", "zstd"] } +arrow-schema = { version = "57.2", features = ["serde"] } async-stream = "0.3" async-trait = "0.1" # Remember to update axum-extra, axum-macros when updating axum @@ -126,19 +126,19 @@ config = "0.13.0" const_format = "0.2" crossbeam-utils = "0.8" dashmap = "6.1" -datafusion = "51.0" -datafusion-common = "51.0" -datafusion-datasource = "51.0" -datafusion-expr = "51.0" -datafusion-functions = "51.0" -datafusion-functions-aggregate-common = "51.0" -datafusion-optimizer = "51.0" -datafusion-orc = { git = "https://github.com/GreptimeTeam/datafusion-orc.git", rev = "35f2e04bf81f2ab7b6f86c0450d6a77b7098d43e" } -datafusion-pg-catalog = "0.13" -datafusion-physical-expr = "51.0" -datafusion-physical-plan = "51.0" -datafusion-sql = "51.0" -datafusion-substrait = "51.0" +datafusion = "52.0" +datafusion-common = "52.0" +datafusion-datasource = "52.0" +datafusion-expr = "52.0" +datafusion-functions = "52.0" +datafusion-functions-aggregate-common = "52.0" +datafusion-optimizer = "52.0" +datafusion-orc = "0.7" +datafusion-pg-catalog = "0.14" +datafusion-physical-expr = "52.0" +datafusion-physical-plan = "52.0" +datafusion-sql = "52.0" +datafusion-substrait = "52.0" deadpool = "0.12" deadpool-postgres = "0.14" derive_builder = "0.20" @@ -185,7 +185,7 @@ otel-arrow-rust = { git = "https://github.com/GreptimeTeam/otel-arrow", rev = "5 "server", ] } parking_lot = "0.12" -parquet = { version = "57.0", default-features = false, features = ["arrow", "async", "object_store"] } +parquet = { version = "57.2", default-features = false, features = ["arrow", "async", "object_store"] } paste = "1.0" pin-project = "1.0" pretty_assertions = "1.4.0" @@ -224,7 +224,7 @@ simd-json = "0.15" similar-asserts = "1.6.0" smallvec = { version = "1", features = ["serde"] } snafu = "0.8" -sqlparser = { version = "0.59.0", default-features = false, features = ["std", "visitor", "serde"] } +sqlparser = { version = "0.60.0", default-features = false, features = ["std", "visitor", "serde"] } sqlx = { version = "0.8", default-features = false, features = ["any", "macros", "json", "runtime-tokio-rustls"] } strum = { version = "0.27", features = ["derive"] } sysinfo = "0.33" @@ -323,20 +323,19 @@ git = "https://github.com/GreptimeTeam/greptime-meter.git" rev = "5618e779cf2bb4755b499c630fba4c35e91898cb" [patch.crates-io] -datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } -datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } -datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } -datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } -datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } -datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } -datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } -datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } -datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } -datafusion-pg-catalog = { git = "https://github.com/GreptimeTeam/datafusion-postgres.git", rev = "74ac8e2806be6de91ff192b97f64735392539d16" } -datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } -datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } -datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } -sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "d7d95a44889e099e32d78e9bad9bc00598faef28" } # on branch v0.59.x +datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" } +datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" } +datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" } +datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" } +datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" } +datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" } +datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" } +datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" } +datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" } +datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" } +datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" } +datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "dc4129282abc8a74eb1674690115ec899d4e5741" } +sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "e8a3ac44693768965abf7030cd38ebcff53a4d29" } # on branch v0.60.x [profile.release] debug = 1 diff --git a/src/common/datasource/Cargo.toml b/src/common/datasource/Cargo.toml index 7b13a5bdac..6ec9a14733 100644 --- a/src/common/datasource/Cargo.toml +++ b/src/common/datasource/Cargo.toml @@ -10,7 +10,7 @@ workspace = true [dependencies] arrow.workspace = true arrow-schema.workspace = true -async-compression = { version = "0.3", features = [ +async-compression = { version = "0.4", features = [ "bzip2", "gzip", "xz", diff --git a/src/common/datasource/src/error.rs b/src/common/datasource/src/error.rs index a8aa08e55c..4b514b5b20 100644 --- a/src/common/datasource/src/error.rs +++ b/src/common/datasource/src/error.rs @@ -203,8 +203,8 @@ pub enum Error { error: parquet::errors::ParquetError, }, - #[snafu(display("Failed to build file stream"))] - BuildFileStream { + #[snafu(transparent)] + DataFusion { #[snafu(implicit)] location: Location, #[snafu(source)] @@ -241,8 +241,9 @@ impl ErrorExt for Error { | ReadRecordBatch { .. } | WriteRecordBatch { .. } | EncodeRecordBatch { .. } - | BuildFileStream { .. } | OrcReader { .. } => StatusCode::Unexpected, + + DataFusion { .. } => StatusCode::Internal, } } diff --git a/src/common/datasource/src/file_format.rs b/src/common/datasource/src/file_format.rs index ba58fad143..2a7c2c41d4 100644 --- a/src/common/datasource/src/file_format.rs +++ b/src/common/datasource/src/file_format.rs @@ -313,16 +313,13 @@ pub async fn file_to_stream( filename.to_string(), 0, )])) - .with_projection_indices(projection) + .with_projection_indices(projection)? .with_file_compression_type(df_compression) .build(); let store = Arc::new(OpendalStore::new(store.clone())); - let file_opener = file_source - .with_projection(&config) - .create_file_opener(store, &config, 0); - let stream = FileStream::new(&config, 0, file_opener, &ExecutionPlanMetricsSet::new()) - .context(error::BuildFileStreamSnafu)?; + let file_opener = file_source.create_file_opener(store, &config, 0)?; + let stream = FileStream::new(&config, 0, file_opener, &ExecutionPlanMetricsSet::new())?; Ok(Box::pin(stream)) } diff --git a/src/common/datasource/src/file_format/tests.rs b/src/common/datasource/src/file_format/tests.rs index 29a23c07da..945ec5320c 100644 --- a/src/common/datasource/src/file_format/tests.rs +++ b/src/common/datasource/src/file_format/tests.rs @@ -46,7 +46,10 @@ struct Test<'a> { impl Test<'_> { async fn run(self, store: &ObjectStore) { let store = Arc::new(object_store_opendal::OpendalStore::new(store.clone())); - let file_opener = self.file_source.create_file_opener(store, &self.config, 0); + let file_opener = self + .file_source + .create_file_opener(store, &self.config, 0) + .unwrap(); let result = FileStream::new( &self.config, diff --git a/src/common/datasource/src/test_util.rs b/src/common/datasource/src/test_util.rs index f647d424ad..808bfdf5c6 100644 --- a/src/common/datasource/src/test_util.rs +++ b/src/common/datasource/src/test_util.rs @@ -154,11 +154,13 @@ pub async fn setup_stream_to_csv_test( let config = scan_config(None, origin_path, csv_source.clone()); let size = store.read(origin_path).await.unwrap().len(); - let csv_opener = csv_source.create_file_opener( - Arc::new(object_store_opendal::OpendalStore::new(store.clone())), - &config, - 0, - ); + let csv_opener = csv_source + .create_file_opener( + Arc::new(object_store_opendal::OpendalStore::new(store.clone())), + &config, + 0, + ) + .unwrap(); let stream = FileStream::new(&config, 0, csv_opener, &ExecutionPlanMetricsSet::new()).unwrap(); let (tmp_store, dir) = test_tmp_store("test_stream_to_csv"); diff --git a/src/common/function/src/aggrs/count_hash.rs b/src/common/function/src/aggrs/count_hash.rs index e181131834..bd00aeb012 100644 --- a/src/common/function/src/aggrs/count_hash.rs +++ b/src/common/function/src/aggrs/count_hash.rs @@ -428,7 +428,7 @@ impl Accumulator for CountHashAccumulator { &self.random_state, &mut self.batch_hashes, )?; - for hash in hashes.as_slice() { + for hash in hashes { self.values.insert(*hash); } Ok(()) diff --git a/src/common/query/src/logical_plan.rs b/src/common/query/src/logical_plan.rs index ca555a2552..41a46d0462 100644 --- a/src/common/query/src/logical_plan.rs +++ b/src/common/query/src/logical_plan.rs @@ -48,7 +48,7 @@ pub fn rename_logical_plan_columns( plan.schema().qualified_field_from_column(&old_column)?; for (qualifier, field) in plan.schema().iter() { - if qualifier.eq(&qualifier_rename) && field.as_ref() == field_rename { + if qualifier.eq(&qualifier_rename) && field == field_rename { projection.push(col(Column::from((qualifier, field))).alias(new_name)); } } diff --git a/src/file-engine/src/error.rs b/src/file-engine/src/error.rs index 3179d0d0fd..877b493cc6 100644 --- a/src/file-engine/src/error.rs +++ b/src/file-engine/src/error.rs @@ -178,8 +178,8 @@ pub enum Error { location: Location, }, - #[snafu(display("Failed to extract column from filter"))] - ExtractColumnFromFilter { + #[snafu(transparent)] + DataFusion { #[snafu(source)] error: DataFusionError, #[snafu(implicit)] @@ -225,8 +225,9 @@ impl ErrorExt for Error { | ManifestExists { .. } | BuildStream { .. } | ParquetScanPlan { .. } - | UnexpectedEngine { .. } - | ExtractColumnFromFilter { .. } => StatusCode::Unexpected, + | UnexpectedEngine { .. } => StatusCode::Unexpected, + + DataFusion { .. } => StatusCode::Internal, } } diff --git a/src/file-engine/src/query.rs b/src/file-engine/src/query.rs index 75d40c4608..2182978631 100644 --- a/src/file-engine/src/query.rs +++ b/src/file-engine/src/query.rs @@ -37,10 +37,7 @@ use snafu::{GenerateImplicitData, ResultExt, ensure}; use store_api::storage::ScanRequest; use self::file_stream::ScanPlanConfig; -use crate::error::{ - BuildBackendSnafu, ExtractColumnFromFilterSnafu, ProjectSchemaSnafu, - ProjectionOutOfBoundsSnafu, Result, -}; +use crate::error::{BuildBackendSnafu, ProjectSchemaSnafu, ProjectionOutOfBoundsSnafu, Result}; use crate::region::FileRegion; impl FileRegion { @@ -126,8 +123,7 @@ impl FileRegion { let mut aux_column_set = HashSet::new(); for scan_filter in scan_filters { - df_logical_expr_utils::expr_to_columns(scan_filter, &mut aux_column_set) - .context(ExtractColumnFromFilterSnafu)?; + df_logical_expr_utils::expr_to_columns(scan_filter, &mut aux_column_set)?; let all_file_columns = aux_column_set .iter() diff --git a/src/file-engine/src/query/file_stream.rs b/src/file-engine/src/query/file_stream.rs index fa0fcdcf73..34d85dd32a 100644 --- a/src/file-engine/src/query/file_stream.rs +++ b/src/file-engine/src/query/file_stream.rs @@ -56,7 +56,7 @@ fn build_record_batch_stream( let config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source.clone()) - .with_projection_indices(scan_plan_config.projection.cloned()) + .with_projection_indices(scan_plan_config.projection.cloned())? .with_limit(limit) .with_file_group(FileGroup::new(files)) .build(); @@ -65,7 +65,7 @@ fn build_record_batch_stream( scan_plan_config.store.clone(), )); - let file_opener = file_source.create_file_opener(store, &config, 0); + let file_opener = file_source.create_file_opener(store, &config, 0)?; let stream = FileStream::new( &config, 0, // partition: hard-code @@ -146,7 +146,7 @@ fn new_parquet_stream_with_exec_plan( let file_scan_config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), Arc::new(parquet_source)) .with_file_group(file_group) - .with_projection_indices(projection.cloned()) + .with_projection_indices(projection.cloned())? .with_limit(*limit) .build(); diff --git a/src/mito2/src/memtable/bulk/row_group_reader.rs b/src/mito2/src/memtable/bulk/row_group_reader.rs index 1e9e5dec4d..fccd22db10 100644 --- a/src/mito2/src/memtable/bulk/row_group_reader.rs +++ b/src/mito2/src/memtable/bulk/row_group_reader.rs @@ -21,7 +21,7 @@ use datatypes::arrow::error::ArrowError; use parquet::arrow::arrow_reader::{ParquetRecordBatchReader, RowGroups, RowSelection}; use parquet::arrow::{FieldLevels, ProjectionMask, parquet_to_arrow_field_levels}; use parquet::column::page::{PageIterator, PageReader}; -use parquet::file::metadata::ParquetMetaData; +use parquet::file::metadata::{ParquetMetaData, RowGroupMetaData}; use snafu::ResultExt; use crate::error; @@ -103,6 +103,14 @@ impl RowGroups for MemtableRowGroupPageFetcher<'_> { reader: Some(self.column_page_reader(i)), })) } + + fn row_groups(&self) -> Box + '_> { + Box::new(std::iter::once(self.base.row_group_metadata())) + } + + fn metadata(&self) -> &ParquetMetaData { + self.base.parquet_metadata() + } } impl RowGroupReaderContext for BulkIterContextRef { diff --git a/src/mito2/src/sst/parquet/row_group.rs b/src/mito2/src/sst/parquet/row_group.rs index b8baf7960f..8f3f6c5f62 100644 --- a/src/mito2/src/sst/parquet/row_group.rs +++ b/src/mito2/src/sst/parquet/row_group.rs @@ -205,7 +205,8 @@ impl ParquetFetchMetrics { } pub(crate) struct RowGroupBase<'a> { - metadata: &'a RowGroupMetaData, + parquet_metadata: &'a ParquetMetaData, + row_group_idx: usize, pub(crate) offset_index: Option<&'a [OffsetIndexMetaData]>, /// Compressed page of each column. column_chunks: Vec>>, @@ -225,7 +226,8 @@ impl<'a> RowGroupBase<'a> { .map(|x| x[row_group_idx].as_slice()); Self { - metadata, + parquet_metadata: parquet_meta, + row_group_idx, offset_index, column_chunks: vec![None; metadata.columns().len()], row_count: metadata.num_rows() as usize, @@ -244,7 +246,7 @@ impl<'a> RowGroupBase<'a> { let ranges = self .column_chunks .iter() - .zip(self.metadata.columns()) + .zip(self.row_group_metadata().columns()) .enumerate() .filter(|&(idx, (chunk, _chunk_meta))| chunk.is_none() && projection.leaf_included(idx)) .flat_map(|(idx, (_chunk, chunk_meta))| { @@ -293,8 +295,12 @@ impl<'a> RowGroupBase<'a> { chunks.push(chunk_data.next().unwrap()); } + let column = self + .parquet_metadata + .row_group(self.row_group_idx) + .column(idx); *chunk = Some(Arc::new(ColumnChunkData::Sparse { - length: self.metadata.column(idx).byte_range().1 as usize, + length: column.byte_range().1 as usize, data: offsets.into_iter().zip(chunks).collect(), })) } @@ -307,7 +313,7 @@ impl<'a> RowGroupBase<'a> { .enumerate() .filter(|&(idx, chunk)| chunk.is_none() && projection.leaf_included(idx)) .map(|(idx, _chunk)| { - let column = self.metadata.column(idx); + let column = self.row_group_metadata().column(idx); let (start, length) = column.byte_range(); start..(start + length) }) @@ -333,7 +339,10 @@ impl<'a> RowGroupBase<'a> { continue; }; - let column = self.metadata.column(idx); + let column = self + .parquet_metadata + .row_group(self.row_group_idx) + .column(idx); *chunk = Some(Arc::new(ColumnChunkData::Dense { offset: column.byte_range().0 as usize, data, @@ -360,7 +369,7 @@ impl<'a> RowGroupBase<'a> { .map(|index| index[col_idx].page_locations.clone()); SerializedPageReader::new( data.clone(), - self.metadata.column(col_idx), + self.row_group_metadata().column(col_idx), self.row_count, page_locations, )? @@ -369,6 +378,14 @@ impl<'a> RowGroupBase<'a> { Ok(page_reader) } + + pub(crate) fn parquet_metadata(&self) -> &ParquetMetaData { + self.parquet_metadata + } + + pub(crate) fn row_group_metadata(&self) -> &RowGroupMetaData { + self.parquet_metadata().row_group(self.row_group_idx) + } } /// An in-memory collection of column chunks @@ -599,6 +616,14 @@ impl RowGroups for InMemoryRowGroup<'_> { reader: Some(Ok(Box::new(page_reader))), })) } + + fn row_groups(&self) -> Box + '_> { + Box::new(std::iter::once(self.base.row_group_metadata())) + } + + fn metadata(&self) -> &ParquetMetaData { + self.base.parquet_metadata() + } } /// An in-memory column chunk diff --git a/src/operator/src/expr_helper.rs b/src/operator/src/expr_helper.rs index ff3d00eb0a..7ba2f953b6 100644 --- a/src/operator/src/expr_helper.rs +++ b/src/operator/src/expr_helper.rs @@ -549,15 +549,10 @@ fn find_primary_keys( let columns_pk = columns .iter() .filter_map(|x| { - if x.options().iter().any(|o| { - matches!( - o.option, - ColumnOption::Unique { - is_primary: true, - .. - } - ) - }) { + if x.options() + .iter() + .any(|o| matches!(o.option, ColumnOption::PrimaryKey(_))) + { Some(x.name().value.clone()) } else { None diff --git a/src/promql/src/extension_plan/empty_metric.rs b/src/promql/src/extension_plan/empty_metric.rs index 5514cb1abb..739521df19 100644 --- a/src/promql/src/extension_plan/empty_metric.rs +++ b/src/promql/src/extension_plan/empty_metric.rs @@ -81,7 +81,7 @@ impl EmptyMetric { ) -> DataFusionResult { let qualifier = Some(TableReference::bare("")); let ts_only_schema = build_ts_only_schema(&time_index_column_name); - let mut fields = vec![(qualifier.clone(), Arc::new(ts_only_schema.field(0).clone()))]; + let mut fields = vec![(qualifier.clone(), ts_only_schema.field(0).clone())]; if let Some(field_expr) = &field_expr { let field_data_type = field_expr.get_type(&ts_only_schema)?; fields.push(( diff --git a/src/promql/src/extension_plan/range_manipulate.rs b/src/promql/src/extension_plan/range_manipulate.rs index a3ac1304a5..099f06ec31 100644 --- a/src/promql/src/extension_plan/range_manipulate.rs +++ b/src/promql/src/extension_plan/range_manipulate.rs @@ -129,7 +129,7 @@ impl RangeManipulate { let mut new_columns = Vec::with_capacity(columns.len() + 1); for i in 0..columns.len() { let x = input_schema.qualified_field(i); - new_columns.push((x.0.cloned(), Arc::new(x.1.clone()))); + new_columns.push((x.0.cloned(), x.1.clone())); } // process time index column diff --git a/src/promql/src/extension_plan/scalar_calculate.rs b/src/promql/src/extension_plan/scalar_calculate.rs index 6a5f0fb0d8..c902a04647 100644 --- a/src/promql/src/extension_plan/scalar_calculate.rs +++ b/src/promql/src/extension_plan/scalar_calculate.rs @@ -94,7 +94,7 @@ impl ScalarCalculate { let qualifier = table_name.map(TableReference::bare); let schema = DFSchema::new_with_metadata( vec![ - (qualifier.clone(), Arc::new(ts_field)), + (qualifier.clone(), ts_field), (qualifier, Arc::new(val_field)), ], input_schema.metadata().clone(), diff --git a/src/promql/src/range_array.rs b/src/promql/src/range_array.rs index 4ae67dd2da..4af5217a76 100644 --- a/src/promql/src/range_array.rs +++ b/src/promql/src/range_array.rs @@ -14,9 +14,6 @@ //!An extended "array" based on [DictionaryArray]. -use std::sync::Arc; - -use datafusion::arrow::buffer::NullBuffer; use datafusion::arrow::datatypes::Field; use datatypes::arrow::array::{Array, ArrayData, ArrayRef, DictionaryArray, Int64Array}; use datatypes::arrow::datatypes::{DataType, Int64Type}; @@ -225,52 +222,6 @@ impl RangeArray { } } -impl Array for RangeArray { - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn into_data(self) -> ArrayData { - self.array.into_data() - } - - fn to_data(&self) -> ArrayData { - self.array.to_data() - } - - fn slice(&self, offset: usize, length: usize) -> ArrayRef { - Arc::new(self.array.slice(offset, length)) - } - - fn nulls(&self) -> Option<&NullBuffer> { - self.array.nulls() - } - - fn data_type(&self) -> &DataType { - self.array.data_type() - } - - fn len(&self) -> usize { - self.len() - } - - fn is_empty(&self) -> bool { - self.is_empty() - } - - fn offset(&self) -> usize { - self.array.offset() - } - - fn get_buffer_memory_size(&self) -> usize { - self.array.get_buffer_memory_size() - } - - fn get_array_memory_size(&self) -> usize { - self.array.get_array_memory_size() - } -} - impl std::fmt::Debug for RangeArray { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let ranges = self diff --git a/src/query/src/dist_plan/analyzer/utils.rs b/src/query/src/dist_plan/analyzer/utils.rs index 319f7f2d1e..e9205e33f4 100644 --- a/src/query/src/dist_plan/analyzer/utils.rs +++ b/src/query/src/dist_plan/analyzer/utils.rs @@ -46,7 +46,7 @@ pub(crate) struct PatchOptimizerContext { } impl OptimizerConfig for PatchOptimizerContext { - fn query_execution_start_time(&self) -> DateTime { + fn query_execution_start_time(&self) -> Option> { self.inner.query_execution_start_time() } diff --git a/src/query/src/dist_plan/commutativity.rs b/src/query/src/dist_plan/commutativity.rs index 91aa909622..9f83e5a8e4 100644 --- a/src/query/src/dist_plan/commutativity.rs +++ b/src/query/src/dist_plan/commutativity.rs @@ -279,7 +279,8 @@ impl Categorizer { Expr::Unnest(_) | Expr::GroupingSet(_) | Expr::Placeholder(_) - | Expr::OuterReferenceColumn(_, _) => Commutativity::Unimplemented, + | Expr::OuterReferenceColumn(_, _) + | Expr::SetComparison(_) => Commutativity::Unimplemented, } } diff --git a/src/query/src/optimizer/remove_duplicate.rs b/src/query/src/optimizer/remove_duplicate.rs index 6ed81a75c7..65d191e7b5 100644 --- a/src/query/src/optimizer/remove_duplicate.rs +++ b/src/query/src/optimizer/remove_duplicate.rs @@ -17,13 +17,12 @@ use std::sync::Arc; use datafusion::config::ConfigOptions; use datafusion::physical_optimizer::PhysicalOptimizerRule; use datafusion::physical_plan::ExecutionPlan; -use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec; use datafusion::physical_plan::repartition::RepartitionExec; use datafusion_common::Result as DfResult; use datafusion_common::tree_node::{Transformed, TreeNode}; /// This is [PhysicalOptimizerRule] to remove duplicate physical plans such as two -/// adjoining [CoalesceBatchesExec] or [RepartitionExec]. They won't have any effect +/// adjoining [RepartitionExec]. They won't have any effect /// if one runs right after another. /// /// This rule is expected to be run in the final stage of the optimization process. @@ -52,9 +51,7 @@ impl RemoveDuplicate { fn do_optimize(plan: Arc) -> DfResult> { let result = plan .transform_down(|plan| { - if plan.as_any().is::() - || plan.as_any().is::() - { + if plan.as_any().is::() { // check child let child = plan.children()[0].clone(); if child.as_any().type_id() == plan.as_any().type_id() { @@ -72,49 +69,3 @@ impl RemoveDuplicate { Ok(result) } } - -#[cfg(test)] -mod test { - use std::sync::Arc; - - use arrow_schema::Schema; - use datafusion::physical_plan::displayable; - use datafusion::physical_plan::empty::EmptyExec; - use datafusion_physical_expr::Partitioning; - - use super::*; - - #[test] - fn remove_coalesce_batches() { - let empty = Arc::new(EmptyExec::new(Arc::new(Schema::empty()))); - let coalesce_batches = Arc::new(CoalesceBatchesExec::new(empty, 1024)); - let another_coalesce_batches = Arc::new(CoalesceBatchesExec::new(coalesce_batches, 8192)); - - let optimized = RemoveDuplicate::do_optimize(another_coalesce_batches).unwrap(); - let formatted = displayable(optimized.as_ref()).indent(true).to_string(); - let expected = "CoalesceBatchesExec: target_batch_size=8192\ - \n EmptyExec\n"; - - assert_eq!(expected, formatted); - } - - #[test] - fn non_continuous_coalesce_batches() { - let empty = Arc::new(EmptyExec::new(Arc::new(Schema::empty()))); - let coalesce_batches = Arc::new(CoalesceBatchesExec::new(empty, 1024)); - let repartition = Arc::new( - RepartitionExec::try_new(coalesce_batches, Partitioning::UnknownPartitioning(1)) - .unwrap(), - ); - let another_coalesce_batches = Arc::new(CoalesceBatchesExec::new(repartition, 8192)); - - let optimized = RemoveDuplicate::do_optimize(another_coalesce_batches).unwrap(); - let formatted = displayable(optimized.as_ref()).indent(true).to_string(); - let expected = "CoalesceBatchesExec: target_batch_size=8192\ - \n RepartitionExec: partitioning=UnknownPartitioning(1), input_partitions=1\ - \n CoalesceBatchesExec: target_batch_size=1024\ - \n EmptyExec\n"; - - assert_eq!(expected, formatted); - } -} diff --git a/src/query/src/optimizer/windowed_sort.rs b/src/query/src/optimizer/windowed_sort.rs index 469f4db159..2d5844d689 100644 --- a/src/query/src/optimizer/windowed_sort.rs +++ b/src/query/src/optimizer/windowed_sort.rs @@ -17,7 +17,6 @@ use std::sync::Arc; use datafusion::physical_optimizer::PhysicalOptimizerRule; use datafusion::physical_plan::ExecutionPlan; -use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::coop::CooperativeExec; use datafusion::physical_plan::filter::FilterExec; @@ -80,8 +79,6 @@ impl WindowedSortPhysicalRule { let preserve_partitioning = sort_exec.preserve_partitioning(); let sort_input = remove_repartition(sort_exec.input().clone())?.data; - let sort_input = - remove_coalesce_batches_exec(sort_input, sort_exec.fetch())?.data; // Gets scanner info from the input without repartition before filter. let Some(scanner_info) = fetch_partition_range(sort_input.clone())? else { @@ -164,7 +161,6 @@ fn fetch_partition_range(input: Arc) -> DataFusionResult() { @@ -180,16 +176,9 @@ fn fetch_partition_range(input: Arc) -> DataFusionResult() { - is_batch_coalesced = true; - } - // only a very limited set of plans can exist between region scan and sort exec // other plans might make this optimize wrong, so be safe here by limiting it - if !(plan.as_any().is::() - || plan.as_any().is::() - || plan.as_any().is::()) - { + if !(plan.as_any().is::() || plan.as_any().is::()) { partition_ranges = None; } @@ -218,11 +207,6 @@ fn fetch_partition_range(input: Arc) -> DataFusionResult, - fetch: Option, -) -> DataFusionResult>> { - let Some(fetch) = fetch else { - return Ok(Transformed::no(plan)); - }; - - // Avoid removing multiple coalesce batches - let mut is_done = false; - - plan.transform_down(|plan| { - if let Some(coalesce_batches_exec) = plan.as_any().downcast_ref::() { - let target_batch_size = coalesce_batches_exec.target_batch_size(); - if fetch < target_batch_size && !is_done { - is_done = true; - return Ok(Transformed::yes(coalesce_batches_exec.input().clone())); - } - } - - Ok(Transformed::no(plan)) - }) -} - /// Resolves alias of the time index column. /// /// i.e if a is time index, alias= {a:b, b:c}, then result should be {a, b}(not {a, c}) because projection is not transitive diff --git a/src/query/src/range_select/plan.rs b/src/query/src/range_select/plan.rs index 663acca0e0..2ee97a39c5 100644 --- a/src/query/src/range_select/plan.rs +++ b/src/query/src/range_select/plan.rs @@ -414,7 +414,7 @@ impl RangeSelect { .iter() .map(|i| { let f = schema_before_project.qualified_field(*i); - (f.0.cloned(), Arc::new(f.1.clone())) + (f.0.cloned(), f.1.clone()) }) .collect(); Arc::new(DFSchema::new_with_metadata( diff --git a/src/query/src/range_select/plan_rewrite.rs b/src/query/src/range_select/plan_rewrite.rs index 87b036df65..26d354b123 100644 --- a/src/query/src/range_select/plan_rewrite.rs +++ b/src/query/src/range_select/plan_rewrite.rs @@ -19,7 +19,6 @@ use std::time::Duration; use arrow_schema::DataType; use async_recursion::async_recursion; use catalog::table_source::DfTableSourceProvider; -use chrono::Utc; use common_time::interval::{MS_PER_DAY, NANOS_PER_MILLI}; use common_time::timestamp::TimeUnit; use common_time::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth, Timestamp, Timezone}; @@ -28,7 +27,6 @@ use datafusion::prelude::Column; use datafusion::scalar::ScalarValue; use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion, TreeNodeRewriter}; use datafusion_common::{DFSchema, DataFusionError, Result as DFResult}; -use datafusion_expr::execution_props::ExecutionProps; use datafusion_expr::expr::WildcardOptions; use datafusion_expr::simplify::SimplifyContext; use datafusion_expr::{ @@ -146,8 +144,7 @@ fn evaluate_expr_to_millisecond(args: &[Expr], i: usize, interval_only: bool) -> if interval_only && !interval_only_in_expr(expr) { return Err(dispose_parse_error(Some(expr))); } - let execution_props = ExecutionProps::new().with_query_execution_start_time(Utc::now()); - let info = SimplifyContext::new(&execution_props).with_schema(Arc::new(DFSchema::empty())); + let info = SimplifyContext::default().with_current_time(); let simplify_expr = ExprSimplifier::new(info).simplify(expr.clone())?; match simplify_expr { Expr::Literal(ScalarValue::TimestampNanosecond(ts_nanos, _), _) diff --git a/src/sql/src/parsers/create_parser.rs b/src/sql/src/parsers/create_parser.rs index fe68a07669..f22add2ad6 100644 --- a/src/sql/src/parsers/create_parser.rs +++ b/src/sql/src/parsers/create_parser.rs @@ -25,7 +25,10 @@ use datatypes::arrow::datatypes::{DataType as ArrowDataType, IntervalUnit}; use datatypes::data_type::ConcreteDataType; use itertools::Itertools; use snafu::{OptionExt, ResultExt, ensure}; -use sqlparser::ast::{ColumnOption, ColumnOptionDef, DataType, Expr}; +use sqlparser::ast::{ + ColumnOption, ColumnOptionDef, DataType, Expr, KeyOrIndexDisplay, NullsDistinctOption, + PrimaryKeyConstraint, UniqueConstraint, +}; use sqlparser::dialect::keywords::Keyword; use sqlparser::keywords::ALL_KEYWORDS; use sqlparser::parser::IsOptional::Mandatory; @@ -727,15 +730,25 @@ impl<'a> ParserContext<'a> { parser.parse_expr().context(SyntaxSnafu)?, ))) } else if parser.parse_keywords(&[Keyword::PRIMARY, Keyword::KEY]) { - Ok(Some(ColumnOption::Unique { - is_primary: true, + Ok(Some(ColumnOption::PrimaryKey(PrimaryKeyConstraint { + name: None, + index_name: None, + index_type: None, + columns: vec![], + index_options: vec![], characteristics: None, - })) + }))) } else if parser.parse_keyword(Keyword::UNIQUE) { - Ok(Some(ColumnOption::Unique { - is_primary: false, + Ok(Some(ColumnOption::Unique(UniqueConstraint { + name: None, + index_name: None, + index_type_display: KeyOrIndexDisplay::None, + index_type: None, + columns: vec![], + index_options: vec![], characteristics: None, - })) + nulls_distinct: NullsDistinctOption::None, + }))) } else if parser.parse_keywords(&[Keyword::TIME, Keyword::INDEX]) { // Use a DialectSpecific option for time index Ok(Some(ColumnOption::DialectSpecific(vec![ diff --git a/src/sql/src/parsers/delete_parser.rs b/src/sql/src/parsers/delete_parser.rs index b26b709374..3f13d18f97 100644 --- a/src/sql/src/parsers/delete_parser.rs +++ b/src/sql/src/parsers/delete_parser.rs @@ -23,8 +23,11 @@ use crate::statements::statement::Statement; /// DELETE statement parser implementation impl ParserContext<'_> { pub(crate) fn parse_delete(&mut self) -> Result { - let _ = self.parser.next_token(); - let spstatement = self.parser.parse_delete().context(error::SyntaxSnafu)?; + let token = self.parser.next_token(); + let spstatement = self + .parser + .parse_delete(token) + .context(error::SyntaxSnafu)?; match spstatement { SpStatement::Delete { .. } => { diff --git a/src/sql/src/parsers/insert_parser.rs b/src/sql/src/parsers/insert_parser.rs index a369928456..a7d5f311f6 100644 --- a/src/sql/src/parsers/insert_parser.rs +++ b/src/sql/src/parsers/insert_parser.rs @@ -23,8 +23,11 @@ use crate::statements::statement::Statement; /// INSERT/REPLACE statement parser implementation impl ParserContext<'_> { pub(crate) fn parse_insert(&mut self) -> Result { - let _ = self.parser.next_token(); - let spstatement = self.parser.parse_insert().context(error::SyntaxSnafu)?; + let token = self.parser.next_token(); + let spstatement = self + .parser + .parse_insert(token) + .context(error::SyntaxSnafu)?; match spstatement { SpStatement::Insert { .. } => { @@ -38,8 +41,11 @@ impl ParserContext<'_> { } pub(crate) fn parse_replace(&mut self) -> Result { - let _ = self.parser.next_token(); - let spstatement = self.parser.parse_insert().context(error::SyntaxSnafu)?; + let token = self.parser.next_token(); + let spstatement = self + .parser + .parse_insert(token) + .context(error::SyntaxSnafu)?; match spstatement { SpStatement::Insert(mut insert_stmt) => { diff --git a/src/sql/src/parsers/utils.rs b/src/sql/src/parsers/utils.rs index 679466da2f..356c615a6c 100644 --- a/src/sql/src/parsers/utils.rs +++ b/src/sql/src/parsers/utils.rs @@ -14,7 +14,6 @@ use std::sync::Arc; -use chrono::Utc; use datafusion::config::ConfigOptions; use datafusion::error::Result as DfResult; use datafusion::execution::SessionStateBuilder; @@ -22,7 +21,6 @@ use datafusion::execution::context::SessionState; use datafusion::optimizer::simplify_expressions::ExprSimplifier; use datafusion_common::tree_node::{TreeNode, TreeNodeVisitor}; use datafusion_common::{DFSchema, ScalarValue}; -use datafusion_expr::execution_props::ExecutionProps; use datafusion_expr::simplify::SimplifyContext; use datafusion_expr::{AggregateUDF, Expr, ScalarUDF, TableSource, WindowUDF}; use datafusion_sql::TableReference; @@ -124,10 +122,7 @@ pub fn parser_expr_to_scalar_value_literal( } // 2. simplify logical expr - let execution_props = ExecutionProps::new().with_query_execution_start_time(Utc::now()); - let info = - SimplifyContext::new(&execution_props).with_schema(Arc::new(empty_df_schema.clone())); - + let info = SimplifyContext::default().with_current_time(); let simplifier = ExprSimplifier::new(info); // Coerce the logical expression so simplifier can handle it correctly. This is necessary for const eval with possible type mismatch. i.e.: `now() - now() + '15s'::interval` which is `TimestampNanosecond - TimestampNanosecond + IntervalMonthDayNano`. @@ -274,8 +269,6 @@ pub fn convert_month_day_nano_to_duration( #[cfg(test)] mod tests { - use std::sync::Arc; - use chrono::DateTime; use datafusion::functions::datetime::expr_fn::now; use datafusion_expr::lit; @@ -324,9 +317,7 @@ mod tests { ), ]; - let execution_props = ExecutionProps::new().with_query_execution_start_time(now_time); - let info = SimplifyContext::new(&execution_props).with_schema(Arc::new(DFSchema::empty())); - + let info = SimplifyContext::default().with_query_execution_start_time(Some(now_time)); let simplifier = ExprSimplifier::new(info); for (expr, expected) in testcases { let expr_name = expr.schema_name().to_string(); diff --git a/src/sql/src/parsers/with_tql_parser.rs b/src/sql/src/parsers/with_tql_parser.rs index e52e3103a4..d15b34658b 100644 --- a/src/sql/src/parsers/with_tql_parser.rs +++ b/src/sql/src/parsers/with_tql_parser.rs @@ -114,6 +114,7 @@ impl ParserContext<'_> { match cte.content { CteContent::Sql(body) => sql_cte_tables.push(Cte { alias: TableAlias { + explicit: false, name: cte.name, columns: cte .columns diff --git a/src/sql/src/statements.rs b/src/sql/src/statements.rs index 09b4f1d337..211fc5598e 100644 --- a/src/sql/src/statements.rs +++ b/src/sql/src/statements.rs @@ -91,10 +91,7 @@ pub fn has_primary_key_option(column_def: &ColumnDef) -> bool { column_def .options .iter() - .any(|options| match options.option { - ColumnOption::Unique { is_primary, .. } => is_primary, - _ => false, - }) + .any(|options| matches!(options.option, ColumnOption::PrimaryKey(..))) } /// Create a `ColumnSchema` from `Column`. @@ -198,15 +195,10 @@ pub fn sql_column_def_to_grpc_column_def( .context(ConvertToGrpcDataTypeSnafu)? .to_parts(); - let is_primary_key = col.options.iter().any(|o| { - matches!( - o.option, - ColumnOption::Unique { - is_primary: true, - .. - } - ) - }); + let is_primary_key = col + .options + .iter() + .any(|o| matches!(o.option, ColumnOption::PrimaryKey(..))); let semantic_type = if is_primary_key { SemanticType::Tag @@ -375,7 +367,7 @@ mod tests { use datatypes::schema::{ COLUMN_FULLTEXT_OPT_KEY_ANALYZER, COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE, FulltextAnalyzer, }; - use sqlparser::ast::{ColumnOptionDef, Expr}; + use sqlparser::ast::{ColumnOptionDef, Expr, PrimaryKeyConstraint}; use super::*; use crate::ast::TimezoneInfo; @@ -505,10 +497,14 @@ mod tests { data_type: SqlDataType::Double(ExactNumberInfo::None), options: vec![ColumnOptionDef { name: None, - option: ColumnOption::Unique { - is_primary: true, + option: ColumnOption::PrimaryKey(PrimaryKeyConstraint { + name: None, + index_name: None, + index_type: None, + columns: vec![], + index_options: vec![], characteristics: None, - }, + }), }], }; @@ -583,10 +579,14 @@ mod tests { data_type: SqlDataType::Double(ExactNumberInfo::None), options: vec![ColumnOptionDef { name: None, - option: ColumnOption::Unique { - is_primary: true, + option: ColumnOption::PrimaryKey(PrimaryKeyConstraint { + name: None, + index_name: None, + index_type: None, + columns: vec![], + index_options: vec![], characteristics: None, - }, + }), }], }; assert!(has_primary_key_option(&column_def));