From 53e45b6b90ce2441fdf089e06ac03d44fd39bf13 Mon Sep 17 00:00:00 2001 From: Hengfei Yang Date: Fri, 14 Jun 2024 11:43:43 +0800 Subject: [PATCH] chore(deps): update to datafusion 39 (#3734) --- Cargo.lock | 539 ++++++++++++++---- Cargo.toml | 16 +- src/config/src/meta/sql.rs | 46 +- src/infra/src/storage/local.rs | 26 +- src/infra/src/storage/mod.rs | 2 +- src/infra/src/storage/remote.rs | 27 +- src/service/search/datafusion/rewrite.rs | 47 +- .../search/datafusion/storage/memory.rs | 26 +- .../search/datafusion/storage/tmpfs.rs | 26 +- src/service/search/datafusion/storage/wal.rs | 24 +- 10 files changed, 562 insertions(+), 217 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9572086180..054ad8f4cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,7 +54,7 @@ dependencies = [ "encoding_rs", "flate2", "futures-core", - "h2", + "h2 0.3.26", "http 0.2.12", "httparse", "httpdate", @@ -139,9 +139,9 @@ dependencies = [ [[package]] name = "actix-rt" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" +checksum = "24eda4e2a6e042aa4e55ac438a2ae052d3b5da0ecf83d7411e1a368946925208" dependencies = [ "futures-core", "tokio", @@ -149,9 +149,9 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eb13e7eef0423ea6eab0e59f6c72e7cb46d33691ad56a726b3cd07ddec2c2d4" +checksum = "b02303ce8d4e8be5b855af6cf3c3a08f3eff26880faad82bab679c22d3650cb5" dependencies = [ "actix-rt", "actix-service", @@ -259,13 +259,13 @@ dependencies = [ [[package]] name = "actix-web-httpauth" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d613edf08a42ccc6864c941d30fe14e1b676a77d16f1dbadc1174d065a0a775" +checksum = "456348ed9dcd72a13a1f4a660449fafdecee9ac8205552e286809eb5b0b29bd3" dependencies = [ "actix-utils", "actix-web", - "base64 0.21.7", + "base64 0.22.1", "futures-core", "futures-util", "log", @@ -579,9 +579,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" +checksum = "7ae9728f104939be6d8d9b368a354b4929b0569160ea1641f0721b55a861ce38" dependencies = [ "arrow-arith", "arrow-array", @@ -600,9 +600,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" +checksum = "a7029a5b3efbeafbf4a12d12dc16b8f9e9bff20a410b8c25c5d28acc089e1043" dependencies = [ "arrow-array", "arrow-buffer", @@ -615,16 +615,16 @@ dependencies = [ [[package]] name = "arrow-array" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" +checksum = "d33238427c60271710695f17742f45b1a5dc5bcfc5c15331c25ddfe7abf70d97" dependencies = [ "ahash 0.8.11", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "chrono-tz", + "chrono-tz 0.9.0", "half", "hashbrown 0.14.5", "num", @@ -632,9 +632,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27" +checksum = "fe9b95e825ae838efaf77e366c00d3fc8cca78134c9db497d6bda425f2e7b7c1" dependencies = [ "bytes", "half", @@ -643,9 +643,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" +checksum = "87cf8385a9d5b5fcde771661dd07652b79b9139fea66193eda6a88664400ccab" dependencies = [ "arrow-array", "arrow-buffer", @@ -664,9 +664,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2" +checksum = "cea5068bef430a86690059665e40034625ec323ffa4dd21972048eebb0127adc" dependencies = [ "arrow-array", "arrow-buffer", @@ -683,9 +683,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" +checksum = "cb29be98f987bcf217b070512bb7afba2f65180858bca462edf4a39d84a23e10" dependencies = [ "arrow-buffer", "arrow-schema", @@ -695,9 +695,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d" +checksum = "ffc68f6523970aa6f7ce1dc9a33a7d9284cfb9af77d4ad3e617dbe5d79cc6ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -711,9 +711,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49" +checksum = "2041380f94bd6437ab648e6c2085a045e45a0c44f91a1b9a4fe3fed3d379bfb1" dependencies = [ "arrow-array", "arrow-buffer", @@ -731,9 +731,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" +checksum = "fcb56ed1547004e12203652f12fe12e824161ff9d1e5cf2a7dc4ff02ba94f413" dependencies = [ "arrow-array", "arrow-buffer", @@ -746,9 +746,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" +checksum = "575b42f1fc588f2da6977b94a5ca565459f5ab07b60545e17243fb9a7ed6d43e" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -761,18 +761,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" +checksum = "32aae6a60458a2389c0da89c9de0b7932427776127da1a738e2efc21d32f3393" dependencies = [ "serde", ] [[package]] name = "arrow-select" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" +checksum = "de36abaef8767b4220d7b4a8c2fe5ffc78b47db81b03d77e2136091c3ba39102" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -784,9 +784,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" +checksum = "e435ada8409bcafc910bc3e0077f532a4daa20e99060a496685c0e3e53cc2597" dependencies = [ "arrow-array", "arrow-buffer", @@ -1062,7 +1062,7 @@ dependencies = [ "derive_more", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "itoa", "log", @@ -1441,9 +1441,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.72" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", @@ -2003,7 +2003,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" dependencies = [ "chrono", - "chrono-tz-build", + "chrono-tz-build 0.2.1", + "phf", +] + +[[package]] +name = "chrono-tz" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +dependencies = [ + "chrono", + "chrono-tz-build 0.3.0", "phf", ] @@ -2018,6 +2029,17 @@ dependencies = [ "phf_codegen", ] +[[package]] +name = "chrono-tz-build" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + [[package]] name = "chumsky" version = "0.9.3" @@ -2104,9 +2126,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.6" +version = "4.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9689a29b593160de5bc4aacab7b5d54fb52231de70122626c178e6a368994c7" +checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" dependencies = [ "clap_builder", "clap_derive 4.5.5", @@ -2114,9 +2136,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.6" +version = "4.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5387378c84f6faa26890ebf9f0a92989f8873d4d380467bcd0d8d8620424df" +checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" dependencies = [ "anstream", "anstyle", @@ -2477,7 +2499,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.5.6", + "clap 4.5.7", "criterion-plot", "is-terminal", "itertools 0.10.5", @@ -2703,9 +2725,9 @@ checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "datafusion" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05fb4eeeb7109393a0739ac5b8fd892f95ccef691421491c85544f7997366f68" +checksum = "2f92d2d7a9cba4580900b32b009848d9eb35f1028ac84cdd6ddcf97612cd0068" dependencies = [ "ahash 0.8.11", "arrow", @@ -2727,6 +2749,7 @@ dependencies = [ "datafusion-functions-array", "datafusion-optimizer", "datafusion-physical-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-sql", "flate2", @@ -2741,6 +2764,7 @@ dependencies = [ "object_store", "parking_lot", "parquet", + "paste", "pin-project-lite", "rand", "sqlparser", @@ -2755,9 +2779,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "741aeac15c82f239f2fc17deccaab19873abbd62987be20023689b15fa72fa09" +checksum = "effed030d2c1667eb1e11df5372d4981eaf5d11a521be32220b3985ae5ba6971" dependencies = [ "ahash 0.8.11", "arrow", @@ -2766,6 +2790,7 @@ dependencies = [ "arrow-schema", "chrono", "half", + "hashbrown 0.14.5", "instant", "libc", "num_cpus", @@ -2776,18 +2801,18 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8ddfb8d8cb51646a30da0122ecfffb81ca16919ae9a3495a9e7468bdcd52b8" +checksum = "d0091318129dad1359f08e4c6c71f855163c35bba05d1dbf983196f727857894" dependencies = [ "tokio", ] [[package]] name = "datafusion-execution" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282122f90b20e8f98ebfa101e4bf20e718fd2684cf81bef4e8c6366571c64404" +checksum = "8385aba84fc4a06d3ebccfbcbf9b4f985e80c762fac634b49079f7cc14933fb1" dependencies = [ "arrow", "chrono", @@ -2806,13 +2831,14 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5478588f733df0dfd87a62671c7478f590952c95fa2fa5c137e3ff2929491e22" +checksum = "ebb192f0055d2ce64e38ac100abc18e4e6ae9734d3c28eee522bbbd6a32108a3" dependencies = [ "ahash 0.8.11", "arrow", "arrow-array", + "arrow-buffer", "chrono", "datafusion-common", "paste", @@ -2824,9 +2850,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4afd261cea6ac9c3ca1192fd5e9f940596d8e9208c5b1333f4961405db53185" +checksum = "27c081ae5b7edd712b92767fb8ed5c0e32755682f8075707666cd70835807c0b" dependencies = [ "arrow", "base64 0.22.1", @@ -2851,11 +2877,13 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b36a6c4838ab94b5bf8f7a96ce6ce059d805c5d1dcaa6ace49e034eb65cd999" +checksum = "feb28a4ea52c28a26990646986a27c4052829a2a2572386258679e19263f8b78" dependencies = [ + "ahash 0.8.11", "arrow", + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -2867,9 +2895,9 @@ dependencies = [ [[package]] name = "datafusion-functions-array" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fdd200a6233f48d3362e7ccb784f926f759100e44ae2137a5e2dcb986a59c4" +checksum = "89b17c02a74cdc87380a56758ec27e7d417356bf806f33062700908929aedb8a" dependencies = [ "arrow", "arrow-array", @@ -2887,9 +2915,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54f2820938810e8a2d71228fd6f59f33396aebc5f5f687fcbf14de5aab6a7e1a" +checksum = "12172f2a6c9eb4992a51e62d709eeba5dedaa3b5369cce37ff6c2260e100ba76" dependencies = [ "arrow", "async-trait", @@ -2906,9 +2934,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9adf8eb12716f52ddf01e09eb6c94d3c9b291e062c05c91b839a448bddba2ff8" +checksum = "7a3fce531b623e94180f6cd33d620ef01530405751b6ddd2fd96250cdbd78e2e" dependencies = [ "ahash 0.8.11", "arrow", @@ -2937,20 +2965,21 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5472c3230584c150197b3f2c23f2392b9dc54dbfb62ad41e7e36447cfce4be" +checksum = "046400b6a2cc3ed57a7c576f5ae6aecc77804ac8e0186926b278b189305b2a77" dependencies = [ "arrow", "datafusion-common", "datafusion-expr", + "rand", ] [[package]] name = "datafusion-physical-plan" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18ae750c38389685a8b62e5b899bbbec488950755ad6d218f3662d35b800c4fe" +checksum = "4aed47f5a2ad8766260befb375b201592e86a08b260256e168ae4311426a2bff" dependencies = [ "ahash 0.8.11", "arrow", @@ -2982,9 +3011,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "befc67a3cdfbfa76853f43b10ac27337821bb98e519ab6baf431fcc0bcfcafdb" +checksum = "7fa92bb1fd15e46ce5fb6f1c85f3ac054592560f294429a28e392b5f9cd4255e" dependencies = [ "arrow", "arrow-array", @@ -2992,6 +3021,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "log", + "regex", "sqlparser", "strum 0.26.2", ] @@ -3119,6 +3149,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "dissimilar" version = "1.0.9" @@ -3447,9 +3488,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" -version = "23.5.26" +version = "24.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -3777,6 +3818,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", + "indexmap 2.1.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.4.1" @@ -3978,12 +4038,12 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", - "futures-core", + "futures-util", "http 1.1.0", "http-body 1.0.0", "pin-project-lite", @@ -3991,9 +4051,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "d0e7a4dd27b9476dc40cb050d3632d3bba3a70ddbff012285f7f8559a1e7e545" [[package]] name = "httpdate" @@ -4026,7 +4086,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -4049,6 +4109,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", + "h2 0.4.5", "http 1.1.0", "http-body 1.0.0", "httparse", @@ -4147,6 +4208,124 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -4163,6 +4342,18 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "idna" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed" +dependencies = [ + "icu_normalizer", + "icu_properties", + "smallvec", + "utf8_iter", +] + [[package]] name = "impl-more" version = "0.1.6" @@ -4486,7 +4677,7 @@ dependencies = [ "futures-util", "hostname 0.4.0", "httpdate", - "idna", + "idna 0.5.0", "mime", "nom", "percent-encoding", @@ -4644,6 +4835,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "local-channel" version = "0.1.5" @@ -5141,33 +5338,33 @@ dependencies = [ [[package]] name = "object" -version = "0.35.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" +checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" +checksum = "fbebfd32c213ba1907fa7a9c9138015a8de2b43e30c5aa45b18f7deb46786ad6" dependencies = [ "async-trait", - "base64 0.21.7", + "base64 0.22.1", "bytes", "chrono", "futures", "humantime", - "hyper 0.14.29", + "hyper 1.3.1", "itertools 0.12.1", "md-5", "parking_lot", "percent-encoding", "quick-xml", "rand", - "reqwest 0.11.27", + "reqwest 0.12.4", "ring 0.17.8", "rustls-pemfile 2.1.2", "serde", @@ -5254,7 +5451,7 @@ dependencies = [ "bytes", "chromiumoxide", "chrono", - "clap 4.5.6", + "clap 4.5.7", "cloudevents-sdk", "config", "console-subscriber", @@ -5526,9 +5723,9 @@ dependencies = [ [[package]] name = "parquet" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "096795d4f47f65fd3ee1ec5a98b77ab26d602f2cc785b0e4be5443add17ecc32" +checksum = "29c3b5322cc1bbf67f11c079c42be41a55949099b78732f7dba9e15edde40eab" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -5539,7 +5736,7 @@ dependencies = [ "arrow-schema", "arrow-select", "base64 0.22.1", - "brotli 3.5.0", + "brotli 6.0.0", "bytes", "chrono", "flate2", @@ -5557,6 +5754,7 @@ dependencies = [ "tokio", "twox-hash", "zstd", + "zstd-sys", ] [[package]] @@ -6366,7 +6564,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.29", @@ -6379,7 +6577,6 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls 0.21.12", - "rustls-native-certs 0.6.3", "rustls-pemfile 1.0.4", "serde", "serde_json", @@ -6388,12 +6585,10 @@ dependencies = [ "system-configuration", "tokio", "tokio-rustls 0.24.1", - "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", "web-sys", "webpki-roots 0.25.4", "winreg 0.50.0", @@ -6409,6 +6604,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", + "h2 0.4.5", "http 1.1.0", "http-body 1.0.0", "http-body-util", @@ -7240,20 +7436,19 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" +checksum = "f895e3734318cc55f1fe66258926c9b910c124d47520339efecbb6c59cec7c1f" dependencies = [ - "itertools 0.12.1", "nom", "unicode_categories", ] [[package]] name = "sqlparser" -version = "0.45.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7bbffee862a796d67959a89859d6b1046bb5016d63e23835ad0da182777bbe0" +checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" dependencies = [ "log", "serde", @@ -7674,6 +7869,17 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "sysinfo" version = "0.29.11" @@ -7875,6 +8081,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -8036,7 +8252,7 @@ dependencies = [ "axum", "base64 0.21.7", "bytes", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.29", @@ -8067,7 +8283,7 @@ dependencies = [ "base64 0.21.7", "bytes", "flate2", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.29", @@ -8309,9 +8525,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "uaparser" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4d8fcdf9685cad74cecf1553af084ab4c494e833c47d3c50ca32cba8035545" +checksum = "a4c9e1c3f893758f154004195fc2d2c52fbda462df725220ceaef830ac29affa" dependencies = [ "derive_more", "lazy_static", @@ -8417,12 +8633,12 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56" dependencies = [ "form_urlencoded", - "idna", + "idna 1.0.0", "percent-encoding", "serde", ] @@ -8439,12 +8655,24 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + [[package]] name = "utf8-width" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -8569,9 +8797,9 @@ dependencies = [ "chacha20poly1305", "charset", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "cidr-utils", - "clap 4.5.6", + "clap 4.5.7", "codespan-reporting", "community-id", "crypto_secretbox", @@ -9176,6 +9404,18 @@ dependencies = [ "regex", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "wyz" version = "0.5.1" @@ -9200,6 +9440,30 @@ dependencies = [ "lzma-sys", ] +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.34" @@ -9220,12 +9484,55 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerovec" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "zip" version = "0.6.6" @@ -9240,27 +9547,27 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.1.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index cc71abf34a..199108953a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -215,13 +215,13 @@ chromiumoxide = { git = "https://github.com/mattsse/chromiumoxide", features = [ chrono = { version = "0.4", default-features = false, features = ["clock"] } cityhasher = { version = "0.1", default-features = false } dashmap = { version = "5.5", features = ["serde"] } -datafusion = "38" -datafusion-expr = "38" -arrow = { version = "51.0.0", features = ["ipc_compression", "prettyprint"] } -arrow-json = "51.0.0" -arrow-schema = { version = "51.0.0", features = ["serde"] } -parquet = { version = "51.0.0", features = ["arrow", "async", "object_store"] } -object_store = { version = "0.9.1", features = ["aws", "azure", "gcp"] } +datafusion = "39" +datafusion-expr = "39" +arrow = { version = "52.0.0", features = ["ipc_compression", "prettyprint"] } +arrow-json = "52.0.0" +arrow-schema = { version = "52.0.0", features = ["serde"] } +parquet = { version = "52.0.0", features = ["arrow", "async", "object_store"] } +object_store = { version = "0.10.1", features = ["aws", "azure", "gcp"] } dotenv_config = "0.1" dotenvy = "0.15" env_logger = "0.10" @@ -274,7 +274,7 @@ sqlx = { version = "0.7", features = [ "sqlite", "chrono", ] } -sqlparser = { version = "0.45", features = ["serde", "visitor"] } +sqlparser = { version = "0.47", features = ["serde", "visitor"] } svix-ksuid = { version = "0.8", features = ["serde"] } sysinfo = "0.29" tempfile = "3" diff --git a/src/config/src/meta/sql.rs b/src/config/src/meta/sql.rs index 170707c6bd..82f4cc356e 100644 --- a/src/config/src/meta/sql.rs +++ b/src/config/src/meta/sql.rs @@ -18,9 +18,9 @@ use regex::Regex; use serde::Serialize; use sqlparser::{ ast::{ - BinaryOperator, Expr as SqlExpr, Function, FunctionArg, FunctionArgExpr, GroupByExpr, - Offset as SqlOffset, OrderByExpr, Select, SelectItem, SetExpr, Statement, TableFactor, - TableWithJoins, Value, + BinaryOperator, Expr as SqlExpr, Function, FunctionArg, FunctionArgExpr, FunctionArguments, + GroupByExpr, Offset as SqlOffset, OrderByExpr, Select, SelectItem, SetExpr, Statement, + TableFactor, TableWithJoins, Value, }, parser::Parser, }; @@ -557,7 +557,7 @@ fn parse_expr_like( _negated: &bool, expr: &SqlExpr, pattern: &SqlExpr, - _escape_char: &Option, + _escape_char: &Option, next_op: &SqlOperator, field: &str, fields: &mut Vec<(String, SqlValue, SqlOperator, SqlOperator)>, @@ -660,16 +660,21 @@ fn parse_expr_function( return parse_expr_fun_time_range(f, field, fields); } - if f.args.len() < 2 { + let args = match &f.args { + FunctionArguments::None => return Ok(()), + FunctionArguments::Subquery(_) => return Ok(()), + FunctionArguments::List(args) => &args.args, + }; + if args.len() < 2 { return Ok(()); } let nop = SqlOperator::And; let next_op = SqlOperator::And; - let field_name = f.args.first().unwrap().to_string(); + let field_name = args.first().unwrap().to_string(); let field_name = field_name.trim_matches(|c: char| c == '\'' || c == '"'); if parse_expr_check_field_name(field_name, field) { - match f.args.get(1).unwrap() { + match args.get(1).unwrap() { FunctionArg::Named { name: _name, arg, @@ -709,18 +714,23 @@ fn parse_expr_fun_time_range( field: &str, fields: &mut Vec<(String, SqlValue, SqlOperator, SqlOperator)>, ) -> Result<(), anyhow::Error> { - if f.args.len() != 3 { + let args = match &f.args { + FunctionArguments::None => return Ok(()), + FunctionArguments::Subquery(_) => return Ok(()), + FunctionArguments::List(args) => &args.args, + }; + if args.len() != 3 { return Err(anyhow::anyhow!( "SqlExpr::Function: time_range function must have 3 arguments" )); } let next_op = SqlOperator::And; - let field_name = f.args.first().unwrap().to_string(); + let field_name = args.first().unwrap().to_string(); let field_name = field_name.trim_matches(|c: char| c == '\'' || c == '"'); if parse_expr_check_field_name(field_name, field) { let mut vals = Vec::new(); - for arg in f.args.iter() { + for arg in args.iter() { let val = match arg { FunctionArg::Named { name: _name, @@ -799,8 +809,13 @@ fn get_field_name_from_expr(expr: &SqlExpr) -> Option> { (!fields.is_empty()).then_some(fields) } SqlExpr::Function(f) => { - let mut fields = Vec::with_capacity(f.args.len()); - for arg in f.args.iter() { + let args = match &f.args { + FunctionArguments::None => return None, + FunctionArguments::Subquery(_) => return None, + FunctionArguments::List(args) => &args.args, + }; + let mut fields = Vec::with_capacity(args.len()); + for arg in args.iter() { match arg { FunctionArg::Named { name: _name, @@ -842,9 +857,7 @@ fn get_field_name_from_expr(expr: &SqlExpr) -> Option> { } (!fields.is_empty()).then_some(fields) } - SqlExpr::Cast { expr, .. } | SqlExpr::TryCast { expr, .. } => { - get_field_name_from_expr(expr) - } + SqlExpr::Cast { expr, .. } => get_field_name_from_expr(expr), SqlExpr::Case { operand: _, conditions, @@ -862,7 +875,8 @@ fn get_field_name_from_expr(expr: &SqlExpr) -> Option> { SqlExpr::AtTimeZone { timestamp, .. } => get_field_name_from_expr(timestamp), SqlExpr::Extract { expr, .. } => get_field_name_from_expr(expr), SqlExpr::MapAccess { column, .. } => get_field_name_from_expr(column), - SqlExpr::ArrayIndex { obj, .. } => get_field_name_from_expr(obj), + SqlExpr::CompositeAccess { expr, .. } => get_field_name_from_expr(expr), + SqlExpr::Subscript { expr, .. } => get_field_name_from_expr(expr), _ => None, } } diff --git a/src/infra/src/storage/local.rs b/src/infra/src/storage/local.rs index 8d4805e733..f175080930 100644 --- a/src/infra/src/storage/local.rs +++ b/src/infra/src/storage/local.rs @@ -21,9 +21,9 @@ use config::metrics; use futures::stream::BoxStream; use object_store::{ limit::LimitStore, local::LocalFileSystem, path::Path, Error, GetOptions, GetResult, - ListResult, MultipartId, ObjectMeta, ObjectStore, PutOptions, PutResult, Result, + ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOpts, PutOptions, PutPayload, + PutResult, Result, }; -use tokio::io::AsyncWrite; use crate::storage::{format_key, CONCURRENT_REQUESTS}; @@ -61,13 +61,18 @@ impl std::fmt::Display for Local { #[async_trait] impl ObjectStore for Local { - async fn put_opts(&self, location: &Path, bytes: Bytes, opts: PutOptions) -> Result { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> Result { let start = std::time::Instant::now(); let file = location.to_string(); - let data_size = bytes.len(); + let data_size = payload.content_length(); match self .client - .put_opts(&(format_key(&file, self.with_prefix).into()), bytes, opts) + .put_opts(&(format_key(&file, self.with_prefix).into()), payload, opts) .await { Ok(_output) => { @@ -97,14 +102,15 @@ impl ObjectStore for Local { } } - async fn put_multipart( - &self, - _location: &Path, - ) -> Result<(MultipartId, Box)> { + async fn put_multipart(&self, _location: &Path) -> Result> { Err(Error::NotImplemented) } - async fn abort_multipart(&self, _location: &Path, _multipart_id: &MultipartId) -> Result<()> { + async fn put_multipart_opts( + &self, + _location: &Path, + _opts: PutMultipartOpts, + ) -> Result> { Err(Error::NotImplemented) } diff --git a/src/infra/src/storage/mod.rs b/src/infra/src/storage/mod.rs index f9d236a215..db926b59d7 100644 --- a/src/infra/src/storage/mod.rs +++ b/src/infra/src/storage/mod.rs @@ -77,7 +77,7 @@ pub async fn get(file: &str) -> Result { } pub async fn put(file: &str, data: bytes::Bytes) -> Result<(), anyhow::Error> { - DEFAULT.put(&file.into(), data).await?; + DEFAULT.put(&file.into(), data.into()).await?; Ok(()) } diff --git a/src/infra/src/storage/remote.rs b/src/infra/src/storage/remote.rs index 11f1452663..43beb119a0 100644 --- a/src/infra/src/storage/remote.rs +++ b/src/infra/src/storage/remote.rs @@ -20,10 +20,9 @@ use bytes::Bytes; use config::{get_config, metrics}; use futures::stream::BoxStream; use object_store::{ - limit::LimitStore, path::Path, Error, GetOptions, GetResult, ListResult, MultipartId, - ObjectMeta, ObjectStore, PutOptions, PutResult, Result, + limit::LimitStore, path::Path, Error, GetOptions, GetResult, ListResult, MultipartUpload, + ObjectMeta, ObjectStore, PutMultipartOpts, PutOptions, PutPayload, PutResult, Result, }; -use tokio::io::AsyncWrite; use crate::storage::{format_key, CONCURRENT_REQUESTS}; @@ -53,13 +52,18 @@ impl std::fmt::Display for Remote { #[async_trait] impl ObjectStore for Remote { - async fn put_opts(&self, location: &Path, bytes: Bytes, opts: PutOptions) -> Result { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> Result { let start = std::time::Instant::now(); let file = location.to_string(); - let data_size = bytes.len(); + let data_size = payload.content_length(); match self .client - .put_opts(&(format_key(&file, true).into()), bytes, opts) + .put_opts(&(format_key(&file, true).into()), payload, opts) .await { Ok(_) => { @@ -89,14 +93,15 @@ impl ObjectStore for Remote { } } - async fn put_multipart( - &self, - _location: &Path, - ) -> Result<(MultipartId, Box)> { + async fn put_multipart(&self, _location: &Path) -> Result> { Err(Error::NotImplemented) } - async fn abort_multipart(&self, _location: &Path, _multipart_id: &MultipartId) -> Result<()> { + async fn put_multipart_opts( + &self, + _location: &Path, + _opts: PutMultipartOpts, + ) -> Result> { Err(Error::NotImplemented) } diff --git a/src/service/search/datafusion/rewrite.rs b/src/service/search/datafusion/rewrite.rs index b5d5415f98..7ef9a55edf 100644 --- a/src/service/search/datafusion/rewrite.rs +++ b/src/service/search/datafusion/rewrite.rs @@ -21,7 +21,7 @@ use config::FxIndexSet; use datafusion::error::Result; use itertools::Itertools; use sqlparser::{ - ast::{Expr, Function, GroupByExpr, Ident, Query, VisitMut, VisitorMut}, + ast::{Expr, Function, FunctionArguments, GroupByExpr, Ident, Query, VisitMut, VisitorMut}, dialect::GenericDialect, parser::Parser, }; @@ -86,23 +86,29 @@ impl VisitorMut for Rewrite { } let mut field_names = FxIndexSet::default(); let mut delete_idx = Vec::new(); + let empty_vec = vec![]; for (idx, select_item) in &mut select.projection.iter_mut().enumerate() { match select_item { sqlparser::ast::SelectItem::ExprWithAlias { expr, alias } => { match expr { - Expr::Function(Function { - name, - args, - distinct, - .. - }) => { + Expr::Function(Function { name, args, .. }) => { // extract field name from the aggregate function if AGGREGATE_UDF_LIST .contains(&name.to_string().to_lowercase().as_str()) { - // check if aggregate function is count and distinct is - // false - if name.to_string().to_lowercase() == "count" && !(*distinct) { + let distinct = match &args { + FunctionArguments::List(args) => matches!( + args.duplicate_treatment, + Some(sqlparser::ast::DuplicateTreatment::Distinct,) + ), + _ => false, + }; + let args = match &args { + FunctionArguments::List(args) => &args.args, + _ => &empty_vec, + }; + // check if aggregate function is count and distinct is false + if name.to_string().to_lowercase() == "count" && !(distinct) { select.distinct = None; } // check if the field name is already in the projection list @@ -133,13 +139,7 @@ impl VisitorMut for Rewrite { } // extrace function paraments from the projection list sqlparser::ast::SelectItem::UnnamedExpr(expr) => { - if let Expr::Function(Function { - name, - args, - distinct, - .. - }) = expr - { + if let Expr::Function(Function { name, args, .. }) = expr { if !AGGREGATE_UDF_LIST .contains(&name.to_string().to_lowercase().as_str()) { @@ -154,7 +154,18 @@ impl VisitorMut for Rewrite { } } } else { - if name.to_string().to_lowercase() == "count" && !(*distinct) { + let distinct = match &args { + FunctionArguments::List(args) => matches!( + args.duplicate_treatment, + Some(sqlparser::ast::DuplicateTreatment::Distinct) + ), + _ => false, + }; + let args = match &args { + FunctionArguments::List(args) => &args.args, + _ => &empty_vec, + }; + if name.to_string().to_lowercase() == "count" && !(distinct) { select.distinct = None; } let field_name = remove_brackets(format!("{}", args[0]).as_str()); diff --git a/src/service/search/datafusion/storage/memory.rs b/src/service/search/datafusion/storage/memory.rs index de90779aa5..734080c8c0 100644 --- a/src/service/search/datafusion/storage/memory.rs +++ b/src/service/search/datafusion/storage/memory.rs @@ -21,10 +21,9 @@ use config::utils::time::BASE_TIME; use futures::{stream::BoxStream, StreamExt}; use infra::{cache::file_data, storage}; use object_store::{ - path::Path, GetOptions, GetResult, GetResultPayload, ListResult, MultipartId, ObjectMeta, - ObjectStore, PutOptions, PutResult, Result, + path::Path, Attributes, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, + ObjectMeta, ObjectStore, PutMultipartOpts, PutOptions, PutPayload, PutResult, Result, }; -use tokio::io::AsyncWrite; use super::GetRangeExt; @@ -81,6 +80,7 @@ impl ObjectStore for FS { payload: GetResultPayload::Stream( futures::stream::once(async move { Ok(data) }).boxed(), ), + attributes: Attributes::default(), meta, range, }) @@ -116,6 +116,7 @@ impl ObjectStore for FS { payload: GetResultPayload::Stream( futures::stream::once(async move { Ok(data) }).boxed(), ), + attributes: Attributes::default(), meta, range, }) @@ -224,24 +225,23 @@ impl ObjectStore for FS { async fn put_opts( &self, location: &Path, - _bytes: Bytes, + _payload: PutPayload, _opts: PutOptions, ) -> Result { log::error!("NotImplemented put_opts: {}", location); Err(object_store::Error::NotImplemented {}) } - async fn put_multipart( - &self, - location: &Path, - ) -> Result<(MultipartId, Box)> { - log::error!("NotImplemented put_multipart: {}", location); - Err(object_store::Error::NotImplemented {}) + async fn put_multipart(&self, _location: &Path) -> Result> { + Err(object_store::Error::NotImplemented) } - async fn abort_multipart(&self, location: &Path, _multipart_id: &MultipartId) -> Result<()> { - log::error!("NotImplemented abort_multipart: {}", location); - Err(object_store::Error::NotImplemented {}) + async fn put_multipart_opts( + &self, + _location: &Path, + _opts: PutMultipartOpts, + ) -> Result> { + Err(object_store::Error::NotImplemented) } async fn delete(&self, location: &Path) -> Result<()> { diff --git a/src/service/search/datafusion/storage/tmpfs.rs b/src/service/search/datafusion/storage/tmpfs.rs index 6cbde1ee40..7a1527730f 100644 --- a/src/service/search/datafusion/storage/tmpfs.rs +++ b/src/service/search/datafusion/storage/tmpfs.rs @@ -22,11 +22,10 @@ use config::utils::time::BASE_TIME; use futures::{stream::BoxStream, StreamExt}; use infra::cache::tmpfs; use object_store::{ - path::Path, GetOptions, GetResult, GetResultPayload, ListResult, MultipartId, ObjectMeta, - ObjectStore, PutOptions, PutResult, Result, + path::Path, Attributes, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, + ObjectMeta, ObjectStore, PutMultipartOpts, PutOptions, PutPayload, PutResult, Result, }; use thiserror::Error as ThisError; -use tokio::io::AsyncWrite; use super::GetRangeExt; @@ -80,6 +79,7 @@ impl ObjectStore for Tmpfs { payload: GetResultPayload::Stream( futures::stream::once(async move { Ok(data) }).boxed(), ), + attributes: Attributes::default(), meta, range, }) @@ -108,6 +108,7 @@ impl ObjectStore for Tmpfs { payload: GetResultPayload::Stream( futures::stream::once(async move { Ok(data) }).boxed(), ), + attributes: Attributes::default(), meta, range, }) @@ -198,24 +199,25 @@ impl ObjectStore for Tmpfs { async fn put_opts( &self, location: &Path, - _bytes: Bytes, + _payload: PutPayload, _opts: PutOptions, ) -> Result { log::error!("NotImplemented put_opts: {}", location); Err(object_store::Error::NotImplemented {}) } - async fn put_multipart( - &self, - location: &Path, - ) -> Result<(MultipartId, Box)> { + async fn put_multipart(&self, location: &Path) -> Result> { log::error!("NotImplemented put_multipart: {}", location); - Err(object_store::Error::NotImplemented {}) + Err(object_store::Error::NotImplemented) } - async fn abort_multipart(&self, location: &Path, _multipart_id: &MultipartId) -> Result<()> { - log::error!("NotImplemented abort_multipart: {}", location); - Err(object_store::Error::NotImplemented {}) + async fn put_multipart_opts( + &self, + location: &Path, + _opts: PutMultipartOpts, + ) -> Result> { + log::error!("NotImplemented put_multipart_opts: {}", location); + Err(object_store::Error::NotImplemented) } async fn delete(&self, location: &Path) -> Result<()> { diff --git a/src/service/search/datafusion/storage/wal.rs b/src/service/search/datafusion/storage/wal.rs index 737ec8dc09..c50813e9e4 100644 --- a/src/service/search/datafusion/storage/wal.rs +++ b/src/service/search/datafusion/storage/wal.rs @@ -20,10 +20,9 @@ use bytes::Bytes; use futures::{stream::BoxStream, StreamExt}; use infra::storage; use object_store::{ - path::Path, GetOptions, GetResult, ListResult, MultipartId, ObjectMeta, ObjectStore, - PutOptions, PutResult, Result, + path::Path, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, + PutMultipartOpts, PutOptions, PutPayload, PutResult, Result, }; -use tokio::io::AsyncWrite; /// File system for local wal #[derive(Debug, Default)] @@ -99,24 +98,25 @@ impl ObjectStore for FS { async fn put_opts( &self, location: &Path, - _bytes: Bytes, + _payload: PutPayload, _opts: PutOptions, ) -> Result { log::error!("NotImplemented put_opts: {}", location); Err(object_store::Error::NotImplemented {}) } - async fn put_multipart( - &self, - location: &Path, - ) -> Result<(MultipartId, Box)> { + async fn put_multipart(&self, location: &Path) -> Result> { log::error!("NotImplemented put_multipart: {}", location); - Err(object_store::Error::NotImplemented {}) + Err(object_store::Error::NotImplemented) } - async fn abort_multipart(&self, location: &Path, _multipart_id: &MultipartId) -> Result<()> { - log::error!("NotImplemented abort_multipart: {}", location); - Err(object_store::Error::NotImplemented {}) + async fn put_multipart_opts( + &self, + location: &Path, + _opts: PutMultipartOpts, + ) -> Result> { + log::error!("NotImplemented put_multipart_opts: {}", location); + Err(object_store::Error::NotImplemented) } async fn delete(&self, location: &Path) -> Result<()> {