diff --git a/Cargo.lock b/Cargo.lock index 93d8bf4e1b..0a2d21cfd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -214,7 +214,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" name = "api" version = "1.0.0-beta.4" dependencies = [ - "arrow-schema", + "arrow-schema 57.0.0", "common-base", "common-decimal", "common-error", @@ -223,7 +223,7 @@ dependencies = [ "datatypes", "greptime-proto", "paste", - "prost 0.13.5", + "prost 0.14.1", "serde_json", "snafu 0.8.6", ] @@ -257,7 +257,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -299,19 +299,40 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-csv", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-csv 56.2.0", + "arrow-data 56.2.0", + "arrow-ipc 56.2.0", + "arrow-json 56.2.0", + "arrow-ord 56.2.0", + "arrow-row 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "arrow-string 56.2.0", +] + +[[package]] +name = "arrow" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae" +dependencies = [ + "arrow-arith 57.0.0", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-cast 57.0.0", + "arrow-csv 57.0.0", + "arrow-data 57.0.0", + "arrow-ipc 57.0.0", + "arrow-json 57.0.0", + "arrow-ord 57.0.0", + "arrow-row 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", + "arrow-string 57.0.0", ] [[package]] @@ -320,14 +341,28 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "num", ] +[[package]] +name = "arrow-arith" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "chrono", + "num-traits", +] + [[package]] name = "arrow-array" version = "56.2.0" @@ -335,14 +370,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" dependencies = [ "ahash 0.8.12", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "chrono", + "half", + "hashbrown 0.16.0", + "num", +] + +[[package]] +name = "arrow-array" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31" +dependencies = [ + "ahash 0.8.12", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", "chrono", "chrono-tz", "half", "hashbrown 0.16.0", - "num", + "num-complex", + "num-integer", + "num-traits", ] [[package]] @@ -356,24 +409,56 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76885a2697a7edf6b59577f568b456afc94ce0e2edc15b784ce3685b6c3c5c27" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + [[package]] name = "arrow-cast" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "atoi", + "base64 0.22.1", + "chrono", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-cast" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", "atoi", "base64 0.22.1", "chrono", "comfy-table", "half", "lexical-core", - "num", + "num-traits", "ryu", ] @@ -383,9 +468,24 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-cast 56.2.0", + "arrow-schema 56.2.0", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-csv" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c" +dependencies = [ + "arrow-array 57.0.0", + "arrow-cast 57.0.0", + "arrow-schema 57.0.0", "chrono", "csv", "csv-core", @@ -398,29 +498,43 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 56.2.0", + "arrow-schema 56.2.0", "half", "num", ] [[package]] -name = "arrow-flight" -version = "56.2.0" +name = "arrow-data" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c8b0ba0784d56bc6266b79f5de7a24b47024e7b3a0045d2ad4df3d9b686099f" +checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ipc", - "arrow-schema", + "arrow-buffer 57.0.0", + "arrow-schema 57.0.0", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-flight" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f70bb56412a007b0cfc116d15f24dda6adeed9611a213852a004cda20085a3b9" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-cast 57.0.0", + "arrow-ipc 57.0.0", + "arrow-schema 57.0.0", "base64 0.22.1", "bytes", "futures", - "prost 0.13.5", - "prost-types 0.13.5", - "tonic 0.13.1", + "prost 0.14.1", + "prost-types 0.14.1", + "tonic 0.14.2", + "tonic-prost", ] [[package]] @@ -429,11 +543,26 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "flatbuffers", + "zstd 0.13.3", +] + +[[package]] +name = "arrow-ipc" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", "flatbuffers", "lz4_flex", "zstd 0.13.3", @@ -445,14 +574,14 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "half", - "indexmap 2.11.4", + "indexmap 2.12.0", "lexical-core", "memchr", "num", @@ -461,17 +590,54 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "arrow-json" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-cast 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "chrono", + "half", + "indexmap 2.12.0", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + [[package]] name = "arrow-ord" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", +] + +[[package]] +name = "arrow-ord" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", ] [[package]] @@ -480,10 +646,23 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "half", +] + +[[package]] +name = "arrow-row" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", "half", ] @@ -492,8 +671,15 @@ name = "arrow-schema" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" + +[[package]] +name = "arrow-schema" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5" dependencies = [ "serde", + "serde_core", "serde_json", ] @@ -504,30 +690,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" dependencies = [ "ahash 0.8.12", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "num", ] +[[package]] +name = "arrow-select" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47" +dependencies = [ + "ahash 0.8.12", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "num-traits", +] + [[package]] name = "arrow-string" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "memchr", "num", "regex", "regex-syntax 0.8.7", ] +[[package]] +name = "arrow-string" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", + "memchr", + "num-traits", + "regex", + "regex-syntax 0.8.7", +] + [[package]] name = "ascii-canvas" version = "4.0.0" @@ -639,7 +856,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -661,7 +878,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -678,7 +895,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -758,7 +975,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -913,7 +1130,7 @@ checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -1061,7 +1278,7 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -1190,7 +1407,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -1213,7 +1430,7 @@ dependencies = [ "proc-macro-crate 3.3.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -1364,9 +1581,9 @@ dependencies = [ [[package]] name = "bzip2" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" dependencies = [ "libbz2-rs-sys", ] @@ -1421,8 +1638,8 @@ name = "catalog" version = "1.0.0-beta.4" dependencies = [ "api", - "arrow", - "arrow-schema", + "arrow 57.0.0", + "arrow-schema 57.0.0", "async-stream", "async-trait", "bytes", @@ -1555,7 +1772,7 @@ version = "0.13.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fe45e18904af7af10e4312df7c97251e98af98c70f42f1f2587aecfcbee56bf" dependencies = [ - "indexmap 2.11.4", + "indexmap 2.12.0", "lazy_static", "num-traits", "regex", @@ -1623,25 +1840,25 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efdce149c370f133a071ca8ef6ea340b7b88748ab0810097a9e2976eaa34b4f3" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" dependencies = [ "chrono", "chrono-tz-build", - "phf 0.11.3", + "phf 0.12.1", "uncased", ] [[package]] name = "chrono-tz-build" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f10f8c9340e31fc120ff885fcdb54a0b48e474bbd77cab557f0c30a3e569402" +checksum = "c7d8d1efd5109b9c1cd3b7966bd071cdfb53bb6eb0b22a473a68c2f70a11a1eb" dependencies = [ "parse-zoneinfo", - "phf_codegen 0.11.3", - "phf_shared 0.11.3", + "phf_codegen", + "phf_shared 0.12.1", "uncased", ] @@ -1743,7 +1960,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -1844,7 +2061,7 @@ dependencies = [ "moka", "parking_lot 0.12.4", "prometheus", - "prost 0.13.5", + "prost 0.14.1", "query", "rand 0.9.1", "serde_json", @@ -1854,7 +2071,7 @@ dependencies = [ "substrait 1.0.0-beta.4", "tokio", "tokio-stream", - "tonic 0.13.1", + "tonic 0.14.2", "tracing", ] @@ -1944,7 +2161,7 @@ dependencies = [ "plugins", "pprof", "prometheus", - "prost 0.13.5", + "prost 0.14.1", "query", "rand 0.9.1", "regex", @@ -1964,7 +2181,7 @@ dependencies = [ "tikv-jemallocator", "tokio", "toml 0.8.23", - "tonic 0.13.1", + "tonic 0.14.2", "tracing-appender", ] @@ -2083,8 +2300,8 @@ dependencies = [ name = "common-datasource" version = "1.0.0-beta.4" dependencies = [ - "arrow", - "arrow-schema", + "arrow 57.0.0", + "arrow-schema 57.0.0", "async-compression 0.3.15", "async-trait", "bytes", @@ -2096,6 +2313,7 @@ dependencies = [ "common-telemetry", "common-test-util", "datafusion", + "datafusion-datasource", "datafusion-orc", "datatypes", "futures", @@ -2135,7 +2353,7 @@ dependencies = [ "http 1.3.1", "snafu 0.8.6", "strum 0.27.1", - "tonic 0.13.1", + "tonic 0.14.2", ] [[package]] @@ -2179,7 +2397,7 @@ dependencies = [ "session", "snafu 0.8.6", "tokio", - "tonic 0.13.1", + "tonic 0.14.2", ] [[package]] @@ -2190,9 +2408,9 @@ dependencies = [ "api", "approx 0.5.1", "arc-swap", - "arrow", - "arrow-cast", - "arrow-schema", + "arrow 57.0.0", + "arrow-cast 57.0.0", + "arrow-schema 57.0.0", "async-trait", "bincode", "catalog", @@ -2283,7 +2501,7 @@ dependencies = [ "hyper 1.6.0", "hyper-util", "lazy_static", - "prost 0.13.5", + "prost 0.14.1", "rand 0.9.1", "serde", "serde_json", @@ -2291,7 +2509,7 @@ dependencies = [ "tempfile", "tokio", "tokio-util", - "tonic 0.13.1", + "tonic 0.14.2", "tower 0.5.2", "vec1", ] @@ -2310,7 +2528,7 @@ dependencies = [ "common-time", "datatypes", "paste", - "prost 0.13.5", + "prost 0.14.1", "snafu 0.8.6", "store-api", "table", @@ -2324,7 +2542,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -2402,8 +2620,8 @@ dependencies = [ "moka", "object-store", "prometheus", - "prost 0.13.5", - "prost-types 0.13.5", + "prost 0.14.1", + "prost-types 0.14.1", "rand 0.9.1", "regex", "rskafka", @@ -2422,7 +2640,7 @@ dependencies = [ "tokio", "tokio-postgres", "tokio-postgres-rustls", - "tonic 0.13.1", + "tonic 0.14.2", "tracing", "typetag", "uuid", @@ -2448,7 +2666,6 @@ dependencies = [ "common-error", "common-macro", "pprof", - "prost 0.13.5", "snafu 0.8.6", "tokio", ] @@ -2583,7 +2800,7 @@ dependencies = [ name = "common-sql" version = "1.0.0-beta.4" dependencies = [ - "arrow-schema", + "arrow-schema 57.0.0", "common-base", "common-decimal", "common-error", @@ -2627,10 +2844,10 @@ dependencies = [ "humantime-serde", "lazy_static", "once_cell", - "opentelemetry", + "opentelemetry 0.30.0", "opentelemetry-otlp", "opentelemetry-semantic-conventions", - "opentelemetry_sdk", + "opentelemetry_sdk 0.30.0", "parking_lot 0.12.4", "prometheus", "serde", @@ -2660,7 +2877,7 @@ dependencies = [ name = "common-time" version = "1.0.0-beta.4" dependencies = [ - "arrow", + "arrow 57.0.0", "chrono", "chrono-tz", "common-error", @@ -2806,6 +3023,18 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "const-hex" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bb320cac8a0750d7f25280aa97b09c26edfe161164238ecbbb31092b079e735" +dependencies = [ + "cfg-if", + "cpufeatures", + "proptest", + "serde_core", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -3209,11 +3438,11 @@ checksum = "7a9bc1a22964ff6a355fbec24cf68266a0ed28f8b84c0864c386474ea3d0e479" dependencies = [ "cc", "codespan-reporting 0.13.1", - "indexmap 2.11.4", + "indexmap 2.12.0", "proc-macro2", "quote", "scratch", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -3224,10 +3453,10 @@ checksum = "b1f29a879d35f7906e3c9b77d7a1005a6a0787d330c09dfe4ffb5f617728cb44" dependencies = [ "clap 4.5.40", "codespan-reporting 0.13.1", - "indexmap 2.11.4", + "indexmap 2.12.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -3242,10 +3471,10 @@ version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d187e019e7b05a1f3e69a8396b70800ee867aa9fc2ab972761173ccee03742df" dependencies = [ - "indexmap 2.11.4", + "indexmap 2.12.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -3303,7 +3532,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -3317,7 +3546,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -3339,7 +3568,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -3350,7 +3579,7 @@ checksum = "2b5be8a7a562d315a5b92a630c30cec6bcf663e6673f00fbb69cca66a6f521b9" dependencies = [ "darling_core 0.21.1", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -3375,21 +3604,21 @@ dependencies = [ [[package]] name = "datafusion" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", - "arrow-ipc", - "arrow-schema", + "arrow 57.0.0", + "arrow-schema 57.0.0", "async-trait", "bytes", - "bzip2 0.6.0", + "bzip2 0.6.1", "chrono", "datafusion-catalog", "datafusion-catalog-listing", "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", + "datafusion-datasource-arrow", "datafusion-datasource-csv", "datafusion-datasource-json", "datafusion-datasource-parquet", @@ -3418,7 +3647,7 @@ dependencies = [ "parquet", "rand 0.9.1", "regex", - "rstest", + "rstest 0.26.1", "sqlparser", "tempfile", "tokio", @@ -3430,10 +3659,10 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "async-trait", "dashmap", "datafusion-common", @@ -3454,10 +3683,10 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -3465,10 +3694,11 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", - "datafusion-session", "futures", + "itertools 0.14.0", "log", "object_store", "tokio", @@ -3476,16 +3706,16 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ "ahash 0.8.12", - "arrow", - "arrow-ipc", + "arrow 57.0.0", + "arrow-ipc 57.0.0", "chrono", "half", "hashbrown 0.14.5", - "indexmap 2.11.4", + "indexmap 2.12.0", "libc", "log", "object_store", @@ -3499,8 +3729,8 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ "futures", "log", @@ -3509,14 +3739,14 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "async-compression 0.4.19", "async-trait", "bytes", - "bzip2 0.6.0", + "bzip2 0.6.1", "chrono", "datafusion-common", "datafusion-common-runtime", @@ -3542,11 +3772,34 @@ dependencies = [ ] [[package]] -name = "datafusion-datasource-csv" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +name = "datafusion-datasource-arrow" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", + "arrow-ipc 57.0.0", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.14.0", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" +dependencies = [ + "arrow 57.0.0", "async-trait", "bytes", "datafusion-common", @@ -3565,10 +3818,10 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "async-trait", "bytes", "datafusion-common", @@ -3586,10 +3839,10 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "async-trait", "bytes", "datafusion-common", @@ -3615,15 +3868,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" [[package]] name = "datafusion-execution" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "async-trait", "dashmap", "datafusion-common", @@ -3639,10 +3892,10 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "async-trait", "chrono", "datafusion-common", @@ -3651,7 +3904,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.11.4", + "indexmap 2.12.0", "itertools 0.14.0", "paste", "recursive", @@ -3661,23 +3914,23 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "datafusion-common", - "indexmap 2.11.4", + "indexmap 2.12.0", "itertools 0.14.0", "paste", ] [[package]] name = "datafusion-functions" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", - "arrow-buffer", + "arrow 57.0.0", + "arrow-buffer 57.0.0", "base64 0.22.1", "blake2", "blake3", @@ -3692,6 +3945,7 @@ dependencies = [ "itertools 0.14.0", "log", "md-5", + "num-traits", "rand 0.9.1", "regex", "sha2", @@ -3701,11 +3955,11 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 57.0.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -3721,11 +3975,11 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 57.0.0", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", @@ -3733,11 +3987,11 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", - "arrow-ord", + "arrow 57.0.0", + "arrow-ord 57.0.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -3755,10 +4009,10 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -3770,10 +4024,10 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "datafusion-common", "datafusion-doc", "datafusion-expr", @@ -3787,8 +4041,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -3796,26 +4050,26 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] name = "datafusion-optimizer" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "chrono", "datafusion-common", "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", - "indexmap 2.11.4", + "indexmap 2.12.0", "itertools 0.14.0", "log", "recursive", @@ -3825,11 +4079,9 @@ dependencies = [ [[package]] name = "datafusion-orc" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2088adcf23fad3b1430ba95e7782c74e49c9ce5b0965151d96b295d4d538fb17" +version = "0.6.0" +source = "git+https://github.com/GreptimeTeam/datafusion-orc.git?rev=35f2e04bf81f2ab7b6f86c0450d6a77b7098d43e#35f2e04bf81f2ab7b6f86c0450d6a77b7098d43e" dependencies = [ - "arrow", "async-trait", "bytes", "datafusion", @@ -3843,9 +4095,8 @@ dependencies = [ [[package]] name = "datafusion-pg-catalog" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09bfd1feed7ed335227af0b65955ed825e467cf67fad6ecd089123202024cfd1" +version = "0.13.1" +source = "git+https://github.com/GreptimeTeam/datafusion-postgres.git?rev=74ac8e2806be6de91ff192b97f64735392539d16#74ac8e2806be6de91ff192b97f64735392539d16" dependencies = [ "async-trait", "datafusion", @@ -3857,11 +4108,11 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 57.0.0", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -3869,7 +4120,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", - "indexmap 2.11.4", + "indexmap 2.12.0", "itertools 0.14.0", "parking_lot 0.12.4", "paste", @@ -3878,10 +4129,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "datafusion-common", "datafusion-expr", "datafusion-functions", @@ -3892,11 +4143,11 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 57.0.0", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", @@ -3905,10 +4156,10 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -3923,13 +4174,13 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ "ahash 0.8.12", - "arrow", - "arrow-ord", - "arrow-schema", + "arrow 57.0.0", + "arrow-ord 57.0.0", + "arrow-schema 57.0.0", "async-trait", "chrono", "datafusion-common", @@ -3943,7 +4194,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap 2.11.4", + "indexmap 2.12.0", "itertools 0.14.0", "log", "parking_lot 0.12.4", @@ -3953,10 +4204,10 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "datafusion-common", "datafusion-datasource", "datafusion-expr-common", @@ -3969,8 +4220,8 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ "async-trait", "datafusion-common", @@ -3982,15 +4233,15 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ - "arrow", + "arrow 57.0.0", "bigdecimal 0.4.8", "chrono", "datafusion-common", "datafusion-expr", - "indexmap 2.11.4", + "indexmap 2.12.0", "log", "recursive", "regex", @@ -3999,18 +4250,19 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "50.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=fd4b2abcf3c3e43e94951bda452c9fd35243aab0#fd4b2abcf3c3e43e94951bda452c9fd35243aab0" +version = "51.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=7143b2fc4492a7970774583ed0997a459f3e5c05#7143b2fc4492a7970774583ed0997a459f3e5c05" dependencies = [ "async-recursion", "async-trait", "chrono", "datafusion", + "half", "itertools 0.14.0", "object_store", "pbjson-types", - "prost 0.13.5", - "substrait 0.58.0", + "prost 0.14.1", + "substrait 0.62.2", "tokio", "url", "uuid", @@ -4065,7 +4317,7 @@ dependencies = [ "object-store", "partition", "prometheus", - "prost 0.13.5", + "prost 0.14.1", "query", "reqwest", "serde", @@ -4077,7 +4329,7 @@ dependencies = [ "table", "tokio", "toml 0.8.23", - "tonic 0.13.1", + "tonic 0.14.2", "tracing", ] @@ -4085,9 +4337,9 @@ dependencies = [ name = "datatypes" version = "1.0.0-beta.4" dependencies = [ - "arrow", - "arrow-array", - "arrow-schema", + "arrow 57.0.0", + "arrow-array 57.0.0", + "arrow-schema 57.0.0", "common-base", "common-decimal", "common-error", @@ -4209,7 +4461,7 @@ checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -4220,7 +4472,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -4262,7 +4514,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -4282,7 +4534,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core 0.20.2", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -4304,7 +4556,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.106", + "syn 2.0.114", "unicode-xid", ] @@ -4316,7 +4568,7 @@ checksum = "ccfae181bab5ab6c5478b2ccb69e4c68a02f8c3ec72f6616bfec9dbc599d2ee0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -4375,7 +4627,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -4450,7 +4702,7 @@ checksum = "0e197fdfd2cdb5fdeb7f8ddcf3aed5d5d04ecde2890d448b14ffb716f7376b70" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -4558,7 +4810,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -4570,7 +4822,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -4609,7 +4861,7 @@ checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -4640,16 +4892,18 @@ dependencies = [ [[package]] name = "etcd-client" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88365f1a5671eb2f7fc240adb216786bc6494b38ce15f1d26ad6eaa303d5e822" +checksum = "8acfe553027cd07fc5fafa81a84f19a7a87eaffaccd2162b6db05e8d6ce98084" dependencies = [ "http 1.3.1", - "prost 0.13.5", + "prost 0.14.1", "tokio", "tokio-stream", - "tonic 0.13.1", + "tonic 0.14.2", "tonic-build", + "tonic-prost", + "tonic-prost-build", "tower 0.5.2", "tower-service", ] @@ -4893,8 +5147,8 @@ name = "flow" version = "1.0.0-beta.4" dependencies = [ "api", - "arrow", - "arrow-schema", + "arrow 57.0.0", + "arrow-schema 57.0.0", "async-recursion", "async-trait", "auth", @@ -4947,7 +5201,7 @@ dependencies = [ "partition", "pretty_assertions", "prometheus", - "prost 0.13.5", + "prost 0.14.1", "query", "rand 0.9.1", "serde", @@ -4962,7 +5216,7 @@ dependencies = [ "substrait 1.0.0-beta.4", "table", "tokio", - "tonic 0.13.1", + "tonic 0.14.2", ] [[package]] @@ -5069,14 +5323,14 @@ dependencies = [ "meta-client", "meta-srv", "num_cpus", - "opentelemetry-proto", + "opentelemetry-proto 0.31.0", "operator", "otel-arrow-rust", "partition", "pipeline", "prometheus", "promql-parser", - "prost 0.13.5", + "prost 0.14.1", "query", "rand 0.9.1", "reqwest", @@ -5093,7 +5347,7 @@ dependencies = [ "tokio", "tokio-util", "toml 0.8.23", - "tonic 0.13.1", + "tonic 0.14.2", "tower 0.5.2", "tracing", "uuid", @@ -5128,7 +5382,7 @@ checksum = "a0b4095fc99e1d858e5b8c7125d2638372ec85aa0fe6c807105cf10b0265ca6c" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -5140,7 +5394,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -5152,7 +5406,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -5276,7 +5530,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -5468,16 +5722,17 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=58aeee49267fb1eafa6f9123f9d0c47dd0f62722#58aeee49267fb1eafa6f9123f9d0c47dd0f62722" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=1353b0ada9e17890c7ba0e402ba29b2b57816ff1#1353b0ada9e17890c7ba0e402ba29b2b57816ff1" dependencies = [ - "prost 0.13.5", - "prost-types 0.13.5", + "prost 0.14.1", + "prost-types 0.14.1", "serde", "serde_json", "strum 0.25.0", "strum_macros 0.25.3", - "tonic 0.13.1", - "tonic-build", + "tonic 0.14.2", + "tonic-prost", + "tonic-prost-build", ] [[package]] @@ -5502,7 +5757,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.11.4", + "indexmap 2.12.0", "slab", "tokio", "tokio-util", @@ -5521,7 +5776,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.3.1", - "indexmap 2.11.4", + "indexmap 2.12.0", "slab", "tokio", "tokio-util", @@ -5549,13 +5804,14 @@ checksum = "6fb45e8060378c0353781abf67e1917b545a6b710d0342d85b70c125af7ef320" [[package]] name = "half" -version = "2.6.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", "num-traits", + "zerocopy", ] [[package]] @@ -5612,6 +5868,11 @@ name = "hashbrown" version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "hashlink" @@ -6212,7 +6473,7 @@ dependencies = [ "libflate", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -6260,7 +6521,7 @@ dependencies = [ "mockall", "nalgebra", "pin-project", - "prost 0.13.5", + "prost 0.14.1", "puffin", "rand 0.9.1", "rand_chacha 0.9.0", @@ -6293,9 +6554,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.4" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" +checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" dependencies = [ "equivalent", "hashbrown 0.16.0", @@ -6314,7 +6575,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash 0.8.12", - "indexmap 2.11.4", + "indexmap 2.12.0", "is-terminal", "itoa", "log", @@ -6337,7 +6598,7 @@ dependencies = [ "crossbeam-utils", "dashmap", "env_logger", - "indexmap 2.11.4", + "indexmap 2.12.0", "itoa", "log", "num-format", @@ -6379,7 +6640,7 @@ checksum = "6c38228f24186d9cc68c729accb4d413be9eaed6ad07ff79e0270d9e56f3de13" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -6556,7 +6817,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6105f38f083bb1a79ad523bd32fa0d8ffcb6abd2fc4da9da203c32bca5b6ace3" dependencies = [ - "phf_codegen 0.12.1", + "phf_codegen", ] [[package]] @@ -6713,7 +6974,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.11.4", + "indexmap 2.12.0", ] [[package]] @@ -6814,7 +7075,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -6900,7 +7161,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -6923,7 +7184,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -7171,9 +7432,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.27" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "log-query" @@ -7231,12 +7492,13 @@ dependencies = [ [[package]] name = "loki-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/loki-proto.git?rev=3b7cd33234358b18ece977bf689dc6fb760f29ab#3b7cd33234358b18ece977bf689dc6fb760f29ab" +source = "git+https://github.com/GreptimeTeam/loki-proto.git?rev=f69c8924c4babe516373e26a4118be82d976629c#f69c8924c4babe516373e26a4118be82d976629c" dependencies = [ - "prost 0.13.5", - "prost-types 0.13.5", - "tonic 0.13.1", - "tonic-build", + "prost 0.14.1", + "prost-types 0.14.1", + "tonic 0.14.2", + "tonic-prost", + "tonic-prost-build", ] [[package]] @@ -7280,7 +7542,7 @@ dependencies = [ "cactus", "cfgrammar", "filetime", - "indexmap 2.11.4", + "indexmap 2.12.0", "lazy_static", "lrtable", "num-traits", @@ -7510,7 +7772,7 @@ dependencies = [ "snafu 0.8.6", "tokio", "tokio-stream", - "tonic 0.13.1", + "tonic 0.14.2", "tower 0.5.2", "tracing", "tracing-subscriber", @@ -7570,7 +7832,7 @@ dependencies = [ "parking_lot 0.12.4", "partition", "prometheus", - "prost 0.13.5", + "prost 0.14.1", "rand 0.9.1", "regex", "rskafka", @@ -7587,7 +7849,7 @@ dependencies = [ "tokio-postgres", "tokio-stream", "toml 0.8.23", - "tonic 0.13.1", + "tonic 0.14.2", "tower 0.5.2", "tower-http 0.6.6", "tracing", @@ -7794,14 +8056,14 @@ dependencies = [ "paste", "pin-project", "prometheus", - "prost 0.13.5", + "prost 0.14.1", "puffin", "rand 0.9.1", "rayon", "regex", "roaring", "rskafka", - "rstest", + "rstest 0.25.0", "rstest_reuse", "scopeguard", "serde", @@ -7845,7 +8107,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -7944,7 +8206,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", "termcolor", "thiserror 1.0.69", ] @@ -7962,7 +8224,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", "termcolor", "thiserror 2.0.17", ] @@ -8098,7 +8360,7 @@ checksum = "254a5372af8fc138e36684761d3c0cdb758a4410e938babcff1c860ce14ddbfc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -8325,7 +8587,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -8438,7 +8700,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -8450,7 +8712,7 @@ dependencies = [ "proc-macro-crate 3.3.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -8694,6 +8956,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror 2.0.17", + "tracing", +] + [[package]] name = "opentelemetry-http" version = "0.30.0" @@ -8703,7 +8979,7 @@ dependencies = [ "async-trait", "bytes", "http 1.3.1", - "opentelemetry", + "opentelemetry 0.30.0", "reqwest", ] @@ -8714,10 +8990,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbee664a43e07615731afc539ca60c6d9f1a9425e25ca09c57bc36c87c55852b" dependencies = [ "http 1.3.1", - "opentelemetry", + "opentelemetry 0.30.0", "opentelemetry-http", - "opentelemetry-proto", - "opentelemetry_sdk", + "opentelemetry-proto 0.30.0", + "opentelemetry_sdk 0.30.0", "prost 0.13.5", "reqwest", "thiserror 2.0.17", @@ -8732,15 +9008,29 @@ version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e046fd7660710fe5a05e8748e70d9058dc15c94ba914e7c4faa7c728f0e8ddc" dependencies = [ - "base64 0.22.1", - "hex", - "opentelemetry", - "opentelemetry_sdk", + "opentelemetry 0.30.0", + "opentelemetry_sdk 0.30.0", "prost 0.13.5", - "serde", "tonic 0.13.1", ] +[[package]] +name = "opentelemetry-proto" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +dependencies = [ + "base64 0.22.1", + "const-hex", + "opentelemetry 0.31.0", + "opentelemetry_sdk 0.31.0", + "prost 0.14.1", + "serde", + "serde_json", + "tonic 0.14.2", + "tonic-prost", +] + [[package]] name = "opentelemetry-semantic-conventions" version = "0.30.0" @@ -8756,7 +9046,7 @@ dependencies = [ "futures-channel", "futures-executor", "futures-util", - "opentelemetry", + "opentelemetry 0.30.0", "percent-encoding", "rand 0.9.1", "serde_json", @@ -8765,14 +9055,29 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "opentelemetry_sdk" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry 0.31.0", + "percent-encoding", + "rand 0.9.1", + "thiserror 2.0.17", +] + [[package]] name = "operator" version = "1.0.0-beta.4" dependencies = [ "ahash 0.8.12", "api", - "arrow", - "arrow-ipc", + "arrow 57.0.0", + "arrow-ipc 57.0.0", "async-stream", "async-trait", "bytes", @@ -8817,8 +9122,8 @@ dependencies = [ "partition", "path-slash", "prometheus", - "prost 0.13.5", - "prost-types 0.13.5", + "prost 0.14.1", + "prost-types 0.14.1", "query", "regex", "serde_json", @@ -8831,17 +9136,17 @@ dependencies = [ "table", "tokio", "tokio-util", - "tonic 0.13.1", + "tonic 0.14.2", "tracing", ] [[package]] name = "orc-rust" -version = "0.6.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcd2b5295acfa0f64a14a1577a1f74c8a7dcc806396baaadcfa770509bed6a00" +checksum = "c8f1a357fb58dd9aab45ca21166d852f7894320d67ebeaa6d6b3fe557a9356c8" dependencies = [ - "arrow", + "arrow 57.0.0", "async-trait", "bytemuck", "bytes", @@ -8922,45 +9227,49 @@ checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" [[package]] name = "otel-arrow-rust" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/otel-arrow?rev=2d64b7c0fa95642028a8205b36fe9ea0b023ec59#2d64b7c0fa95642028a8205b36fe9ea0b023ec59" +source = "git+https://github.com/GreptimeTeam/otel-arrow?rev=5da284414e9b14f678344b51e5292229e4b5f8d2#5da284414e9b14f678344b51e5292229e4b5f8d2" dependencies = [ - "arrow", - "arrow-ipc", + "ahash 0.8.12", + "arrow 56.2.0", + "arrow-ipc 56.2.0", "ciborium", + "itertools 0.14.0", "lazy_static", "num_enum 0.7.4", "otlp-derive", "otlp-model", "paste", - "prost 0.13.5", - "prost-build 0.13.5", + "prost 0.14.1", + "prost-build 0.14.1", "rand 0.9.1", "replace_with", "serde", + "smallvec", "snafu 0.8.6", "tokio", - "tonic 0.13.1", - "tonic-build", + "tonic 0.14.2", + "tonic-prost", + "tonic-prost-build", ] [[package]] name = "otlp-derive" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/otel-arrow?rev=2d64b7c0fa95642028a8205b36fe9ea0b023ec59#2d64b7c0fa95642028a8205b36fe9ea0b023ec59" +source = "git+https://github.com/GreptimeTeam/otel-arrow?rev=5da284414e9b14f678344b51e5292229e4b5f8d2#5da284414e9b14f678344b51e5292229e4b5f8d2" dependencies = [ "convert_case 0.8.0", "otlp-model", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] name = "otlp-model" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/otel-arrow?rev=2d64b7c0fa95642028a8205b36fe9ea0b023ec59#2d64b7c0fa95642028a8205b36fe9ea0b023ec59" +source = "git+https://github.com/GreptimeTeam/otel-arrow?rev=5da284414e9b14f678344b51e5292229e4b5f8d2#5da284414e9b14f678344b51e5292229e4b5f8d2" dependencies = [ - "tonic-build", + "tonic-prost-build", ] [[package]] @@ -9062,18 +9371,18 @@ dependencies = [ [[package]] name = "parquet" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" +checksum = "7a0f31027ef1af7549f7cec603a9a21dce706d3f8d7c2060a68f43c1773be95a" dependencies = [ "ahash 0.8.12", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-cast 57.0.0", + "arrow-data 57.0.0", + "arrow-ipc 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", "base64 0.22.1", "brotli 8.0.1", "bytes", @@ -9083,8 +9392,9 @@ dependencies = [ "half", "hashbrown 0.16.0", "lz4_flex", - "num", "num-bigint", + "num-integer", + "num-traits", "object_store", "paste", "seq-macro", @@ -9104,12 +9414,9 @@ checksum = "487f2ccd1e17ce8c1bfab3a65c89525af41cfad4c8659021a1e9a2aacd73b89b" [[package]] name = "parse-zoneinfo" -version = "0.3.1" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" -dependencies = [ - "regex", -] +checksum = "e3c406c9e2aa74554e662d2c2ee11cd3e73756988800be7e6f5eddb16fed4699" [[package]] name = "partition" @@ -9158,38 +9465,38 @@ checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "pbjson" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e6349fa080353f4a597daffd05cb81572a9c031a6d4fff7e504947496fcc68" +checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "serde", ] [[package]] name = "pbjson-build" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eea3058763d6e656105d1403cb04e0a41b7bbac6362d413e7c33be0c32279c9" +checksum = "af22d08a625a2213a78dbb0ffa253318c5c79ce3133d32d296655a7bdfb02095" dependencies = [ "heck 0.5.0", - "itertools 0.13.0", - "prost 0.13.5", - "prost-types 0.13.5", + "itertools 0.14.0", + "prost 0.14.1", + "prost-types 0.14.1", ] [[package]] name = "pbjson-types" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54e5e7bfb1652f95bc361d76f3c780d8e526b134b85417e774166ee941f0887" +checksum = "8e748e28374f10a330ee3bb9f29b828c0ac79831a32bab65015ad9b661ead526" dependencies = [ "bytes", "chrono", "pbjson", "pbjson-build", - "prost 0.13.5", - "prost-build 0.13.5", + "prost 0.14.1", + "prost-build 0.14.1", "serde", ] @@ -9271,7 +9578,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -9291,7 +9598,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset 0.4.2", - "indexmap 2.11.4", + "indexmap 2.12.0", ] [[package]] @@ -9301,7 +9608,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset 0.5.7", - "indexmap 2.11.4", + "indexmap 2.12.0", ] [[package]] @@ -9312,7 +9619,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset 0.5.7", "hashbrown 0.15.4", - "indexmap 2.11.4", + "indexmap 2.12.0", "serde", ] @@ -9374,40 +9681,21 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ + "phf_macros", "phf_shared 0.12.1", "serde", ] -[[package]] -name = "phf_codegen" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" -dependencies = [ - "phf_generator 0.11.3", - "phf_shared 0.11.3", -] - [[package]] name = "phf_codegen" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efbdcb6f01d193b17f0b9c3360fa7e0e620991b193ff08702f78b3ce365d7e61" dependencies = [ - "phf_generator 0.12.1", + "phf_generator", "phf_shared 0.12.1", ] -[[package]] -name = "phf_generator" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" -dependencies = [ - "phf_shared 0.11.3", - "rand 0.8.5", -] - [[package]] name = "phf_generator" version = "0.12.1" @@ -9418,6 +9706,20 @@ dependencies = [ "phf_shared 0.12.1", ] +[[package]] +name = "phf_macros" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d713258393a82f091ead52047ca779d37e5766226d009de21696c4e667044368" +dependencies = [ + "phf_generator", + "phf_shared 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.114", + "uncased", +] + [[package]] name = "phf_shared" version = "0.11.3" @@ -9425,7 +9727,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ "siphasher", - "uncased", ] [[package]] @@ -9435,6 +9736,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" dependencies = [ "siphasher", + "uncased", ] [[package]] @@ -9454,7 +9756,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -9475,8 +9777,8 @@ version = "1.0.0-beta.4" dependencies = [ "ahash 0.8.12", "api", - "arrow", - "arrow-schema", + "arrow 57.0.0", + "arrow-schema 57.0.0", "async-trait", "catalog", "chrono", @@ -9592,7 +9894,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d77244ce2d584cd84f6a15f86195b8c9b2a0dfbfd817c09e0464244091a58ed" dependencies = [ "base64 0.22.1", - "indexmap 2.11.4", + "indexmap 2.12.0", "quick-xml 0.37.5", "serde", "time", @@ -9818,12 +10120,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.35" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061c1221631e079b26479d25bbf2275bfe5917ae8419cd7e34f13bfc2aa7539a" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -9864,14 +10166,14 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] name = "proc-macro2" -version = "1.0.95" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7" dependencies = [ "unicode-ident", ] @@ -9947,7 +10249,7 @@ dependencies = [ "greptime-proto", "lazy_static", "prometheus", - "prost 0.13.5", + "prost 0.14.1", "snafu 0.8.6", "tokio", ] @@ -9968,6 +10270,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "proptest" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40" +dependencies = [ + "bitflags 2.9.1", + "num-traits", + "rand 0.9.1", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax 0.8.7", + "unarray", +] + [[package]] name = "prost" version = "0.11.9" @@ -9998,6 +10315,16 @@ dependencies = [ "prost-derive 0.13.5", ] +[[package]] +name = "prost" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" +dependencies = [ + "bytes", + "prost-derive 0.14.1", +] + [[package]] name = "prost-build" version = "0.12.6" @@ -10015,7 +10342,7 @@ dependencies = [ "prost 0.12.6", "prost-types 0.12.6", "regex", - "syn 2.0.106", + "syn 2.0.114", "tempfile", ] @@ -10031,11 +10358,31 @@ dependencies = [ "multimap", "once_cell", "petgraph 0.7.1", - "prettyplease", "prost 0.13.5", "prost-types 0.13.5", "regex", - "syn 2.0.106", + "tempfile", +] + +[[package]] +name = "prost-build" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph 0.7.1", + "prettyplease", + "prost 0.14.1", + "prost-types 0.14.1", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn 2.0.114", "tempfile", ] @@ -10062,7 +10409,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -10075,7 +10422,20 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", +] + +[[package]] +name = "prost-derive" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] @@ -10116,6 +10476,15 @@ dependencies = [ "prost 0.13.5", ] +[[package]] +name = "prost-types" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" +dependencies = [ + "prost 0.14.1", +] + [[package]] name = "protobuf" version = "2.28.0" @@ -10242,6 +10611,26 @@ dependencies = [ "uuid", ] +[[package]] +name = "pulldown-cmark" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" +dependencies = [ + "bitflags 2.9.1", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "21.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8246feae3db61428fd0bb94285c690b460e4517d83152377543ca802357785f1" +dependencies = [ + "pulldown-cmark", +] + [[package]] name = "pusherator" version = "0.0.12" @@ -10259,8 +10648,8 @@ dependencies = [ "ahash 0.8.12", "api", "arc-swap", - "arrow", - "arrow-schema", + "arrow 57.0.0", + "arrow-schema 57.0.0", "async-recursion", "async-stream", "async-trait", @@ -10311,7 +10700,7 @@ dependencies = [ "prometheus", "promql", "promql-parser", - "prost 0.13.5", + "prost 0.14.1", "rand 0.9.1", "regex", "serde", @@ -10540,6 +10929,15 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.3", +] + [[package]] name = "ratelimit" version = "0.10.0" @@ -10594,7 +10992,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -10632,7 +11030,7 @@ checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -10674,7 +11072,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c11639076bf147be211b90e47790db89f4c22b6c8a9ca6e960833869da67166" dependencies = [ "aho-corasick", - "indexmap 2.11.4", + "indexmap 2.12.0", "itertools 0.13.0", "nohash", "regex", @@ -10711,11 +11109,11 @@ dependencies = [ [[package]] name = "regress" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ef7fa9ed0256d64a688a3747d0fef7a88851c18a5e1d57f115f38ec2e09366" +checksum = "2057b2325e68a893284d1538021ab90279adac1139957ca2a74426c6f118fb48" dependencies = [ - "hashbrown 0.15.4", + "hashbrown 0.16.0", "memchr", ] @@ -11003,10 +11401,21 @@ checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d" dependencies = [ "futures-timer", "futures-util", - "rstest_macros", + "rstest_macros 0.25.0", "rustc_version", ] +[[package]] +name = "rstest" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros 0.26.1", +] + [[package]] name = "rstest_macros" version = "0.25.0" @@ -11021,7 +11430,25 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.106", + "syn 2.0.114", + "unicode-ident", +] + +[[package]] +name = "rstest_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate 3.3.0", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.114", "unicode-ident", ] @@ -11033,7 +11460,7 @@ checksum = "b3a8fb4672e840a587a66fc577a5491375df51ddb88f2a2c2a792598c326fe14" dependencies = [ "quote", "rand 0.8.5", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -11056,7 +11483,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.106", + "syn 2.0.114", "walkdir", ] @@ -11377,7 +11804,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -11429,7 +11856,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", "thiserror 2.0.17", ] @@ -11448,7 +11875,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -11499,11 +11926,12 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" dependencies = [ "serde", + "serde_core", ] [[package]] @@ -11549,7 +11977,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -11560,19 +11988,20 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] name = "serde_json" -version = "1.0.140" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", - "ryu", "serde", + "serde_core", + "zmij", ] [[package]] @@ -11593,7 +12022,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -11614,7 +12043,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -11654,7 +12083,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -11663,7 +12092,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.11.4", + "indexmap 2.12.0", "itoa", "ryu", "serde", @@ -11676,10 +12105,10 @@ version = "1.0.0-beta.4" dependencies = [ "ahash 0.8.12", "api", - "arrow", + "arrow 57.0.0", "arrow-flight", - "arrow-ipc", - "arrow-schema", + "arrow-ipc 57.0.0", + "arrow-schema 57.0.0", "async-trait", "auth", "axum 0.8.4", @@ -11729,7 +12158,7 @@ dependencies = [ "humantime", "humantime-serde", "hyper 1.6.0", - "indexmap 2.11.4", + "indexmap 2.12.0", "influxdb_line_protocol", "itertools 0.14.0", "json5", @@ -11745,7 +12174,7 @@ dependencies = [ "once_cell", "openmetrics-parser", "opensrv-mysql", - "opentelemetry-proto", + "opentelemetry-proto 0.31.0", "operator", "otel-arrow-rust", "parking_lot 0.12.4", @@ -11758,7 +12187,7 @@ dependencies = [ "pprof", "prometheus", "promql-parser", - "prost 0.13.5", + "prost 0.14.1", "query", "quoted-string", "rand 0.9.1", @@ -11787,7 +12216,7 @@ dependencies = [ "tokio-rustls", "tokio-stream", "tokio-util", - "tonic 0.13.1", + "tonic 0.14.2", "tonic-reflection", "tower 0.5.2", "tower-http 0.6.6", @@ -12071,7 +12500,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -12148,7 +12577,7 @@ name = "sql" version = "1.0.0-beta.4" dependencies = [ "api", - "arrow-buffer", + "arrow-buffer 57.0.0", "chrono", "common-base", "common-catalog", @@ -12235,8 +12664,8 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.58.0" -source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=a0ce2bc6eb3e804532932f39833c32432f5c9a39#a0ce2bc6eb3e804532932f39833c32432f5c9a39" +version = "0.59.0" +source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=d7d95a44889e099e32d78e9bad9bc00598faef28#d7d95a44889e099e32d78e9bad9bc00598faef28" dependencies = [ "lazy_static", "log", @@ -12260,11 +12689,11 @@ dependencies = [ [[package]] name = "sqlparser_derive" version = "0.3.0" -source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=a0ce2bc6eb3e804532932f39833c32432f5c9a39#a0ce2bc6eb3e804532932f39833c32432f5c9a39" +source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=d7d95a44889e099e32d78e9bad9bc00598faef28#d7d95a44889e099e32d78e9bad9bc00598faef28" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -12299,7 +12728,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.4", "hashlink", - "indexmap 2.11.4", + "indexmap 2.12.0", "log", "memchr", "once_cell", @@ -12327,7 +12756,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -12350,7 +12779,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.106", + "syn 2.0.114", "tokio", "url", ] @@ -12552,7 +12981,7 @@ dependencies = [ "lazy_static", "num_enum 0.7.4", "prometheus", - "prost 0.13.5", + "prost 0.14.1", "serde", "serde_json", "snafu 0.8.6", @@ -12652,7 +13081,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -12665,7 +13094,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -12678,7 +13107,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -12707,33 +13136,33 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.106", + "syn 2.0.114", "typify 0.1.0", "walkdir", ] [[package]] name = "substrait" -version = "0.58.0" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de6d24c270c6c672a86c183c3a8439ba46c1936f93cf7296aa692de3b0ff0228" +checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0" dependencies = [ "heck 0.5.0", "pbjson", "pbjson-build", "pbjson-types", "prettyplease", - "prost 0.13.5", - "prost-build 0.13.5", - "prost-types 0.13.5", - "regress 0.10.3", + "prost 0.14.1", + "prost-build 0.14.1", + "prost-types 0.14.1", + "regress 0.10.5", "schemars", "semver", "serde", "serde_json", "serde_yaml", - "syn 2.0.106", - "typify 0.4.2", + "syn 2.0.114", + "typify 0.5.0", "walkdir", ] @@ -12753,7 +13182,7 @@ dependencies = [ "datafusion-substrait", "datatypes", "promql", - "prost 0.13.5", + "prost 0.14.1", "snafu 0.8.6", "substrait 0.37.3", "tokio", @@ -12801,9 +13230,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.106" +version = "2.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" dependencies = [ "proc-macro2", "quote", @@ -12833,7 +13262,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -13230,16 +13659,16 @@ dependencies = [ "moka", "mysql_async", "object-store", - "opentelemetry-proto", + "opentelemetry-proto 0.31.0", "operator", "otel-arrow-rust", "partition", "paste", "pipeline", - "prost 0.13.5", + "prost 0.14.1", "query", "rand 0.9.1", - "rstest", + "rstest 0.25.0", "rstest_reuse", "sea-query", "serde_json", @@ -13258,7 +13687,7 @@ dependencies = [ "tokio", "tokio-postgres", "tokio-stream", - "tonic 0.13.1", + "tonic 0.14.2", "tower 0.5.2", "url", "urlencoding", @@ -13299,7 +13728,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -13310,7 +13739,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -13487,7 +13916,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -13620,7 +14049,7 @@ version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ - "indexmap 2.11.4", + "indexmap 2.12.0", "serde", "serde_spanned", "toml_datetime", @@ -13642,7 +14071,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.11.4", + "indexmap 2.12.0", "toml_datetime", "winnow 0.5.40", ] @@ -13653,7 +14082,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.11.4", + "indexmap 2.12.0", "serde", "serde_spanned", "toml_datetime", @@ -13700,6 +14129,32 @@ name = "tonic" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.6.0", + "hyper-timeout 0.5.2", + "hyper-util", + "percent-encoding", + "pin-project", + "prost 0.13.5", + "tokio", + "tokio-stream", + "tower 0.5.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" dependencies = [ "async-trait", "axum 0.8.4", @@ -13715,9 +14170,9 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost 0.13.5", "rustls-native-certs 0.8.1", - "socket2 0.5.10", + "socket2 0.6.0", + "sync_wrapper 1.0.2", "tokio", "tokio-rustls", "tokio-stream", @@ -13730,29 +14185,55 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.13.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" +checksum = "4c40aaccc9f9eccf2cd82ebc111adc13030d23e887244bc9cfa5d1d636049de3" dependencies = [ "prettyplease", "proc-macro2", - "prost-build 0.13.5", - "prost-types 0.13.5", "quote", - "syn 2.0.106", + "syn 2.0.114", +] + +[[package]] +name = "tonic-prost" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" +dependencies = [ + "bytes", + "prost 0.14.1", + "tonic 0.14.2", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a16cba4043dc3ff43fcb3f96b4c5c154c64cbd18ca8dce2ab2c6a451d058a2" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build 0.14.1", + "prost-types 0.14.1", + "quote", + "syn 2.0.114", + "tempfile", + "tonic-build", ] [[package]] name = "tonic-reflection" -version = "0.13.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9687bd5bfeafebdded2356950f278bba8226f0b32109537c4253406e09aafe1" +checksum = "34da53e8387581d66db16ff01f98a70b426b091fdf76856e289d5c1bd386ed7b" dependencies = [ - "prost 0.13.5", - "prost-types 0.13.5", + "prost 0.14.1", + "prost-types 0.14.1", "tokio", "tokio-stream", - "tonic 0.13.1", + "tonic 0.14.2", + "tonic-prost", ] [[package]] @@ -13784,7 +14265,7 @@ dependencies = [ "futures-core", "futures-util", "hdrhistogram", - "indexmap 2.11.4", + "indexmap 2.12.0", "pin-project-lite", "slab", "sync_wrapper 1.0.2", @@ -13889,7 +14370,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -13921,8 +14402,8 @@ checksum = "ddcf5959f39507d0d04d6413119c04f33b623f4f951ebcbdddddfad2d0623a9c" dependencies = [ "js-sys", "once_cell", - "opentelemetry", - "opentelemetry_sdk", + "opentelemetry 0.30.0", + "opentelemetry_sdk 0.30.0", "smallvec", "tracing", "tracing-core", @@ -14013,7 +14494,7 @@ checksum = "35f5380909ffc31b4de4f4bdf96b877175a016aa2ca98cee39fcfd8c4d53d952" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -14028,12 +14509,12 @@ dependencies = [ [[package]] name = "typify" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c6c647a34e851cf0260ccc14687f17cdcb8302ff1a8a687a24b97ca0f82406f" +checksum = "e6d5bcc6f62eb1fa8aa4098f39b29f93dcb914e17158b76c50360911257aa629" dependencies = [ - "typify-impl 0.4.2", - "typify-macro 0.4.2", + "typify-impl 0.5.0", + "typify-macro 0.5.0", ] [[package]] @@ -14051,27 +14532,27 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.106", + "syn 2.0.114", "thiserror 1.0.69", "unicode-ident", ] [[package]] name = "typify-impl" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "741b7f1e2e1338c0bee5ad5a7d3a9bbd4e24c33765c08b7691810e68d879365d" +checksum = "a1eb359f7ffa4f9ebe947fa11a1b2da054564502968db5f317b7e37693cb2240" dependencies = [ "heck 0.5.0", "log", "proc-macro2", "quote", - "regress 0.10.3", + "regress 0.10.5", "schemars", "semver", "serde", "serde_json", - "syn 2.0.106", + "syn 2.0.114", "thiserror 2.0.17", "unicode-ident", ] @@ -14089,15 +14570,15 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.106", + "syn 2.0.114", "typify-impl 0.1.0", ] [[package]] name = "typify-macro" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7560adf816a1e8dad7c63d8845ef6e31e673e39eab310d225636779230cbedeb" +checksum = "911c32f3c8514b048c1b228361bebb5e6d73aeec01696e8cc0e82e2ffef8ab7a" dependencies = [ "proc-macro2", "quote", @@ -14106,8 +14587,8 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.106", - "typify-impl 0.4.2", + "syn 2.0.114", + "typify-impl 0.5.0", ] [[package]] @@ -14161,6 +14642,12 @@ dependencies = [ "serde", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "uncased" version = "0.9.10" @@ -14184,9 +14671,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unicode-normalization" @@ -14351,7 +14838,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", "variadics", ] @@ -14436,7 +14923,7 @@ dependencies = [ "hostname 0.4.1", "iana-time-zone", "idna", - "indexmap 2.11.4", + "indexmap 2.12.0", "indoc", "influxdb-line-protocol", "itertools 0.14.0", @@ -14565,7 +15052,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", "wasm-bindgen-shared", ] @@ -14600,7 +15087,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -14803,7 +15290,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -14814,7 +15301,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -14825,7 +15312,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -14836,7 +15323,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -15219,7 +15706,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", "synstructure", ] @@ -15240,7 +15727,7 @@ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -15260,7 +15747,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", "synstructure", ] @@ -15281,7 +15768,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -15314,7 +15801,7 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -15323,6 +15810,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" +[[package]] +name = "zmij" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc5a66a20078bf1251bde995aa2fdcc4b800c70b5d92dd2c62abc5c60f679f8" + [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" diff --git a/Cargo.toml b/Cargo.toml index 491dbeea17..077e22abe0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -100,13 +100,13 @@ rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] } # See for more detaiils: https://github.com/rust-lang/cargo/issues/11329 ahash = { version = "0.8", features = ["compile-time-rng"] } aquamarine = "0.6" -arrow = { version = "56.2", features = ["prettyprint"] } -arrow-array = { version = "56.2", default-features = false, features = ["chrono-tz"] } -arrow-buffer = "56.2" -arrow-cast = "56.2" -arrow-flight = "56.2" -arrow-ipc = { version = "56.2", default-features = false, features = ["lz4", "zstd"] } -arrow-schema = { version = "56.2", features = ["serde"] } +arrow = { version = "57.0", features = ["prettyprint"] } +arrow-array = { version = "57.0", default-features = false, features = ["chrono-tz"] } +arrow-buffer = "57.0" +arrow-cast = "57.0" +arrow-flight = "57.0" +arrow-ipc = { version = "57.0", default-features = false, features = ["lz4", "zstd"] } +arrow-schema = { version = "57.0", features = ["serde"] } async-stream = "0.3" async-trait = "0.1" # Remember to update axum-extra, axum-macros when updating axum @@ -120,38 +120,39 @@ bitflags = "2.4.1" bytemuck = "1.12" bytes = { version = "1.7", features = ["serde"] } chrono = { version = "0.4", features = ["serde"] } -chrono-tz = { version = "0.10.1", features = ["case-insensitive"] } +chrono-tz = { version = "0.10", features = ["case-insensitive"] } clap = { version = "4.4", features = ["derive"] } config = "0.13.0" const_format = "0.2" crossbeam-utils = "0.8" dashmap = "6.1" -datafusion = "50" -datafusion-common = "50" -datafusion-expr = "50" -datafusion-functions = "50" -datafusion-functions-aggregate-common = "50" -datafusion-optimizer = "50" -datafusion-orc = "0.5" -datafusion-pg-catalog = "0.12.3" -datafusion-physical-expr = "50" -datafusion-physical-plan = "50" -datafusion-sql = "50" -datafusion-substrait = "50" +datafusion = "51.0" +datafusion-common = "51.0" +datafusion-datasource = "51.0" +datafusion-expr = "51.0" +datafusion-functions = "51.0" +datafusion-functions-aggregate-common = "51.0" +datafusion-optimizer = "51.0" +datafusion-orc = { git = "https://github.com/GreptimeTeam/datafusion-orc.git", rev = "35f2e04bf81f2ab7b6f86c0450d6a77b7098d43e" } +datafusion-pg-catalog = "0.13" +datafusion-physical-expr = "51.0" +datafusion-physical-plan = "51.0" +datafusion-sql = "51.0" +datafusion-substrait = "51.0" deadpool = "0.12" deadpool-postgres = "0.14" derive_builder = "0.20" derive_more = { version = "2.1", features = ["full"] } dotenv = "0.15" either = "1.15" -etcd-client = { version = "0.16.1", features = [ +etcd-client = { version = "0.17", features = [ "tls", "tls-roots", ] } fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "58aeee49267fb1eafa6f9123f9d0c47dd0f62722" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "1353b0ada9e17890c7ba0e402ba29b2b57816ff1" } hex = "0.4" http = "1" humantime = "2.1" @@ -162,7 +163,7 @@ itertools = "0.14" jsonb = { git = "https://github.com/databendlabs/jsonb.git", rev = "8c8d2fc294a39f3ff08909d60f718639cfba3875", default-features = false } lazy_static = "1.4" local-ip-address = "0.6" -loki-proto = { git = "https://github.com/GreptimeTeam/loki-proto.git", rev = "3b7cd33234358b18ece977bf689dc6fb760f29ab" } +loki-proto = { git = "https://github.com/GreptimeTeam/loki-proto.git", rev = "f69c8924c4babe516373e26a4118be82d976629c" } meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "5618e779cf2bb4755b499c630fba4c35e91898cb" } mockall = "0.13" moka = "0.12" @@ -172,7 +173,7 @@ notify = "8.0" num_cpus = "1.16" object_store_opendal = "0.54" once_cell = "1.18" -opentelemetry-proto = { version = "0.30", features = [ +opentelemetry-proto = { version = "0.31", features = [ "gen-tonic", "metrics", "trace", @@ -180,18 +181,18 @@ opentelemetry-proto = { version = "0.30", features = [ "logs", ] } ordered-float = { version = "4.3", features = ["serde"] } -otel-arrow-rust = { git = "https://github.com/GreptimeTeam/otel-arrow", rev = "2d64b7c0fa95642028a8205b36fe9ea0b023ec59", features = [ +otel-arrow-rust = { git = "https://github.com/GreptimeTeam/otel-arrow", rev = "5da284414e9b14f678344b51e5292229e4b5f8d2", features = [ "server", ] } parking_lot = "0.12" -parquet = { version = "56.2", default-features = false, features = ["arrow", "async", "object_store"] } +parquet = { version = "57.0", default-features = false, features = ["arrow", "async", "object_store"] } paste = "1.0" pin-project = "1.0" pretty_assertions = "1.4.0" prometheus = { version = "0.13.3", features = ["process"] } promql-parser = { version = "0.7.1", features = ["ser"] } -prost = { version = "0.13", features = ["no-recursion-limit"] } -prost-types = "0.13" +prost = { version = "0.14", features = ["no-recursion-limit"] } +prost-types = "0.14" raft-engine = { version = "0.4.1", default-features = false } rand = "0.9" ratelimit = "0.10" @@ -223,7 +224,7 @@ simd-json = "0.15" similar-asserts = "1.6.0" smallvec = { version = "1", features = ["serde"] } snafu = "0.8" -sqlparser = { version = "0.58.0", default-features = false, features = ["std", "visitor", "serde"] } +sqlparser = { version = "0.59.0", default-features = false, features = ["std", "visitor", "serde"] } sqlx = { version = "0.8", default-features = false, features = ["any", "macros", "json", "runtime-tokio-rustls"] } strum = { version = "0.27", features = ["derive"] } sysinfo = "0.33" @@ -234,7 +235,7 @@ tokio-rustls = { version = "0.26.2", default-features = false } tokio-stream = "0.1" tokio-util = { version = "0.7", features = ["io-util", "compat"] } toml = "0.8.8" -tonic = { version = "0.13", features = ["tls-ring", "gzip", "zstd"] } +tonic = { version = "0.14", features = ["tls-ring", "gzip", "zstd"] } tower = "0.5" tower-http = "0.6" tracing = "0.1" @@ -322,19 +323,20 @@ git = "https://github.com/GreptimeTeam/greptime-meter.git" rev = "5618e779cf2bb4755b499c630fba4c35e91898cb" [patch.crates-io] -datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fd4b2abcf3c3e43e94951bda452c9fd35243aab0" } -datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fd4b2abcf3c3e43e94951bda452c9fd35243aab0" } -datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fd4b2abcf3c3e43e94951bda452c9fd35243aab0" } -datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fd4b2abcf3c3e43e94951bda452c9fd35243aab0" } -datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fd4b2abcf3c3e43e94951bda452c9fd35243aab0" } -datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fd4b2abcf3c3e43e94951bda452c9fd35243aab0" } -datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fd4b2abcf3c3e43e94951bda452c9fd35243aab0" } -datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fd4b2abcf3c3e43e94951bda452c9fd35243aab0" } -datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fd4b2abcf3c3e43e94951bda452c9fd35243aab0" } -datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fd4b2abcf3c3e43e94951bda452c9fd35243aab0" } -datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fd4b2abcf3c3e43e94951bda452c9fd35243aab0" } -datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "fd4b2abcf3c3e43e94951bda452c9fd35243aab0" } -sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "a0ce2bc6eb3e804532932f39833c32432f5c9a39" } # branch = "v0.58.x" +datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } +datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } +datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } +datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } +datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } +datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } +datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } +datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } +datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } +datafusion-pg-catalog = { git = "https://github.com/GreptimeTeam/datafusion-postgres.git", rev = "74ac8e2806be6de91ff192b97f64735392539d16" } +datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } +datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } +datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "7143b2fc4492a7970774583ed0997a459f3e5c05" } +sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "d7d95a44889e099e32d78e9bad9bc00598faef28" } # on branch v0.59.x [profile.release] debug = 1 diff --git a/src/cmd/src/datanode/objbench.rs b/src/cmd/src/datanode/objbench.rs index 1b06bf449a..59e9a58d42 100644 --- a/src/cmd/src/datanode/objbench.rs +++ b/src/cmd/src/datanode/objbench.rs @@ -35,6 +35,7 @@ use mito2::sst::parquet::reader::ParquetReaderBuilder; use mito2::sst::parquet::{PARQUET_METADATA_KEY, WriteOptions}; use mito2::worker::write_cache_from_config; use object_store::ObjectStore; +use parquet::file::metadata::{FooterTail, KeyValue}; use regex::Regex; use snafu::OptionExt; use store_api::metadata::{RegionMetadata, RegionMetadataRef}; @@ -463,7 +464,6 @@ fn extract_region_metadata( file_path: &str, meta: &parquet::file::metadata::ParquetMetaData, ) -> error::Result { - use parquet::format::KeyValue; let kvs: Option<&Vec> = meta.file_metadata().key_value_metadata(); let Some(kvs) = kvs else { return Err(error::IllegalConfigSnafu { @@ -608,7 +608,7 @@ async fn load_parquet_metadata( let buffer_len = buffer.len(); let mut footer = [0; 8]; footer.copy_from_slice(&buffer[buffer_len - FOOTER_SIZE..]); - let footer = ParquetMetaDataReader::decode_footer_tail(&footer)?; + let footer = FooterTail::try_new(&footer)?; let metadata_len = footer.metadata_length() as u64; if actual_size - (FOOTER_SIZE as u64) < metadata_len { return Err("invalid footer/metadata length".into()); diff --git a/src/common/datasource/Cargo.toml b/src/common/datasource/Cargo.toml index 99afa9d6b9..7b13a5bdac 100644 --- a/src/common/datasource/Cargo.toml +++ b/src/common/datasource/Cargo.toml @@ -27,13 +27,14 @@ common-recordbatch.workspace = true common-runtime.workspace = true common-telemetry.workspace = true datafusion.workspace = true +datafusion-datasource.workspace = true datafusion-orc.workspace = true datatypes.workspace = true futures.workspace = true lazy_static.workspace = true object-store.workspace = true object_store_opendal.workspace = true -orc-rust = { version = "0.6.3", default-features = false, features = ["async"] } +orc-rust = { version = "0.7", default-features = false, features = ["async"] } parquet.workspace = true paste.workspace = true regex.workspace = true diff --git a/src/common/datasource/src/buffered_writer.rs b/src/common/datasource/src/buffered_writer.rs index 953715b223..63f99d3c3f 100644 --- a/src/common/datasource/src/buffered_writer.rs +++ b/src/common/datasource/src/buffered_writer.rs @@ -14,7 +14,7 @@ use arrow::record_batch::RecordBatch; use async_trait::async_trait; -use datafusion::parquet::format::FileMetaData; +use parquet::file::metadata::ParquetMetaData; use crate::error::Result; @@ -24,5 +24,5 @@ pub trait DfRecordBatchEncoder { #[async_trait] pub trait ArrowWriterCloser { - async fn close(mut self) -> Result; + async fn close(mut self) -> Result; } diff --git a/src/common/datasource/src/file_format.rs b/src/common/datasource/src/file_format.rs index 614be170e8..ba58fad143 100644 --- a/src/common/datasource/src/file_format.rs +++ b/src/common/datasource/src/file_format.rs @@ -40,7 +40,6 @@ use datafusion::datasource::physical_plan::{ use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; -use datatypes::arrow::datatypes::SchemaRef; use futures::{StreamExt, TryStreamExt}; use object_store::ObjectStore; use object_store_opendal::OpendalStore; @@ -303,24 +302,20 @@ where pub async fn file_to_stream( store: &ObjectStore, filename: &str, - file_schema: SchemaRef, file_source: Arc, projection: Option>, compression_type: CompressionType, ) -> Result { let df_compression: DfCompressionType = compression_type.into(); - let config = FileScanConfigBuilder::new( - ObjectStoreUrl::local_filesystem(), - file_schema, - file_source.clone(), - ) - .with_file_group(FileGroup::new(vec![PartitionedFile::new( - filename.to_string(), - 0, - )])) - .with_projection(projection) - .with_file_compression_type(df_compression) - .build(); + let config = + FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source.clone()) + .with_file_group(FileGroup::new(vec![PartitionedFile::new( + filename.to_string(), + 0, + )])) + .with_projection_indices(projection) + .with_file_compression_type(df_compression) + .build(); let store = Arc::new(OpendalStore::new(store.clone())); let file_opener = file_source diff --git a/src/common/datasource/src/file_format/csv.rs b/src/common/datasource/src/file_format/csv.rs index 392f3ff49d..7832c7eae9 100644 --- a/src/common/datasource/src/file_format/csv.rs +++ b/src/common/datasource/src/file_format/csv.rs @@ -440,14 +440,11 @@ mod tests { .await .unwrap(), ); - let csv_source = CsvSource::new(true, b',', b'"') - .with_schema(schema.clone()) - .with_batch_size(8192); + let csv_source = CsvSource::new(schema).with_batch_size(8192); let stream = file_to_stream( &store, compressed_file_path_str, - schema.clone(), csv_source.clone(), None, compression_type, diff --git a/src/common/datasource/src/file_format/json.rs b/src/common/datasource/src/file_format/json.rs index cafcd71372..86376933bd 100644 --- a/src/common/datasource/src/file_format/json.rs +++ b/src/common/datasource/src/file_format/json.rs @@ -347,14 +347,11 @@ mod tests { .await .unwrap(), ); - let json_source = JsonSource::new() - .with_schema(schema.clone()) - .with_batch_size(8192); + let json_source = JsonSource::new(schema).with_batch_size(8192); let stream = file_to_stream( &store, compressed_file_path_str, - schema.clone(), json_source.clone(), None, compression_type, diff --git a/src/common/datasource/src/file_format/parquet.rs b/src/common/datasource/src/file_format/parquet.rs index f2c849b2f6..c2c14b4680 100644 --- a/src/common/datasource/src/file_format/parquet.rs +++ b/src/common/datasource/src/file_format/parquet.rs @@ -18,15 +18,15 @@ use std::sync::Arc; use arrow::record_batch::RecordBatch; use arrow_schema::Schema; use async_trait::async_trait; -use datafusion::datasource::physical_plan::{FileMeta, ParquetFileReaderFactory}; +use datafusion::datasource::physical_plan::ParquetFileReaderFactory; use datafusion::error::Result as DatafusionResult; use datafusion::parquet::arrow::async_reader::AsyncFileReader; use datafusion::parquet::arrow::{ArrowWriter, parquet_to_arrow_schema}; use datafusion::parquet::errors::{ParquetError, Result as ParquetResult}; use datafusion::parquet::file::metadata::ParquetMetaData; -use datafusion::parquet::format::FileMetaData; use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; +use datafusion_datasource::PartitionedFile; use datatypes::schema::SchemaRef; use futures::StreamExt; use futures::future::BoxFuture; @@ -100,11 +100,11 @@ impl ParquetFileReaderFactory for DefaultParquetFileReaderFactory { fn create_reader( &self, _partition_index: usize, - file_meta: FileMeta, + partitioned_file: PartitionedFile, _metadata_size_hint: Option, _metrics: &ExecutionPlanMetricsSet, ) -> DatafusionResult> { - let path = file_meta.location().to_string(); + let path = partitioned_file.path().to_string(); let object_store = self.object_store.clone(); Ok(Box::new(LazyParquetFileReader::new(object_store, path))) @@ -180,7 +180,7 @@ impl DfRecordBatchEncoder for ArrowWriter { #[async_trait] impl ArrowWriterCloser for ArrowWriter { - async fn close(self) -> Result { + async fn close(self) -> Result { self.close().context(error::EncodeRecordBatchSnafu) } } diff --git a/src/common/datasource/src/file_format/tests.rs b/src/common/datasource/src/file_format/tests.rs index d3e5adf5a3..29a23c07da 100644 --- a/src/common/datasource/src/file_format/tests.rs +++ b/src/common/datasource/src/file_format/tests.rs @@ -67,14 +67,14 @@ impl Test<'_> { async fn test_json_opener() { let store = test_store("/"); let schema = test_basic_schema(); - let file_source = Arc::new(JsonSource::new()).with_batch_size(test_util::TEST_BATCH_SIZE); + let file_source = Arc::new(JsonSource::new(schema)).with_batch_size(test_util::TEST_BATCH_SIZE); let path = &find_workspace_path("/src/common/datasource/tests/json/basic.json") .display() .to_string(); let tests = [ Test { - config: scan_config(schema.clone(), None, path, file_source.clone()), + config: scan_config(None, path, file_source.clone()), file_source: file_source.clone(), expected: vec![ "+-----+-------+", @@ -87,7 +87,7 @@ async fn test_json_opener() { ], }, Test { - config: scan_config(schema, Some(1), path, file_source.clone()), + config: scan_config(Some(1), path, file_source.clone()), file_source, expected: vec![ "+-----+------+", @@ -112,13 +112,11 @@ async fn test_csv_opener() { .display() .to_string(); - let file_source = CsvSource::new(true, b',', b'"') - .with_batch_size(test_util::TEST_BATCH_SIZE) - .with_schema(schema.clone()); + let file_source = CsvSource::new(schema).with_batch_size(test_util::TEST_BATCH_SIZE); let tests = [ Test { - config: scan_config(schema.clone(), None, path, file_source.clone()), + config: scan_config(None, path, file_source.clone()), file_source: file_source.clone(), expected: vec![ "+-----+-------+---------------------+----------+------------+", @@ -131,7 +129,7 @@ async fn test_csv_opener() { ], }, Test { - config: scan_config(schema, Some(1), path, file_source.clone()), + config: scan_config(Some(1), path, file_source.clone()), file_source, expected: vec![ "+-----+------+---------------------+----------+------------+", @@ -158,10 +156,10 @@ async fn test_parquet_exec() { .display() .to_string(); - let parquet_source = ParquetSource::default() + let parquet_source = ParquetSource::new(schema) .with_parquet_file_reader_factory(Arc::new(DefaultParquetFileReaderFactory::new(store))); - let config = scan_config(schema, None, path, Arc::new(parquet_source)); + let config = scan_config(None, path, Arc::new(parquet_source)); let exec = DataSourceExec::from_data_source(config); let ctx = SessionContext::new(); @@ -197,11 +195,11 @@ async fn test_orc_opener() { let store = test_store("/"); let schema = Arc::new(OrcFormat.infer_schema(&store, path).await.unwrap()); - let file_source = Arc::new(OrcSource::default()); + let file_source = Arc::new(OrcSource::new(schema.into())); let tests = [ Test { - config: scan_config(schema.clone(), None, path, file_source.clone()), + config: scan_config(None, path, file_source.clone()), file_source: file_source.clone(), expected: vec![ "+----------+-----+-------+------------+-----+-----+-------+--------------------+------------------------+-----------+---------------+------------+----------------+---------------+-------------------+--------------+---------------+---------------+----------------------------+-------------+", @@ -216,7 +214,7 @@ async fn test_orc_opener() { ], }, Test { - config: scan_config(schema.clone(), Some(1), path, file_source.clone()), + config: scan_config(Some(1), path, file_source.clone()), file_source, expected: vec![ "+----------+-----+------+------------+---+-----+-------+--------------------+------------------------+-----------+---------------+------------+----------------+---------------+-------------------+--------------+---------------+---------------+-------------------------+-------------+", diff --git a/src/common/datasource/src/test_util.rs b/src/common/datasource/src/test_util.rs index 244df3b7a5..f647d424ad 100644 --- a/src/common/datasource/src/test_util.rs +++ b/src/common/datasource/src/test_util.rs @@ -80,7 +80,6 @@ pub fn csv_basic_schema() -> SchemaRef { } pub(crate) fn scan_config( - file_schema: SchemaRef, limit: Option, filename: &str, file_source: Arc, @@ -89,7 +88,7 @@ pub(crate) fn scan_config( let filename = &filename.replace('\\', "/"); let file_group = FileGroup::new(vec![PartitionedFile::new(filename.clone(), 4096)]); - FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_schema, file_source) + FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source) .with_file_group(file_group) .with_limit(limit) .build() @@ -109,7 +108,7 @@ pub async fn setup_stream_to_json_test(origin_path: &str, threshold: impl Fn(usi let size = store.read(origin_path).await.unwrap().len(); - let config = scan_config(schema, None, origin_path, Arc::new(JsonSource::new())); + let config = scan_config(None, origin_path, Arc::new(JsonSource::new(schema))); let stream = FileStream::new( &config, 0, @@ -151,10 +150,8 @@ pub async fn setup_stream_to_csv_test( let schema = csv_basic_schema(); - let csv_source = CsvSource::new(true, b',', b'"') - .with_schema(schema.clone()) - .with_batch_size(TEST_BATCH_SIZE); - let config = scan_config(schema, None, origin_path, csv_source.clone()); + let csv_source = CsvSource::new(schema).with_batch_size(TEST_BATCH_SIZE); + let config = scan_config(None, origin_path, csv_source.clone()); let size = store.read(origin_path).await.unwrap().len(); let csv_opener = csv_source.create_file_opener( diff --git a/src/common/function/src/admin/flush_compact_region.rs b/src/common/function/src/admin/flush_compact_region.rs index 3b5be3dc0c..60fd19ef5a 100644 --- a/src/common/function/src/admin/flush_compact_region.rs +++ b/src/common/function/src/admin/flush_compact_region.rs @@ -104,7 +104,8 @@ mod tests { assert!(matches!(f.signature(), datafusion_expr::Signature { type_signature: datafusion_expr::TypeSignature::Uniform(1, valid_types), - volatility: datafusion_expr::Volatility::Immutable + volatility: datafusion_expr::Volatility::Immutable, + .. } if valid_types == &ConcreteDataType::numerics().into_iter().map(|dt| { use datatypes::data_type::DataType; dt.as_arrow_type() }).collect::>())); } diff --git a/src/common/function/src/admin/flush_compact_table.rs b/src/common/function/src/admin/flush_compact_table.rs index e45f2f0f84..3298a95061 100644 --- a/src/common/function/src/admin/flush_compact_table.rs +++ b/src/common/function/src/admin/flush_compact_table.rs @@ -331,7 +331,8 @@ mod tests { assert!(matches!(f.signature(), datafusion_expr::Signature { type_signature: datafusion_expr::TypeSignature::Uniform(1, valid_types), - volatility: datafusion_expr::Volatility::Immutable + volatility: datafusion_expr::Volatility::Immutable, + .. } if valid_types == &vec![ArrowDataType::Utf8])); } diff --git a/src/common/function/src/admin/migrate_region.rs b/src/common/function/src/admin/migrate_region.rs index b980cb4cb9..91b5540b1a 100644 --- a/src/common/function/src/admin/migrate_region.rs +++ b/src/common/function/src/admin/migrate_region.rs @@ -145,7 +145,8 @@ mod tests { assert!(matches!(f.signature(), datafusion_expr::Signature { type_signature: datafusion_expr::TypeSignature::OneOf(sigs), - volatility: datafusion_expr::Volatility::Immutable + volatility: datafusion_expr::Volatility::Immutable, + .. } if sigs.len() == 2)); } diff --git a/src/common/function/src/aggrs/aggr_wrapper.rs b/src/common/function/src/aggrs/aggr_wrapper.rs index 54dc1ac78e..ab0e13e1c4 100644 --- a/src/common/function/src/aggrs/aggr_wrapper.rs +++ b/src/common/function/src/aggrs/aggr_wrapper.rs @@ -341,6 +341,7 @@ impl AggregateUDFImpl for StateWrapper { name: acc_args.name, is_distinct: acc_args.is_distinct, exprs: acc_args.exprs, + expr_fields: acc_args.expr_fields, }; self.inner.accumulator(acc_args)? }; diff --git a/src/common/function/src/aggrs/aggr_wrapper/tests.rs b/src/common/function/src/aggrs/aggr_wrapper/tests.rs index 97a5a792d9..f23277f818 100644 --- a/src/common/function/src/aggrs/aggr_wrapper/tests.rs +++ b/src/common/function/src/aggrs/aggr_wrapper/tests.rs @@ -650,7 +650,7 @@ async fn test_last_value_order_by_udaf() { DataType::Timestamp(arrow_schema::TimeUnit::Millisecond, None), true ), // ordering field is added to state fields too - Field::new("is_set", DataType::Boolean, true) + Field::new("last_value[last_value_is_set]", DataType::Boolean, true) ] .into() ), @@ -735,7 +735,7 @@ async fn test_last_value_order_by_udaf() { DataType::Timestamp(arrow_schema::TimeUnit::Millisecond, None), true, ), - Field::new("is_set", DataType::Boolean, true), + Field::new("last_value[last_value_is_set]", DataType::Boolean, true), ] .into(), vec![ diff --git a/src/common/function/src/scalars/date/date_add.rs b/src/common/function/src/scalars/date/date_add.rs index 37d1fe12b1..34fc11fe56 100644 --- a/src/common/function/src/scalars/date/date_add.rs +++ b/src/common/function/src/scalars/date/date_add.rs @@ -122,7 +122,8 @@ mod tests { matches!(f.signature(), Signature { type_signature: TypeSignature::OneOf(sigs), - volatility: Volatility::Immutable + volatility: Volatility::Immutable, + .. } if sigs.len() == 15), "{:?}", f.signature() diff --git a/src/common/function/src/scalars/date/date_format.rs b/src/common/function/src/scalars/date/date_format.rs index dfa5a444ca..ed2d67fa60 100644 --- a/src/common/function/src/scalars/date/date_format.rs +++ b/src/common/function/src/scalars/date/date_format.rs @@ -193,7 +193,8 @@ mod tests { assert!(matches!(f.signature(), Signature { type_signature: TypeSignature::OneOf(sigs), - volatility: Volatility::Immutable + volatility: Volatility::Immutable, + .. } if sigs.len() == 6)); } diff --git a/src/common/function/src/scalars/date/date_sub.rs b/src/common/function/src/scalars/date/date_sub.rs index 12fb9125a9..32e86bdc8b 100644 --- a/src/common/function/src/scalars/date/date_sub.rs +++ b/src/common/function/src/scalars/date/date_sub.rs @@ -120,7 +120,8 @@ mod tests { matches!(f.signature(), Signature { type_signature: TypeSignature::OneOf(sigs), - volatility: Volatility::Immutable + volatility: Volatility::Immutable, + .. } if sigs.len() == 15), "{:?}", f.signature() diff --git a/src/common/function/src/scalars/json/json_get.rs b/src/common/function/src/scalars/json/json_get.rs index 40bd78cbc4..ce2ee08f91 100644 --- a/src/common/function/src/scalars/json/json_get.rs +++ b/src/common/function/src/scalars/json/json_get.rs @@ -25,7 +25,6 @@ use datafusion_common::arrow::array::{ }; use datafusion_common::arrow::datatypes::DataType; use datafusion_common::{DataFusionError, Result}; -use datafusion_expr::type_coercion::aggregates::STRINGS; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, Volatility}; use datatypes::arrow_array::{int_array_value_at_index, string_array_value_at_index}; use datatypes::json::JsonStructureSettings; @@ -519,7 +518,7 @@ impl Default for JsonGetObject { DataType::LargeBinary, DataType::BinaryView, ], - STRINGS.to_vec(), + vec![DataType::UInt8, DataType::LargeUtf8, DataType::Utf8View], ), } } diff --git a/src/common/function/src/scalars/math/rate.rs b/src/common/function/src/scalars/math/rate.rs index c8be471e0e..b2a45b2036 100644 --- a/src/common/function/src/scalars/math/rate.rs +++ b/src/common/function/src/scalars/math/rate.rs @@ -99,7 +99,8 @@ mod tests { assert!(matches!(rate.signature(), Signature { type_signature: TypeSignature::Uniform(2, valid_types), - volatility: Volatility::Immutable + volatility: Volatility::Immutable, + .. } if valid_types == NUMERICS )); let values = vec![1.0, 3.0, 6.0]; diff --git a/src/common/function/src/scalars/vector/convert/vector_to_string.rs b/src/common/function/src/scalars/vector/convert/vector_to_string.rs index 1143abd4b9..5acbec6cab 100644 --- a/src/common/function/src/scalars/vector/convert/vector_to_string.rs +++ b/src/common/function/src/scalars/vector/convert/vector_to_string.rs @@ -19,8 +19,10 @@ use datafusion_common::DataFusionError; use datafusion_common::arrow::array::{Array, AsArray, StringViewBuilder}; use datafusion_common::arrow::compute; use datafusion_common::arrow::datatypes::DataType; -use datafusion_expr::type_coercion::aggregates::BINARYS; -use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, TypeSignature, Volatility}; +use datafusion_common::types::logical_binary; +use datafusion_expr::{ + Coercion, ColumnarValue, ScalarFunctionArgs, Signature, TypeSignatureClass, Volatility, +}; use datatypes::types::vector_type_value_to_string; use crate::function::{Function, extract_args}; @@ -35,11 +37,10 @@ pub struct VectorToStringFunction { impl Default for VectorToStringFunction { fn default() -> Self { Self { - signature: Signature::one_of( - vec![ - TypeSignature::Uniform(1, vec![DataType::BinaryView]), - TypeSignature::Uniform(1, BINARYS.to_vec()), - ], + signature: Signature::coercible( + vec![Coercion::new_exact(TypeSignatureClass::Native( + logical_binary(), + ))], Volatility::Immutable, ), } diff --git a/src/common/function/src/scalars/vector/elem_avg.rs b/src/common/function/src/scalars/vector/elem_avg.rs index 7ebee3ad41..f3ffcac3ae 100644 --- a/src/common/function/src/scalars/vector/elem_avg.rs +++ b/src/common/function/src/scalars/vector/elem_avg.rs @@ -15,10 +15,10 @@ use std::fmt::Display; use datafusion::arrow::datatypes::DataType; -use datafusion::logical_expr::ColumnarValue; +use datafusion::logical_expr::{Coercion, ColumnarValue, TypeSignature, TypeSignatureClass}; use datafusion_common::ScalarValue; -use datafusion_expr::type_coercion::aggregates::{BINARYS, STRINGS}; -use datafusion_expr::{ScalarFunctionArgs, Signature, TypeSignature, Volatility}; +use datafusion_common::types::{logical_binary, logical_string}; +use datafusion_expr::{ScalarFunctionArgs, Signature, Volatility}; use nalgebra::DVectorView; use crate::function::Function; @@ -36,9 +36,12 @@ impl Default for ElemAvgFunction { Self { signature: Signature::one_of( vec![ - TypeSignature::Uniform(1, STRINGS.to_vec()), - TypeSignature::Uniform(1, BINARYS.to_vec()), - TypeSignature::Uniform(1, vec![DataType::BinaryView]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Native(logical_binary()), + )]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Native(logical_string()), + )]), ], Volatility::Immutable, ), diff --git a/src/common/function/src/scalars/vector/elem_product.rs b/src/common/function/src/scalars/vector/elem_product.rs index 5a3a7416fb..140299d380 100644 --- a/src/common/function/src/scalars/vector/elem_product.rs +++ b/src/common/function/src/scalars/vector/elem_product.rs @@ -15,10 +15,10 @@ use std::fmt::Display; use datafusion::arrow::datatypes::DataType; -use datafusion::logical_expr::ColumnarValue; -use datafusion::logical_expr_common::type_coercion::aggregates::{BINARYS, STRINGS}; +use datafusion::logical_expr::{Coercion, ColumnarValue, TypeSignature, TypeSignatureClass}; use datafusion_common::ScalarValue; -use datafusion_expr::{ScalarFunctionArgs, Signature, TypeSignature, Volatility}; +use datafusion_common::types::{logical_binary, logical_string}; +use datafusion_expr::{ScalarFunctionArgs, Signature, Volatility}; use nalgebra::DVectorView; use crate::function::Function; @@ -49,9 +49,12 @@ impl Default for ElemProductFunction { Self { signature: Signature::one_of( vec![ - TypeSignature::Uniform(1, STRINGS.to_vec()), - TypeSignature::Uniform(1, BINARYS.to_vec()), - TypeSignature::Uniform(1, vec![DataType::BinaryView]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Native(logical_binary()), + )]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Native(logical_string()), + )]), ], Volatility::Immutable, ), diff --git a/src/common/function/src/scalars/vector/elem_sum.rs b/src/common/function/src/scalars/vector/elem_sum.rs index ff44e49ccf..eaad10cb90 100644 --- a/src/common/function/src/scalars/vector/elem_sum.rs +++ b/src/common/function/src/scalars/vector/elem_sum.rs @@ -15,9 +15,9 @@ use std::fmt::Display; use datafusion::arrow::datatypes::DataType; -use datafusion::logical_expr::ColumnarValue; +use datafusion::logical_expr::{Coercion, ColumnarValue, TypeSignatureClass}; use datafusion_common::ScalarValue; -use datafusion_expr::type_coercion::aggregates::{BINARYS, STRINGS}; +use datafusion_common::types::{logical_binary, logical_string}; use datafusion_expr::{ScalarFunctionArgs, Signature, TypeSignature, Volatility}; use nalgebra::DVectorView; @@ -36,9 +36,12 @@ impl Default for ElemSumFunction { Self { signature: Signature::one_of( vec![ - TypeSignature::Uniform(1, STRINGS.to_vec()), - TypeSignature::Uniform(1, BINARYS.to_vec()), - TypeSignature::Uniform(1, vec![DataType::BinaryView]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Native(logical_binary()), + )]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Native(logical_string()), + )]), ], Volatility::Immutable, ), diff --git a/src/common/function/src/scalars/vector/vector_dim.rs b/src/common/function/src/scalars/vector/vector_dim.rs index 6b42095185..7feaab4c28 100644 --- a/src/common/function/src/scalars/vector/vector_dim.rs +++ b/src/common/function/src/scalars/vector/vector_dim.rs @@ -15,9 +15,9 @@ use std::fmt::Display; use datafusion::arrow::datatypes::DataType; -use datafusion::logical_expr::ColumnarValue; -use datafusion::logical_expr_common::type_coercion::aggregates::{BINARYS, STRINGS}; +use datafusion::logical_expr::{Coercion, ColumnarValue, TypeSignatureClass}; use datafusion_common::ScalarValue; +use datafusion_common::types::{logical_binary, logical_string}; use datafusion_expr::{ScalarFunctionArgs, Signature, TypeSignature, Volatility}; use crate::function::Function; @@ -49,8 +49,12 @@ impl Default for VectorDimFunction { Self { signature: Signature::one_of( vec![ - TypeSignature::Uniform(1, STRINGS.to_vec()), - TypeSignature::Uniform(1, BINARYS.to_vec()), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Native(logical_binary()), + )]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Native(logical_string()), + )]), ], Volatility::Immutable, ), diff --git a/src/common/function/src/scalars/vector/vector_norm.rs b/src/common/function/src/scalars/vector/vector_norm.rs index 31beea64d2..97e028cbf4 100644 --- a/src/common/function/src/scalars/vector/vector_norm.rs +++ b/src/common/function/src/scalars/vector/vector_norm.rs @@ -15,9 +15,9 @@ use std::fmt::Display; use datafusion::arrow::datatypes::DataType; -use datafusion::logical_expr::ColumnarValue; -use datafusion::logical_expr_common::type_coercion::aggregates::{BINARYS, STRINGS}; +use datafusion::logical_expr::{Coercion, ColumnarValue, TypeSignatureClass}; use datafusion_common::ScalarValue; +use datafusion_common::types::{logical_binary, logical_string}; use datafusion_expr::{ScalarFunctionArgs, Signature, TypeSignature, Volatility}; use nalgebra::DVectorView; @@ -52,9 +52,12 @@ impl Default for VectorNormFunction { Self { signature: Signature::one_of( vec![ - TypeSignature::Uniform(1, STRINGS.to_vec()), - TypeSignature::Uniform(1, BINARYS.to_vec()), - TypeSignature::Uniform(1, vec![DataType::BinaryView]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Native(logical_binary()), + )]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Native(logical_string()), + )]), ], Volatility::Immutable, ), diff --git a/src/common/function/src/system/procedure_state.rs b/src/common/function/src/system/procedure_state.rs index 8c2856c636..7f7388edae 100644 --- a/src/common/function/src/system/procedure_state.rs +++ b/src/common/function/src/system/procedure_state.rs @@ -106,7 +106,8 @@ mod tests { assert!(matches!(f.signature(), datafusion_expr::Signature { type_signature: datafusion_expr::TypeSignature::Uniform(1, valid_types), - volatility: datafusion_expr::Volatility::Immutable + volatility: datafusion_expr::Volatility::Immutable, + .. } if valid_types == &vec![ArrowDataType::Utf8])); } diff --git a/src/common/grpc/src/flight.rs b/src/common/grpc/src/flight.rs index ae06b02558..5fc115a60e 100644 --- a/src/common/grpc/src/flight.rs +++ b/src/common/grpc/src/flight.rs @@ -103,10 +103,11 @@ impl FlightEncoder { FlightMessage::RecordBatch(record_batch) => { let (encoded_dictionaries, encoded_batch) = self .data_gen - .encoded_batch( + .encode( &record_batch, &mut self.dictionary_tracker, &self.write_options, + &mut Default::default(), ) .expect("DictionaryTracker configured above to not fail on replacement"); diff --git a/src/common/pprof/Cargo.toml b/src/common/pprof/Cargo.toml index 1e7d008034..6d96b26d9f 100644 --- a/src/common/pprof/Cargo.toml +++ b/src/common/pprof/Cargo.toml @@ -7,7 +7,6 @@ license.workspace = true [dependencies] common-error.workspace = true common-macro.workspace = true -prost.workspace = true snafu.workspace = true tokio.workspace = true diff --git a/src/datatypes/src/value.rs b/src/datatypes/src/value.rs index 1c7dc35de6..2211d24262 100644 --- a/src/datatypes/src/value.rs +++ b/src/datatypes/src/value.rs @@ -1145,10 +1145,11 @@ impl TryFrom for Value { ScalarValue::List(array) => { // this is for item type let datatype = ConcreteDataType::try_from(&array.value_type())?; - let items = ScalarValue::convert_array_to_scalar_vec(array.as_ref()) - .context(ConvertArrowArrayToScalarsSnafu)? + let scalar_values = ScalarValue::convert_array_to_scalar_vec(array.as_ref()) + .context(ConvertArrowArrayToScalarsSnafu)?; + let items = scalar_values .into_iter() - .flatten() + .flat_map(|v| v.unwrap_or_else(|| vec![ScalarValue::Null])) .map(|x| x.try_into()) .collect::>>()?; Value::List(ListValue::new(items, Arc::new(datatype))) @@ -2997,6 +2998,7 @@ pub(crate) mod tests { .unwrap() .into_iter() .flatten() + .flatten() .collect::>(); assert_eq!( vs, diff --git a/src/datatypes/src/vectors/boolean.rs b/src/datatypes/src/vectors/boolean.rs index 6013762133..4ea7a4591b 100644 --- a/src/datatypes/src/vectors/boolean.rs +++ b/src/datatypes/src/vectors/boolean.rs @@ -13,7 +13,6 @@ // limitations under the License. use std::any::Any; -use std::borrow::Borrow; use std::sync::Arc; use arrow::array::{Array, ArrayBuilder, ArrayIter, ArrayRef, BooleanArray, BooleanBuilder}; @@ -69,8 +68,8 @@ impl From>> for BooleanVector { } } -impl>> FromIterator for BooleanVector { - fn from_iter>(iter: I) -> Self { +impl FromIterator> for BooleanVector { + fn from_iter>>(iter: T) -> Self { BooleanVector { array: BooleanArray::from_iter(iter), } @@ -303,7 +302,7 @@ mod tests { #[test] fn test_boolean_vector_from_iter() { let input = vec![Some(false), Some(true), Some(false), Some(true)]; - let vec = input.iter().collect::(); + let vec = input.iter().cloned().collect::(); assert_eq!(4, vec.len()); for (i, v) in input.into_iter().enumerate() { assert_eq!(v, vec.get_data(i), "Failed at {i}") diff --git a/src/datatypes/src/vectors/decimal.rs b/src/datatypes/src/vectors/decimal.rs index 9a1478dfff..46fa73297a 100644 --- a/src/datatypes/src/vectors/decimal.rs +++ b/src/datatypes/src/vectors/decimal.rs @@ -83,8 +83,6 @@ impl Decimal128Vector { /// For example: /// value = 12345, precision = 3, return error. pub fn with_precision_and_scale(self, precision: u8, scale: i8) -> Result { - // validate if precision is too small - self.validate_decimal_precision(precision)?; let array = self .array .with_precision_and_scale(precision, scale) @@ -124,7 +122,7 @@ impl Decimal128Vector { } /// Validate decimal precision, if precision is invalid, return error. - fn validate_decimal_precision(&self, precision: u8) -> Result<()> { + pub fn validate_decimal_precision(&self, precision: u8) -> Result<()> { self.array .validate_decimal_precision(precision) .context(ValueExceedsPrecisionSnafu { precision }) @@ -564,7 +562,9 @@ pub mod tests { let decimal_vector = decimal_builder.finish(); assert_eq!(decimal_vector.precision(), 38); assert_eq!(decimal_vector.scale(), 10); - let result = decimal_vector.with_precision_and_scale(3, 2); + let result = decimal_vector + .with_precision_and_scale(3, 2) + .and_then(|x| x.validate_decimal_precision(3)); assert_eq!( "Value exceeds the precision 3 bound", result.unwrap_err().to_string() diff --git a/src/datatypes/src/vectors/helper.rs b/src/datatypes/src/vectors/helper.rs index 1bc6951ce3..39d0c324e7 100644 --- a/src/datatypes/src/vectors/helper.rs +++ b/src/datatypes/src/vectors/helper.rs @@ -170,10 +170,11 @@ impl Helper { ScalarValue::List(array) => { let item_type = Arc::new(ConcreteDataType::try_from(&array.value_type())?); let mut builder = ListVectorBuilder::with_type_capacity(item_type.clone(), 1); - let values = ScalarValue::convert_array_to_scalar_vec(array.as_ref()) - .context(ConvertArrowArrayToScalarsSnafu)? + let scalar_values = ScalarValue::convert_array_to_scalar_vec(array.as_ref()) + .context(ConvertArrowArrayToScalarsSnafu)?; + let values = scalar_values .into_iter() - .flatten() + .flat_map(|v| v.unwrap_or_else(|| vec![ScalarValue::Null])) .map(ScalarValue::try_into) .collect::>>()?; builder.push(Some(ListValueRef::Ref { diff --git a/src/file-engine/src/query/file_stream.rs b/src/file-engine/src/query/file_stream.rs index 199bb5e0bd..fa0fcdcf73 100644 --- a/src/file-engine/src/query/file_stream.rs +++ b/src/file-engine/src/query/file_stream.rs @@ -18,6 +18,7 @@ use common_datasource::file_format::Format; use common_datasource::file_format::csv::CsvFormat; use common_datasource::file_format::parquet::DefaultParquetFileReaderFactory; use datafusion::common::ToDFSchema; +use datafusion::config::CsvOptions; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::object_store::ObjectStoreUrl; use datafusion::datasource::physical_plan::{ @@ -34,7 +35,6 @@ use datafusion::prelude::SessionContext; use datafusion_expr::expr::Expr; use datafusion_expr::utils::conjunction; use datafusion_orc::OrcSource; -use datatypes::arrow::datatypes::Schema as ArrowSchema; use datatypes::schema::SchemaRef; use object_store::ObjectStore; use snafu::ResultExt; @@ -45,7 +45,6 @@ const DEFAULT_BATCH_SIZE: usize = 8192; fn build_record_batch_stream( scan_plan_config: &ScanPlanConfig, - file_schema: Arc, limit: Option, file_source: Arc, ) -> Result { @@ -55,15 +54,12 @@ fn build_record_batch_stream( .map(|filename| PartitionedFile::new(filename.clone(), 0)) .collect::>(); - let config = FileScanConfigBuilder::new( - ObjectStoreUrl::local_filesystem(), - file_schema, - file_source.clone(), - ) - .with_projection(scan_plan_config.projection.cloned()) - .with_limit(limit) - .with_file_group(FileGroup::new(files)) - .build(); + let config = + FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source.clone()) + .with_projection_indices(scan_plan_config.projection.cloned()) + .with_limit(limit) + .with_file_group(FileGroup::new(files)) + .build(); let store = Arc::new(object_store_opendal::OpendalStore::new( scan_plan_config.store.clone(), @@ -89,11 +85,14 @@ fn new_csv_stream( // push down limit only if there is no filter let limit = config.filters.is_empty().then_some(config.limit).flatten(); - let csv_source = CsvSource::new(format.has_header, format.delimiter, b'"') - .with_schema(file_schema.clone()) + let options = CsvOptions::default() + .with_has_header(format.has_header) + .with_delimiter(format.delimiter); + let csv_source = CsvSource::new(file_schema) + .with_csv_options(options) .with_batch_size(DEFAULT_BATCH_SIZE); - build_record_batch_stream(config, file_schema, limit, csv_source) + build_record_batch_stream(config, limit, csv_source) } fn new_json_stream(config: &ScanPlanConfig) -> Result { @@ -102,8 +101,8 @@ fn new_json_stream(config: &ScanPlanConfig) -> Result>(), ); - let mut parquet_source = ParquetSource::default().with_parquet_file_reader_factory(Arc::new( - DefaultParquetFileReaderFactory::new(store.clone()), - )); + let mut parquet_source = ParquetSource::new(file_schema.clone()) + .with_parquet_file_reader_factory(Arc::new(DefaultParquetFileReaderFactory::new( + store.clone(), + ))); // build predicate filter let filters = filters.to_vec(); @@ -143,15 +143,12 @@ fn new_parquet_stream_with_exec_plan( parquet_source = parquet_source.with_predicate(filters); }; - let file_scan_config = FileScanConfigBuilder::new( - ObjectStoreUrl::local_filesystem(), - file_schema, - Arc::new(parquet_source), - ) - .with_file_group(file_group) - .with_projection(projection.cloned()) - .with_limit(*limit) - .build(); + let file_scan_config = + FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), Arc::new(parquet_source)) + .with_file_group(file_group) + .with_projection_indices(projection.cloned()) + .with_limit(*limit) + .build(); // TODO(ruihang): get this from upper layer let task_ctx = SessionContext::default().task_ctx(); @@ -170,8 +167,8 @@ fn new_orc_stream(config: &ScanPlanConfig) -> Result Option> { match self { - CacheStrategy::EnableAll(cache_manager) => { - cache_manager.get_parquet_meta_data(file_id, metrics).await - } - CacheStrategy::Compaction(cache_manager) => { - cache_manager.get_parquet_meta_data(file_id, metrics).await + CacheStrategy::EnableAll(cache_manager) | CacheStrategy::Compaction(cache_manager) => { + cache_manager + .get_parquet_meta_data(file_id, metrics, page_index_policy) + .await } CacheStrategy::Disabled => { metrics.cache_miss += 1; @@ -340,6 +340,7 @@ impl CacheManager { &self, file_id: RegionFileId, metrics: &mut MetadataCacheMetrics, + page_index_policy: PageIndexPolicy, ) -> Option> { // Try to get metadata from sst meta cache if let Some(metadata) = self.get_parquet_meta_data_from_mem_cache(file_id) { @@ -352,7 +353,7 @@ impl CacheManager { if let Some(write_cache) = &self.write_cache && let Some(metadata) = write_cache .file_cache() - .get_parquet_meta_data(key, metrics) + .get_parquet_meta_data(key, metrics, page_index_policy) .await { metrics.file_cache_hit += 1; @@ -893,7 +894,7 @@ mod tests { cache.put_parquet_meta_data(file_id, metadata); assert!( cache - .get_parquet_meta_data(file_id, &mut metrics) + .get_parquet_meta_data(file_id, &mut metrics, Default::default()) .await .is_none() ); @@ -923,7 +924,7 @@ mod tests { let file_id = RegionFileId::new(region_id, FileId::random()); assert!( cache - .get_parquet_meta_data(file_id, &mut metrics) + .get_parquet_meta_data(file_id, &mut metrics, Default::default()) .await .is_none() ); @@ -931,14 +932,14 @@ mod tests { cache.put_parquet_meta_data(file_id, metadata); assert!( cache - .get_parquet_meta_data(file_id, &mut metrics) + .get_parquet_meta_data(file_id, &mut metrics, Default::default()) .await .is_some() ); cache.remove_parquet_meta_data(file_id); assert!( cache - .get_parquet_meta_data(file_id, &mut metrics) + .get_parquet_meta_data(file_id, &mut metrics, Default::default()) .await .is_none() ); diff --git a/src/mito2/src/cache/cache_size.rs b/src/mito2/src/cache/cache_size.rs index 63e7b88cfa..0383aec025 100644 --- a/src/mito2/src/cache/cache_size.rs +++ b/src/mito2/src/cache/cache_size.rs @@ -16,11 +16,13 @@ use std::mem; +use parquet::basic::ColumnOrder; use parquet::file::metadata::{ - FileMetaData, ParquetColumnIndex, ParquetMetaData, ParquetOffsetIndex, RowGroupMetaData, + FileMetaData, KeyValue, ParquetColumnIndex, ParquetMetaData, ParquetOffsetIndex, + RowGroupMetaData, }; -use parquet::file::page_index::index::Index; -use parquet::format::{ColumnOrder, KeyValue, PageLocation}; +use parquet::file::page_index::column_index::ColumnIndexMetaData as Index; +use parquet::file::page_index::offset_index::PageLocation; use parquet::schema::types::{ColumnDescriptor, SchemaDescriptor, Type}; /// Returns estimated size of [ParquetMetaData]. diff --git a/src/mito2/src/cache/file_cache.rs b/src/mito2/src/cache/file_cache.rs index aeea238fdc..32a276d0e4 100644 --- a/src/mito2/src/cache/file_cache.rs +++ b/src/mito2/src/cache/file_cache.rs @@ -28,7 +28,7 @@ use moka::notification::RemovalCause; use moka::policy::EvictionPolicy; use object_store::util::join_path; use object_store::{ErrorKind, ObjectStore, Reader}; -use parquet::file::metadata::ParquetMetaData; +use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData}; use snafu::ResultExt; use store_api::storage::{FileId, RegionId}; use tokio::sync::mpsc::{Sender, UnboundedReceiver}; @@ -571,6 +571,7 @@ impl FileCache { &self, key: IndexKey, cache_metrics: &mut MetadataCacheMetrics, + page_index_policy: PageIndexPolicy, ) -> Option { // Check if file cache contains the key if let Some(index_value) = self.inner.parquet_index.get(&key).await { @@ -578,7 +579,8 @@ impl FileCache { let local_store = self.local_store(); let file_path = self.inner.cache_file_path(key); let file_size = index_value.file_size as u64; - let metadata_loader = MetadataLoader::new(local_store, &file_path, file_size); + let mut metadata_loader = MetadataLoader::new(local_store, &file_path, file_size); + metadata_loader.with_page_index_policy(page_index_policy); match metadata_loader.load(cache_metrics).await { Ok(metadata) => { diff --git a/src/mito2/src/cache/test_util.rs b/src/mito2/src/cache/test_util.rs index a891ae7f72..65ad9d87eb 100644 --- a/src/mito2/src/cache/test_util.rs +++ b/src/mito2/src/cache/test_util.rs @@ -24,6 +24,7 @@ use object_store::services::Fs; use parquet::arrow::ArrowWriter; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use parquet::file::metadata::ParquetMetaData; +use parquet::file::statistics::Statistics; /// Returns a parquet meta data. pub(crate) fn parquet_meta() -> Arc { @@ -49,3 +50,60 @@ pub(crate) fn new_fs_store(path: &str) -> ObjectStore { let builder = Fs::default(); ObjectStore::new(builder.root(path)).unwrap().finish() } + +pub(crate) fn assert_parquet_metadata_equal(x: Arc, y: Arc) { + // Normalize the statistics in parquet metadata because the flag "min_max_backwards_compatible" + // is not persisted across parquet metadata writer and reader. + fn normalize_statistics(metadata: ParquetMetaData) -> ParquetMetaData { + let unset_min_max_backwards_compatible_flag = |stats: Statistics| -> Statistics { + match stats { + Statistics::Boolean(stats) => { + Statistics::Boolean(stats.with_backwards_compatible_min_max(false)) + } + Statistics::Int32(stats) => { + Statistics::Int32(stats.with_backwards_compatible_min_max(false)) + } + Statistics::Int64(stats) => { + Statistics::Int64(stats.with_backwards_compatible_min_max(false)) + } + Statistics::Int96(stats) => { + Statistics::Int96(stats.with_backwards_compatible_min_max(false)) + } + Statistics::Float(stats) => { + Statistics::Float(stats.with_backwards_compatible_min_max(false)) + } + Statistics::Double(stats) => { + Statistics::Double(stats.with_backwards_compatible_min_max(false)) + } + Statistics::ByteArray(stats) => { + Statistics::ByteArray(stats.with_backwards_compatible_min_max(false)) + } + Statistics::FixedLenByteArray(stats) => { + Statistics::FixedLenByteArray(stats.with_backwards_compatible_min_max(false)) + } + } + }; + + let mut metadata_builder = metadata.into_builder(); + for rg in metadata_builder.take_row_groups() { + let mut rg_builder = rg.into_builder(); + for col in rg_builder.take_columns() { + let stats = col + .statistics() + .cloned() + .map(unset_min_max_backwards_compatible_flag); + let mut col_builder = col.into_builder().clear_statistics(); + if let Some(stats) = stats { + col_builder = col_builder.set_statistics(stats); + } + rg_builder = rg_builder.add_column_metadata(col_builder.build().unwrap()); + } + metadata_builder = metadata_builder.add_row_group(rg_builder.build().unwrap()); + } + metadata_builder.build() + } + + let x = normalize_statistics(Arc::unwrap_or_clone(x)); + let y = normalize_statistics(Arc::unwrap_or_clone(y)); + assert_eq!(x, y); +} diff --git a/src/mito2/src/cache/write_cache.rs b/src/mito2/src/cache/write_cache.rs index e074b7de7b..accb7dfcff 100644 --- a/src/mito2/src/cache/write_cache.rs +++ b/src/mito2/src/cache/write_cache.rs @@ -470,11 +470,12 @@ impl UploadTracker { mod tests { use common_test_util::temp_dir::create_temp_dir; use object_store::ATOMIC_WRITE_DIR; + use parquet::file::metadata::PageIndexPolicy; use store_api::region_request::PathType; use super::*; use crate::access_layer::OperationType; - use crate::cache::test_util::new_fs_store; + use crate::cache::test_util::{assert_parquet_metadata_equal, new_fs_store}; use crate::cache::{CacheManager, CacheStrategy}; use crate::error::InvalidBatchSnafu; use crate::read::Source; @@ -482,8 +483,7 @@ mod tests { use crate::sst::parquet::reader::ParquetReaderBuilder; use crate::test_util::TestEnv; use crate::test_util::sst_util::{ - assert_parquet_metadata_eq, new_batch_by_range, new_source, sst_file_handle_with_file_id, - sst_region_metadata, + new_batch_by_range, new_source, sst_file_handle_with_file_id, sst_region_metadata, }; #[tokio::test] @@ -652,11 +652,12 @@ mod tests { handle.clone(), mock_store.clone(), ) - .cache(CacheStrategy::EnableAll(cache_manager.clone())); + .cache(CacheStrategy::EnableAll(cache_manager.clone())) + .page_index_policy(PageIndexPolicy::Optional); let reader = builder.build().await.unwrap(); // Check parquet metadata - assert_parquet_metadata_eq(write_parquet_metadata, reader.parquet_metadata()); + assert_parquet_metadata_equal(write_parquet_metadata, reader.parquet_metadata()); } #[tokio::test] diff --git a/src/mito2/src/error.rs b/src/mito2/src/error.rs index 3756ff6805..f4fd786df3 100644 --- a/src/mito2/src/error.rs +++ b/src/mito2/src/error.rs @@ -601,14 +601,6 @@ pub enum Error { location: Location, }, - #[snafu(display("Invalid file metadata"))] - ConvertMetaData { - #[snafu(implicit)] - location: Location, - #[snafu(source)] - error: parquet::errors::ParquetError, - }, - #[snafu(display("Column not found, column: {column}"))] ColumnNotFound { column: String, @@ -1284,7 +1276,6 @@ impl ErrorExt for Error { | Join { .. } | WorkerStopped { .. } | Recv { .. } - | ConvertMetaData { .. } | DecodeWal { .. } | ComputeArrow { .. } | BiErrors { .. } diff --git a/src/mito2/src/memtable/bulk/part.rs b/src/mito2/src/memtable/bulk/part.rs index dc8414aaad..8bbd24ff07 100644 --- a/src/mito2/src/memtable/bulk/part.rs +++ b/src/mito2/src/memtable/bulk/part.rs @@ -71,7 +71,6 @@ use crate::sst::index::IndexOutput; use crate::sst::parquet::file_range::{PreFilterMode, row_group_contains_delete}; use crate::sst::parquet::flat_format::primary_key_column_index; use crate::sst::parquet::format::{PrimaryKeyArray, PrimaryKeyArrayBuilder, ReadFormat}; -use crate::sst::parquet::helper::parse_parquet_metadata; use crate::sst::parquet::{PARQUET_METADATA_KEY, SstInfo}; use crate::sst::{SeriesEstimator, to_sst_arrow_schema}; @@ -1197,7 +1196,7 @@ impl BulkPartEncoder { metrics.num_rows += total_rows; let buf = Bytes::from(buf); - let parquet_metadata = Arc::new(parse_parquet_metadata(file_metadata)?); + let parquet_metadata = Arc::new(file_metadata); let num_series = series_estimator.finish(); Ok(Some(EncodedBulkPart { @@ -1232,7 +1231,7 @@ impl BulkPartEncoder { }; let buf = Bytes::from(buf); - let parquet_metadata = Arc::new(parse_parquet_metadata(file_metadata)?); + let parquet_metadata = Arc::new(file_metadata); Ok(Some(EncodedBulkPart { data: buf, diff --git a/src/mito2/src/sst/parquet.rs b/src/mito2/src/sst/parquet.rs index ce8bd63c0a..989aeb812b 100644 --- a/src/mito2/src/sst/parquet.rs +++ b/src/mito2/src/sst/parquet.rs @@ -115,7 +115,7 @@ mod tests { use object_store::ObjectStore; use parquet::arrow::AsyncArrowWriter; use parquet::basic::{Compression, Encoding, ZstdLevel}; - use parquet::file::metadata::KeyValue; + use parquet::file::metadata::{KeyValue, PageIndexPolicy}; use parquet::file::properties::WriterProperties; use store_api::codec::PrimaryKeyEncoding; use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataBuilder}; @@ -126,6 +126,7 @@ mod tests { use super::*; use crate::access_layer::{FilePathProvider, Metrics, RegionFilePathFactory, WriteType}; + use crate::cache::test_util::assert_parquet_metadata_equal; use crate::cache::{CacheManager, CacheStrategy, PageKey}; use crate::config::IndexConfig; use crate::read::{BatchBuilder, BatchReader, FlatSource}; @@ -143,9 +144,9 @@ mod tests { DEFAULT_WRITE_CONCURRENCY, FlatSchemaOptions, location, to_flat_sst_arrow_schema, }; use crate::test_util::sst_util::{ - assert_parquet_metadata_eq, build_test_binary_test_region_metadata, new_batch_by_range, - new_batch_with_binary, new_batch_with_custom_sequence, new_primary_key, new_source, - new_sparse_primary_key, sst_file_handle, sst_file_handle_with_file_id, sst_region_metadata, + build_test_binary_test_region_metadata, new_batch_by_range, new_batch_with_binary, + new_batch_with_custom_sequence, new_primary_key, new_source, new_sparse_primary_key, + sst_file_handle, sst_file_handle_with_file_id, sst_region_metadata, sst_region_metadata_with_encoding, }; use crate::test_util::{TestEnv, check_reader_result}; @@ -377,11 +378,12 @@ mod tests { PathType::Bare, handle.clone(), object_store, - ); + ) + .page_index_policy(PageIndexPolicy::Optional); let reader = builder.build().await.unwrap(); let reader_metadata = reader.parquet_metadata(); - assert_parquet_metadata_eq(writer_metadata, reader_metadata) + assert_parquet_metadata_equal(writer_metadata, reader_metadata); } #[tokio::test] diff --git a/src/mito2/src/sst/parquet/helper.rs b/src/mito2/src/sst/parquet/helper.rs index ee7bc2a8b8..890dde26a1 100644 --- a/src/mito2/src/sst/parquet/helper.rs +++ b/src/mito2/src/sst/parquet/helper.rs @@ -13,82 +13,11 @@ // limitations under the License. use std::ops::Range; -use std::sync::Arc; use std::time::Instant; use bytes::Bytes; use common_telemetry::trace; use object_store::ObjectStore; -use parquet::basic::ColumnOrder; -use parquet::file::metadata::{FileMetaData, ParquetMetaData, RowGroupMetaData}; -use parquet::format; -use parquet::schema::types::{SchemaDescriptor, from_thrift}; -use snafu::ResultExt; - -use crate::error; -use crate::error::Result; - -// Refer to https://github.com/apache/arrow-rs/blob/7e134f4d277c0b62c27529fc15a4739de3ad0afd/parquet/src/file/footer.rs#L74-L90 -/// Convert [format::FileMetaData] to [ParquetMetaData] -pub fn parse_parquet_metadata(t_file_metadata: format::FileMetaData) -> Result { - let schema = from_thrift(&t_file_metadata.schema).context(error::ConvertMetaDataSnafu)?; - let schema_desc_ptr = Arc::new(SchemaDescriptor::new(schema)); - - let mut row_groups = Vec::with_capacity(t_file_metadata.row_groups.len()); - for rg in t_file_metadata.row_groups { - row_groups.push( - RowGroupMetaData::from_thrift(schema_desc_ptr.clone(), rg) - .context(error::ConvertMetaDataSnafu)?, - ); - } - let column_orders = parse_column_orders(t_file_metadata.column_orders, &schema_desc_ptr); - - let file_metadata = FileMetaData::new( - t_file_metadata.version, - t_file_metadata.num_rows, - t_file_metadata.created_by, - t_file_metadata.key_value_metadata, - schema_desc_ptr, - column_orders, - ); - // There may be a problem owing to lacking of column_index and offset_index, - // if we open page index in the future. - Ok(ParquetMetaData::new(file_metadata, row_groups)) -} - -// Port from https://github.com/apache/arrow-rs/blob/7e134f4d277c0b62c27529fc15a4739de3ad0afd/parquet/src/file/footer.rs#L106-L137 -/// Parses column orders from Thrift definition. -/// If no column orders are defined, returns `None`. -fn parse_column_orders( - t_column_orders: Option>, - schema_descr: &SchemaDescriptor, -) -> Option> { - match t_column_orders { - Some(orders) => { - // Should always be the case - assert_eq!( - orders.len(), - schema_descr.num_columns(), - "Column order length mismatch" - ); - let mut res = Vec::with_capacity(schema_descr.num_columns()); - for (i, column) in schema_descr.columns().iter().enumerate() { - match orders[i] { - format::ColumnOrder::TYPEORDER(_) => { - let sort_order = ColumnOrder::get_sort_order( - column.logical_type(), - column.converted_type(), - column.physical_type(), - ); - res.push(ColumnOrder::TYPE_DEFINED_ORDER(sort_order)); - } - } - } - Some(res) - } - None => None, - } -} const FETCH_PARALLELISM: usize = 8; pub(crate) const MERGE_GAP: usize = 512 * 1024; diff --git a/src/mito2/src/sst/parquet/metadata.rs b/src/mito2/src/sst/parquet/metadata.rs index 1e85b77801..d13f318fb5 100644 --- a/src/mito2/src/sst/parquet/metadata.rs +++ b/src/mito2/src/sst/parquet/metadata.rs @@ -21,7 +21,7 @@ use futures::future::BoxFuture; use object_store::ObjectStore; use parquet::arrow::async_reader::MetadataFetch; use parquet::errors::{ParquetError, Result as ParquetResult}; -use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader}; +use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader}; use snafu::{IntoError as _, ResultExt}; use crate::error::{self, Result}; @@ -37,6 +37,7 @@ pub(crate) struct MetadataLoader<'a> { file_path: &'a str, // The size of parquet file file_size: u64, + page_index_policy: PageIndexPolicy, } impl<'a> MetadataLoader<'a> { @@ -50,9 +51,14 @@ impl<'a> MetadataLoader<'a> { object_store, file_path, file_size, + page_index_policy: Default::default(), } } + pub(crate) fn with_page_index_policy(&mut self, page_index_policy: PageIndexPolicy) { + self.page_index_policy = page_index_policy; + } + /// Get the size of parquet file. If file_size is 0, stat the object store to get the size. async fn get_file_size(&self) -> Result { let file_size = match self.file_size { @@ -70,8 +76,9 @@ impl<'a> MetadataLoader<'a> { pub async fn load(&self, cache_metrics: &mut MetadataCacheMetrics) -> Result { let path = self.file_path; let file_size = self.get_file_size().await?; - let reader = - ParquetMetaDataReader::new().with_prefetch_hint(Some(DEFAULT_PREFETCH_SIZE as usize)); + let reader = ParquetMetaDataReader::new() + .with_prefetch_hint(Some(DEFAULT_PREFETCH_SIZE as usize)) + .with_page_index_policy(self.page_index_policy); let num_reads = AtomicUsize::new(0); let bytes_read = AtomicU64::new(0); diff --git a/src/mito2/src/sst/parquet/reader.rs b/src/mito2/src/sst/parquet/reader.rs index ad0f093c8f..1545a240cc 100644 --- a/src/mito2/src/sst/parquet/reader.rs +++ b/src/mito2/src/sst/parquet/reader.rs @@ -33,8 +33,7 @@ use mito_codec::row_converter::build_primary_key_codec; use object_store::ObjectStore; use parquet::arrow::arrow_reader::{ParquetRecordBatchReader, RowSelection}; use parquet::arrow::{FieldLevels, ProjectionMask, parquet_to_arrow_field_levels}; -use parquet::file::metadata::ParquetMetaData; -use parquet::format::KeyValue; +use parquet::file::metadata::{KeyValue, PageIndexPolicy, ParquetMetaData}; use snafu::{OptionExt, ResultExt}; use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef}; use store_api::region_request::PathType; @@ -142,6 +141,7 @@ pub struct ParquetReaderBuilder { pre_filter_mode: PreFilterMode, /// Whether to decode primary key values eagerly when reading primary key format SSTs. decode_primary_key_values: bool, + page_index_policy: PageIndexPolicy, } impl ParquetReaderBuilder { @@ -172,6 +172,7 @@ impl ParquetReaderBuilder { compaction: false, pre_filter_mode: PreFilterMode::All, decode_primary_key_values: false, + page_index_policy: Default::default(), } } @@ -276,6 +277,12 @@ impl ParquetReaderBuilder { self } + #[must_use] + pub fn page_index_policy(mut self, page_index_policy: PageIndexPolicy) -> Self { + self.page_index_policy = page_index_policy; + self + } + /// Builds a [ParquetReader]. /// /// This needs to perform IO operation. @@ -314,7 +321,12 @@ impl ParquetReaderBuilder { // Loads parquet metadata of the file. let (parquet_meta, cache_miss) = self - .read_parquet_metadata(&file_path, file_size, &mut metrics.metadata_cache_metrics) + .read_parquet_metadata( + &file_path, + file_size, + &mut metrics.metadata_cache_metrics, + self.page_index_policy, + ) .await?; // Decodes region metadata. let key_value_meta = parquet_meta.file_metadata().key_value_metadata(); @@ -479,6 +491,7 @@ impl ParquetReaderBuilder { file_path: &str, file_size: u64, cache_metrics: &mut MetadataCacheMetrics, + page_index_policy: PageIndexPolicy, ) -> Result<(Arc, bool)> { let start = Instant::now(); let _t = READ_STAGE_ELAPSED @@ -489,7 +502,7 @@ impl ParquetReaderBuilder { // Tries to get from cache with metrics tracking. if let Some(metadata) = self .cache_strategy - .get_parquet_meta_data(file_id, cache_metrics) + .get_parquet_meta_data(file_id, cache_metrics, page_index_policy) .await { cache_metrics.metadata_load_cost += start.elapsed(); @@ -497,7 +510,9 @@ impl ParquetReaderBuilder { } // Cache miss, load metadata directly. - let metadata_loader = MetadataLoader::new(self.object_store.clone(), file_path, file_size); + let mut metadata_loader = + MetadataLoader::new(self.object_store.clone(), file_path, file_size); + metadata_loader.with_page_index_policy(page_index_policy); let metadata = metadata_loader.load(cache_metrics).await?; let metadata = Arc::new(metadata); diff --git a/src/mito2/src/sst/parquet/writer.rs b/src/mito2/src/sst/parquet/writer.rs index 8c03a51368..568b54415c 100644 --- a/src/mito2/src/sst/parquet/writer.rs +++ b/src/mito2/src/sst/parquet/writer.rs @@ -55,7 +55,6 @@ use crate::sst::file::RegionFileId; use crate::sst::index::{IndexOutput, Indexer, IndexerBuilder}; use crate::sst::parquet::flat_format::{FlatWriteFormat, time_index_column_index}; use crate::sst::parquet::format::PrimaryKeyWriteFormat; -use crate::sst::parquet::helper::parse_parquet_metadata; use crate::sst::parquet::{PARQUET_METADATA_KEY, SstInfo, WriteOptions}; use crate::sst::{ DEFAULT_WRITE_BUFFER_SIZE, DEFAULT_WRITE_CONCURRENCY, FlatSchemaOptions, SeriesEstimator, @@ -205,14 +204,12 @@ where } current_writer.flush().await.context(WriteParquetSnafu)?; - let file_meta = current_writer.close().await.context(WriteParquetSnafu)?; + let parquet_metadata = current_writer.close().await.context(WriteParquetSnafu)?; let file_size = self.bytes_written.load(Ordering::Relaxed) as u64; // Safety: num rows > 0 so we must have min/max. let time_range = stats.time_range.unwrap(); - // convert FileMetaData to ParquetMetaData - let parquet_metadata = parse_parquet_metadata(file_meta)?; let max_row_group_uncompressed_size: u64 = parquet_metadata .row_groups() .iter() diff --git a/src/mito2/src/test_util/sst_util.rs b/src/mito2/src/test_util/sst_util.rs index d7b45b9237..389d9bf107 100644 --- a/src/mito2/src/test_util/sst_util.rs +++ b/src/mito2/src/test_util/sst_util.rs @@ -23,7 +23,6 @@ use datatypes::prelude::ConcreteDataType; use datatypes::schema::{ColumnSchema, SkippingIndexOptions}; use datatypes::value::ValueRef; use mito_codec::row_converter::{DensePrimaryKeyCodec, PrimaryKeyCodecExt, SortField}; -use parquet::file::metadata::ParquetMetaData; use store_api::metadata::{ ColumnMetadata, RegionMetadata, RegionMetadataBuilder, RegionMetadataRef, }; @@ -277,30 +276,6 @@ pub fn new_batch_with_binary(tags: &[&str], start: usize, end: usize) -> Batch { builder.build().unwrap() } -/// ParquetMetaData doesn't implement `PartialEq` trait, check internal fields manually -pub fn assert_parquet_metadata_eq(a: Arc, b: Arc) { - macro_rules! assert_metadata { - ( $a:expr, $b:expr, $($method:ident,)+ ) => { - $( - assert_eq!($a.$method(), $b.$method()); - )+ - } - } - - assert_metadata!( - a.file_metadata(), - b.file_metadata(), - version, - num_rows, - created_by, - key_value_metadata, - schema_descr, - column_orders, - ); - - assert_metadata!(a, b, row_groups, column_index, offset_index,); -} - /// Creates a new region metadata for testing SSTs with binary datatype. /// /// Schema: tag_0(string), field_0(binary), ts diff --git a/src/mito2/src/worker/handle_manifest.rs b/src/mito2/src/worker/handle_manifest.rs index 92e7ca9b9d..e7708f7b65 100644 --- a/src/mito2/src/worker/handle_manifest.rs +++ b/src/mito2/src/worker/handle_manifest.rs @@ -21,6 +21,7 @@ use std::num::NonZeroU64; use std::sync::Arc; use common_telemetry::{info, warn}; +use parquet::file::metadata::PageIndexPolicy; use store_api::logstore::LogStore; use store_api::metadata::RegionMetadataRef; use store_api::storage::RegionId; @@ -523,7 +524,11 @@ async fn edit_region( let mut cache_metrics = Default::default(); let _ = write_cache .file_cache() - .get_parquet_meta_data(index_key, &mut cache_metrics) + .get_parquet_meta_data( + index_key, + &mut cache_metrics, + PageIndexPolicy::Optional, + ) .await; listener.on_file_cache_filled(index_key.file_id); diff --git a/src/operator/src/statement/copy_table_from.rs b/src/operator/src/statement/copy_table_from.rs index 35cfdc7830..6f58603247 100644 --- a/src/operator/src/statement/copy_table_from.rs +++ b/src/operator/src/statement/copy_table_from.rs @@ -33,6 +33,7 @@ use common_telemetry::{debug, tracing}; use datafusion::datasource::physical_plan::{CsvSource, FileSource, JsonSource}; use datafusion::parquet::arrow::ParquetRecordBatchStreamBuilder; use datafusion::parquet::arrow::arrow_reader::ArrowReaderMetadata; +use datafusion_common::config::CsvOptions; use datafusion_expr::Expr; use datatypes::arrow::compute::can_cast_types; use datatypes::arrow::datatypes::{DataType as ArrowDataType, Schema, SchemaRef}; @@ -214,13 +215,15 @@ impl StatementExecutor { .context(error::ProjectSchemaSnafu)?, ); - let csv_source = CsvSource::new(format.has_header, format.delimiter, b'"') - .with_schema(schema.clone()) + let options = CsvOptions::default() + .with_has_header(format.has_header) + .with_delimiter(format.delimiter); + let csv_source = CsvSource::new(schema.clone()) + .with_csv_options(options) .with_batch_size(DEFAULT_BATCH_SIZE); let stream = file_to_stream( object_store, path, - schema.clone(), csv_source, Some(projection), format.compression_type, @@ -247,13 +250,11 @@ impl StatementExecutor { .context(error::ProjectSchemaSnafu)?, ); - let json_source = JsonSource::new() - .with_schema(schema.clone()) - .with_batch_size(DEFAULT_BATCH_SIZE); + let json_source = + JsonSource::new(schema.clone()).with_batch_size(DEFAULT_BATCH_SIZE); let stream = file_to_stream( object_store, path, - schema.clone(), json_source, Some(projection), format.compression_type, diff --git a/src/promql/src/extension_plan/range_manipulate.rs b/src/promql/src/extension_plan/range_manipulate.rs index f8ffd87782..a3ac1304a5 100644 --- a/src/promql/src/extension_plan/range_manipulate.rs +++ b/src/promql/src/extension_plan/range_manipulate.rs @@ -995,7 +995,7 @@ mod test { #[tokio::test] async fn interval_30s_range_90s() { let expected = String::from( - "PrimitiveArray\n[\n \ + "PrimitiveArray\n[\n \ 1970-01-01T00:00:00,\n \ 1970-01-01T00:00:30,\n \ 1970-01-01T00:01:00,\n \ @@ -1015,7 +1015,7 @@ mod test { ranges: [Some(0..1), Some(0..2), Some(0..3), Some(0..4), Some(1..5), Some(2..5), Some(3..6), Some(4..6), Some(5..7), Some(5..8), Some(6..10)] \ }\nStringArray\n[\n \"foo\",\n \"foo\",\n \"foo\",\n \"foo\",\n \"foo\",\n \"foo\",\n \"foo\",\n \"foo\",\n \"foo\",\n \"foo\",\n \"foo\",\n]\n\ RangeArray { \ - base array: PrimitiveArray\n[\n 1970-01-01T00:00:00,\n 1970-01-01T00:00:30,\n 1970-01-01T00:01:00,\n 1970-01-01T00:01:30,\n 1970-01-01T00:02:00,\n 1970-01-01T00:03:00,\n 1970-01-01T00:04:00,\n 1970-01-01T00:04:01,\n 1970-01-01T00:04:31,\n 1970-01-01T00:04:51,\n], \ + base array: PrimitiveArray\n[\n 1970-01-01T00:00:00,\n 1970-01-01T00:00:30,\n 1970-01-01T00:01:00,\n 1970-01-01T00:01:30,\n 1970-01-01T00:02:00,\n 1970-01-01T00:03:00,\n 1970-01-01T00:04:00,\n 1970-01-01T00:04:01,\n 1970-01-01T00:04:31,\n 1970-01-01T00:04:51,\n], \ ranges: [Some(0..1), Some(0..2), Some(0..3), Some(0..4), Some(1..5), Some(2..5), Some(3..6), Some(4..6), Some(5..7), Some(5..8), Some(6..10)] \ }", ); @@ -1028,7 +1028,7 @@ mod test { #[tokio::test] async fn small_empty_range() { let expected = String::from( - "PrimitiveArray\n[\n \ + "PrimitiveArray\n[\n \ 1970-01-01T00:00:00.001,\n \ 1970-01-01T00:00:03.001,\n \ 1970-01-01T00:00:06.001,\n \ @@ -1041,7 +1041,7 @@ mod test { ranges: [Some(0..1), Some(0..0), Some(0..0), Some(0..0)] \ }\nStringArray\n[\n \"foo\",\n \"foo\",\n \"foo\",\n \"foo\",\n]\n\ RangeArray { \ - base array: PrimitiveArray\n[\n 1970-01-01T00:00:00,\n 1970-01-01T00:00:30,\n 1970-01-01T00:01:00,\n 1970-01-01T00:01:30,\n 1970-01-01T00:02:00,\n 1970-01-01T00:03:00,\n 1970-01-01T00:04:00,\n 1970-01-01T00:04:01,\n 1970-01-01T00:04:31,\n 1970-01-01T00:04:51,\n], \ + base array: PrimitiveArray\n[\n 1970-01-01T00:00:00,\n 1970-01-01T00:00:30,\n 1970-01-01T00:01:00,\n 1970-01-01T00:01:30,\n 1970-01-01T00:02:00,\n 1970-01-01T00:03:00,\n 1970-01-01T00:04:00,\n 1970-01-01T00:04:01,\n 1970-01-01T00:04:31,\n 1970-01-01T00:04:51,\n], \ ranges: [Some(0..1), Some(0..0), Some(0..0), Some(0..0)] \ }", ); diff --git a/src/query/src/analyze.rs b/src/query/src/analyze.rs index 26da084852..776b1bd1fb 100644 --- a/src/query/src/analyze.rs +++ b/src/query/src/analyze.rs @@ -255,9 +255,9 @@ fn metrics_to_string(metrics: RecordBatchMetrics, format: AnalyzeFormat) -> DfRe match format { AnalyzeFormat::JSON => Ok(JsonMetrics::from_record_batch_metrics(metrics).to_string()), AnalyzeFormat::TEXT => Ok(metrics.to_string()), - AnalyzeFormat::GRAPHVIZ => Err(DataFusionError::NotImplemented( - "GRAPHVIZ format is not supported for metrics output".to_string(), - )), + format => Err(DataFusionError::NotImplemented(format!( + "AnalyzeFormat {format}", + ))), } } diff --git a/src/query/src/datafusion.rs b/src/query/src/datafusion.rs index 391490aa55..ef597ecc38 100644 --- a/src/query/src/datafusion.rs +++ b/src/query/src/datafusion.rs @@ -316,18 +316,15 @@ impl DatafusionQueryEngine { return state .create_physical_plan(logical_plan) .await - .context(error::DatafusionSnafu) - .map_err(BoxedError::new) - .context(QueryExecutionSnafu); + .map_err(Into::into); } // analyze first - let analyzed_plan = state - .analyzer() - .execute_and_check(logical_plan.clone(), state.config_options(), |_, _| {}) - .context(error::DatafusionSnafu) - .map_err(BoxedError::new) - .context(QueryExecutionSnafu)?; + let analyzed_plan = state.analyzer().execute_and_check( + logical_plan.clone(), + state.config_options(), + |_, _| {}, + )?; logger.after_analyze = Some(analyzed_plan.clone()); @@ -341,10 +338,7 @@ impl DatafusionQueryEngine { } else { state .optimizer() - .optimize(analyzed_plan, state, |_, _| {}) - .context(error::DatafusionSnafu) - .map_err(BoxedError::new) - .context(QueryExecutionSnafu)? + .optimize(analyzed_plan, state, |_, _| {})? }; common_telemetry::debug!("Create physical plan, optimized plan: {optimized_plan}"); @@ -371,19 +365,10 @@ impl DatafusionQueryEngine { // Optimized by extension rules let optimized_plan = self .state - .optimize_by_extension_rules(plan.clone(), context) - .context(error::DatafusionSnafu) - .map_err(BoxedError::new) - .context(QueryExecutionSnafu)?; + .optimize_by_extension_rules(plan.clone(), context)?; // Optimized by datafusion optimizer - let optimized_plan = self - .state - .session_state() - .optimize(&optimized_plan) - .context(error::DatafusionSnafu) - .map_err(BoxedError::new) - .context(QueryExecutionSnafu)?; + let optimized_plan = self.state.session_state().optimize(&optimized_plan)?; Ok(optimized_plan) } @@ -516,11 +501,7 @@ impl QueryEngine for DatafusionQueryEngine { } fn read_table(&self, table: TableRef) -> Result { - self.state - .read_table(table) - .context(error::DatafusionSnafu) - .map_err(BoxedError::new) - .context(QueryExecutionSnafu) + self.state.read_table(table).map_err(Into::into) } fn engine_context(&self, query_ctx: QueryContextRef) -> QueryEngineContext { @@ -543,7 +524,8 @@ impl QueryEngine for DatafusionQueryEngine { } // configure execution options - state.config_mut().options_mut().execution.time_zone = query_ctx.timezone().to_string(); + state.config_mut().options_mut().execution.time_zone = + Some(query_ctx.timezone().to_string()); // usually it's impossible to have both `set variable` set by sql client and // hint in header by grpc client, so only need to deal with them separately @@ -619,11 +601,7 @@ impl QueryExecutor for DatafusionQueryEngine { Ok(Box::pin(EmptyRecordBatchStream::new(schema))) } 1 => { - let df_stream = plan - .execute(0, task_ctx) - .context(error::DatafusionSnafu) - .map_err(BoxedError::new) - .context(QueryExecutionSnafu)?; + let df_stream = plan.execute(0, task_ctx)?; let mut stream = RecordBatchStreamAdapter::try_new_with_span(df_stream, span) .context(error::ConvertDfRecordBatchStreamSnafu) .map_err(BoxedError::new) @@ -652,11 +630,7 @@ impl QueryExecutor for DatafusionQueryEngine { .output_partitioning() .partition_count() ); - let df_stream = merged_plan - .execute(0, task_ctx) - .context(error::DatafusionSnafu) - .map_err(BoxedError::new) - .context(QueryExecutionSnafu)?; + let df_stream = merged_plan.execute(0, task_ctx)?; let mut stream = RecordBatchStreamAdapter::try_new_with_span(df_stream, span) .context(error::ConvertDfRecordBatchStreamSnafu) .map_err(BoxedError::new) diff --git a/src/query/src/datafusion/error.rs b/src/query/src/datafusion/error.rs index c6952c6e4b..5b85d7619d 100644 --- a/src/query/src/datafusion/error.rs +++ b/src/query/src/datafusion/error.rs @@ -25,7 +25,7 @@ use snafu::{Location, Snafu}; #[snafu(visibility(pub))] #[stack_trace_debug] pub enum InnerError { - #[snafu(display("DataFusion error"))] + #[snafu(transparent)] Datafusion { #[snafu(source)] error: DataFusionError, diff --git a/src/query/src/dist_plan/analyzer/test.rs b/src/query/src/dist_plan/analyzer/test.rs index 166fb89fdd..7d4578f3dd 100644 --- a/src/query/src/dist_plan/analyzer/test.rs +++ b/src/query/src/dist_plan/analyzer/test.rs @@ -1170,7 +1170,7 @@ fn test_simplify_select_now_expression() { let expected = [ "Projection: now()", " MergeScan [is_placeholder=false, remote_input=[", - r#"Projection: TimestampNanosecond(