diff --git a/.github/workflows/apidoc.yml b/.github/workflows/apidoc.yml index ca1befa52c..75979fbce8 100644 --- a/.github/workflows/apidoc.yml +++ b/.github/workflows/apidoc.yml @@ -13,7 +13,7 @@ on: name: Build API docs env: - RUST_TOOLCHAIN: nightly-2023-12-19 + RUST_TOOLCHAIN: nightly-2024-04-18 jobs: apidoc: diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 4dc909ae51..626decd81a 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -30,7 +30,7 @@ concurrency: cancel-in-progress: true env: - RUST_TOOLCHAIN: nightly-2023-12-19 + RUST_TOOLCHAIN: nightly-2024-04-18 jobs: check-typos-and-docs: diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 3ea1812314..b635ab16d6 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -12,7 +12,7 @@ concurrency: cancel-in-progress: true env: - RUST_TOOLCHAIN: nightly-2023-12-19 + RUST_TOOLCHAIN: nightly-2024-04-18 jobs: sqlness: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77705c21a6..37dd3c70a8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -82,7 +82,7 @@ on: # Use env variables to control all the release process. env: # The arguments of building greptime. - RUST_TOOLCHAIN: nightly-2023-12-19 + RUST_TOOLCHAIN: nightly-2024-04-18 CARGO_PROFILE: nightly # Controls whether to run tests, include unit-test, integration-test and sqlness. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dcbef7b530..94bfb60ac3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,7 +50,7 @@ GreptimeDB uses the [Apache 2.0 license](https://github.com/GreptimeTeam/greptim - To ensure that community is free and confident in its ability to use your contributions, please sign the Contributor License Agreement (CLA) which will be incorporated in the pull request process. - Make sure all files have proper license header (running `docker run --rm -v $(pwd):/github/workspace ghcr.io/korandoru/hawkeye-native:v3 format` from the project root). -- Make sure all your codes are formatted and follow the [coding style](https://pingcap.github.io/style-guide/rust/). +- Make sure all your codes are formatted and follow the [coding style](https://pingcap.github.io/style-guide/rust/) and [style guide](http://github.com/greptimeTeam/docs/style-guide.md). - Make sure all unit tests are passed (using `cargo test --workspace` or [nextest](https://nexte.st/index.html) `cargo nextest run`). - Make sure all clippy warnings are fixed (you can check it locally by running `cargo clippy --workspace --all-targets -- -D warnings`). diff --git a/Cargo.lock b/Cargo.lock index 9f7f378c5e..2c17075f47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -42,9 +42,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ "getrandom", "once_cell", @@ -53,9 +53,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if 1.0.0", "const-random", @@ -67,9 +67,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -111,9 +111,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "android-tzdata" @@ -147,9 +147,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.5" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" dependencies = [ "anstyle", "anstyle-parse", @@ -161,9 +161,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.76" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59d2a3357dde987206219e78ecfbbb6e8dad06cbb65292758d3270e6254f7355" +checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" [[package]] name = "anymap" @@ -217,7 +217,7 @@ dependencies = [ "datatypes", "greptime-proto", "paste", - "prost 0.12.3", + "prost 0.12.4", "snafu", "tonic-build 0.9.2", ] @@ -262,9 +262,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "array-init" @@ -286,11 +286,10 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fab9e93ba8ce88a37d5a30dce4b9913b75413dc1ac56cb5d72e5a840543f829" +checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" dependencies = [ - "ahash 0.8.6", "arrow-arith", "arrow-array", "arrow-buffer", @@ -309,70 +308,73 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1d4e368e87ad9ee64f28b9577a3834ce10fe2703a26b28417d485bbbdff956" +checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "half 2.3.1", + "half 2.4.1", "num", ] [[package]] name = "arrow-array" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d02efa7253ede102d45a4e802a129e83bcc3f49884cab795b1ac223918e4318d" +checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.11", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", "chrono-tz", - "half 2.3.1", + "half 2.4.1", "hashbrown 0.14.3", "num", ] [[package]] name = "arrow-buffer" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda119225204141138cb0541c692fbfef0e875ba01bfdeaed09e9d354f9d6195" +checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27" dependencies = [ "bytes", - "half 2.3.1", + "half 2.4.1", "num", ] [[package]] name = "arrow-cast" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d825d51b9968868d50bc5af92388754056796dbc62a4e25307d588a1fc84dee" +checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "atoi 2.0.0", + "base64 0.22.0", "chrono", "comfy-table", - "half 2.3.1", + "half 2.4.1", "lexical-core", "num", + "ryu", ] [[package]] name = "arrow-csv" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43ef855dc6b126dc197f43e061d4de46b9d4c033aa51c2587657f7508242cef1" +checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2" dependencies = [ "arrow-array", "arrow-buffer", @@ -389,41 +391,42 @@ dependencies = [ [[package]] name = "arrow-data" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "475a4c3699c8b4095ca61cecf15da6f67841847a5f5aac983ccb9a377d02f73a" +checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" dependencies = [ "arrow-buffer", "arrow-schema", - "half 2.3.1", + "half 2.4.1", "num", ] [[package]] name = "arrow-flight" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd938ea4a0e8d0db2b9f47ebba792f73f6188f4289707caeaf93a3be705e5ed5" +checksum = "3241ce691192d789b7b94f56a10e166ee608bdc3932c759eb0b85f09235352bb" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", "arrow-ipc", "arrow-schema", - "base64 0.21.5", + "base64 0.22.0", "bytes", "futures", "paste", - "prost 0.12.3", + "prost 0.12.4", + "prost-types 0.12.4", "tokio", - "tonic 0.10.2", + "tonic 0.11.0", ] [[package]] name = "arrow-ipc" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1248005c8ac549f869b7a840859d942bf62471479c1a2d82659d453eebcd166a" +checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d" dependencies = [ "arrow-array", "arrow-buffer", @@ -431,14 +434,14 @@ dependencies = [ "arrow-data", "arrow-schema", "flatbuffers", - "lz4", + "lz4_flex 0.11.3", ] [[package]] name = "arrow-json" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f03d7e3b04dd688ccec354fe449aed56b831679f03e44ee2c1cfc4045067b69c" +checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49" dependencies = [ "arrow-array", "arrow-buffer", @@ -446,8 +449,8 @@ dependencies = [ "arrow-data", "arrow-schema", "chrono", - "half 2.3.1", - "indexmap 2.1.0", + "half 2.4.1", + "indexmap 2.2.6", "lexical-core", "num", "serde", @@ -456,51 +459,51 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b87aa408ea6a6300e49eb2eba0c032c88ed9dc19e0a9948489c55efdca71f4" +checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", - "half 2.3.1", + "half 2.4.1", "num", ] [[package]] name = "arrow-row" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "114a348ab581e7c9b6908fcab23cb39ff9f060eb19e72b13f8fb8eaa37f65d22" +checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.11", "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", - "half 2.3.1", + "half 2.4.1", "hashbrown 0.14.3", ] [[package]] name = "arrow-schema" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1d179c117b158853e0101bfbed5615e86fe97ee356b4af901f1c5001e1ce4b" +checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "serde", ] [[package]] name = "arrow-select" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5c71e003202e67e9db139e5278c79f5520bb79922261dfe140e4637ee8b6108" +checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.11", "arrow-array", "arrow-buffer", "arrow-data", @@ -510,18 +513,19 @@ dependencies = [ [[package]] name = "arrow-string" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cebbb282d6b9244895f4a9a912e55e57bce112554c7fa91fcec5459cb421ab" +checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "memchr", "num", "regex", - "regex-syntax 0.7.5", + "regex-syntax 0.8.3", ] [[package]] @@ -546,7 +550,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" dependencies = [ "concurrent-queue", - "event-listener", + "event-listener 2.5.3", "futures-core", ] @@ -570,11 +574,11 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.5" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc2d0cfb2a7388d34f590e76686704c494ed7aaceed62ee1ba35cbf363abc2a5" +checksum = "07dbbf24db18d609b1462965249abdf49129ccad073ec257da372adc83259c60" dependencies = [ - "brotli", + "brotli 4.0.0", "bzip2", "flate2", "futures-core", @@ -583,28 +587,30 @@ dependencies = [ "pin-project-lite", "tokio", "xz2", - "zstd 0.13.0", - "zstd-safe 7.0.0", + "zstd 0.13.1", + "zstd-safe 7.1.0", ] [[package]] name = "async-lock" -version = "2.8.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" dependencies = [ - "event-listener", + "event-listener 4.0.3", + "event-listener-strategy", + "pin-project-lite", ] [[package]] name = "async-recursion" -version = "1.0.5" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" +checksum = "30c5ef0ede93efbf733c1a727f3b6b5a1060bbedd5600183e66f6e4be4af0ec5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -626,18 +632,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] name = "async-trait" -version = "0.1.75" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -662,6 +668,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic" version = "0.5.3" @@ -700,14 +715,13 @@ dependencies = [ [[package]] name = "auto_impl" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fee3da8ef1276b0bee5dd1c7258010d8fffd31801447323115a25560e1327b89" +checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ - "proc-macro-error", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.60", ] [[package]] @@ -718,15 +732,15 @@ checksum = "7460f7dd8e100147b82a63afca1a20eb6c231ee36b90ba7272e14951cb58af59" [[package]] name = "autocfg" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" [[package]] name = "autotools" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aef8da1805e028a172334c3b680f93e71126f2327622faef2ec3d893c0a4ad77" +checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf" dependencies = [ "cc", ] @@ -787,19 +801,19 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdca6a10ecad987bda04e95606ef85a5417dcaac1a78455242d72e031e2b6b62" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] name = "backon" -version = "0.4.1" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9" +checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0" dependencies = [ - "fastrand 1.9.0", + "fastrand", "futures-core", "pin-project", "tokio", @@ -807,9 +821,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" dependencies = [ "addr2line", "cc", @@ -828,9 +842,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" @@ -846,9 +860,9 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bcder" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf16bec990f8ea25cab661199904ef452fcf11f565c404ce6cffbdf3f8cbbc47" +checksum = "c627747a6774aab38beb35990d88309481378558875a41da1a4b2e373c906ef0" dependencies = [ "bytes", "smallvec", @@ -861,7 +875,7 @@ dependencies = [ "api", "arrow", "chrono", - "clap 4.4.11", + "clap 4.5.4", "client", "common-base", "common-telemetry", @@ -884,15 +898,15 @@ dependencies = [ "serde", "store-api", "tokio", - "toml 0.8.8", + "toml 0.8.12", "uuid", ] [[package]] name = "bigdecimal" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06619be423ea5bb86c95f087d5707942791a08a85530df0db2209a3ecfb8bc9" +checksum = "9324c8014cd04590682b34f1e9448d38f0674d0f7b2dc553331016ef0e4e9ebc" dependencies = [ "autocfg", "libm", @@ -912,22 +926,22 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.1" +version = "0.69.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ffcebc3849946a7170a05992aac39da343a90676ab392c51a4280981d6379c2" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "cexpr", "clang-sys", + "itertools 0.12.1", "lazy_static", "lazycell", - "peeking_take_while", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -953,9 +967,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "bitvec" @@ -980,9 +994,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" +checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" dependencies = [ "arrayref", "arrayvec", @@ -1011,9 +1025,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d4d6dafc1a3bb54687538972158f07b2c948bc57d5890df22c0739098b3028" +checksum = "0901fc8eb0aca4c83be0106d6f2db17d86a08dfc2c25f0e84464bf381158add6" dependencies = [ "borsh-derive", "cfg_aliases", @@ -1021,27 +1035,38 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4918709cc4dd777ad2b6303ed03cb37f3ca0ccede8c1b0d28ac6db8f4710e0" +checksum = "51670c3aa053938b0ee3bd67c3817e471e626151131b934038e83c5bf8de48f5" dependencies = [ "once_cell", - "proc-macro-crate 2.0.0", + "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", "syn_derive", ] [[package]] name = "brotli" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor", + "brotli-decompressor 2.5.1", +] + +[[package]] +name = "brotli" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "125740193d7fee5cc63ab9e16c2fdc4e07c74ba755cc53b327d6ea029e9fc569" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor 3.0.0", ] [[package]] @@ -1054,6 +1079,16 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "brotli-decompressor" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65622a320492e09b5e0ac436b14c54ff68199bac392d0e89a6832c4518eea525" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bstr" version = "0.2.17" @@ -1087,15 +1122,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytecheck" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" dependencies = [ "bytecheck_derive", "ptr_meta", @@ -1104,26 +1139,20 @@ dependencies = [ [[package]] name = "bytecheck_derive" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" dependencies = [ "proc-macro2", "quote", "syn 1.0.109", ] -[[package]] -name = "bytecount" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" - [[package]] name = "bytemuck" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" +checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" [[package]] name = "byteorder" @@ -1133,9 +1162,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" dependencies = [ "serde", ] @@ -1183,40 +1212,9 @@ dependencies = [ [[package]] name = "cactus" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf034765b7d19a011c6d619e880582bf95e8186b580e6fab56589872dd87dcf5" - -[[package]] -name = "camino" -version = "1.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" -dependencies = [ - "serde", -] - -[[package]] -name = "cargo-platform" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34637b3140142bdf929fb439e8aa4ebad7651ebf7b1080b3930aa16ac1459ff" -dependencies = [ - "serde", -] - -[[package]] -name = "cargo_metadata" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" -dependencies = [ - "camino", - "cargo-platform", - "semver", - "serde", - "serde_json", -] +checksum = "acbc26382d871df4b7442e3df10a9402bf3cf5e55cbd66f12be38861425f0564" [[package]] name = "caseless" @@ -1289,9 +1287,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" dependencies = [ "jobserver", "libc", @@ -1352,9 +1350,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.33" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1362,14 +1360,14 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.0", + "windows-targets 0.52.5", ] [[package]] name = "chrono-tz" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e23185c0e21df6ed832a12e2bda87c7d1def6842881fb634a8511ced741b0d76" +checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" dependencies = [ "chrono", "chrono-tz-build", @@ -1395,9 +1393,9 @@ checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" [[package]] name = "ciborium" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" dependencies = [ "ciborium-io", "ciborium-ll", @@ -1406,18 +1404,18 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" [[package]] name = "ciborium-ll" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", - "half 1.8.2", + "half 2.4.1", ] [[package]] @@ -1432,13 +1430,13 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" dependencies = [ "glob", "libc", - "libloading", + "libloading 0.8.3", ] [[package]] @@ -1465,14 +1463,14 @@ dependencies = [ "bitflags 1.3.2", "clap_lex 0.2.4", "indexmap 1.9.3", - "textwrap 0.16.0", + "textwrap 0.16.1", ] [[package]] name = "clap" -version = "4.4.11" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" dependencies = [ "clap_builder", "clap_derive", @@ -1480,26 +1478,26 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.11" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" dependencies = [ "anstream", "anstyle", - "clap_lex 0.6.0", - "strsim 0.10.0", + "clap_lex 0.7.0", + "strsim 0.11.1", ] [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -1513,9 +1511,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "client" @@ -1543,7 +1541,7 @@ dependencies = [ "moka", "parking_lot 0.12.1", "prometheus", - "prost 0.12.3", + "prost 0.12.4", "rand", "serde_json", "snafu", @@ -1551,7 +1549,7 @@ dependencies = [ "substrait 0.7.2", "tokio", "tokio-stream", - "tonic 0.10.2", + "tonic 0.11.0", "tracing", "tracing-subscriber", ] @@ -1584,7 +1582,7 @@ dependencies = [ "auth", "catalog", "chrono", - "clap 4.4.11", + "clap 4.5.4", "client", "common-base", "common-catalog", @@ -1617,7 +1615,7 @@ dependencies = [ "nu-ansi-term", "plugins", "prometheus", - "prost 0.12.3", + "prost 0.12.4", "query", "rand", "regex", @@ -1635,7 +1633,7 @@ dependencies = [ "tempfile", "tikv-jemallocator", "tokio", - "toml 0.8.8", + "toml 0.8.12", ] [[package]] @@ -1646,9 +1644,9 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "combine" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ "bytes", "memchr", @@ -1656,12 +1654,12 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.1.0" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" +checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7" dependencies = [ - "strum 0.25.0", - "strum_macros 0.25.3", + "strum 0.26.2", + "strum_macros 0.26.2", "unicode-width", ] @@ -1683,7 +1681,7 @@ dependencies = [ "paste", "serde", "snafu", - "toml 0.8.8", + "toml 0.8.12", ] [[package]] @@ -1828,11 +1826,11 @@ dependencies = [ "datatypes", "flatbuffers", "lazy_static", - "prost 0.12.3", + "prost 0.12.4", "rand", "snafu", "tokio", - "tonic 0.10.2", + "tonic 0.11.0", "tower", ] @@ -1865,7 +1863,7 @@ dependencies = [ "snafu", "static_assertions", "syn 1.0.109", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -1888,7 +1886,7 @@ dependencies = [ "api", "async-recursion", "async-trait", - "base64 0.21.5", + "base64 0.21.7", "bytes", "chrono", "common-base", @@ -1913,7 +1911,7 @@ dependencies = [ "itertools 0.10.5", "lazy_static", "prometheus", - "prost 0.12.3", + "prost 0.12.4", "rand", "regex", "rskafka", @@ -1925,7 +1923,7 @@ dependencies = [ "strum 0.25.0", "table", "tokio", - "tonic 0.10.2", + "tonic 0.11.0", "typetag", "uuid", ] @@ -1984,8 +1982,8 @@ dependencies = [ "datatypes", "serde", "snafu", - "sqlparser 0.38.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=6a93567ae38d42be5c8d08b13c8ff4dde26502ef)", - "sqlparser_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sqlparser 0.44.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=c919990bf62ad38d2b0c0a3bc90b26ad919d51b0)", + "sqlparser_derive 0.1.1", "statrs", "tokio", ] @@ -1997,6 +1995,7 @@ dependencies = [ "arc-swap", "common-error", "common-macro", + "common-telemetry", "datafusion", "datafusion-common", "datatypes", @@ -2039,10 +2038,10 @@ dependencies = [ "greptime-proto", "lazy_static", "once_cell", - "opentelemetry 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)", + "opentelemetry 0.21.0", "opentelemetry-otlp", "opentelemetry-semantic-conventions", - "opentelemetry_sdk 0.21.1", + "opentelemetry_sdk 0.21.2", "parking_lot 0.12.1", "prometheus", "serde", @@ -2108,7 +2107,7 @@ dependencies = [ "serde_with", "snafu", "tokio", - "toml 0.8.8", + "toml 0.8.12", ] [[package]] @@ -2141,15 +2140,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.7" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] @@ -2202,9 +2201,9 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const-random" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" dependencies = [ "const-random-macro", ] @@ -2253,18 +2252,18 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] [[package]] name = "crc" -version = "3.0.1" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" dependencies = [ "crc-catalog", ] @@ -2277,9 +2276,9 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32c" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74" +checksum = "89254598aa9b9fa608de44b3ae54c810f0f06d755e24c50177f1f8f31ff50ce2" dependencies = [ "rustc_version", ] @@ -2333,11 +2332,10 @@ dependencies = [ [[package]] name = "crossbeam" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eb9105919ca8e40d437fc9cbb8f1975d916f1bd28afe795a48aae32a2cc8920" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" dependencies = [ - "cfg-if 1.0.0", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", @@ -2347,54 +2345,46 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.10" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a9b73a36529d9c47029b9fb3a6f0ea3cc916a261195352ba19e770fc1748b2" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" dependencies = [ - "cfg-if 1.0.0", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if 1.0.0", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.17" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if 1.0.0", "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc6598521bb5a83d491e8c1fe51db7296019d2ca3cb93cc6c2a20369a4d78a2" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" dependencies = [ - "cfg-if 1.0.0", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.18" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crunchy" @@ -2455,12 +2445,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.3" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" dependencies = [ - "darling_core 0.20.3", - "darling_macro 0.20.3", + "darling_core 0.20.8", + "darling_macro 0.20.8", ] [[package]] @@ -2479,16 +2469,16 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.3" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim 0.10.0", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -2504,13 +2494,13 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.3" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ - "darling_core 0.20.3", + "darling_core 0.20.8", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -2534,22 +2524,27 @@ checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" [[package]] name = "datafusion" -version = "32.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9" +version = "37.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=34eda15b73a9e278af8844b30ed2f1c21c10359c#34eda15b73a9e278af8844b30ed2f1c21c10359c" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.11", "arrow", "arrow-array", + "arrow-ipc", "arrow-schema", - "async-compression 0.4.5", + "async-compression 0.4.8", "async-trait", "bytes", "bzip2", "chrono", "dashmap", "datafusion-common", + "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-array", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-plan", @@ -2557,50 +2552,59 @@ dependencies = [ "flate2", "futures", "glob", - "half 2.3.1", + "half 2.4.1", "hashbrown 0.14.3", - "indexmap 2.1.0", - "itertools 0.11.0", + "indexmap 2.2.6", + "itertools 0.12.1", "log", "num_cpus", "object_store", "parking_lot 0.12.1", "parquet", - "percent-encoding", "pin-project-lite", "rand", - "sqlparser 0.38.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sqlparser 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile", "tokio", "tokio-util", "url", "uuid", "xz2", - "zstd 0.13.0", + "zstd 0.13.1", ] [[package]] name = "datafusion-common" -version = "32.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9" +version = "37.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=34eda15b73a9e278af8844b30ed2f1c21c10359c#34eda15b73a9e278af8844b30ed2f1c21c10359c" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.11", "arrow", "arrow-array", "arrow-buffer", "arrow-schema", "chrono", - "half 2.3.1", + "half 2.4.1", + "instant", + "libc", "num_cpus", "object_store", "parquet", - "sqlparser 0.38.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sqlparser 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "datafusion-common-runtime" +version = "37.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=34eda15b73a9e278af8844b30ed2f1c21c10359c#34eda15b73a9e278af8844b30ed2f1c21c10359c" +dependencies = [ + "tokio", ] [[package]] name = "datafusion-execution" -version = "32.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9" +version = "37.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=34eda15b73a9e278af8844b30ed2f1c21c10359c#34eda15b73a9e278af8844b30ed2f1c21c10359c" dependencies = [ "arrow", "chrono", @@ -2619,61 +2623,40 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "32.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9" +version = "37.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=34eda15b73a9e278af8844b30ed2f1c21c10359c#34eda15b73a9e278af8844b30ed2f1c21c10359c" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.11", "arrow", "arrow-array", - "datafusion-common", - "sqlparser 0.38.0 (registry+https://github.com/rust-lang/crates.io-index)", - "strum 0.25.0", - "strum_macros 0.25.3", -] - -[[package]] -name = "datafusion-optimizer" -version = "32.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9" -dependencies = [ - "arrow", - "async-trait", "chrono", "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", - "hashbrown 0.14.3", - "itertools 0.11.0", - "log", - "regex-syntax 0.8.2", + "paste", + "serde_json", + "sqlparser 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)", + "strum 0.26.2", + "strum_macros 0.26.2", ] [[package]] -name = "datafusion-physical-expr" -version = "32.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9" +name = "datafusion-functions" +version = "37.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=34eda15b73a9e278af8844b30ed2f1c21c10359c#34eda15b73a9e278af8844b30ed2f1c21c10359c" dependencies = [ - "ahash 0.8.6", "arrow", - "arrow-array", - "arrow-buffer", - "arrow-schema", - "base64 0.21.5", + "base64 0.22.0", "blake2", "blake3", "chrono", "datafusion-common", + "datafusion-execution", "datafusion-expr", - "half 2.3.1", + "datafusion-physical-expr", "hashbrown 0.14.3", "hex", - "indexmap 2.1.0", - "itertools 0.11.0", - "libc", + "itertools 0.12.1", "log", "md-5", - "paste", - "petgraph", "rand", "regex", "sha2", @@ -2682,62 +2665,156 @@ dependencies = [ ] [[package]] -name = "datafusion-physical-plan" -version = "32.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9" +name = "datafusion-functions-aggregate" +version = "37.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=34eda15b73a9e278af8844b30ed2f1c21c10359c#34eda15b73a9e278af8844b30ed2f1c21c10359c" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-array" +version = "37.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=34eda15b73a9e278af8844b30ed2f1c21c10359c#34eda15b73a9e278af8844b30ed2f1c21c10359c" dependencies = [ - "ahash 0.8.6", "arrow", "arrow-array", "arrow-buffer", + "arrow-ord", "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "itertools 0.12.1", + "log", + "paste", +] + +[[package]] +name = "datafusion-optimizer" +version = "37.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=34eda15b73a9e278af8844b30ed2f1c21c10359c#34eda15b73a9e278af8844b30ed2f1c21c10359c" +dependencies = [ + "arrow", "async-trait", "chrono", "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "hashbrown 0.14.3", + "itertools 0.12.1", + "log", + "regex-syntax 0.8.3", +] + +[[package]] +name = "datafusion-physical-expr" +version = "37.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=34eda15b73a9e278af8844b30ed2f1c21c10359c#34eda15b73a9e278af8844b30ed2f1c21c10359c" +dependencies = [ + "ahash 0.8.11", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "arrow-string", + "base64 0.22.0", + "chrono", + "datafusion-common", "datafusion-execution", "datafusion-expr", - "datafusion-physical-expr", - "futures", - "half 2.3.1", + "datafusion-functions-aggregate", + "datafusion-physical-expr-common", + "half 2.4.1", "hashbrown 0.14.3", - "indexmap 2.1.0", - "itertools 0.11.0", + "hex", + "indexmap 2.2.6", + "itertools 0.12.1", + "log", + "paste", + "petgraph", + "regex", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "37.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=34eda15b73a9e278af8844b30ed2f1c21c10359c#34eda15b73a9e278af8844b30ed2f1c21c10359c" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", +] + +[[package]] +name = "datafusion-physical-plan" +version = "37.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=34eda15b73a9e278af8844b30ed2f1c21c10359c#34eda15b73a9e278af8844b30ed2f1c21c10359c" +dependencies = [ + "ahash 0.8.11", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half 2.4.1", + "hashbrown 0.14.3", + "indexmap 2.2.6", + "itertools 0.12.1", "log", "once_cell", "parking_lot 0.12.1", "pin-project-lite", "rand", "tokio", - "uuid", ] [[package]] name = "datafusion-sql" -version = "32.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9" +version = "37.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=34eda15b73a9e278af8844b30ed2f1c21c10359c#34eda15b73a9e278af8844b30ed2f1c21c10359c" dependencies = [ "arrow", + "arrow-array", "arrow-schema", "datafusion-common", "datafusion-expr", "log", - "sqlparser 0.38.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sqlparser 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)", + "strum 0.26.2", ] [[package]] name = "datafusion-substrait" -version = "32.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=26e43acac3a96cec8dd4c8365f22dfb1a84306e9#26e43acac3a96cec8dd4c8365f22dfb1a84306e9" +version = "37.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=34eda15b73a9e278af8844b30ed2f1c21c10359c#34eda15b73a9e278af8844b30ed2f1c21c10359c" dependencies = [ "async-recursion", "chrono", "datafusion", - "itertools 0.11.0", + "itertools 0.12.1", "object_store", - "prost 0.12.3", - "prost-types 0.12.3", - "substrait 0.17.1", - "tokio", + "prost 0.12.4", + "prost-types 0.12.4", + "substrait 0.30.0", ] [[package]] @@ -2781,7 +2858,7 @@ dependencies = [ "mito2", "object-store", "prometheus", - "prost 0.12.3", + "prost 0.12.4", "query", "reqwest", "secrecy", @@ -2793,8 +2870,8 @@ dependencies = [ "substrait 0.7.2", "table", "tokio", - "toml 0.8.8", - "tonic 0.10.2", + "toml 0.8.12", + "tonic 0.11.0", ] [[package]] @@ -2843,9 +2920,9 @@ dependencies = [ [[package]] name = "der" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" dependencies = [ "const-oid 0.9.6", "pem-rfc7468 0.7.0", @@ -2854,9 +2931,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", "serde", @@ -2881,7 +2958,7 @@ checksum = "d150dea618e920167e5973d70ae6ece4385b7164e0d799fe7c122dd0a5d912ad" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -2892,7 +2969,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -3090,15 +3167,15 @@ checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" [[package]] name = "dyn-clone" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" [[package]] name = "either" -version = "1.9.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" [[package]] name = "ena" @@ -3117,9 +3194,9 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if 1.0.0", ] @@ -3132,34 +3209,34 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "enum-iterator" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7add3873b5dd076766ee79c8e406ad1a472c385476b9e38849f8eec24f1be689" +checksum = "9fd242f399be1da0a5354aa462d57b4ab2b4ee0683cc552f7c007d2d12d36e94" dependencies = [ "enum-iterator-derive", ] [[package]] name = "enum-iterator-derive" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" +checksum = "03cdc46ec28bd728e67540c528013c6a10eb69a02eb31078a1bda695438cbfb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] name = "enum_dispatch" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f33313078bb8d4d05a2733a94ac4c2d8a0df9a2b84424ebf4f33bfc224a890e" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -3170,9 +3247,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "erased-serde" -version = "0.4.1" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4adbf0983fe06bd3a5c19c8477a637c2389feb0994eca7a59e3b961054aa7c0a" +checksum = "2b73807008a3c7f171cc40312f37d95ef0396e048b5848d775f54b1a4dd4a0d3" dependencies = [ "serde", ] @@ -3187,15 +3264,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "error-chain" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" -dependencies = [ - "version_check", -] - [[package]] name = "error-code" version = "2.3.1" @@ -3208,15 +3276,14 @@ dependencies = [ [[package]] name = "etcd-client" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5231ad671c74ee5dc02753a0a9c855fe6e90de2a07acb2582f8a702470e04d1" +version = "0.12.4" +source = "git+https://github.com/MichaelScofield/etcd-client.git?rev=4c371e9b3ea8e0a8ee2f9cbd7ded26e54a45df3b#4c371e9b3ea8e0a8ee2f9cbd7ded26e54a45df3b" dependencies = [ "http", - "prost 0.12.3", + "prost 0.12.4", "tokio", "tokio-stream", - "tonic 0.10.2", + "tonic 0.11.0", "tonic-build 0.10.2", "tower", "tower-service", @@ -3228,6 +3295,38 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +[[package]] +name = "event-listener" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.3", + "pin-project-lite", +] + [[package]] name = "exitcode" version = "1.1.2" @@ -3259,18 +3358,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - -[[package]] -name = "fastrand" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" [[package]] name = "fd-lock" @@ -3279,7 +3369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5" dependencies = [ "cfg-if 1.0.0", - "rustix 0.38.28", + "rustix 0.38.32", "windows-sys 0.48.0", ] @@ -3301,6 +3391,7 @@ dependencies = [ "common-test-util", "common-time", "datafusion", + "datafusion-expr", "datatypes", "futures", "object-store", @@ -3350,9 +3441,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flagset" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a7e408202050813e6f1d9addadcaafef3dca7530c7ddfb005d4081cce6779" +checksum = "cdeb3aa5e95cf9aabc17f060cfa0ced7b83f042390760ca53bf09df9968acaa1" [[package]] name = "flatbuffers" @@ -3404,7 +3495,7 @@ dependencies = [ "hydroflow", "itertools 0.10.5", "num-traits", - "prost 0.12.3", + "prost 0.12.4", "query", "serde", "serde_json", @@ -3416,7 +3507,7 @@ dependencies = [ "substrait 0.7.2", "table", "tokio", - "tonic 0.10.2", + "tonic 0.11.0", ] [[package]] @@ -3480,11 +3571,11 @@ dependencies = [ "log-store", "meta-client", "meta-srv", - "opentelemetry-proto 0.3.0", + "opentelemetry-proto 0.5.0", "operator", "partition", "prometheus", - "prost 0.12.3", + "prost 0.12.4", "query", "raft-engine", "script", @@ -3493,13 +3584,13 @@ dependencies = [ "session", "snafu", "sql", - "sqlparser 0.38.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=6a93567ae38d42be5c8d08b13c8ff4dde26502ef)", + "sqlparser 0.44.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=c919990bf62ad38d2b0c0a3bc90b26ad919d51b0)", "store-api", "strfmt", "table", "tokio", - "toml 0.8.8", - "tonic 0.10.2", + "toml 0.8.12", + "tonic 0.11.0", "tower", "uuid", ] @@ -3529,7 +3620,7 @@ checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -3541,7 +3632,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -3553,7 +3644,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -3654,7 +3745,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -3671,9 +3762,9 @@ checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" @@ -3724,9 +3815,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3755,11 +3846,11 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "git2" -version = "0.18.1" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd" +checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "libc", "libgit2-sys", "log", @@ -3775,15 +3866,15 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=04d78b6e025ceb518040fdd10858c2a9d9345820#04d78b6e025ceb518040fdd10858c2a9d9345820" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=73ac0207ab71dfea48f30259ffdb611501b5ecb8#73ac0207ab71dfea48f30259ffdb611501b5ecb8" dependencies = [ - "prost 0.12.3", + "prost 0.12.4", "serde", "serde_json", "strum 0.25.0", "strum_macros 0.25.3", - "tonic 0.10.2", - "tonic-build 0.10.2", + "tonic 0.11.0", + "tonic-build 0.11.0", ] [[package]] @@ -3798,7 +3889,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.1.0", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -3807,15 +3898,15 @@ dependencies = [ [[package]] name = "half" -version = "1.8.2" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" [[package]] name = "half" -version = "2.3.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ "cfg-if 1.0.0", "crunchy", @@ -3828,7 +3919,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash 0.7.7", + "ahash 0.7.8", ] [[package]] @@ -3837,7 +3928,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.11", ] [[package]] @@ -3846,7 +3937,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.11", "allocator-api2", ] @@ -3865,7 +3956,7 @@ version = "7.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "byteorder", "flate2", "nom", @@ -3878,7 +3969,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "bytes", "headers-core", "http", @@ -3905,6 +3996,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -3916,9 +4013,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hex" @@ -3981,9 +4078,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -4021,9 +4118,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "human-panic" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a79a67745be0cb8dd2771f03b24c2f25df98d5471fe7a595d668cfa2e6f843d" +checksum = "c4f016c89920bbb30951a8405ecacbb4540db5524313b9445736e7e1855cf370" dependencies = [ "anstream", "anstyle", @@ -4031,7 +4128,7 @@ dependencies = [ "os_info", "serde", "serde_derive", - "toml 0.8.8", + "toml 0.8.12", "uuid", ] @@ -4092,7 +4189,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -4107,7 +4204,7 @@ dependencies = [ "rust-sitter", "rust-sitter-tool", "slotmap", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -4116,17 +4213,17 @@ version = "0.6.0" source = "git+https://github.com/GreptimeTeam/hydroflow.git?rev=ba2df44efd42b7c4d37ebefbf82e77c6f1d4cb94#ba2df44efd42b7c4d37ebefbf82e77c6f1d4cb94" dependencies = [ "auto_impl", - "clap 4.4.11", + "clap 4.5.4", "data-encoding", "itertools 0.10.5", - "prettyplease 0.2.15", + "prettyplease 0.2.19", "proc-macro2", "quote", "regex", "serde", "serde_json", "slotmap", - "syn 2.0.43", + "syn 2.0.60", "webbrowser", ] @@ -4140,7 +4237,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -4160,7 +4257,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.5", + "socket2 0.5.6", "tokio", "tower-service", "tracing", @@ -4195,16 +4292,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core 0.51.1", + "windows-core", ] [[package]] @@ -4274,10 +4371,10 @@ dependencies = [ "greptime-proto", "mockall", "pin-project", - "prost 0.12.3", + "prost 0.12.4", "rand", "regex", - "regex-automata 0.4.3", + "regex-automata 0.4.6", "snafu", "tempfile", "tokio", @@ -4297,9 +4394,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -4308,9 +4405,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" dependencies = [ "console", "instant", @@ -4321,9 +4418,9 @@ dependencies = [ [[package]] name = "indoc" -version = "1.0.9" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "inferno" @@ -4331,8 +4428,8 @@ version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "321f0f839cd44a4686e9504b0a62b4d69a50b62072144c71c68f5873c167b8d9" dependencies = [ - "ahash 0.8.6", - "indexmap 2.1.0", + "ahash 0.8.11", + "indexmap 2.2.6", "is-terminal", "itoa", "log", @@ -4410,9 +4507,9 @@ checksum = "924df4f0e24e2e7f9cdd90babb0b96f93b20f3ecfa949ea9e6613756b8c8e1bf" [[package]] name = "inventory" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8573b2b1fb643a372c73b23f4da5f888677feef3305146d68a539250a9bccc7" +checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767" [[package]] name = "io-lifetimes" @@ -4420,7 +4517,7 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi 0.3.9", "libc", "windows-sys 0.48.0", ] @@ -4433,9 +4530,9 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "iri-string" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21859b667d66a4c1dacd9df0863b3efb65785474255face87f5bca39dd8407c0" +checksum = "7f5f6c2df22c009ac44f6f1499308e7a3ac7ba42cd2378475cc691510e1eef1b" dependencies = [ "memchr", "serde", @@ -4456,13 +4553,13 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ - "hermit-abi 0.3.3", - "rustix 0.38.28", - "windows-sys 0.48.0", + "hermit-abi 0.3.9", + "libc", + "windows-sys 0.52.0", ] [[package]] @@ -4476,27 +4573,18 @@ dependencies = [ [[package]] name = "itertools" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jni" @@ -4522,18 +4610,18 @@ checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" [[package]] name = "jobserver" -version = "0.1.27" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -4551,14 +4639,14 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "9.2.0" +version = "9.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4" +checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "js-sys", "pem", - "ring 0.17.7", + "ring 0.17.8", "serde", "serde_json", "simple_asn1", @@ -4566,9 +4654,9 @@ dependencies = [ [[package]] name = "keccak" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" dependencies = [ "cpufeatures", ] @@ -4579,7 +4667,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.6", ] [[package]] @@ -4761,6 +4849,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "libloading" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +dependencies = [ + "cfg-if 1.0.0", + "windows-targets 0.52.5", +] + [[package]] name = "libm" version = "0.2.8" @@ -4769,13 +4867,12 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libredox" -version = "0.0.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "libc", - "redox_syscall 0.4.1", ] [[package]] @@ -4791,9 +4888,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "5e143b5e666b2695d28f6bca6497720813f699c9602dd7f5cac91008b8ada7f9" dependencies = [ "cc", "libc", @@ -4815,9 +4912,9 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -4831,9 +4928,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "log-store" @@ -4925,9 +5022,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2994eeba8ed550fd9b47a0b38f0242bc3344e496483c6180b69139cc2fa5d1d7" +checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" dependencies = [ "hashbrown 0.14.3", ] @@ -4961,6 +5058,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lz4_flex" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" +dependencies = [ + "twox-hash", +] + [[package]] name = "lzma-sys" version = "0.1.20" @@ -4974,23 +5080,14 @@ dependencies = [ [[package]] name = "mac_address" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4863ee94f19ed315bf3bc00299338d857d4b5bc856af375cc97d237382ad3856" +checksum = "5aa12182b93606fff55b70a5cfe6130eaf7407c2ea4f2c2bcc8b113b67c9928f" dependencies = [ - "nix 0.23.2", + "nix 0.28.0", "winapi", ] -[[package]] -name = "mach2" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" -dependencies = [ - "libc", -] - [[package]] name = "malloc_buf" version = "0.0.6" @@ -5061,9 +5158,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "memcomparable" @@ -5087,9 +5184,9 @@ dependencies = [ [[package]] name = "memmap2" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45fd3a57831bf88bc63f8cebc0cf956116276e97fef3966103e96416209f7c92" +checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" dependencies = [ "libc", ] @@ -5114,9 +5211,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" dependencies = [ "autocfg", ] @@ -5141,7 +5238,7 @@ dependencies = [ "snafu", "tokio", "tokio-stream", - "tonic 0.10.2", + "tonic 0.11.0", "tower", "tracing", "tracing-subscriber", @@ -5182,7 +5279,7 @@ dependencies = [ "once_cell", "parking_lot 0.12.1", "prometheus", - "prost 0.12.3", + "prost 0.12.4", "rand", "regex", "serde", @@ -5194,8 +5291,8 @@ dependencies = [ "table", "tokio", "tokio-stream", - "toml 0.8.8", - "tonic 0.10.2", + "toml 0.8.12", + "tonic 0.11.0", "tower", "tracing", "tracing-subscriber", @@ -5228,7 +5325,7 @@ dependencies = [ "api", "aquamarine", "async-trait", - "base64 0.21.5", + "base64 0.21.7", "common-error", "common-macro", "common-query", @@ -5274,9 +5371,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -5336,7 +5433,7 @@ dependencies = [ "paste", "pin-project", "prometheus", - "prost 0.12.3", + "prost 0.12.4", "puffin", "rand", "regex", @@ -5351,7 +5448,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-util", - "toml 0.8.8", + "toml 0.8.12", "uuid", ] @@ -5384,21 +5481,21 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.1" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8017ec3548ffe7d4cef7ac0e12b044c01164a74c0f3119420faeaf13490ad8b" +checksum = "87bfd249f570638bfb0b4f9d258e6b8cddd2a5a7d0ed47e8bb8b176bfc0e7a17" dependencies = [ "async-lock", "async-trait", "crossbeam-channel", "crossbeam-epoch", "crossbeam-utils", + "event-listener 5.3.0", "futures-util", "once_cell", "parking_lot 0.12.1", "quanta", "rustc_version", - "skeptic", "smallvec", "tagptr", "thiserror", @@ -5421,6 +5518,12 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" + [[package]] name = "mur3" version = "0.1.0" @@ -5433,32 +5536,32 @@ version = "0.30.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56b0d8a0db9bf6d2213e11f2c701cb91387b0614361625ab7b9743b41aa4938f" dependencies = [ - "darling 0.20.3", - "heck", + "darling 0.20.8", + "heck 0.4.1", "num-bigint", "proc-macro-crate 1.3.1", "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", "termcolor", "thiserror", ] [[package]] name = "mysql-common-derive" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c60492b5eb751e55b42d716b6b26dceb66767996cd7a5560a842fbf613ca2e92" +checksum = "afe0450cc9344afff34915f8328600ab5ae19260802a334d0f72d2d5bdda3bfe" dependencies = [ - "darling 0.20.3", - "heck", + "darling 0.20.8", + "heck 0.4.1", "num-bigint", "proc-macro-crate 3.1.0", "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", "termcolor", "thiserror", ] @@ -5489,7 +5592,7 @@ dependencies = [ "rustls-pemfile 1.0.4", "serde", "serde_json", - "socket2 0.5.5", + "socket2 0.5.6", "thiserror", "tokio", "tokio-rustls 0.24.1", @@ -5497,7 +5600,7 @@ dependencies = [ "twox-hash", "url", "webpki", - "webpki-roots 0.25.3", + "webpki-roots 0.25.4", ] [[package]] @@ -5506,10 +5609,10 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06f19e4cfa0ab5a76b627cec2d81331c49b034988eaf302c3bafeada684eadef" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "bigdecimal", "bindgen", - "bitflags 2.4.1", + "bitflags 2.5.0", "bitvec", "btoi", "byteorder", @@ -5541,14 +5644,14 @@ dependencies = [ [[package]] name = "mysql_common" -version = "0.32.0" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73aacd01475af6d2efbdf489efd60fc519515ffe94edfd74236f954d521e31b" +checksum = "0ccdc1fe2bb3ef97e07ba4397327ed45509a1e2e499e2f8265243879cbc7313c" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "bigdecimal", "bindgen", - "bitflags 2.4.1", + "bitflags 2.5.0", "bitvec", "btoi", "byteorder", @@ -5560,7 +5663,7 @@ dependencies = [ "flate2", "frunk", "lazy_static", - "mysql-common-derive 0.31.0", + "mysql-common-derive 0.31.1", "num-bigint", "num-traits", "rand", @@ -5576,7 +5679,7 @@ dependencies = [ "thiserror", "time", "uuid", - "zstd 0.13.0", + "zstd 0.13.1", ] [[package]] @@ -5616,9 +5719,9 @@ checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" [[package]] name = "new_debug_unreachable" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" [[package]] name = "nibble_vec" @@ -5629,19 +5732,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "nix" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" -dependencies = [ - "bitflags 1.3.2", - "cc", - "cfg-if 1.0.0", - "libc", - "memoffset 0.6.5", -] - [[package]] name = "nix" version = "0.25.1" @@ -5669,6 +5759,19 @@ dependencies = [ "pin-utils", ] +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.5.0", + "cfg-if 1.0.0", + "cfg_aliases", + "libc", + "memoffset 0.9.1", +] + [[package]] name = "nom" version = "7.1.3" @@ -5691,7 +5794,7 @@ version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "crossbeam-channel", "filetime", "fsevent-sys", @@ -5725,9 +5828,9 @@ dependencies = [ [[package]] name = "num" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" +checksum = "3135b08af27d103b0a51f2ae0f8632117b7b185ccf931445affa8df530576a41" dependencies = [ "num-bigint", "num-complex", @@ -5767,22 +5870,28 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214" +checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" dependencies = [ "num-traits", ] [[package]] -name = "num-derive" -version = "0.4.1" +name = "num-conv" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -5797,19 +5906,18 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" dependencies = [ "autocfg", "num-integer", @@ -5830,9 +5938,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", "libm", @@ -5844,7 +5952,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi 0.3.9", "libc", ] @@ -5923,16 +6031,16 @@ dependencies = [ [[package]] name = "object_store" -version = "0.7.1" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f930c88a43b1c3f6e776dfe495b4afab89882dbc81530c632db2ed65451ebcb4" +checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" dependencies = [ "async-trait", "bytes", "chrono", "futures", "humantime", - "itertools 0.11.0", + "itertools 0.12.1", "parking_lot 0.12.1", "percent-encoding", "snafu", @@ -5963,7 +6071,7 @@ dependencies = [ "anyhow", "async-trait", "backon", - "base64 0.21.5", + "base64 0.21.7", "bytes", "chrono", "flagset", @@ -6005,7 +6113,7 @@ dependencies = [ "async-trait", "byteorder", "chrono", - "mysql_common 0.32.0", + "mysql_common 0.32.2", "nom", "pin-project-lite", "tokio", @@ -6026,7 +6134,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.1.0", + "indexmap 2.2.6", "js-sys", "once_cell", "pin-project-lite", @@ -6036,12 +6144,12 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.21.0" -source = "git+https://github.com/waynexia/opentelemetry-rust.git?rev=33841b38dda79b15f2024952be5f32533325ca02#33841b38dda79b15f2024952be5f32533325ca02" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d57987be3f2aeb70d385fff9b27fb74c5723cc9a52d904d4f9c807a0667bf" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.1.0", "js-sys", "once_cell", "pin-project-lite", @@ -6058,72 +6166,54 @@ dependencies = [ "async-trait", "futures-core", "http", - "opentelemetry 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)", + "opentelemetry 0.21.0", "opentelemetry-proto 0.4.0", "opentelemetry-semantic-conventions", - "opentelemetry_sdk 0.21.1", + "opentelemetry_sdk 0.21.2", "prost 0.11.9", "thiserror", "tokio", "tonic 0.9.2", ] -[[package]] -name = "opentelemetry-proto" -version = "0.3.0" -source = "git+https://github.com/waynexia/opentelemetry-rust.git?rev=33841b38dda79b15f2024952be5f32533325ca02#33841b38dda79b15f2024952be5f32533325ca02" -dependencies = [ - "opentelemetry 0.21.0 (git+https://github.com/waynexia/opentelemetry-rust.git?rev=33841b38dda79b15f2024952be5f32533325ca02)", - "opentelemetry_sdk 0.20.0", - "prost 0.12.3", - "tonic 0.10.2", -] - [[package]] name = "opentelemetry-proto" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2e155ce5cc812ea3d1dffbd1539aed653de4bf4882d60e6e04dcf0901d674e1" dependencies = [ - "opentelemetry 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)", - "opentelemetry_sdk 0.21.1", + "opentelemetry 0.21.0", + "opentelemetry_sdk 0.21.2", "prost 0.11.9", "tonic 0.9.2", ] +[[package]] +name = "opentelemetry-proto" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8fddc9b68f5b80dae9d6f510b88e02396f006ad48cac349411fbecc80caae4" +dependencies = [ + "opentelemetry 0.22.0", + "opentelemetry_sdk 0.22.1", + "prost 0.12.4", + "tonic 0.11.0", +] + [[package]] name = "opentelemetry-semantic-conventions" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" dependencies = [ - "opentelemetry 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)", + "opentelemetry 0.21.0", ] [[package]] name = "opentelemetry_sdk" -version = "0.20.0" -source = "git+https://github.com/waynexia/opentelemetry-rust.git?rev=33841b38dda79b15f2024952be5f32533325ca02#33841b38dda79b15f2024952be5f32533325ca02" -dependencies = [ - "async-trait", - "crossbeam-channel", - "futures-channel", - "futures-executor", - "futures-util", - "glob", - "once_cell", - "opentelemetry 0.21.0 (git+https://github.com/waynexia/opentelemetry-rust.git?rev=33841b38dda79b15f2024952be5f32533325ca02)", - "ordered-float 4.2.0", - "percent-encoding", - "rand", - "thiserror", -] - -[[package]] -name = "opentelemetry_sdk" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "968ba3f2ca03e90e5187f5e4f46c791ef7f2c163ae87789c8ce5f5ca3b7b7de5" +checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" dependencies = [ "async-trait", "crossbeam-channel", @@ -6132,7 +6222,7 @@ dependencies = [ "futures-util", "glob", "once_cell", - "opentelemetry 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)", + "opentelemetry 0.21.0", "ordered-float 4.2.0", "percent-encoding", "rand", @@ -6141,6 +6231,26 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "opentelemetry_sdk" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e90c7113be649e31e9a0f8b5ee24ed7a16923b322c3c5ab6367469c049d6b7e" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry 0.22.0", + "ordered-float 4.2.0", + "percent-encoding", + "rand", + "thiserror", +] + [[package]] name = "operator" version = "0.7.2" @@ -6186,12 +6296,12 @@ dependencies = [ "session", "snafu", "sql", - "sqlparser 0.38.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=6a93567ae38d42be5c8d08b13c8ff4dde26502ef)", + "sqlparser 0.44.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=c919990bf62ad38d2b0c0a3bc90b26ad919d51b0)", "store-api", "substrait 0.7.2", "table", "tokio", - "tonic 0.10.2", + "tonic 0.11.0", ] [[package]] @@ -6203,8 +6313,7 @@ checksum = "978aa494585d3ca4ad74929863093e87cac9790d81fe7aba2b3dc2890643a0fc" [[package]] name = "orc-rust" version = "0.2.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900310981898f6e3877286f1272b75f5c4a604628594a0a7026311b93a2aa5e6" +source = "git+https://github.com/MichaelScofield/orc-rs.git?rev=17347f5f084ac937863317df882218055c4ea8c1#17347f5f084ac937863317df882218055c4ea8c1" dependencies = [ "arrow", "bytes", @@ -6263,9 +6372,9 @@ dependencies = [ [[package]] name = "ordered-multimap" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4d6a8c22fc714f0c2373e6091bf6f5e9b37b1bc0b1184874b7e0a4e303d318f" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" dependencies = [ "dlv-list 0.5.2", "hashbrown 0.14.3", @@ -6273,13 +6382,13 @@ dependencies = [ [[package]] name = "os_info" -version = "3.7.0" +version = "3.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "006e42d5b888366f1880eda20371fedde764ed2213dc8496f49622fa0c99cd5e" +checksum = "ae99c7fa6dd38c7cafe1ec085e804f8f555a2f8659b0dbe03f1f9963a9b51092" dependencies = [ "log", "serde", - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -6323,6 +6432,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" + [[package]] name = "parking_lot" version = "0.11.2" @@ -6376,11 +6491,11 @@ dependencies = [ [[package]] name = "parquet" -version = "47.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0463cc3b256d5f50408c49a4be3a16674f4c8ceef60941709620a062b1f6bf4d" +checksum = "096795d4f47f65fd3ee1ec5a98b77ab26d602f2cc785b0e4be5443add17ecc32" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.11", "arrow-array", "arrow-buffer", "arrow-cast", @@ -6388,14 +6503,15 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.21.5", - "brotli", + "base64 0.22.0", + "brotli 3.5.0", "bytes", "chrono", "flate2", "futures", + "half 2.4.1", "hashbrown 0.14.3", - "lz4", + "lz4_flex 0.11.3", "num", "num-bigint", "object_store", @@ -6405,7 +6521,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd 0.12.4", + "zstd 0.13.1", ] [[package]] @@ -6435,7 +6551,7 @@ dependencies = [ "serde_json", "snafu", "sql", - "sqlparser 0.38.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=6a93567ae38d42be5c8d08b13c8ff4dde26502ef)", + "sqlparser 0.44.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=c919990bf62ad38d2b0c0a3bc90b26ad919d51b0)", "store-api", "table", ] @@ -6468,19 +6584,13 @@ dependencies = [ "hmac", ] -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "pem" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" +checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" dependencies = [ - "base64 0.21.5", + "base64 0.22.0", "serde", ] @@ -6516,9 +6626,9 @@ checksum = "df202b0b0f5b8e389955afd5f27b007b00fb948162953f1db9c70d2c7e3157d7" [[package]] name = "pest" -version = "2.7.5" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" +checksum = "311fb059dee1a7b802f036316d790138c613a4e8b180c822e3925a662e9f0c95" dependencies = [ "memchr", "thiserror", @@ -6527,9 +6637,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.5" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" +checksum = "f73541b156d32197eecda1a4014d7f868fd2bcb3c550d5386087cfba442bf69c" dependencies = [ "pest", "pest_generator", @@ -6537,22 +6647,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.5" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" +checksum = "c35eeed0a3fab112f75165fdc026b3913f4183133f19b49be773ac9ea966e8bd" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] name = "pest_meta" -version = "2.7.5" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" +checksum = "2adbf29bb9776f28caece835398781ab24435585fe0d4dc1374a61db5accedca" dependencies = [ "once_cell", "pest", @@ -6566,7 +6676,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.1.0", + "indexmap 2.2.6", ] [[package]] @@ -6586,7 +6696,7 @@ dependencies = [ "md5", "postgres-types", "rand", - "ring 0.17.7", + "ring 0.17.8", "stringprep", "thiserror", "time", @@ -6645,29 +6755,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -6692,7 +6802,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der 0.7.8", + "der 0.7.9", "pkcs8 0.10.2", "spki 0.7.3", ] @@ -6705,7 +6815,7 @@ checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" dependencies = [ "aes", "cbc", - "der 0.7.8", + "der 0.7.9", "pbkdf2", "scrypt", "sha2", @@ -6729,7 +6839,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.8", + "der 0.7.9", "pkcs5", "rand_core", "spki 0.7.3", @@ -6737,9 +6847,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plotters" @@ -6804,7 +6914,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "byteorder", "bytes", "fallible-iterator", @@ -6850,9 +6960,9 @@ dependencies = [ "nix 0.26.4", "once_cell", "parking_lot 0.12.1", - "prost 0.12.3", - "prost-build 0.12.3", - "prost-derive 0.12.3", + "prost 0.12.4", + "prost-build 0.12.4", + "prost-derive 0.12.4", "protobuf", "sha2", "smallvec", @@ -6925,12 +7035,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.15" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +checksum = "5ac2cf0f2e4f42b49f5ffd07dae8d746508ef7526c13940e5f524012ae6c6550" dependencies = [ "proc-macro2", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -6943,22 +7053,13 @@ dependencies = [ "toml_edit 0.19.15", ] -[[package]] -name = "proc-macro-crate" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8" -dependencies = [ - "toml_edit 0.20.7", -] - [[package]] name = "proc-macro-crate" version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" dependencies = [ - "toml_edit 0.21.0", + "toml_edit 0.21.1", ] [[package]] @@ -6987,9 +7088,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.71" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" +checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" dependencies = [ "unicode-ident", ] @@ -7040,7 +7141,7 @@ dependencies = [ name = "promql" version = "0.7.2" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.11", "async-recursion", "async-trait", "bytemuck", @@ -7052,13 +7153,16 @@ dependencies = [ "common-recordbatch", "common-telemetry", "datafusion", + "datafusion-expr", + "datafusion-functions", "datatypes", "futures", "greptime-proto", + "itertools 0.10.5", "lazy_static", "prometheus", "promql-parser", - "prost 0.12.3", + "prost 0.12.4", "query", "session", "snafu", @@ -7091,12 +7195,12 @@ dependencies = [ [[package]] name = "prost" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" +checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" dependencies = [ "bytes", - "prost-derive 0.12.3", + "prost-derive 0.12.4", ] [[package]] @@ -7106,11 +7210,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ "bytes", - "heck", + "heck 0.4.1", "itertools 0.10.5", "lazy_static", "log", - "multimap", + "multimap 0.8.3", "petgraph", "prettyplease 0.1.25", "prost 0.11.9", @@ -7123,24 +7227,23 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" +checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1" dependencies = [ "bytes", - "heck", - "itertools 0.11.0", + "heck 0.5.0", + "itertools 0.12.1", "log", - "multimap", + "multimap 0.10.0", "once_cell", "petgraph", - "prettyplease 0.2.15", - "prost 0.12.3", - "prost-types 0.12.3", + "prettyplease 0.2.19", + "prost 0.12.4", + "prost-types 0.12.4", "regex", - "syn 2.0.43", + "syn 2.0.60", "tempfile", - "which", ] [[package]] @@ -7158,15 +7261,15 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" +checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -7180,11 +7283,11 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" +checksum = "3235c33eb02c1f1e212abdbe34c78b264b038fb58ca612664343271e36e55ffe" dependencies = [ - "prost 0.12.3", + "prost 0.12.4", ] [[package]] @@ -7252,7 +7355,7 @@ name = "puffin" version = "0.7.2" dependencies = [ "async-trait", - "bitflags 2.4.1", + "bitflags 2.5.0", "common-error", "common-macro", "derive_builder 0.12.0", @@ -7265,17 +7368,6 @@ dependencies = [ "tokio-util", ] -[[package]] -name = "pulldown-cmark" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" -dependencies = [ - "bitflags 1.3.2", - "memchr", - "unicase", -] - [[package]] name = "puruspe" version = "0.1.5" @@ -7293,15 +7385,16 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.19.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e681a6cfdc4adcc93b4d3cf993749a4552018ee0a9b65fc0ccfad74352c72a38" +checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" dependencies = [ "cfg-if 1.0.0", "indoc", "libc", - "memoffset 0.9.0", + "memoffset 0.9.1", "parking_lot 0.12.1", + "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", @@ -7310,9 +7403,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.19.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076c73d0bc438f7a4ef6fdd0c3bb4732149136abd952b110ac93e4edb13a6ba5" +checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" dependencies = [ "once_cell", "target-lexicon", @@ -7320,9 +7413,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.19.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9" +checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" dependencies = [ "libc", "pyo3-build-config", @@ -7330,36 +7423,37 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.19.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfeb4c99597e136528c6dd7d5e3de5434d1ceaf487436a3f03b2d56b6fc9efd1" +checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 1.0.109", + "syn 2.0.60", ] [[package]] name = "pyo3-macros-backend" -version = "0.19.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "947dc12175c254889edc0c02e399476c2f652b4b9ebd123aa655c224de259536" +checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" dependencies = [ + "heck 0.4.1", "proc-macro2", + "pyo3-build-config", "quote", - "syn 1.0.109", + "syn 2.0.60", ] [[package]] name = "quanta" -version = "0.11.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +checksum = "8e5167a477619228a0b284fac2674e3c388cba90631d7b7de620e6f1fcd08da5" dependencies = [ "crossbeam-utils", "libc", - "mach2", "once_cell", "raw-cpuid", "wasi", @@ -7371,7 +7465,7 @@ dependencies = [ name = "query" version = "0.7.2" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.11", "api", "approx_eq", "arc-swap", @@ -7453,9 +7547,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -7553,11 +7647,11 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "10.7.0" +version = "11.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", ] [[package]] @@ -7574,9 +7668,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -7584,9 +7678,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -7612,9 +7706,9 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ "getrandom", "libredox", @@ -7623,34 +7717,34 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53313ec9f12686aeeffb43462c3ac77aa25f590a5f630eb2cde0de59417b29c7" +checksum = "c4846d4c50d1721b1a3bef8af76924eef20d5e723647333798c1b519b3a9473f" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2566c4bf6845f2c2e83b27043c3f5dfcd5ba8f2937d6c00dc009bfb51a079dc4" +checksum = "5fddb4f8d99b0a2ebafc65a87a69a7b9875e4b1ae1f00db265d300ef7f28bccc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] name = "regex" -version = "1.10.2" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.3", - "regex-syntax 0.8.2", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", ] [[package]] @@ -7664,13 +7758,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax 0.8.3", ] [[package]] @@ -7681,15 +7775,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" - -[[package]] -name = "regex-syntax" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "regress" @@ -7702,10 +7790,20 @@ dependencies = [ ] [[package]] -name = "rend" -version = "0.4.1" +name = "regress" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2571463863a6bd50c32f94402933f03457a3fbaf697a707c5be741e459f08fd" +checksum = "4f5f39ba4513916c1b2657b72af6ec671f091cd637992f58d0ede5cae4e5dea0" +dependencies = [ + "hashbrown 0.14.3", + "memchr", +] + +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" dependencies = [ "bytecheck", ] @@ -7718,7 +7816,7 @@ checksum = "43e319d9de9ff4d941abf4ac718897118b0fe04577ea3f8e0f5788971784eef5" dependencies = [ "anyhow", "async-trait", - "base64 0.21.5", + "base64 0.21.7", "chrono", "form_urlencoded", "getrandom", @@ -7743,11 +7841,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.23" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -7771,6 +7869,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-rustls 0.24.1", @@ -7845,23 +7944,24 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", + "cfg-if 1.0.0", "getrandom", "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "rkyv" -version = "0.7.43" +version = "0.7.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527a97cdfef66f65998b5f3b637c26f5a5ec09cc52a3f9932313ac645f4190f5" +checksum = "5cba464629b3394fc4dbc6f940ff8f5b4ff5c7aef40f29166fd4ad12acbc99c0" dependencies = [ "bitvec", "bytecheck", @@ -7877,9 +7977,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.43" +version = "0.7.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5c462a1328c8e67e4d6dbad1eb0355dd43e8ab432c6e227a43657f16ade5033" +checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65" dependencies = [ "proc-macro2", "quote", @@ -8020,7 +8120,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.43", + "syn 2.0.60", "walkdir", ] @@ -8051,7 +8151,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e0698206bcb8882bf2a9ecb4c1e7785db57ff052297085a6efd4fe42302068a" dependencies = [ "cfg-if 1.0.0", - "ordered-multimap 0.7.1", + "ordered-multimap 0.7.3", ] [[package]] @@ -8105,9 +8205,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.33.1" +version = "1.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06676aec5ccb8fc1da723cc8c0f9a46549f21ebb8753d3915c6c41db1e7f1dc4" +checksum = "1790d1c4c0ca81211399e0e0af16333276f375209e71a37b67698a373db5b47a" dependencies = [ "arrayvec", "borsh", @@ -8156,14 +8256,14 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "errno", "libc", - "linux-raw-sys 0.4.12", + "linux-raw-sys 0.4.13", "windows-sys 0.52.0", ] @@ -8186,21 +8286,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring 0.17.7", + "ring 0.17.8", "rustls-webpki 0.101.7", "sct", ] [[package]] name = "rustls" -version = "0.22.1" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe6b63262c9fcac8659abfaa96cac103d28166d3ff3eaf8f412e19f3ae9e5a48" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" dependencies = [ "log", - "ring 0.17.7", + "ring 0.17.8", "rustls-pki-types", - "rustls-webpki 0.102.0", + "rustls-webpki 0.102.2", "subtle", "zeroize", ] @@ -8223,24 +8323,24 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", ] [[package]] name = "rustls-pemfile" -version = "2.0.0" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64 0.21.5", + "base64 0.22.0", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.1.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" +checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247" [[package]] name = "rustls-webpki" @@ -8248,17 +8348,17 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "untrusted 0.9.0", ] [[package]] name = "rustls-webpki" -version = "0.102.0" +version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de2635c8bc2b88d367767c5de8ea1d8db9af3f6219eba28442242d9ab81d1b89" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "rustls-pki-types", "untrusted 0.9.0", ] @@ -8278,7 +8378,7 @@ name = "rustpython-codegen" version = "0.2.0" source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ - "ahash 0.7.7", + "ahash 0.7.8", "bitflags 1.3.2", "indexmap 1.9.3", "itertools 0.10.5", @@ -8333,7 +8433,7 @@ dependencies = [ "bitflags 1.3.2", "bstr", "itertools 0.10.5", - "lz4_flex", + "lz4_flex 0.9.5", "num-bigint", "num-complex", ] @@ -8379,7 +8479,7 @@ name = "rustpython-parser" version = "0.2.0" source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ - "ahash 0.7.7", + "ahash 0.7.8", "anyhow", "itertools 0.10.5", "lalrpop", @@ -8414,7 +8514,7 @@ version = "0.2.0" source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ "adler32", - "ahash 0.7.7", + "ahash 0.7.8", "ascii", "base64 0.13.1", "blake2", @@ -8480,7 +8580,7 @@ name = "rustpython-vm" version = "0.2.0" source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ - "ahash 0.7.7", + "ahash 0.7.8", "ascii", "atty", "bitflags 1.3.2", @@ -8492,7 +8592,7 @@ dependencies = [ "exitcode", "getrandom", "glob", - "half 1.8.2", + "half 1.8.3", "hex", "indexmap 1.9.3", "is-macro", @@ -8548,9 +8648,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" [[package]] name = "rustyline" @@ -8600,9 +8700,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "safe-lock" @@ -8630,18 +8730,18 @@ dependencies = [ [[package]] name = "safe-regex" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15289bf322e0673d52756a18194167f2378ec1a15fe884af6e2d2cb934822b0" +checksum = "e6ab4bc484ef480a9ce79b381efd7b6767700f514d47bc599036e9d6f7f3c49d" dependencies = [ "safe-regex-macro", ] [[package]] name = "safe-regex-compiler" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fba76fae590a2aa665279deb1f57b5098cbace01a0c5e60e262fcf55f7c51542" +checksum = "6d71f8c78bffb07962595e1bfa5ed11d24dd855eedc50b6a735f5ef648ce621b" dependencies = [ "safe-proc-macro2", "safe-quote", @@ -8649,9 +8749,9 @@ dependencies = [ [[package]] name = "safe-regex-macro" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c2e96b5c03f158d1b16ba79af515137795f4ad4e8de3f790518aae91f1d127" +checksum = "0909ab4b77511df24201cd66541d6a028887c77ecc065f277c68a12a663274ef" dependencies = [ "safe-proc-macro2", "safe-regex-compiler", @@ -8692,11 +8792,11 @@ checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -8755,6 +8855,7 @@ dependencies = [ "datafusion", "datafusion-common", "datafusion-expr", + "datafusion-functions", "datafusion-physical-expr", "datatypes", "futures", @@ -8802,7 +8903,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -8818,10 +8919,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4a8caec23b7800fb97971a1c6ae365b6239aaeddfb934d6265f8505e795699d" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -8836,9 +8937,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.2" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" dependencies = [ "bitflags 1.3.2", "core-foundation", @@ -8849,9 +8950,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" dependencies = [ "core-foundation-sys", "libc", @@ -8859,12 +8960,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" -dependencies = [ - "serde", -] +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" [[package]] name = "seq-macro" @@ -8874,22 +8972,22 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.193" +version = "1.0.198" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.198" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -8905,11 +9003,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -8917,9 +9015,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.14" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" dependencies = [ "itoa", "serde", @@ -8927,13 +9025,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" +checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -8954,7 +9052,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -8971,16 +9069,17 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.4.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" +checksum = "ee80b0e361bbf88fd2f6e242ccd19cfda072cb0faa6ae694ecee08199938569a" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.1.0", + "indexmap 2.2.6", "serde", + "serde_derive", "serde_json", "serde_with_macros", "time", @@ -8988,23 +9087,23 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.4.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" +checksum = "6561dc161a9224638a31d876ccdfefbc1df91d3f3a8342eddb35f055d48c7655" dependencies = [ - "darling 0.20.3", + "darling 0.20.8", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] name = "serde_yaml" -version = "0.9.29" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15e0ef66bf939a7c890a0bf6d5a733c70202225f9888a89ed5c62298b019129" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -9025,7 +9124,7 @@ dependencies = [ "auth", "axum", "axum-macros", - "base64 0.21.5", + "base64 0.21.7", "bytes", "catalog", "chrono", @@ -9069,7 +9168,7 @@ dependencies = [ "once_cell", "openmetrics-parser", "opensrv-mysql", - "opentelemetry-proto 0.3.0", + "opentelemetry-proto 0.5.0", "parking_lot 0.12.1", "permutation", "pgwire", @@ -9078,14 +9177,14 @@ dependencies = [ "pprof", "prometheus", "promql-parser", - "prost 0.12.3", + "prost 0.12.4", "query", "rand", "regex", "reqwest", "rust-embed", - "rustls 0.22.1", - "rustls-pemfile 2.0.0", + "rustls 0.22.4", + "rustls-pemfile 2.1.2", "rustls-pki-types", "schemars", "script", @@ -9106,12 +9205,12 @@ dependencies = [ "tokio-rustls 0.25.0", "tokio-stream", "tokio-test", - "tonic 0.10.2", + "tonic 0.11.0", "tonic-reflection", "tower", "tower-http", "urlencoding", - "zstd 0.13.0", + "zstd 0.13.1", ] [[package]] @@ -9245,21 +9344,6 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" -[[package]] -name = "skeptic" -version = "0.13.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8" -dependencies = [ - "bytecount", - "cargo_metadata", - "error-chain", - "glob", - "pulldown-cmark", - "tempfile", - "walkdir", -] - [[package]] name = "slab" version = "0.4.9" @@ -9281,15 +9365,15 @@ dependencies = [ [[package]] name = "smallbitvec" -version = "2.5.1" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ce4f9dc4a41b4c3476cc925f1efb11b66df373a8fde5d4b8915fa91b5d995e" +checksum = "fcc3fc564a4b53fd1e8589628efafe57602d91bde78be18186b5f61e8faea470" [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" dependencies = [ "serde", ] @@ -9310,7 +9394,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "syn 1.0.109", @@ -9334,12 +9418,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -9383,7 +9467,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der 0.7.8", + "der 0.7.9", ] [[package]] @@ -9411,8 +9495,8 @@ dependencies = [ "lazy_static", "regex", "snafu", - "sqlparser 0.38.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=6a93567ae38d42be5c8d08b13c8ff4dde26502ef)", - "sqlparser_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sqlparser 0.44.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=c919990bf62ad38d2b0c0a3bc90b26ad919d51b0)", + "sqlparser_derive 0.1.1", "table", ] @@ -9422,7 +9506,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" dependencies = [ - "itertools 0.12.0", + "itertools 0.12.1", "nom", "unicode_categories", ] @@ -9447,7 +9531,7 @@ name = "sqlness-runner" version = "0.7.2" dependencies = [ "async-trait", - "clap 4.4.11", + "clap 4.5.4", "client", "common-error", "common-query", @@ -9462,24 +9546,24 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.38.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272b7bb0a225320170c99901b4b5fb3a4384e255a7f2cc228f61e2ba3893e75" +checksum = "aaf9c7ff146298ffda83a200f8d5084f08dcee1edfc135fcc1d646a45d50ffd6" dependencies = [ "log", - "sqlparser_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sqlparser_derive 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "sqlparser" -version = "0.38.0" -source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=6a93567ae38d42be5c8d08b13c8ff4dde26502ef#6a93567ae38d42be5c8d08b13c8ff4dde26502ef" +version = "0.44.0" +source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=c919990bf62ad38d2b0c0a3bc90b26ad919d51b0#c919990bf62ad38d2b0c0a3bc90b26ad919d51b0" dependencies = [ "lazy_static", "log", "regex", - "sqlparser 0.38.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sqlparser_derive 0.1.1 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=6a93567ae38d42be5c8d08b13c8ff4dde26502ef)", + "sqlparser 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sqlparser_derive 0.2.2 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=c919990bf62ad38d2b0c0a3bc90b26ad919d51b0)", ] [[package]] @@ -9495,12 +9579,23 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.1.1" -source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=6a93567ae38d42be5c8d08b13c8ff4dde26502ef#6a93567ae38d42be5c8d08b13c8ff4dde26502ef" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.60", +] + +[[package]] +name = "sqlparser_derive" +version = "0.2.2" +source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=c919990bf62ad38d2b0c0a3bc90b26ad919d51b0#c919990bf62ad38d2b0c0a3bc90b26ad919d51b0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", ] [[package]] @@ -9519,8 +9614,8 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029" dependencies = [ - "ahash 0.7.7", - "atoi", + "ahash 0.7.8", + "atoi 1.0.0", "base64 0.13.1", "bitflags 1.3.2", "byteorder", @@ -9532,7 +9627,7 @@ dependencies = [ "dirs 4.0.0", "dotenvy", "either", - "event-listener", + "event-listener 2.5.3", "futures-channel", "futures-core", "futures-intrusive", @@ -9579,7 +9674,7 @@ checksum = "9966e64ae989e7e575b19d7265cb79d7fc3cbbdf179835cb0d716f294c2049c9" dependencies = [ "dotenvy", "either", - "heck", + "heck 0.4.1", "once_cell", "proc-macro2", "quote", @@ -9607,7 +9702,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1120e6a8cbd4d85d5532d2e8a245aef2128e1853981f8b6d9943264184843102" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "num_enum", "optional", ] @@ -9734,6 +9829,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "strum" version = "0.24.1" @@ -9749,13 +9850,22 @@ dependencies = [ "strum_macros 0.25.3", ] +[[package]] +name = "strum" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" +dependencies = [ + "strum_macros 0.26.2", +] + [[package]] name = "strum_macros" version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "rustversion", @@ -9768,11 +9878,24 @@ version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "rustversion", - "syn 2.0.43", + "syn 2.0.60", +] + +[[package]] +name = "strum_macros" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.60", ] [[package]] @@ -9793,14 +9916,17 @@ dependencies = [ "bytes", "catalog", "common-error", + "common-function", "common-macro", + "common-telemetry", "datafusion", "datafusion-common", "datafusion-expr", "datafusion-substrait", "datatypes", "promql", - "prost 0.12.3", + "prost 0.12.4", + "session", "snafu", "substrait 0.17.1", "tokio", @@ -9813,18 +9939,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1e8440a1c9b95a7c9a00a19f78b980749e8c945eb880687a5d673cea83729c5" dependencies = [ "git2", - "heck", - "prettyplease 0.2.15", - "prost 0.12.3", - "prost-build 0.12.3", - "prost-types 0.12.3", + "heck 0.4.1", + "prettyplease 0.2.19", + "prost 0.12.4", + "prost-build 0.12.4", + "prost-types 0.12.4", "schemars", "semver", "serde", "serde_json", "serde_yaml", - "syn 2.0.43", - "typify", + "syn 2.0.60", + "typify 0.0.14", + "walkdir", +] + +[[package]] +name = "substrait" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba959c71b2a1a341a94e1f362615d7e5f1a4de9d25d82fceea8160f79f1e1dfb" +dependencies = [ + "heck 0.5.0", + "prettyplease 0.2.19", + "prost 0.12.4", + "prost-build 0.12.4", + "prost-types 0.12.4", + "schemars", + "semver", + "serde", + "serde_json", + "serde_yaml", + "syn 2.0.60", + "typify 0.0.16", "walkdir", ] @@ -9841,7 +9988,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cccfffbc6bb3bb2d3a26cd2077f4d055f6808d266f9d4d158797a4c60510dfe" dependencies = [ "debugid", - "memmap2 0.9.3", + "memmap2 0.9.4", "stable_deref_trait", "uuid", ] @@ -9870,9 +10017,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.43" +version = "2.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" +checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" dependencies = [ "proc-macro2", "quote", @@ -9907,7 +10054,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -9918,9 +10065,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sysinfo" -version = "0.30.5" +version = "0.30.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb4f3438c8f6389c864e61221cbc97e9bca98b4daf39a5beb7bea660f528bb2" +checksum = "26d7c217777061d5a2d652aea771fb9ba98b6dade657204b08c4b9604d11555b" dependencies = [ "cfg-if 1.0.0", "core-foundation-sys", @@ -10001,9 +10148,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "target-lexicon" -version = "0.12.12" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" +checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" [[package]] name = "temp-env" @@ -10016,15 +10163,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.1" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if 1.0.0", - "fastrand 2.0.1", - "redox_syscall 0.4.1", - "rustix 0.38.28", - "windows-sys 0.48.0", + "fastrand", + "rustix 0.38.32", + "windows-sys 0.52.0", ] [[package]] @@ -10040,9 +10186,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -10086,7 +10232,7 @@ dependencies = [ "serde_json", "snafu", "sql", - "sqlparser 0.38.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=6a93567ae38d42be5c8d08b13c8ff4dde26502ef)", + "sqlparser 0.44.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=c919990bf62ad38d2b0c0a3bc90b26ad919d51b0)", "sqlx", "tokio", ] @@ -10129,11 +10275,11 @@ dependencies = [ "meta-srv", "mysql_async", "object-store", - "opentelemetry-proto 0.3.0", + "opentelemetry-proto 0.5.0", "operator", "partition", "paste", - "prost 0.12.3", + "prost 0.12.4", "query", "rand", "rstest", @@ -10152,10 +10298,10 @@ dependencies = [ "time", "tokio", "tokio-postgres", - "tonic 0.10.2", + "tonic 0.11.0", "tower", "uuid", - "zstd 0.13.0", + "zstd 0.13.1", ] [[package]] @@ -10175,28 +10321,28 @@ checksum = "b7b3e525a49ec206798b40326a44121291b530c963cfb01018f63e135bac543d" [[package]] name = "textwrap" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.51" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.51" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -10211,9 +10357,9 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if 1.0.0", "once_cell", @@ -10263,12 +10409,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.31" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", + "num-conv", "powerfmt", "serde", "time-core", @@ -10283,10 +10430,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -10344,9 +10492,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.1" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ "backtrace", "bytes", @@ -10356,7 +10504,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.5", + "socket2 0.5.6", "tokio-macros", "tracing", "windows-sys 0.48.0", @@ -10380,7 +10528,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -10428,7 +10576,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand", - "socket2 0.5.5", + "socket2 0.5.6", "tokio", "tokio-util", "whoami", @@ -10441,8 +10589,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ea13f22eda7127c827983bdaf0d7fff9df21c8817bab02815ac277a21143677" dependencies = [ "futures", - "ring 0.17.7", - "rustls 0.22.1", + "ring 0.17.8", + "rustls 0.22.4", "tokio", "tokio-postgres", "tokio-rustls 0.25.0", @@ -10476,16 +10624,16 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ - "rustls 0.22.1", + "rustls 0.22.4", "rustls-pki-types", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" dependencies = [ "futures-core", "pin-project-lite", @@ -10495,9 +10643,9 @@ dependencies = [ [[package]] name = "tokio-test" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89b3cbabd3ae862100094ae433e1def582cf86451b4e9bf83aa7ac1d8a7d719" +checksum = "2468baabc3311435b55dd935f702f42cd1b8abb7e754fb7dfb16bd36aa88f9f7" dependencies = [ "async-stream", "bytes", @@ -10532,14 +10680,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.8" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.21.0", + "toml_edit 0.22.9", ] [[package]] @@ -10557,33 +10705,33 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.6", "toml_datetime", - "winnow", + "winnow 0.5.40", ] [[package]] name = "toml_edit" -version = "0.20.7" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.6", "toml_datetime", - "winnow", + "winnow 0.5.40", ] [[package]] name = "toml_edit" -version = "0.21.0" +version = "0.22.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +checksum = "8e40bb779c5187258fd7aad0eb68cb8706a0a81fa712fbea808ab43c4b8374c4" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", - "winnow", + "winnow 0.6.6", ] [[package]] @@ -10594,7 +10742,7 @@ checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" dependencies = [ "async-trait", "axum", - "base64 0.21.5", + "base64 0.21.7", "bytes", "futures-core", "futures-util", @@ -10616,14 +10764,14 @@ dependencies = [ [[package]] name = "tonic" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" dependencies = [ "async-stream", "async-trait", "axum", - "base64 0.21.5", + "base64 0.21.7", "bytes", "h2", "http", @@ -10632,11 +10780,11 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost 0.12.3", - "rustls 0.21.10", - "rustls-pemfile 1.0.4", + "prost 0.12.4", + "rustls-pemfile 2.1.2", + "rustls-pki-types", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls 0.25.0", "tokio-stream", "tower", "tower-layer", @@ -10663,24 +10811,37 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" dependencies = [ - "prettyplease 0.2.15", + "prettyplease 0.2.19", "proc-macro2", - "prost-build 0.12.3", + "prost-build 0.12.4", "quote", - "syn 2.0.43", + "syn 2.0.60", +] + +[[package]] +name = "tonic-build" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" +dependencies = [ + "prettyplease 0.2.19", + "proc-macro2", + "prost-build 0.12.4", + "quote", + "syn 2.0.60", ] [[package]] name = "tonic-reflection" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fa37c513df1339d197f4ba21d28c918b9ef1ac1768265f11ecb6b7f1cba1b76" +checksum = "548c227bd5c0fae5925812c4ec6c66ffcfced23ea370cb823f4d18f0fc1cb6a7" dependencies = [ - "prost 0.12.3", - "prost-types 0.12.3", + "prost 0.12.4", + "prost-types 0.12.4", "tokio", "tokio-stream", - "tonic 0.10.2", + "tonic 0.11.0", ] [[package]] @@ -10710,9 +10871,9 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "async-compression 0.4.5", - "base64 0.21.5", - "bitflags 2.4.1", + "async-compression 0.4.8", + "base64 0.21.7", + "bitflags 2.5.0", "bytes", "futures-core", "futures-util", @@ -10778,7 +10939,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -10821,8 +10982,8 @@ checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" dependencies = [ "js-sys", "once_cell", - "opentelemetry 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)", - "opentelemetry_sdk 0.21.1", + "opentelemetry 0.21.0", + "opentelemetry_sdk 0.21.2", "smallvec", "tracing", "tracing-core", @@ -10938,7 +11099,7 @@ dependencies = [ "anyhow", "cc", "dirs 3.0.2", - "libloading", + "libloading 0.7.4", "once_cell", "regex", "serde", @@ -11000,9 +11161,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "typetag" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196976efd4a62737b3a2b662cda76efb448d099b1049613d7a5d72743c611ce0" +checksum = "661d18414ec032a49ece2d56eee03636e43c4e8d577047ab334c0ba892e29aaf" dependencies = [ "erased-serde", "inventory", @@ -11013,13 +11174,13 @@ dependencies = [ [[package]] name = "typetag-impl" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eea6765137e2414c44c7b1e07c73965a118a72c46148e1e168b3fc9d3ccf3aa" +checksum = "ac73887f47b9312552aa90ef477927ff014d63d1920ca8037c6c1951eab64bb1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -11028,8 +11189,18 @@ version = "0.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2e3b707a653e2915a2fc2c4ee96a3d30b9554b9435eb4cc8b5c6c74bbdd3044" dependencies = [ - "typify-impl", - "typify-macro", + "typify-impl 0.0.14", + "typify-macro 0.0.14", +] + +[[package]] +name = "typify" +version = "0.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c61e9db210bbff218e6535c664b37ec47da449169b98e7866d0580d0db75529" +dependencies = [ + "typify-impl 0.0.16", + "typify-macro 0.0.16", ] [[package]] @@ -11038,14 +11209,32 @@ version = "0.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d9c752192779f666e4c868672dee56a652b82c08032c7e9d23f6a845b282298" dependencies = [ - "heck", + "heck 0.4.1", "log", "proc-macro2", "quote", - "regress", + "regress 0.7.1", "schemars", "serde_json", - "syn 2.0.43", + "syn 2.0.60", + "thiserror", + "unicode-ident", +] + +[[package]] +name = "typify-impl" +version = "0.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95e32f38493804f88e2dc7a5412eccd872ea5452b4db9b0a77de4df180f2a87e" +dependencies = [ + "heck 0.4.1", + "log", + "proc-macro2", + "quote", + "regress 0.8.0", + "schemars", + "serde_json", + "syn 2.0.60", "thiserror", "unicode-ident", ] @@ -11062,8 +11251,24 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.43", - "typify-impl", + "syn 2.0.60", + "typify-impl 0.0.14", +] + +[[package]] +name = "typify-macro" +version = "0.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc09508b72f63d521d68e42c7f172c7416d67986df44b3c7d1f7f9963948ed32" +dependencies = [ + "proc-macro2", + "quote", + "schemars", + "serde", + "serde_json", + "serde_tokenstream", + "syn 2.0.60", + "typify-impl 0.0.16", ] [[package]] @@ -11214,9 +11419,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-casing" @@ -11232,18 +11437,18 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" @@ -11273,15 +11478,15 @@ dependencies = [ [[package]] name = "unindent" -version = "0.1.11" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "unsafe-libyaml" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "untrusted" @@ -11326,9 +11531,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.6.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ "atomic", "getrandom", @@ -11339,13 +11544,13 @@ dependencies = [ [[package]] name = "uuid-macro-internal" -version = "1.6.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f49e7f3f3db8040a100710a11932239fd30697115e2ba4107080d8252939845e" +checksum = "9881bea7cbe687e36c9ab3b778c36cd0487402e270304e8b1296d5085303c1a2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -11414,9 +11619,9 @@ checksum = "f8e76fae08f03f96e166d2dfda232190638c10e0383841252416f9cfe2ae60e6" [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -11445,9 +11650,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -11455,24 +11660,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -11482,9 +11687,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -11492,28 +11697,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -11524,9 +11729,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -11544,9 +11749,9 @@ dependencies = [ [[package]] name = "webbrowser" -version = "0.8.12" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b2391658b02c27719fc5a0a73d6e696285138e8b12fba9d4baa70451023c71" +checksum = "db67ae75a9405634f5882791678772c94ff5f16a66535aae186e26aa0841fc8b" dependencies = [ "core-foundation", "home", @@ -11565,7 +11770,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -11580,9 +11785,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.3" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "which" @@ -11593,7 +11798,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.28", + "rustix 0.38.32", ] [[package]] @@ -11609,9 +11814,9 @@ dependencies = [ [[package]] name = "wide" -version = "0.7.13" +version = "0.7.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c68938b57b33da363195412cfc5fc37c9ed49aa9cfe2156fde64b8d2c9498242" +checksum = "81a1851a719f11d1d2fea40e15c72f6c00de8c142d7ac47c1441cc7e4d0d5bc6" dependencies = [ "bytemuck", "safe_arch", @@ -11673,17 +11878,8 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core 0.52.0", - "windows-targets 0.52.0", -] - -[[package]] -name = "windows-core" -version = "0.51.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" -dependencies = [ - "windows-targets 0.48.5", + "windows-core", + "windows-targets 0.52.5", ] [[package]] @@ -11692,7 +11888,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.5", ] [[package]] @@ -11719,7 +11915,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.5", ] [[package]] @@ -11754,17 +11950,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -11781,9 +11978,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -11805,9 +12002,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -11829,9 +12026,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -11853,9 +12056,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -11877,9 +12080,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -11895,9 +12098,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -11919,15 +12122,24 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winnow" -version = "0.5.30" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b5c3db89721d50d0e2a673f5043fc4722f76dcc352d7b1ab8b8288bed4ed2c5" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c976aaaa0e1f90dbb21e9587cdaf1d9679a1cde8875c0d6bd83ab96a208352" dependencies = [ "memchr", ] @@ -11969,10 +12181,10 @@ dependencies = [ "bcder", "bytes", "chrono", - "der 0.7.8", + "der 0.7.9", "hex", "pem", - "ring 0.17.7", + "ring 0.17.8", "signature", "spki 0.7.3", "thiserror", @@ -11981,9 +12193,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcb9cbac069e033553e8bb871be2fbdffcab578eb25bd0f7c508cedc6dcd75a" +checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193" [[package]] name = "xz2" @@ -12020,7 +12232,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -12040,7 +12252,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.60", ] [[package]] @@ -12072,11 +12284,11 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" dependencies = [ - "zstd-safe 7.0.0", + "zstd-safe 7.1.0", ] [[package]] @@ -12101,18 +12313,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.0.0" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.9+zstd.1.5.5" +version = "2.0.10+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 788bc68798..62faf6e308 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,16 +70,24 @@ license = "Apache-2.0" clippy.print_stdout = "warn" clippy.print_stderr = "warn" clippy.implicit_clone = "warn" +clippy.readonly_write_lock = "allow" rust.unknown_lints = "deny" +# Remove this after https://github.com/PyO3/pyo3/issues/4094 +rust.non_local_definitions = "allow" [workspace.dependencies] +# We turn off default-features for some dependencies here so the workspaces which inherit them can +# selectively turn them on if needed, since we can override default-features = true (from false) +# for the inherited dependency but cannot do the reverse (override from true to false). +# +# See for more detaiils: https://github.com/rust-lang/cargo/issues/11329 ahash = { version = "0.8", features = ["compile-time-rng"] } aquamarine = "0.3" -arrow = { version = "47.0" } -arrow-array = "47.0" -arrow-flight = "47.0" -arrow-ipc = { version = "47.0", features = ["lz4"] } -arrow-schema = { version = "47.0", features = ["serde"] } +arrow = { version = "51.0.0", features = ["prettyprint"] } +arrow-array = { version = "51.0.0", default-features = false, features = ["chrono-tz"] } +arrow-flight = "51.0" +arrow-ipc = { version = "51.0.0", default-features = false, features = ["lz4"] } +arrow-schema = { version = "51.0", features = ["serde"] } async-stream = "0.3" async-trait = "0.1" axum = { version = "0.6", features = ["headers"] } @@ -91,20 +99,22 @@ bytes = { version = "1.5", features = ["serde"] } chrono = { version = "0.4", features = ["serde"] } clap = { version = "4.4", features = ["derive"] } dashmap = "5.4" -datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" } -datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" } -datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" } -datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" } -datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" } -datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" } -datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "26e43acac3a96cec8dd4c8365f22dfb1a84306e9" } +datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" } +datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" } +datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" } +datafusion-functions = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" } +datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" } +datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" } +datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" } +datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "34eda15b73a9e278af8844b30ed2f1c21c10359c" } derive_builder = "0.12" dotenv = "0.15" -etcd-client = "0.12" +# TODO(LFC): Wait for https://github.com/etcdv3/etcd-client/pull/76 +etcd-client = { git = "https://github.com/MichaelScofield/etcd-client.git", rev = "4c371e9b3ea8e0a8ee2f9cbd7ded26e54a45df3b" } fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "04d78b6e025ceb518040fdd10858c2a9d9345820" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "73ac0207ab71dfea48f30259ffdb611501b5ecb8" } humantime = "2.1" humantime-serde = "1.1" itertools = "0.10" @@ -115,12 +125,12 @@ moka = "0.12" notify = "6.1" num_cpus = "1.16" once_cell = "1.18" -opentelemetry-proto = { git = "https://github.com/waynexia/opentelemetry-rust.git", rev = "33841b38dda79b15f2024952be5f32533325ca02", features = [ +opentelemetry-proto = { version = "0.5", features = [ "gen-tonic", "metrics", "trace", ] } -parquet = "47.0" +parquet = { version = "51.0.0", default-features = false, features = ["arrow", "async", "object_store"] } paste = "1.0" pin-project = "1.0" prometheus = { version = "0.13.3", features = ["process"] } @@ -144,18 +154,18 @@ serde_with = "3" smallvec = { version = "1", features = ["serde"] } snafu = "0.7" sysinfo = "0.30" -# on branch v0.38.x -sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "6a93567ae38d42be5c8d08b13c8ff4dde26502ef", features = [ +# on branch v0.44.x +sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "c919990bf62ad38d2b0c0a3bc90b26ad919d51b0", features = [ "visitor", ] } strum = { version = "0.25", features = ["derive"] } tempfile = "3" -tokio = { version = "1.28", features = ["full"] } +tokio = { version = "1.36", features = ["full"] } tokio-stream = { version = "0.1" } tokio-util = { version = "0.7", features = ["io-util", "compat"] } toml = "0.8.8" -tonic = { version = "0.10", features = ["tls"] } -uuid = { version = "1", features = ["serde", "v4", "fast-rng"] } +tonic = { version = "0.11", features = ["tls"] } +uuid = { version = "1.7", features = ["serde", "v4", "fast-rng"] } zstd = "0.13" ## workspaces members diff --git a/benchmarks/src/bin/nyc-taxi.rs b/benchmarks/src/bin/nyc-taxi.rs index 1e60db69fa..bfc26f3dae 100644 --- a/benchmarks/src/bin/nyc-taxi.rs +++ b/benchmarks/src/bin/nyc-taxi.rs @@ -215,37 +215,7 @@ fn build_values(column: &ArrayRef) -> (Values, ColumnDataType) { ColumnDataType::String, ) } - DataType::Null - | DataType::Boolean - | DataType::Int8 - | DataType::Int16 - | DataType::Int32 - | DataType::UInt8 - | DataType::UInt16 - | DataType::UInt32 - | DataType::UInt64 - | DataType::Float16 - | DataType::Float32 - | DataType::Date32 - | DataType::Date64 - | DataType::Time32(_) - | DataType::Time64(_) - | DataType::Duration(_) - | DataType::Interval(_) - | DataType::Binary - | DataType::FixedSizeBinary(_) - | DataType::LargeBinary - | DataType::LargeUtf8 - | DataType::List(_) - | DataType::FixedSizeList(_, _) - | DataType::LargeList(_) - | DataType::Struct(_) - | DataType::Union(_, _) - | DataType::Dictionary(_, _) - | DataType::Decimal128(_, _) - | DataType::Decimal256(_, _) - | DataType::RunEndEncoded(_, _) - | DataType::Map(_, _) => todo!(), + _ => unimplemented!(), } } @@ -444,7 +414,7 @@ fn create_table_expr(table_name: &str) -> CreateTableExpr { fn query_set(table_name: &str) -> HashMap { HashMap::from([ ( - "count_all".to_string(), + "count_all".to_string(), format!("SELECT COUNT(*) FROM {table_name};"), ), ( diff --git a/docs/style-guide.md b/docs/style-guide.md new file mode 100644 index 0000000000..901bd56efa --- /dev/null +++ b/docs/style-guide.md @@ -0,0 +1,46 @@ +# GreptimeDB Style Guide + +This style guide is intended to help contributors to GreptimeDB write code that is consistent with the rest of the codebase. It is a living document and will be updated as the codebase evolves. + +It's mainly an complement to the [Rust Style Guide](https://pingcap.github.io/style-guide/rust/). + +## Table of Contents + +- Formatting +- Modules +- Comments + +## Formatting + +- Place all `mod` declaration before any `use`. +- Use `unimplemented!()` instead of `todo!()` for things that aren't likely to be implemented. +- Add an empty line before and after declaration blocks. +- Place comment before attributes (`#[]`) and derive (`#[derive]`). + +## Modules + +- Use the file with same name instead of `mod.rs` to define a module. E.g.: + +``` +. +├── cache +│ ├── cache_size.rs +│ └── write_cache.rs +└── cache.rs +``` + +## Comments + +- Add comments for public functions and structs. +- Prefer document comment (`///`) over normal comment (`//`) for structs, fields, functions etc. +- Add link (`[]`) to struct, method, or any other reference. And make sure that link works. + +## Error handling + +- Define a custom error type for the module if needed. +- Prefer `with_context()` over `context()` when allocation is needed to construct an error. +- Use `error!()` or `warn!()` macros in the `common_telemetry` crate to log errors. E.g.: + +```rust +error!(e; "Failed to do something"); +``` diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 81fce26194..b92b3fa74b 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "nightly-2023-12-19" +channel = "nightly-2024-04-18" diff --git a/src/api/src/lib.rs b/src/api/src/lib.rs index 96c7f6a517..45f3c95c99 100644 --- a/src/api/src/lib.rs +++ b/src/api/src/lib.rs @@ -21,6 +21,7 @@ pub mod prom_store { } } +pub mod region; pub mod v1; pub use greptime_proto; diff --git a/src/api/src/region.rs b/src/api/src/region.rs new file mode 100644 index 0000000000..0493378213 --- /dev/null +++ b/src/api/src/region.rs @@ -0,0 +1,42 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; + +use common_base::AffectedRows; +use greptime_proto::v1::region::RegionResponse as RegionResponseV1; + +/// This result struct is derived from [RegionResponseV1] +#[derive(Debug)] +pub struct RegionResponse { + pub affected_rows: AffectedRows, + pub extension: HashMap>, +} + +impl RegionResponse { + pub fn from_region_response(region_response: RegionResponseV1) -> Self { + Self { + affected_rows: region_response.affected_rows as _, + extension: region_response.extension, + } + } + + /// Creates one response without extension + pub fn new(affected_rows: AffectedRows) -> Self { + Self { + affected_rows, + extension: Default::default(), + } + } +} diff --git a/src/auth/src/tests.rs b/src/auth/src/tests.rs index 06a739241a..d3e8a41aa1 100644 --- a/src/auth/src/tests.rs +++ b/src/auth/src/tests.rs @@ -45,9 +45,9 @@ impl Default for MockUserProvider { impl MockUserProvider { pub fn set_authorization_info(&mut self, info: DatabaseAuthInfo) { - self.catalog = info.catalog.to_owned(); - self.schema = info.schema.to_owned(); - self.username = info.username.to_owned(); + info.catalog.clone_into(&mut self.catalog); + info.schema.clone_into(&mut self.schema); + info.username.clone_into(&mut self.username); } } diff --git a/src/catalog/src/information_schema/predicate.rs b/src/catalog/src/information_schema/predicate.rs index 9afc83a389..243ff70533 100644 --- a/src/catalog/src/information_schema/predicate.rs +++ b/src/catalog/src/information_schema/predicate.rs @@ -109,11 +109,7 @@ impl Predicate { }; } Predicate::Not(p) => { - let Some(b) = p.eval(row) else { - return None; - }; - - return Some(!b); + return Some(!p.eval(row)?); } } @@ -125,13 +121,7 @@ impl Predicate { fn from_expr(expr: DfExpr) -> Option { match expr { // NOT expr - DfExpr::Not(expr) => { - let Some(p) = Self::from_expr(*expr) else { - return None; - }; - - Some(Predicate::Not(Box::new(p))) - } + DfExpr::Not(expr) => Some(Predicate::Not(Box::new(Self::from_expr(*expr)?))), // expr LIKE pattern DfExpr::Like(Like { negated, @@ -178,25 +168,15 @@ impl Predicate { } // left AND right (left, Operator::And, right) => { - let Some(left) = Self::from_expr(left) else { - return None; - }; - - let Some(right) = Self::from_expr(right) else { - return None; - }; + let left = Self::from_expr(left)?; + let right = Self::from_expr(right)?; Some(Predicate::And(Box::new(left), Box::new(right))) } // left OR right (left, Operator::Or, right) => { - let Some(left) = Self::from_expr(left) else { - return None; - }; - - let Some(right) = Self::from_expr(right) else { - return None; - }; + let left = Self::from_expr(left)?; + let right = Self::from_expr(right)?; Some(Predicate::Or(Box::new(left), Box::new(right))) } diff --git a/src/catalog/src/kvbackend/client.rs b/src/catalog/src/kvbackend/client.rs index 8c40c43692..e72d2b5275 100644 --- a/src/catalog/src/kvbackend/client.rs +++ b/src/catalog/src/kvbackend/client.rs @@ -17,7 +17,6 @@ use std::fmt::Debug; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; -use std::usize; use common_error::ext::BoxedError; use common_meta::cache_invalidator::KvCacheInvalidator; diff --git a/src/catalog/src/table_source.rs b/src/catalog/src/table_source.rs index d5d0c282e6..58813a460e 100644 --- a/src/catalog/src/table_source.rs +++ b/src/catalog/src/table_source.rs @@ -49,10 +49,7 @@ impl DfTableSourceProvider { } } - pub fn resolve_table_ref<'a>( - &'a self, - table_ref: TableReference<'a>, - ) -> Result> { + pub fn resolve_table_ref(&self, table_ref: TableReference) -> Result { if self.disallow_cross_catalog_query { match &table_ref { TableReference::Bare { .. } => (), @@ -76,7 +73,7 @@ impl DfTableSourceProvider { pub async fn resolve_table( &mut self, - table_ref: TableReference<'_>, + table_ref: TableReference, ) -> Result> { let table_ref = self.resolve_table_ref(table_ref)?; @@ -106,8 +103,6 @@ impl DfTableSourceProvider { #[cfg(test)] mod tests { - use std::borrow::Cow; - use session::context::QueryContext; use super::*; @@ -120,68 +115,37 @@ mod tests { let table_provider = DfTableSourceProvider::new(MemoryCatalogManager::with_default_setup(), true, query_ctx); - let table_ref = TableReference::Bare { - table: Cow::Borrowed("table_name"), - }; + let table_ref = TableReference::bare("table_name"); let result = table_provider.resolve_table_ref(table_ref); assert!(result.is_ok()); - let table_ref = TableReference::Partial { - schema: Cow::Borrowed("public"), - table: Cow::Borrowed("table_name"), - }; + let table_ref = TableReference::partial("public", "table_name"); let result = table_provider.resolve_table_ref(table_ref); assert!(result.is_ok()); - let table_ref = TableReference::Partial { - schema: Cow::Borrowed("wrong_schema"), - table: Cow::Borrowed("table_name"), - }; + let table_ref = TableReference::partial("wrong_schema", "table_name"); let result = table_provider.resolve_table_ref(table_ref); assert!(result.is_ok()); - let table_ref = TableReference::Full { - catalog: Cow::Borrowed("greptime"), - schema: Cow::Borrowed("public"), - table: Cow::Borrowed("table_name"), - }; + let table_ref = TableReference::full("greptime", "public", "table_name"); let result = table_provider.resolve_table_ref(table_ref); assert!(result.is_ok()); - let table_ref = TableReference::Full { - catalog: Cow::Borrowed("wrong_catalog"), - schema: Cow::Borrowed("public"), - table: Cow::Borrowed("table_name"), - }; + let table_ref = TableReference::full("wrong_catalog", "public", "table_name"); let result = table_provider.resolve_table_ref(table_ref); assert!(result.is_err()); - let table_ref = TableReference::Partial { - schema: Cow::Borrowed("information_schema"), - table: Cow::Borrowed("columns"), - }; + let table_ref = TableReference::partial("information_schema", "columns"); let result = table_provider.resolve_table_ref(table_ref); assert!(result.is_ok()); - let table_ref = TableReference::Full { - catalog: Cow::Borrowed("greptime"), - schema: Cow::Borrowed("information_schema"), - table: Cow::Borrowed("columns"), - }; + let table_ref = TableReference::full("greptime", "information_schema", "columns"); assert!(table_provider.resolve_table_ref(table_ref).is_ok()); - let table_ref = TableReference::Full { - catalog: Cow::Borrowed("dummy"), - schema: Cow::Borrowed("information_schema"), - table: Cow::Borrowed("columns"), - }; + let table_ref = TableReference::full("dummy", "information_schema", "columns"); assert!(table_provider.resolve_table_ref(table_ref).is_err()); - let table_ref = TableReference::Full { - catalog: Cow::Borrowed("greptime"), - schema: Cow::Borrowed("greptime_private"), - table: Cow::Borrowed("columns"), - }; + let table_ref = TableReference::full("greptime", "greptime_private", "columns"); assert!(table_provider.resolve_table_ref(table_ref).is_ok()); } } diff --git a/src/client/src/region.rs b/src/client/src/region.rs index 39c5ddddcc..a401fa4348 100644 --- a/src/client/src/region.rs +++ b/src/client/src/region.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use api::region::RegionResponse; use api::v1::region::{QueryRequest, RegionRequest}; use api::v1::ResponseHeader; use arc_swap::ArcSwapOption; @@ -23,7 +24,7 @@ use async_trait::async_trait; use common_error::ext::{BoxedError, ErrorExt}; use common_error::status_code::StatusCode; use common_grpc::flight::{FlightDecoder, FlightMessage}; -use common_meta::datanode_manager::{Datanode, HandleResponse}; +use common_meta::datanode_manager::Datanode; use common_meta::error::{self as meta_error, Result as MetaResult}; use common_recordbatch::error::ExternalSnafu; use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream}; @@ -46,7 +47,7 @@ pub struct RegionRequester { #[async_trait] impl Datanode for RegionRequester { - async fn handle(&self, request: RegionRequest) -> MetaResult { + async fn handle(&self, request: RegionRequest) -> MetaResult { self.handle_inner(request).await.map_err(|err| { if err.should_retry() { meta_error::Error::RetryLater { @@ -165,7 +166,7 @@ impl RegionRequester { Ok(Box::pin(record_batch_stream)) } - async fn handle_inner(&self, request: RegionRequest) -> Result { + async fn handle_inner(&self, request: RegionRequest) -> Result { let request_type = request .body .as_ref() @@ -194,10 +195,10 @@ impl RegionRequester { check_response_header(&response.header)?; - Ok(HandleResponse::from_region_response(response)) + Ok(RegionResponse::from_region_response(response)) } - pub async fn handle(&self, request: RegionRequest) -> Result { + pub async fn handle(&self, request: RegionRequest) -> Result { self.handle_inner(request).await } } diff --git a/src/cmd/src/cli.rs b/src/cmd/src/cli.rs index 409c8e4d54..35dc1e4ba7 100644 --- a/src/cmd/src/cli.rs +++ b/src/cmd/src/cli.rs @@ -84,10 +84,10 @@ impl Command { let mut logging_opts = LoggingOptions::default(); if let Some(dir) = &cli_options.log_dir { - logging_opts.dir = dir.clone(); + logging_opts.dir.clone_from(dir); } - logging_opts.level = cli_options.log_level.clone(); + logging_opts.level.clone_from(&cli_options.log_level); Ok(Options::Cli(Box::new(logging_opts))) } diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs index 162419a3b4..ff0d904090 100644 --- a/src/cmd/src/datanode.rs +++ b/src/cmd/src/datanode.rs @@ -139,19 +139,19 @@ impl StartCommand { )?; if let Some(dir) = &cli_options.log_dir { - opts.logging.dir = dir.clone(); + opts.logging.dir.clone_from(dir); } if cli_options.log_level.is_some() { - opts.logging.level = cli_options.log_level.clone(); + opts.logging.level.clone_from(&cli_options.log_level); } if let Some(addr) = &self.rpc_addr { - opts.rpc_addr = addr.clone(); + opts.rpc_addr.clone_from(addr); } if self.rpc_hostname.is_some() { - opts.rpc_hostname = self.rpc_hostname.clone(); + opts.rpc_hostname.clone_from(&self.rpc_hostname); } if let Some(node_id) = self.node_id { @@ -161,7 +161,8 @@ impl StartCommand { if let Some(metasrv_addrs) = &self.metasrv_addr { opts.meta_client .get_or_insert_with(MetaClientOptions::default) - .metasrv_addrs = metasrv_addrs.clone(); + .metasrv_addrs + .clone_from(metasrv_addrs); opts.mode = Mode::Distributed; } @@ -173,7 +174,7 @@ impl StartCommand { } if let Some(data_home) = &self.data_home { - opts.storage.data_home = data_home.clone(); + opts.storage.data_home.clone_from(data_home); } // `wal_dir` only affects raft-engine config. @@ -191,7 +192,7 @@ impl StartCommand { } if let Some(http_addr) = &self.http_addr { - opts.http.addr = http_addr.clone(); + opts.http.addr.clone_from(http_addr); } if let Some(http_timeout) = self.http_timeout { diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index 3100182f72..0ff3584625 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -157,11 +157,11 @@ impl StartCommand { )?; if let Some(dir) = &cli_options.log_dir { - opts.logging.dir = dir.clone(); + opts.logging.dir.clone_from(dir); } if cli_options.log_level.is_some() { - opts.logging.level = cli_options.log_level.clone(); + opts.logging.level.clone_from(&cli_options.log_level); } let tls_opts = TlsOption::new( @@ -171,7 +171,7 @@ impl StartCommand { ); if let Some(addr) = &self.http_addr { - opts.http.addr = addr.clone() + opts.http.addr.clone_from(addr); } if let Some(http_timeout) = self.http_timeout { @@ -183,24 +183,24 @@ impl StartCommand { } if let Some(addr) = &self.rpc_addr { - opts.grpc.addr = addr.clone() + opts.grpc.addr.clone_from(addr); } if let Some(addr) = &self.mysql_addr { opts.mysql.enable = true; - opts.mysql.addr = addr.clone(); + opts.mysql.addr.clone_from(addr); opts.mysql.tls = tls_opts.clone(); } if let Some(addr) = &self.postgres_addr { opts.postgres.enable = true; - opts.postgres.addr = addr.clone(); + opts.postgres.addr.clone_from(addr); opts.postgres.tls = tls_opts; } if let Some(addr) = &self.opentsdb_addr { opts.opentsdb.enable = true; - opts.opentsdb.addr = addr.clone(); + opts.opentsdb.addr.clone_from(addr); } if let Some(enable) = self.influxdb_enable { @@ -210,11 +210,12 @@ impl StartCommand { if let Some(metasrv_addrs) = &self.metasrv_addr { opts.meta_client .get_or_insert_with(MetaClientOptions::default) - .metasrv_addrs = metasrv_addrs.clone(); + .metasrv_addrs + .clone_from(metasrv_addrs); opts.mode = Mode::Distributed; } - opts.user_provider = self.user_provider.clone(); + opts.user_provider.clone_from(&self.user_provider); Ok(Options::Frontend(Box::new(opts))) } diff --git a/src/cmd/src/metasrv.rs b/src/cmd/src/metasrv.rs index 29b0f517de..bc542ada30 100644 --- a/src/cmd/src/metasrv.rs +++ b/src/cmd/src/metasrv.rs @@ -134,23 +134,23 @@ impl StartCommand { )?; if let Some(dir) = &cli_options.log_dir { - opts.logging.dir = dir.clone(); + opts.logging.dir.clone_from(dir); } if cli_options.log_level.is_some() { - opts.logging.level = cli_options.log_level.clone(); + opts.logging.level.clone_from(&cli_options.log_level); } if let Some(addr) = &self.bind_addr { - opts.bind_addr = addr.clone(); + opts.bind_addr.clone_from(addr); } if let Some(addr) = &self.server_addr { - opts.server_addr = addr.clone(); + opts.server_addr.clone_from(addr); } if let Some(addr) = &self.store_addr { - opts.store_addr = addr.clone(); + opts.store_addr.clone_from(addr); } if let Some(selector_type) = &self.selector { @@ -168,7 +168,7 @@ impl StartCommand { } if let Some(http_addr) = &self.http_addr { - opts.http.addr = http_addr.clone(); + opts.http.addr.clone_from(http_addr); } if let Some(http_timeout) = self.http_timeout { @@ -176,11 +176,11 @@ impl StartCommand { } if let Some(data_home) = &self.data_home { - opts.data_home = data_home.clone(); + opts.data_home.clone_from(data_home); } if !self.store_key_prefix.is_empty() { - opts.store_key_prefix = self.store_key_prefix.clone() + opts.store_key_prefix.clone_from(&self.store_key_prefix) } if let Some(max_txn_ops) = self.max_txn_ops { diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 694d10d689..2c7d4abad2 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -293,11 +293,11 @@ impl StartCommand { opts.mode = Mode::Standalone; if let Some(dir) = &cli_options.log_dir { - opts.logging.dir = dir.clone(); + opts.logging.dir.clone_from(dir); } if cli_options.log_level.is_some() { - opts.logging.level = cli_options.log_level.clone(); + opts.logging.level.clone_from(&cli_options.log_level); } let tls_opts = TlsOption::new( @@ -307,11 +307,11 @@ impl StartCommand { ); if let Some(addr) = &self.http_addr { - opts.http.addr = addr.clone() + opts.http.addr.clone_from(addr); } if let Some(data_home) = &self.data_home { - opts.storage.data_home = data_home.clone(); + opts.storage.data_home.clone_from(data_home); } if let Some(addr) = &self.rpc_addr { @@ -325,31 +325,31 @@ impl StartCommand { } .fail(); } - opts.grpc.addr = addr.clone() + opts.grpc.addr.clone_from(addr) } if let Some(addr) = &self.mysql_addr { opts.mysql.enable = true; - opts.mysql.addr = addr.clone(); + opts.mysql.addr.clone_from(addr); opts.mysql.tls = tls_opts.clone(); } if let Some(addr) = &self.postgres_addr { opts.postgres.enable = true; - opts.postgres.addr = addr.clone(); + opts.postgres.addr.clone_from(addr); opts.postgres.tls = tls_opts; } if let Some(addr) = &self.opentsdb_addr { opts.opentsdb.enable = true; - opts.opentsdb.addr = addr.clone(); + opts.opentsdb.addr.clone_from(addr); } if self.influxdb_enable { opts.influxdb.enable = self.influxdb_enable; } - opts.user_provider = self.user_provider.clone(); + opts.user_provider.clone_from(&self.user_provider); let metadata_store = opts.metadata_store.clone(); let procedure = opts.procedure.clone(); diff --git a/src/common/datasource/Cargo.toml b/src/common/datasource/Cargo.toml index 8f11043afd..1a4792dcb0 100644 --- a/src/common/datasource/Cargo.toml +++ b/src/common/datasource/Cargo.toml @@ -30,7 +30,7 @@ derive_builder.workspace = true futures.workspace = true lazy_static.workspace = true object-store.workspace = true -orc-rust = "0.2" +orc-rust = { git = "https://github.com/MichaelScofield/orc-rs.git", rev = "17347f5f084ac937863317df882218055c4ea8c1" } parquet.workspace = true paste = "1.0" regex = "1.7" diff --git a/src/common/datasource/src/file_format/csv.rs b/src/common/datasource/src/file_format/csv.rs index 0767722d9b..4cf2b9e133 100644 --- a/src/common/datasource/src/file_format/csv.rs +++ b/src/common/datasource/src/file_format/csv.rs @@ -117,7 +117,7 @@ impl CsvConfig { let mut builder = csv::ReaderBuilder::new(self.file_schema.clone()) .with_delimiter(self.delimiter) .with_batch_size(self.batch_size) - .has_header(self.has_header); + .with_header(self.has_header); if let Some(proj) = &self.file_projection { builder = builder.with_projection(proj.clone()); diff --git a/src/common/datasource/src/file_format/tests.rs b/src/common/datasource/src/file_format/tests.rs index be8650c9d5..7be8664b72 100644 --- a/src/common/datasource/src/file_format/tests.rs +++ b/src/common/datasource/src/file_format/tests.rs @@ -19,6 +19,7 @@ use std::vec; use common_test_util::find_workspace_path; use datafusion::assert_batches_eq; +use datafusion::config::TableParquetOptions; use datafusion::datasource::physical_plan::{FileOpener, FileScanConfig, FileStream, ParquetExec}; use datafusion::execution::context::TaskContext; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; @@ -166,7 +167,7 @@ async fn test_parquet_exec() { .to_string(); let base_config = scan_config(schema.clone(), None, path); - let exec = ParquetExec::new(base_config, None, None) + let exec = ParquetExec::new(base_config, None, None, TableParquetOptions::default()) .with_parquet_file_reader_factory(Arc::new(DefaultParquetFileReaderFactory::new(store))); let ctx = SessionContext::new(); diff --git a/src/common/datasource/src/test_util.rs b/src/common/datasource/src/test_util.rs index 04125f1613..8f1af59c90 100644 --- a/src/common/datasource/src/test_util.rs +++ b/src/common/datasource/src/test_util.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use arrow_schema::{DataType, Field, Schema, SchemaRef}; use common_test_util::temp_dir::{create_temp_dir, TempDir}; +use datafusion::common::Statistics; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::object_store::ObjectStoreUrl; use datafusion::datasource::physical_plan::{FileScanConfig, FileStream}; @@ -72,17 +73,16 @@ pub fn test_basic_schema() -> SchemaRef { pub fn scan_config(file_schema: SchemaRef, limit: Option, filename: &str) -> FileScanConfig { // object_store only recognize the Unix style path, so make it happy. let filename = &filename.replace('\\', "/"); - + let statistics = Statistics::new_unknown(file_schema.as_ref()); FileScanConfig { object_store_url: ObjectStoreUrl::parse("empty://").unwrap(), // won't be used file_schema, file_groups: vec![vec![PartitionedFile::new(filename.to_string(), 10)]], - statistics: Default::default(), + statistics, projection: None, limit, table_partition_cols: vec![], output_ordering: vec![], - infinite_source: false, } } diff --git a/src/common/function/src/scalars/aggregate/diff.rs b/src/common/function/src/scalars/aggregate/diff.rs index 9893d6199b..b83ed6d004 100644 --- a/src/common/function/src/scalars/aggregate/diff.rs +++ b/src/common/function/src/scalars/aggregate/diff.rs @@ -56,7 +56,7 @@ where .map(|&n| n.into()) .collect::>(); Ok(vec![Value::List(ListValue::new( - Some(Box::new(nums)), + nums, I::LogicalType::build_data_type(), ))]) } @@ -120,10 +120,7 @@ where O::from_native(native).into() }) .collect::>(); - let diff = Value::List(ListValue::new( - Some(Box::new(diff)), - O::LogicalType::build_data_type(), - )); + let diff = Value::List(ListValue::new(diff, O::LogicalType::build_data_type())); Ok(diff) } } @@ -218,10 +215,7 @@ mod test { let values = vec![Value::from(2_i64), Value::from(1_i64)]; diff.update_batch(&v).unwrap(); assert_eq!( - Value::List(ListValue::new( - Some(Box::new(values)), - ConcreteDataType::int64_datatype() - )), + Value::List(ListValue::new(values, ConcreteDataType::int64_datatype())), diff.evaluate().unwrap() ); @@ -236,10 +230,7 @@ mod test { let values = vec![Value::from(5_i64), Value::from(1_i64)]; diff.update_batch(&v).unwrap(); assert_eq!( - Value::List(ListValue::new( - Some(Box::new(values)), - ConcreteDataType::int64_datatype() - )), + Value::List(ListValue::new(values, ConcreteDataType::int64_datatype())), diff.evaluate().unwrap() ); @@ -252,10 +243,7 @@ mod test { let values = vec![Value::from(0_i64), Value::from(0_i64), Value::from(0_i64)]; diff.update_batch(&v).unwrap(); assert_eq!( - Value::List(ListValue::new( - Some(Box::new(values)), - ConcreteDataType::int64_datatype() - )), + Value::List(ListValue::new(values, ConcreteDataType::int64_datatype())), diff.evaluate().unwrap() ); } diff --git a/src/common/function/src/scalars/aggregate/percentile.rs b/src/common/function/src/scalars/aggregate/percentile.rs index 49b981a7ee..231e0bf43a 100644 --- a/src/common/function/src/scalars/aggregate/percentile.rs +++ b/src/common/function/src/scalars/aggregate/percentile.rs @@ -104,10 +104,7 @@ where .map(|&n| n.into()) .collect::>(); Ok(vec![ - Value::List(ListValue::new( - Some(Box::new(nums)), - T::LogicalType::build_data_type(), - )), + Value::List(ListValue::new(nums, T::LogicalType::build_data_type())), self.p.into(), ]) } diff --git a/src/common/function/src/scalars/aggregate/polyval.rs b/src/common/function/src/scalars/aggregate/polyval.rs index b56a692c8d..ae6ca101c4 100644 --- a/src/common/function/src/scalars/aggregate/polyval.rs +++ b/src/common/function/src/scalars/aggregate/polyval.rs @@ -72,10 +72,7 @@ where .map(|&n| n.into()) .collect::>(); Ok(vec![ - Value::List(ListValue::new( - Some(Box::new(nums)), - T::LogicalType::build_data_type(), - )), + Value::List(ListValue::new(nums, T::LogicalType::build_data_type())), self.x.into(), ]) } diff --git a/src/common/function/src/scalars/aggregate/scipy_stats_norm_cdf.rs b/src/common/function/src/scalars/aggregate/scipy_stats_norm_cdf.rs index 2ec9540513..e6c92225a6 100644 --- a/src/common/function/src/scalars/aggregate/scipy_stats_norm_cdf.rs +++ b/src/common/function/src/scalars/aggregate/scipy_stats_norm_cdf.rs @@ -56,10 +56,7 @@ where .map(|&x| x.into()) .collect::>(); Ok(vec![ - Value::List(ListValue::new( - Some(Box::new(nums)), - T::LogicalType::build_data_type(), - )), + Value::List(ListValue::new(nums, T::LogicalType::build_data_type())), self.x.into(), ]) } diff --git a/src/common/function/src/scalars/aggregate/scipy_stats_norm_pdf.rs b/src/common/function/src/scalars/aggregate/scipy_stats_norm_pdf.rs index d1bf432c99..3045ae8665 100644 --- a/src/common/function/src/scalars/aggregate/scipy_stats_norm_pdf.rs +++ b/src/common/function/src/scalars/aggregate/scipy_stats_norm_pdf.rs @@ -56,10 +56,7 @@ where .map(|&x| x.into()) .collect::>(); Ok(vec![ - Value::List(ListValue::new( - Some(Box::new(nums)), - T::LogicalType::build_data_type(), - )), + Value::List(ListValue::new(nums, T::LogicalType::build_data_type())), self.x.into(), ]) } diff --git a/src/common/function/src/scalars/math.rs b/src/common/function/src/scalars/math.rs index f7d50f881d..6635e70b17 100644 --- a/src/common/function/src/scalars/math.rs +++ b/src/common/function/src/scalars/math.rs @@ -77,7 +77,7 @@ impl Function for RangeFunction { /// `range_fn` will never been used. As long as a legal signature is returned, the specific content of the signature does not matter. /// In fact, the arguments loaded by `range_fn` are very complicated, and it is difficult to use `Signature` to describe fn signature(&self) -> Signature { - Signature::any(0, Volatility::Immutable) + Signature::variadic_any(Volatility::Immutable) } fn eval(&self, _func_ctx: FunctionContext, _columns: &[VectorRef]) -> Result { diff --git a/src/common/macro/src/range_fn.rs b/src/common/macro/src/range_fn.rs index c907f1d0d1..582fff523d 100644 --- a/src/common/macro/src/range_fn.rs +++ b/src/common/macro/src/range_fn.rs @@ -119,15 +119,17 @@ fn build_struct( } pub fn scalar_udf() -> ScalarUDF { - ScalarUDF { - name: Self::name().to_string(), - signature: Signature::new( + // TODO(LFC): Use the new Datafusion UDF impl. + #[allow(deprecated)] + ScalarUDF::new( + Self::name(), + &Signature::new( TypeSignature::Exact(Self::input_type()), Volatility::Immutable, ), - return_type: Arc::new(|_| Ok(Arc::new(Self::return_type()))), - fun: Arc::new(Self::calc), - } + &(Arc::new(|_: &_| Ok(Arc::new(Self::return_type()))) as _), + &(Arc::new(Self::calc) as _), + ) } fn input_type() -> Vec { diff --git a/src/common/meta/src/cache_invalidator.rs b/src/common/meta/src/cache_invalidator.rs index 0713807cf7..2e7afd37ef 100644 --- a/src/common/meta/src/cache_invalidator.rs +++ b/src/common/meta/src/cache_invalidator.rs @@ -18,6 +18,7 @@ use tokio::sync::RwLock; use crate::error::Result; use crate::instruction::CacheIdent; +use crate::key::schema_name::SchemaNameKey; use crate::key::table_info::TableInfoKey; use crate::key::table_name::TableNameKey; use crate::key::table_route::TableRouteKey; @@ -107,6 +108,10 @@ where let key: TableNameKey = (&table_name).into(); self.invalidate_key(&key.as_raw_key()).await } + CacheIdent::SchemaName(schema_name) => { + let key: SchemaNameKey = (&schema_name).into(); + self.invalidate_key(&key.as_raw_key()).await; + } } } Ok(()) diff --git a/src/common/meta/src/datanode_manager.rs b/src/common/meta/src/datanode_manager.rs index 1fdc2f314e..58990ce01d 100644 --- a/src/common/meta/src/datanode_manager.rs +++ b/src/common/meta/src/datanode_manager.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; use std::sync::Arc; -use api::v1::region::{QueryRequest, RegionRequest, RegionResponse}; +use api::region::RegionResponse; +use api::v1::region::{QueryRequest, RegionRequest}; pub use common_base::AffectedRows; use common_recordbatch::SendableRecordBatchStream; @@ -26,7 +26,7 @@ use crate::peer::Peer; #[async_trait::async_trait] pub trait Datanode: Send + Sync { /// Handles DML, and DDL requests. - async fn handle(&self, request: RegionRequest) -> Result; + async fn handle(&self, request: RegionRequest) -> Result; /// Handles query requests async fn handle_query(&self, request: QueryRequest) -> Result; @@ -42,27 +42,3 @@ pub trait DatanodeManager: Send + Sync { } pub type DatanodeManagerRef = Arc; - -/// This result struct is derived from [RegionResponse] -#[derive(Debug)] -pub struct HandleResponse { - pub affected_rows: AffectedRows, - pub extension: HashMap>, -} - -impl HandleResponse { - pub fn from_region_response(region_response: RegionResponse) -> Self { - Self { - affected_rows: region_response.affected_rows as _, - extension: region_response.extension, - } - } - - /// Creates one response without extension - pub fn new(affected_rows: AffectedRows) -> Self { - Self { - affected_rows, - extension: Default::default(), - } - } -} diff --git a/src/common/meta/src/ddl/drop_database/executor.rs b/src/common/meta/src/ddl/drop_database/executor.rs index 76911a4fa8..8b0b967ba3 100644 --- a/src/common/meta/src/ddl/drop_database/executor.rs +++ b/src/common/meta/src/ddl/drop_database/executor.rs @@ -122,12 +122,12 @@ impl State for DropDatabaseExecutor { mod tests { use std::sync::Arc; + use api::region::RegionResponse; use api::v1::region::{QueryRequest, RegionRequest}; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_error::ext::BoxedError; use common_recordbatch::SendableRecordBatchStream; - use crate::datanode_manager::HandleResponse; use crate::ddl::drop_database::cursor::DropDatabaseCursor; use crate::ddl::drop_database::executor::DropDatabaseExecutor; use crate::ddl::drop_database::{DropDatabaseContext, DropTableTarget, State}; @@ -144,8 +144,8 @@ mod tests { #[async_trait::async_trait] impl MockDatanodeHandler for NaiveDatanodeHandler { - async fn handle(&self, _peer: &Peer, _request: RegionRequest) -> Result { - Ok(HandleResponse::new(0)) + async fn handle(&self, _peer: &Peer, _request: RegionRequest) -> Result { + Ok(RegionResponse::new(0)) } async fn handle_query( @@ -291,7 +291,7 @@ mod tests { #[async_trait::async_trait] impl MockDatanodeHandler for RetryErrorDatanodeHandler { - async fn handle(&self, _peer: &Peer, _request: RegionRequest) -> Result { + async fn handle(&self, _peer: &Peer, _request: RegionRequest) -> Result { Err(Error::RetryLater { source: BoxedError::new( error::UnexpectedSnafu { diff --git a/src/common/meta/src/ddl/drop_database/metadata.rs b/src/common/meta/src/ddl/drop_database/metadata.rs index b6e25197ea..f06c51963a 100644 --- a/src/common/meta/src/ddl/drop_database/metadata.rs +++ b/src/common/meta/src/ddl/drop_database/metadata.rs @@ -18,10 +18,12 @@ use common_procedure::Status; use serde::{Deserialize, Serialize}; use super::end::DropDatabaseEnd; +use crate::cache_invalidator::Context; use crate::ddl::drop_database::{DropDatabaseContext, State}; use crate::ddl::DdlContext; use crate::error::Result; -use crate::key::schema_name::SchemaNameKey; +use crate::instruction::CacheIdent; +use crate::key::schema_name::{SchemaName, SchemaNameKey}; #[derive(Debug, Serialize, Deserialize)] pub(crate) struct DropDatabaseRemoveMetadata; @@ -40,7 +42,53 @@ impl State for DropDatabaseRemoveMetadata { .delete(SchemaNameKey::new(&ctx.catalog, &ctx.schema)) .await?; - return Ok((Box::new(DropDatabaseEnd), Status::done())); + return Ok((Box::new(DropMetadataBroadcast), Status::executing(true))); + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct DropMetadataBroadcast; + +impl DropMetadataBroadcast { + /// Invalidates frontend caches + async fn invalidate_schema_cache( + &self, + ddl_ctx: &DdlContext, + db_ctx: &mut DropDatabaseContext, + ) -> Result<()> { + let cache_invalidator = &ddl_ctx.cache_invalidator; + let ctx = Context { + subject: Some("Invalidate schema cache by dropping database".to_string()), + }; + + cache_invalidator + .invalidate( + &ctx, + vec![CacheIdent::SchemaName(SchemaName { + catalog_name: db_ctx.catalog.clone(), + schema_name: db_ctx.schema.clone(), + })], + ) + .await?; + + Ok(()) + } +} + +#[async_trait::async_trait] +#[typetag::serde] +impl State for DropMetadataBroadcast { + async fn next( + &mut self, + ddl_ctx: &DdlContext, + ctx: &mut DropDatabaseContext, + ) -> Result<(Box, Status)> { + self.invalidate_schema_cache(ddl_ctx, ctx).await?; + Ok((Box::new(DropDatabaseEnd), Status::done())) } fn as_any(&self) -> &dyn Any { @@ -53,7 +101,7 @@ mod tests { use std::sync::Arc; use crate::ddl::drop_database::end::DropDatabaseEnd; - use crate::ddl::drop_database::metadata::DropDatabaseRemoveMetadata; + use crate::ddl::drop_database::metadata::{DropDatabaseRemoveMetadata, DropMetadataBroadcast}; use crate::ddl::drop_database::{DropDatabaseContext, State}; use crate::key::schema_name::SchemaNameKey; use crate::test_util::{new_ddl_context, MockDatanodeManager}; @@ -76,14 +124,23 @@ mod tests { tables: None, }; let (state, status) = state.next(&ddl_context, &mut ctx).await.unwrap(); - state.as_any().downcast_ref::().unwrap(); - assert!(status.is_done()); + state + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!status.is_done()); assert!(!ddl_context .table_metadata_manager .schema_manager() .exists(SchemaNameKey::new("foo", "bar")) .await .unwrap()); + + let mut state = DropMetadataBroadcast; + let (state, status) = state.next(&ddl_context, &mut ctx).await.unwrap(); + state.as_any().downcast_ref::().unwrap(); + assert!(status.is_done()); + // Schema not exists let mut state = DropDatabaseRemoveMetadata; let mut ctx = DropDatabaseContext { @@ -93,7 +150,10 @@ mod tests { tables: None, }; let (state, status) = state.next(&ddl_context, &mut ctx).await.unwrap(); - state.as_any().downcast_ref::().unwrap(); - assert!(status.is_done()); + state + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!status.is_done()); } } diff --git a/src/common/meta/src/ddl/drop_table/executor.rs b/src/common/meta/src/ddl/drop_table/executor.rs index 8be3155553..6659ee238f 100644 --- a/src/common/meta/src/ddl/drop_table/executor.rs +++ b/src/common/meta/src/ddl/drop_table/executor.rs @@ -106,19 +106,6 @@ impl DropTableExecutor { ctx: &DdlContext, table_route_value: &TableRouteValue, ) -> Result<()> { - let table_name_key = TableNameKey::new( - &self.table.catalog_name, - &self.table.schema_name, - &self.table.table_name, - ); - if !ctx - .table_metadata_manager - .table_name_manager() - .exists(table_name_key) - .await? - { - return Ok(()); - } ctx.table_metadata_manager .delete_table_metadata(self.table_id, &self.table, table_route_value) .await @@ -152,19 +139,6 @@ impl DropTableExecutor { ctx: &DdlContext, table_route_value: &TableRouteValue, ) -> Result<()> { - let table_name_key = TableNameKey::new( - &self.table.catalog_name, - &self.table.schema_name, - &self.table.table_name, - ); - if ctx - .table_metadata_manager - .table_name_manager() - .exists(table_name_key) - .await? - { - return Ok(()); - } ctx.table_metadata_manager .restore_table_metadata(self.table_id, &self.table, table_route_value) .await diff --git a/src/common/meta/src/ddl/test_util/datanode_handler.rs b/src/common/meta/src/ddl/test_util/datanode_handler.rs index 69d233dd98..1649ebc00d 100644 --- a/src/common/meta/src/ddl/test_util/datanode_handler.rs +++ b/src/common/meta/src/ddl/test_util/datanode_handler.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use api::region::RegionResponse; use api::v1::region::{QueryRequest, RegionRequest}; use common_error::ext::{BoxedError, ErrorExt, StackError}; use common_error::status_code::StatusCode; @@ -20,14 +21,13 @@ use common_telemetry::debug; use snafu::{ResultExt, Snafu}; use tokio::sync::mpsc; -use crate::datanode_manager::HandleResponse; use crate::error::{self, Error, Result}; use crate::peer::Peer; use crate::test_util::MockDatanodeHandler; #[async_trait::async_trait] impl MockDatanodeHandler for () { - async fn handle(&self, _peer: &Peer, _request: RegionRequest) -> Result { + async fn handle(&self, _peer: &Peer, _request: RegionRequest) -> Result { unreachable!() } @@ -45,10 +45,10 @@ pub struct DatanodeWatcher(pub mpsc::Sender<(Peer, RegionRequest)>); #[async_trait::async_trait] impl MockDatanodeHandler for DatanodeWatcher { - async fn handle(&self, peer: &Peer, request: RegionRequest) -> Result { + async fn handle(&self, peer: &Peer, request: RegionRequest) -> Result { debug!("Returning Ok(0) for request: {request:?}, peer: {peer:?}"); self.0.send((peer.clone(), request)).await.unwrap(); - Ok(HandleResponse::new(0)) + Ok(RegionResponse::new(0)) } async fn handle_query( @@ -65,7 +65,7 @@ pub struct RetryErrorDatanodeHandler; #[async_trait::async_trait] impl MockDatanodeHandler for RetryErrorDatanodeHandler { - async fn handle(&self, peer: &Peer, request: RegionRequest) -> Result { + async fn handle(&self, peer: &Peer, request: RegionRequest) -> Result { debug!("Returning retry later for request: {request:?}, peer: {peer:?}"); Err(Error::RetryLater { source: BoxedError::new( @@ -91,7 +91,7 @@ pub struct UnexpectedErrorDatanodeHandler; #[async_trait::async_trait] impl MockDatanodeHandler for UnexpectedErrorDatanodeHandler { - async fn handle(&self, peer: &Peer, request: RegionRequest) -> Result { + async fn handle(&self, peer: &Peer, request: RegionRequest) -> Result { debug!("Returning mock error for request: {request:?}, peer: {peer:?}"); error::UnexpectedSnafu { err_msg: "mock error", @@ -135,7 +135,7 @@ impl ErrorExt for MockRequestOutdatedError { #[async_trait::async_trait] impl MockDatanodeHandler for RequestOutdatedErrorDatanodeHandler { - async fn handle(&self, peer: &Peer, request: RegionRequest) -> Result { + async fn handle(&self, peer: &Peer, request: RegionRequest) -> Result { debug!("Returning mock error for request: {request:?}, peer: {peer:?}"); Err(BoxedError::new(MockRequestOutdatedError)).context(error::ExternalSnafu) } @@ -154,9 +154,9 @@ pub struct NaiveDatanodeHandler; #[async_trait::async_trait] impl MockDatanodeHandler for NaiveDatanodeHandler { - async fn handle(&self, peer: &Peer, request: RegionRequest) -> Result { + async fn handle(&self, peer: &Peer, request: RegionRequest) -> Result { debug!("Returning Ok(0) for request: {request:?}, peer: {peer:?}"); - Ok(HandleResponse::new(0)) + Ok(RegionResponse::new(0)) } async fn handle_query( diff --git a/src/common/meta/src/error.rs b/src/common/meta/src/error.rs index 0d23a1dd26..7346ae780d 100644 --- a/src/common/meta/src/error.rs +++ b/src/common/meta/src/error.rs @@ -421,8 +421,8 @@ pub enum Error { #[snafu(display("Invalid role: {}", role))] InvalidRole { role: i32, location: Location }, - #[snafu(display("Atomic key changed: {err_msg}"))] - CasKeyChanged { err_msg: String, location: Location }, + #[snafu(display("Failed to move values: {err_msg}"))] + MoveValues { err_msg: String, location: Location }, #[snafu(display("Failed to parse {} from utf8", name))] FromUtf8 { @@ -444,7 +444,7 @@ impl ErrorExt for Error { | EtcdFailed { .. } | EtcdTxnFailed { .. } | ConnectEtcd { .. } - | CasKeyChanged { .. } => StatusCode::Internal, + | MoveValues { .. } => StatusCode::Internal, SerdeJson { .. } | ParseOption { .. } diff --git a/src/common/meta/src/instruction.rs b/src/common/meta/src/instruction.rs index caefe2eb22..7981f2449d 100644 --- a/src/common/meta/src/instruction.rs +++ b/src/common/meta/src/instruction.rs @@ -21,6 +21,7 @@ use store_api::storage::{RegionId, RegionNumber}; use strum::Display; use table::metadata::TableId; +use crate::key::schema_name::SchemaName; use crate::table_name::TableName; use crate::{ClusterId, DatanodeId}; @@ -156,6 +157,7 @@ pub struct UpgradeRegion { pub enum CacheIdent { TableId(TableId), TableName(TableName), + SchemaName(SchemaName), } #[derive(Debug, Clone, Serialize, Deserialize, Display, PartialEq)] diff --git a/src/common/meta/src/key.rs b/src/common/meta/src/key.rs index a38387198b..3911399b13 100644 --- a/src/common/meta/src/key.rs +++ b/src/common/meta/src/key.rs @@ -88,13 +88,13 @@ use self::schema_name::{SchemaManager, SchemaNameKey, SchemaNameValue}; use self::table_route::{TableRouteManager, TableRouteValue}; use self::tombstone::TombstoneManager; use crate::ddl::utils::region_storage_path; -use crate::error::{self, Result, SerdeJsonSnafu, UnexpectedSnafu}; +use crate::error::{self, Result, SerdeJsonSnafu}; use crate::key::table_route::TableRouteKey; -use crate::key::tombstone::Key; use crate::key::txn_helper::TxnOpGetResponseSet; -use crate::kv_backend::txn::{Txn, TxnOp, TxnOpResponse}; +use crate::kv_backend::txn::{Txn, TxnOp}; use crate::kv_backend::KvBackendRef; use crate::rpc::router::{region_distribution, RegionRoute, RegionStatus}; +use crate::rpc::store::BatchDeleteRequest; use crate::table_name::TableName; use crate::DatanodeId; @@ -466,17 +466,18 @@ impl TableMetadataManager { txn = txn.merge(create_datanode_table_txn); } - let r = self.kv_backend.txn(txn).await?; + let mut r = self.kv_backend.txn(txn).await?; // Checks whether metadata was already created. if !r.succeeded { - let remote_table_info = on_create_table_info_failure(&r.responses)? + let mut set = TxnOpGetResponseSet::from(&mut r.responses); + let remote_table_info = on_create_table_info_failure(&mut set)? .context(error::UnexpectedSnafu { err_msg: "Reads the empty table info during the create table metadata", })? .into_inner(); - let remote_table_route = on_create_table_route_failure(&r.responses)? + let remote_table_route = on_create_table_route_failure(&mut set)? .context(error::UnexpectedSnafu { err_msg: "Reads the empty table route during the create table metadata", })? @@ -505,8 +506,8 @@ impl TableMetadataManager { let mut txns = Vec::with_capacity(3 * len); struct OnFailure where - F1: FnOnce(&Vec) -> R1, - F2: FnOnce(&Vec) -> R2, + F1: FnOnce(&mut TxnOpGetResponseSet) -> R1, + F2: FnOnce(&mut TxnOpGetResponseSet) -> R2, { table_info_value: TableInfoValue, on_create_table_info_failure: F1, @@ -551,18 +552,19 @@ impl TableMetadataManager { } let txn = Txn::merge_all(txns); - let r = self.kv_backend.txn(txn).await?; + let mut r = self.kv_backend.txn(txn).await?; // Checks whether metadata was already created. if !r.succeeded { + let mut set = TxnOpGetResponseSet::from(&mut r.responses); for on_failure in on_failures { - let remote_table_info = (on_failure.on_create_table_info_failure)(&r.responses)? + let remote_table_info = (on_failure.on_create_table_info_failure)(&mut set)? .context(error::UnexpectedSnafu { err_msg: "Reads the empty table info during the create table metadata", })? .into_inner(); - let remote_table_route = (on_failure.on_create_table_route_failure)(&r.responses)? + let remote_table_route = (on_failure.on_create_table_route_failure)(&mut set)? .context(error::UnexpectedSnafu { err_msg: "Reads the empty table route during the create table metadata", })? @@ -582,7 +584,7 @@ impl TableMetadataManager { table_id: TableId, table_name: &TableName, table_route_value: &TableRouteValue, - ) -> Result> { + ) -> Result>> { // Builds keys let datanode_ids = if table_route_value.is_physical() { region_distribution(table_route_value.region_routes()?) @@ -604,11 +606,11 @@ impl TableMetadataManager { .map(|datanode_id| DatanodeTableKey::new(datanode_id, table_id)) .collect::>(); - keys.push(Key::compare_and_swap(table_name.as_raw_key())); - keys.push(Key::new(table_info_key.as_raw_key())); - keys.push(Key::new(table_route_key.as_raw_key())); + keys.push(table_name.as_raw_key()); + keys.push(table_info_key.as_raw_key()); + keys.push(table_route_key.as_raw_key()); for key in &datanode_table_keys { - keys.push(Key::new(key.as_raw_key())); + keys.push(key.as_raw_key()); } Ok(keys) } @@ -622,8 +624,7 @@ impl TableMetadataManager { table_route_value: &TableRouteValue, ) -> Result<()> { let keys = self.table_metadata_keys(table_id, table_name, table_route_value)?; - self.tombstone_manager.create(keys).await?; - Ok(()) + self.tombstone_manager.create(keys).await } /// Deletes metadata tombstone for table **permanently**. @@ -634,11 +635,7 @@ impl TableMetadataManager { table_name: &TableName, table_route_value: &TableRouteValue, ) -> Result<()> { - let keys = self - .table_metadata_keys(table_id, table_name, table_route_value)? - .into_iter() - .map(|key| key.into_bytes()) - .collect::>(); + let keys = self.table_metadata_keys(table_id, table_name, table_route_value)?; self.tombstone_manager.delete(keys).await } @@ -651,8 +648,7 @@ impl TableMetadataManager { table_route_value: &TableRouteValue, ) -> Result<()> { let keys = self.table_metadata_keys(table_id, table_name, table_route_value)?; - self.tombstone_manager.restore(keys).await?; - Ok(()) + self.tombstone_manager.restore(keys).await } /// Deletes metadata for table **permanently**. @@ -663,21 +659,11 @@ impl TableMetadataManager { table_name: &TableName, table_route_value: &TableRouteValue, ) -> Result<()> { - let operations = self - .table_metadata_keys(table_id, table_name, table_route_value)? - .into_iter() - .map(|key| TxnOp::Delete(key.into_bytes())) - .collect::>(); - - let txn = Txn::new().and_then(operations); - let resp = self.kv_backend.txn(txn).await?; - ensure!( - resp.succeeded, - UnexpectedSnafu { - err_msg: format!("Failed to destroy table metadata: {table_id}") - } - ); - + let keys = self.table_metadata_keys(table_id, table_name, table_route_value)?; + let _ = self + .kv_backend + .batch_delete(BatchDeleteRequest::new().with_keys(keys)) + .await?; Ok(()) } @@ -724,11 +710,12 @@ impl TableMetadataManager { let txn = Txn::merge_all(vec![update_table_name_txn, update_table_info_txn]); - let r = self.kv_backend.txn(txn).await?; + let mut r = self.kv_backend.txn(txn).await?; // Checks whether metadata was already updated. if !r.succeeded { - let remote_table_info = on_update_table_info_failure(&r.responses)? + let mut set = TxnOpGetResponseSet::from(&mut r.responses); + let remote_table_info = on_update_table_info_failure(&mut set)? .context(error::UnexpectedSnafu { err_msg: "Reads the empty table info during the rename table metadata", })? @@ -756,11 +743,12 @@ impl TableMetadataManager { .table_info_manager() .build_update_txn(table_id, current_table_info_value, &new_table_info_value)?; - let r = self.kv_backend.txn(update_table_info_txn).await?; + let mut r = self.kv_backend.txn(update_table_info_txn).await?; // Checks whether metadata was already updated. if !r.succeeded { - let remote_table_info = on_update_table_info_failure(&r.responses)? + let mut set = TxnOpGetResponseSet::from(&mut r.responses); + let remote_table_info = on_update_table_info_failure(&mut set)? .context(error::UnexpectedSnafu { err_msg: "Reads the empty table info during the updating table info", })? @@ -784,7 +772,7 @@ impl TableMetadataManager { let mut txns = Vec::with_capacity(len); struct OnFailure where - F: FnOnce(&Vec) -> R, + F: FnOnce(&mut TxnOpGetResponseSet) -> R, { table_info_value: TableInfoValue, on_update_table_info_failure: F, @@ -812,11 +800,12 @@ impl TableMetadataManager { } let txn = Txn::merge_all(txns); - let r = self.kv_backend.txn(txn).await?; + let mut r = self.kv_backend.txn(txn).await?; if !r.succeeded { + let mut set = TxnOpGetResponseSet::from(&mut r.responses); for on_failure in on_failures { - let remote_table_info = (on_failure.on_update_table_info_failure)(&r.responses)? + let remote_table_info = (on_failure.on_update_table_info_failure)(&mut set)? .context(error::UnexpectedSnafu { err_msg: "Reads the empty table info during the updating table info", })? @@ -863,11 +852,12 @@ impl TableMetadataManager { let txn = Txn::merge_all(vec![update_datanode_table_txn, update_table_route_txn]); - let r = self.kv_backend.txn(txn).await?; + let mut r = self.kv_backend.txn(txn).await?; // Checks whether metadata was already updated. if !r.succeeded { - let remote_table_route = on_update_table_route_failure(&r.responses)? + let mut set = TxnOpGetResponseSet::from(&mut r.responses); + let remote_table_route = on_update_table_route_failure(&mut set)? .context(error::UnexpectedSnafu { err_msg: "Reads the empty table route during the updating table route", })? @@ -914,11 +904,12 @@ impl TableMetadataManager { .table_route_storage() .build_update_txn(table_id, current_table_route_value, &new_table_route_value)?; - let r = self.kv_backend.txn(update_table_route_txn).await?; + let mut r = self.kv_backend.txn(update_table_route_txn).await?; // Checks whether metadata was already updated. if !r.succeeded { - let remote_table_route = on_update_table_route_failure(&r.responses)? + let mut set = TxnOpGetResponseSet::from(&mut r.responses); + let remote_table_route = on_update_table_route_failure(&mut set)? .context(error::UnexpectedSnafu { err_msg: "Reads the empty table route during the updating leader region status", })? @@ -1286,14 +1277,17 @@ mod tests { .delete_table_metadata(table_id, &table_name, table_route_value) .await .unwrap(); - + // Should be ignored. + table_metadata_manager + .delete_table_metadata(table_id, &table_name, table_route_value) + .await + .unwrap(); assert!(table_metadata_manager .table_info_manager() .get(table_id) .await .unwrap() .is_none()); - assert!(table_metadata_manager .table_route_manager() .table_route_storage() @@ -1301,7 +1295,6 @@ mod tests { .await .unwrap() .is_none()); - assert!(table_metadata_manager .datanode_table_manager() .tables(datanode_id) @@ -1316,7 +1309,6 @@ mod tests { .await .unwrap(); assert!(table_info.is_none()); - let table_route = table_metadata_manager .table_route_manager() .table_route_storage() @@ -1792,5 +1784,12 @@ mod tests { .unwrap(); let kvs = mem_kv.dump(); assert_eq!(kvs, expected_result); + // Should be ignored. + table_metadata_manager + .restore_table_metadata(table_id, &table_name, &table_route_value) + .await + .unwrap(); + let kvs = mem_kv.dump(); + assert_eq!(kvs, expected_result); } } diff --git a/src/common/meta/src/key/datanode_table.rs b/src/common/meta/src/key/datanode_table.rs index 82ddebc320..aa63eef09e 100644 --- a/src/common/meta/src/key/datanode_table.rs +++ b/src/common/meta/src/key/datanode_table.rs @@ -240,10 +240,14 @@ impl DatanodeTableManager { // FIXME(weny): add unit tests. let mut new_region_info = region_info.clone(); if need_update_options { - new_region_info.region_options = new_region_options.clone(); + new_region_info + .region_options + .clone_from(new_region_options); } if need_update_wal_options { - new_region_info.region_wal_options = new_region_wal_options.clone(); + new_region_info + .region_wal_options + .clone_from(new_region_wal_options); } let val = DatanodeTableValue::new(table_id, regions, new_region_info) .try_as_raw_value()?; diff --git a/src/common/meta/src/key/schema_name.rs b/src/common/meta/src/key/schema_name.rs index 8b391ca9b3..17fdb6a4c2 100644 --- a/src/common/meta/src/key/schema_name.rs +++ b/src/common/meta/src/key/schema_name.rs @@ -197,6 +197,21 @@ impl SchemaManager { } } +#[derive(Debug, Clone, Hash, Eq, PartialEq, Deserialize, Serialize)] +pub struct SchemaName { + pub catalog_name: String, + pub schema_name: String, +} + +impl<'a> From<&'a SchemaName> for SchemaNameKey<'a> { + fn from(value: &'a SchemaName) -> Self { + Self { + catalog: &value.catalog_name, + schema: &value.schema_name, + } + } +} + #[cfg(test)] mod tests { diff --git a/src/common/meta/src/key/table_info.rs b/src/common/meta/src/key/table_info.rs index f0fe71b826..798dbf1beb 100644 --- a/src/common/meta/src/key/table_info.rs +++ b/src/common/meta/src/key/table_info.rs @@ -18,11 +18,12 @@ use serde::{Deserialize, Serialize}; use table::metadata::{RawTableInfo, TableId}; use table::table_reference::TableReference; +use super::txn_helper::TxnOpGetResponseSet; use crate::error::Result; use crate::key::{ txn_helper, DeserializedValueWithBytes, TableMetaKey, TableMetaValue, TABLE_INFO_KEY_PREFIX, }; -use crate::kv_backend::txn::{Txn, TxnOpResponse}; +use crate::kv_backend::txn::Txn; use crate::kv_backend::KvBackendRef; use crate::rpc::store::BatchGetRequest; use crate::table_name::TableName; @@ -109,7 +110,9 @@ impl TableInfoManager { table_info_value: &TableInfoValue, ) -> Result<( Txn, - impl FnOnce(&Vec) -> Result>>, + impl FnOnce( + &mut TxnOpGetResponseSet, + ) -> Result>>, )> { let key = TableInfoKey::new(table_id); let raw_key = key.as_raw_key(); @@ -119,7 +122,10 @@ impl TableInfoManager { table_info_value.try_as_raw_value()?, ); - Ok((txn, txn_helper::build_txn_response_decoder_fn(raw_key))) + Ok(( + txn, + TxnOpGetResponseSet::decode_with(TxnOpGetResponseSet::filter(raw_key)), + )) } /// Builds a update table info transaction, it expected the remote value equals the `current_current_table_info_value`. @@ -131,7 +137,9 @@ impl TableInfoManager { new_table_info_value: &TableInfoValue, ) -> Result<( Txn, - impl FnOnce(&Vec) -> Result>>, + impl FnOnce( + &mut TxnOpGetResponseSet, + ) -> Result>>, )> { let key = TableInfoKey::new(table_id); let raw_key = key.as_raw_key(); @@ -140,7 +148,10 @@ impl TableInfoManager { let txn = txn_helper::build_compare_and_put_txn(raw_key.clone(), raw_value, new_raw_value); - Ok((txn, txn_helper::build_txn_response_decoder_fn(raw_key))) + Ok(( + txn, + TxnOpGetResponseSet::decode_with(TxnOpGetResponseSet::filter(raw_key)), + )) } pub async fn get( diff --git a/src/common/meta/src/key/table_route.rs b/src/common/meta/src/key/table_route.rs index 15b2c4f69a..a6745f8db2 100644 --- a/src/common/meta/src/key/table_route.rs +++ b/src/common/meta/src/key/table_route.rs @@ -20,13 +20,16 @@ use snafu::{ensure, OptionExt, ResultExt}; use store_api::storage::{RegionId, RegionNumber}; use table::metadata::TableId; -use super::{txn_helper, DeserializedValueWithBytes, TableMetaValue}; use crate::error::{ self, MetadataCorruptionSnafu, Result, SerdeJsonSnafu, TableRouteNotFoundSnafu, UnexpectedLogicalRouteTableSnafu, }; -use crate::key::{RegionDistribution, TableMetaKey, TABLE_ROUTE_PREFIX}; -use crate::kv_backend::txn::{Txn, TxnOpResponse}; +use crate::key::txn_helper::TxnOpGetResponseSet; +use crate::key::{ + txn_helper, DeserializedValueWithBytes, RegionDistribution, TableMetaKey, TableMetaValue, + TABLE_ROUTE_PREFIX, +}; +use crate::kv_backend::txn::Txn; use crate::kv_backend::KvBackendRef; use crate::rpc::router::{region_distribution, RegionRoute}; use crate::rpc::store::BatchGetRequest; @@ -454,7 +457,9 @@ impl TableRouteStorage { table_route_value: &TableRouteValue, ) -> Result<( Txn, - impl FnOnce(&Vec) -> Result>>, + impl FnOnce( + &mut TxnOpGetResponseSet, + ) -> Result>>, )> { let key = TableRouteKey::new(table_id); let raw_key = key.as_raw_key(); @@ -464,7 +469,10 @@ impl TableRouteStorage { table_route_value.try_as_raw_value()?, ); - Ok((txn, txn_helper::build_txn_response_decoder_fn(raw_key))) + Ok(( + txn, + TxnOpGetResponseSet::decode_with(TxnOpGetResponseSet::filter(raw_key)), + )) } /// Builds a update table route transaction, @@ -477,7 +485,9 @@ impl TableRouteStorage { new_table_route_value: &TableRouteValue, ) -> Result<( Txn, - impl FnOnce(&Vec) -> Result>>, + impl FnOnce( + &mut TxnOpGetResponseSet, + ) -> Result>>, )> { let key = TableRouteKey::new(table_id); let raw_key = key.as_raw_key(); @@ -486,7 +496,10 @@ impl TableRouteStorage { let txn = txn_helper::build_compare_and_put_txn(raw_key.clone(), raw_value, new_raw_value); - Ok((txn, txn_helper::build_txn_response_decoder_fn(raw_key))) + Ok(( + txn, + TxnOpGetResponseSet::decode_with(TxnOpGetResponseSet::filter(raw_key)), + )) } /// Returns the [`TableRouteValue`]. diff --git a/src/common/meta/src/key/tombstone.rs b/src/common/meta/src/key/tombstone.rs index 045959b682..38648f2695 100644 --- a/src/common/meta/src/key/tombstone.rs +++ b/src/common/meta/src/key/tombstone.rs @@ -12,13 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -use snafu::{ensure, OptionExt}; +use std::collections::HashMap; + +use snafu::ensure; -use super::TableMetaKeyGetTxnOp; use crate::error::{self, Result}; use crate::key::txn_helper::TxnOpGetResponseSet; use crate::kv_backend::txn::{Compare, CompareOp, Txn, TxnOp}; use crate::kv_backend::KvBackendRef; +use crate::rpc::store::BatchGetRequest; /// [TombstoneManager] provides the ability to: /// - logically delete values @@ -29,307 +31,160 @@ pub(crate) struct TombstoneManager { const TOMBSTONE_PREFIX: &str = "__tombstone/"; -pub(crate) struct TombstoneKey(T); +pub(crate) struct TombstoneKeyValue { + pub(crate) origin_key: Vec, + pub(crate) tombstone_key: Vec, + pub(crate) value: Vec, +} fn to_tombstone(key: &[u8]) -> Vec { [TOMBSTONE_PREFIX.as_bytes(), key].concat() } -impl TombstoneKey<&Vec> { - /// Returns the origin key and tombstone key. - fn to_keys(&self) -> (Vec, Vec) { - let key = self.0; - let tombstone_key = to_tombstone(key); - (key.clone(), tombstone_key) - } - - /// Returns the origin key and tombstone key. - fn into_keys(self) -> (Vec, Vec) { - self.to_keys() - } - - /// Returns the tombstone key. - fn to_tombstone_key(&self) -> Vec { - let key = self.0; - to_tombstone(key) - } -} - -impl TableMetaKeyGetTxnOp for TombstoneKey<&Vec> { - fn build_get_op( - &self, - ) -> ( - TxnOp, - impl FnMut(&'_ mut TxnOpGetResponseSet) -> Option>, - ) { - TxnOpGetResponseSet::build_get_op(to_tombstone(self.0)) - } -} - -/// The key used in the [TombstoneManager]. -pub(crate) struct Key { - bytes: Vec, - // Atomic Key: - // The value corresponding to the key remains consistent between two transactions. - atomic: bool, -} - -impl Key { - /// Returns a new atomic key. - pub(crate) fn compare_and_swap>>(key: T) -> Self { - Self { - bytes: key.into(), - atomic: true, - } - } - - /// Returns a new normal key. - pub(crate) fn new>>(key: T) -> Self { - Self { - bytes: key.into(), - atomic: false, - } - } - - /// Into bytes - pub(crate) fn into_bytes(self) -> Vec { - self.bytes - } - - fn get_inner(&self) -> &Vec { - &self.bytes - } - - fn is_atomic(&self) -> bool { - self.atomic - } -} - -impl TableMetaKeyGetTxnOp for Key { - fn build_get_op( - &self, - ) -> ( - TxnOp, - impl FnMut(&'_ mut TxnOpGetResponseSet) -> Option>, - ) { - let key = self.get_inner().clone(); - (TxnOp::Get(key.clone()), TxnOpGetResponseSet::filter(key)) - } -} - -fn format_on_failure_error_message Option>>( - mut set: TxnOpGetResponseSet, - on_failure_kv_and_filters: Vec<(Vec, Vec, F)>, -) -> String { - on_failure_kv_and_filters - .into_iter() - .flat_map(|(key, value, mut filter)| { - let got = filter(&mut set); - let Some(got) = got else { - return Some(format!( - "For key: {} was expected: {}, but value does not exists", - String::from_utf8_lossy(&key), - String::from_utf8_lossy(&value), - )); - }; - - if got != value { - Some(format!( - "For key: {} was expected: {}, but got: {}", - String::from_utf8_lossy(&key), - String::from_utf8_lossy(&value), - String::from_utf8_lossy(&got), - )) - } else { - None - } - }) - .collect::>() - .join("; ") -} - -fn format_keys(keys: &[Key]) -> String { - keys.iter() - .map(|key| String::from_utf8_lossy(&key.bytes)) - .collect::>() - .join(", ") -} - impl TombstoneManager { /// Returns [TombstoneManager]. pub fn new(kv_backend: KvBackendRef) -> Self { Self { kv_backend } } + /// Moves value to `dest_key`. + /// + /// Puts `value` to `dest_key` if the value of `src_key` equals `value`. + /// + /// Otherwise retrieves the value of `src_key`. + fn build_move_value_txn( + &self, + src_key: Vec, + value: Vec, + dest_key: Vec, + ) -> (Txn, impl FnMut(&mut TxnOpGetResponseSet) -> Option>) { + let txn = Txn::new() + .when(vec![Compare::with_value( + src_key.clone(), + CompareOp::Equal, + value.clone(), + )]) + .and_then(vec![ + TxnOp::Put(dest_key.clone(), value.clone()), + TxnOp::Delete(src_key.clone()), + ]) + .or_else(vec![TxnOp::Get(src_key.clone())]); + + (txn, TxnOpGetResponseSet::filter(src_key)) + } + + async fn move_values_inner(&self, keys: &[Vec], dest_keys: &[Vec]) -> Result<()> { + ensure!( + keys.len() == dest_keys.len(), + error::UnexpectedSnafu { + err_msg: "The length of keys does not match the length of dest_keys." + } + ); + // The key -> dest key mapping. + let lookup_table = keys.iter().zip(dest_keys.iter()).collect::>(); + + let resp = self + .kv_backend + .batch_get(BatchGetRequest::new().with_keys(keys.to_vec())) + .await?; + let mut results = resp + .kvs + .into_iter() + .map(|kv| (kv.key, kv.value)) + .collect::>(); + + const MAX_RETRIES: usize = 8; + for _ in 0..MAX_RETRIES { + let (txns, (keys, filters)): (Vec<_>, (Vec<_>, Vec<_>)) = results + .iter() + .map(|(key, value)| { + let (txn, filter) = self.build_move_value_txn( + key.clone(), + value.clone(), + lookup_table[&key].clone(), + ); + (txn, (key.clone(), filter)) + }) + .unzip(); + let mut resp = self.kv_backend.txn(Txn::merge_all(txns)).await?; + if resp.succeeded { + return Ok(()); + } + let mut set = TxnOpGetResponseSet::from(&mut resp.responses); + // Updates results. + for (idx, mut filter) in filters.into_iter().enumerate() { + if let Some(value) = filter(&mut set) { + results.insert(keys[idx].clone(), value); + } else { + results.remove(&keys[idx]); + } + } + } + + error::MoveValuesSnafu { + err_msg: format!( + "keys: {:?}", + keys.iter().map(|key| String::from_utf8_lossy(key)), + ), + } + .fail() + } + + /// Moves values to `dest_key`. + async fn move_values(&self, keys: Vec>, dest_keys: Vec>) -> Result<()> { + let chunk_size = self.kv_backend.max_txn_ops() / 2; + if keys.len() > chunk_size { + let keys_chunks = keys.chunks(chunk_size).collect::>(); + let dest_keys_chunks = keys.chunks(chunk_size).collect::>(); + for (keys, dest_keys) in keys_chunks.into_iter().zip(dest_keys_chunks) { + self.move_values_inner(keys, dest_keys).await?; + } + + Ok(()) + } else { + self.move_values_inner(&keys, &dest_keys).await + } + } + /// Creates tombstones for keys. /// /// Preforms to: - /// - retrieve all values corresponding `keys`. + /// - deletes origin values. /// - stores tombstone values. - pub(crate) async fn create(&self, keys: Vec) -> Result<()> { - // Builds transaction to retrieve all values - let (operations, mut filters): (Vec<_>, Vec<_>) = - keys.iter().map(|key| key.build_get_op()).unzip(); + pub(crate) async fn create(&self, keys: Vec>) -> Result<()> { + let (keys, dest_keys): (Vec<_>, Vec<_>) = keys + .into_iter() + .map(|key| { + let tombstone_key = to_tombstone(&key); + (key, tombstone_key) + }) + .unzip(); - let txn = Txn::new().and_then(operations); - let mut resp = self.kv_backend.txn(txn).await?; - ensure!( - resp.succeeded, - error::UnexpectedSnafu { - err_msg: format!( - "Failed to retrieves the metadata, keys: {}", - format_keys(&keys) - ), - } - ); - - let mut set = TxnOpGetResponseSet::from(&mut resp.responses); - // Builds the create tombstone transaction. - let mut tombstone_operations = Vec::with_capacity(keys.len() * 2); - let mut tombstone_comparison = vec![]; - let mut on_failure_operations = vec![]; - let mut on_failure_kv_and_filters = vec![]; - for (idx, key) in keys.iter().enumerate() { - let filter = &mut filters[idx]; - let value = filter(&mut set).with_context(|| error::UnexpectedSnafu { - err_msg: format!( - "Missing value, key: {}", - String::from_utf8_lossy(key.get_inner()) - ), - })?; - let (origin_key, tombstone_key) = TombstoneKey(key.get_inner()).into_keys(); - // Compares the atomic key. - if key.is_atomic() { - tombstone_comparison.push(Compare::with_not_exist_value( - tombstone_key.clone(), - CompareOp::Equal, - )); - tombstone_comparison.push(Compare::with_value( - origin_key.clone(), - CompareOp::Equal, - value.clone(), - )); - let (op, filter) = TxnOpGetResponseSet::build_get_op(origin_key.clone()); - on_failure_operations.push(op); - on_failure_kv_and_filters.push((origin_key.clone(), value.clone(), filter)); - } - tombstone_operations.push(TxnOp::Delete(origin_key)); - tombstone_operations.push(TxnOp::Put(tombstone_key, value)); - } - - let txn = if !tombstone_comparison.is_empty() { - Txn::new().when(tombstone_comparison) - } else { - Txn::new() - } - .and_then(tombstone_operations); - - let txn = if !on_failure_operations.is_empty() { - txn.or_else(on_failure_operations) - } else { - txn - }; - - let mut resp = self.kv_backend.txn(txn).await?; - // TODO(weny): add tests for atomic key changed. - if !resp.succeeded { - let set = TxnOpGetResponseSet::from(&mut resp.responses); - let err_msg = format_on_failure_error_message(set, on_failure_kv_and_filters); - return error::CasKeyChangedSnafu { err_msg }.fail(); - } - Ok(()) + self.move_values(keys, dest_keys).await } /// Restores tombstones for keys. /// /// Preforms to: - /// - retrieve all tombstone values corresponding `keys`. - /// - stores tombstone values. - pub(crate) async fn restore(&self, keys: Vec) -> Result<()> { - // Builds transaction to retrieve all tombstone values - let tombstone_keys = keys - .iter() - .map(|key| TombstoneKey(key.get_inner())) - .collect::>(); - let (operations, mut filters): (Vec<_>, Vec<_>) = - tombstone_keys.iter().map(|key| key.build_get_op()).unzip(); + /// - restore origin value. + /// - deletes tombstone values. + pub(crate) async fn restore(&self, keys: Vec>) -> Result<()> { + let (keys, dest_keys): (Vec<_>, Vec<_>) = keys + .into_iter() + .map(|key| { + let tombstone_key = to_tombstone(&key); + (tombstone_key, key) + }) + .unzip(); - let txn = Txn::new().and_then(operations); - let mut resp = self.kv_backend.txn(txn).await?; - ensure!( - resp.succeeded, - error::UnexpectedSnafu { - err_msg: format!( - "Failed to retrieves the metadata, keys: {}", - format_keys(&keys) - ), - } - ); - - let mut set = TxnOpGetResponseSet::from(&mut resp.responses); - - // Builds the restore tombstone transaction. - let mut tombstone_operations = Vec::with_capacity(keys.len() * 2); - let mut tombstone_comparison = vec![]; - let mut on_failure_operations = vec![]; - let mut on_failure_kv_and_filters = vec![]; - for (idx, key) in keys.iter().enumerate() { - let filter = &mut filters[idx]; - let value = filter(&mut set).with_context(|| error::UnexpectedSnafu { - err_msg: format!( - "Missing value, key: {}", - String::from_utf8_lossy(key.get_inner()) - ), - })?; - let (origin_key, tombstone_key) = tombstone_keys[idx].to_keys(); - // Compares the atomic key. - if key.is_atomic() { - tombstone_comparison.push(Compare::with_not_exist_value( - origin_key.clone(), - CompareOp::Equal, - )); - tombstone_comparison.push(Compare::with_value( - tombstone_key.clone(), - CompareOp::Equal, - value.clone(), - )); - let (op, filter) = tombstone_keys[idx].build_get_op(); - on_failure_operations.push(op); - on_failure_kv_and_filters.push((tombstone_key.clone(), value.clone(), filter)); - } - tombstone_operations.push(TxnOp::Delete(tombstone_key)); - tombstone_operations.push(TxnOp::Put(origin_key, value)); - } - - let txn = if !tombstone_comparison.is_empty() { - Txn::new().when(tombstone_comparison) - } else { - Txn::new() - } - .and_then(tombstone_operations); - - let txn = if !on_failure_operations.is_empty() { - txn.or_else(on_failure_operations) - } else { - txn - }; - - let mut resp = self.kv_backend.txn(txn).await?; - // TODO(weny): add tests for atomic key changed. - if !resp.succeeded { - let set = TxnOpGetResponseSet::from(&mut resp.responses); - let err_msg = format_on_failure_error_message(set, on_failure_kv_and_filters); - return error::CasKeyChangedSnafu { err_msg }.fail(); - } - - Ok(()) + self.move_values(keys, dest_keys).await } - /// Deletes tombstones for keys. + /// Deletes tombstones values for the specified `keys`. pub(crate) async fn delete(&self, keys: Vec>) -> Result<()> { let operations = keys .iter() - .map(|key| TxnOp::Delete(TombstoneKey(key).to_tombstone_key())) + .map(|key| TxnOp::Delete(to_tombstone(key))) .collect::>(); let txn = Txn::new().and_then(operations); @@ -342,13 +197,41 @@ impl TombstoneManager { #[cfg(test)] mod tests { + use std::collections::HashMap; use std::sync::Arc; - use crate::key::tombstone::{Key, TombstoneKey, TombstoneManager}; + use super::to_tombstone; + use crate::error::Error; + use crate::key::tombstone::TombstoneManager; use crate::kv_backend::memory::MemoryKvBackend; use crate::kv_backend::KvBackend; use crate::rpc::store::PutRequest; + #[derive(Debug, Clone)] + struct MoveValue { + key: Vec, + dest_key: Vec, + value: Vec, + } + + async fn check_moved_values( + kv_backend: Arc>, + move_values: &[MoveValue], + ) { + for MoveValue { + key, + dest_key, + value, + } in move_values + { + assert!(kv_backend.get(key).await.unwrap().is_none()); + assert_eq!( + &kv_backend.get(dest_key).await.unwrap().unwrap().value, + value, + ); + } + } + #[tokio::test] async fn test_create_tombstone() { let kv_backend = Arc::new(MemoryKvBackend::default()); @@ -362,14 +245,14 @@ mod tests { .await .unwrap(); tombstone_manager - .create(vec![Key::compare_and_swap("bar"), Key::new("foo")]) + .create(vec![b"bar".to_vec(), b"foo".to_vec()]) .await .unwrap(); assert!(!kv_backend.exists(b"bar").await.unwrap()); assert!(!kv_backend.exists(b"foo").await.unwrap()); assert_eq!( kv_backend - .get(&TombstoneKey(&"bar".into()).to_tombstone_key()) + .get(&to_tombstone(b"bar")) .await .unwrap() .unwrap() @@ -378,7 +261,7 @@ mod tests { ); assert_eq!( kv_backend - .get(&TombstoneKey(&"foo".into()).to_tombstone_key()) + .get(&to_tombstone(b"foo")) .await .unwrap() .unwrap() @@ -389,9 +272,10 @@ mod tests { } #[tokio::test] - async fn test_create_tombstone_without_atomic_key() { + async fn test_create_tombstone_with_non_exist_values() { let kv_backend = Arc::new(MemoryKvBackend::default()); let tombstone_manager = TombstoneManager::new(kv_backend.clone()); + kv_backend .put(PutRequest::new().with_key("bar").with_value("baz")) .await @@ -400,52 +284,20 @@ mod tests { .put(PutRequest::new().with_key("foo").with_value("hi")) .await .unwrap(); + tombstone_manager - .create(vec![Key::new("bar"), Key::new("foo")]) + .create(vec![b"bar".to_vec(), b"baz".to_vec()]) .await .unwrap(); - assert!(!kv_backend.exists(b"bar").await.unwrap()); - assert!(!kv_backend.exists(b"foo").await.unwrap()); - assert_eq!( - kv_backend - .get(&TombstoneKey(&"bar".into()).to_tombstone_key()) - .await - .unwrap() - .unwrap() - .value, - b"baz" - ); - assert_eq!( - kv_backend - .get(&TombstoneKey(&"foo".into()).to_tombstone_key()) - .await - .unwrap() - .unwrap() - .value, - b"hi" - ); - assert_eq!(kv_backend.len(), 2); - } - - #[tokio::test] - async fn test_create_tombstone_origin_value_not_found_err() { - let kv_backend = Arc::new(MemoryKvBackend::default()); - let tombstone_manager = TombstoneManager::new(kv_backend.clone()); - - kv_backend - .put(PutRequest::new().with_key("bar").with_value("baz")) - .await - .unwrap(); - kv_backend - .put(PutRequest::new().with_key("foo").with_value("hi")) - .await - .unwrap(); - - let err = tombstone_manager - .create(vec![Key::compare_and_swap("bar"), Key::new("baz")]) - .await - .unwrap_err(); - assert!(err.to_string().contains("Missing value")); + check_moved_values( + kv_backend.clone(), + &[MoveValue { + key: b"bar".to_vec(), + dest_key: to_tombstone(b"bar"), + value: b"baz".to_vec(), + }], + ) + .await; } #[tokio::test] @@ -462,63 +314,16 @@ mod tests { .unwrap(); let expected_kvs = kv_backend.dump(); tombstone_manager - .create(vec![Key::compare_and_swap("bar"), Key::new("foo")]) + .create(vec![b"bar".to_vec(), b"foo".to_vec()]) .await .unwrap(); tombstone_manager - .restore(vec![Key::compare_and_swap("bar"), Key::new("foo")]) + .restore(vec![b"bar".to_vec(), b"foo".to_vec()]) .await .unwrap(); assert_eq!(expected_kvs, kv_backend.dump()); } - #[tokio::test] - async fn test_restore_tombstone_without_atomic_key() { - let kv_backend = Arc::new(MemoryKvBackend::default()); - let tombstone_manager = TombstoneManager::new(kv_backend.clone()); - kv_backend - .put(PutRequest::new().with_key("bar").with_value("baz")) - .await - .unwrap(); - kv_backend - .put(PutRequest::new().with_key("foo").with_value("hi")) - .await - .unwrap(); - let expected_kvs = kv_backend.dump(); - tombstone_manager - .create(vec![Key::compare_and_swap("bar"), Key::new("foo")]) - .await - .unwrap(); - tombstone_manager - .restore(vec![Key::new("bar"), Key::new("foo")]) - .await - .unwrap(); - assert_eq!(expected_kvs, kv_backend.dump()); - } - - #[tokio::test] - async fn test_restore_tombstone_origin_value_not_found_err() { - let kv_backend = Arc::new(MemoryKvBackend::default()); - let tombstone_manager = TombstoneManager::new(kv_backend.clone()); - kv_backend - .put(PutRequest::new().with_key("bar").with_value("baz")) - .await - .unwrap(); - kv_backend - .put(PutRequest::new().with_key("foo").with_value("hi")) - .await - .unwrap(); - tombstone_manager - .create(vec![Key::compare_and_swap("bar"), Key::new("foo")]) - .await - .unwrap(); - let err = tombstone_manager - .restore(vec![Key::new("bar"), Key::new("baz")]) - .await - .unwrap_err(); - assert!(err.to_string().contains("Missing value")); - } - #[tokio::test] async fn test_delete_tombstone() { let kv_backend = Arc::new(MemoryKvBackend::default()); @@ -532,7 +337,7 @@ mod tests { .await .unwrap(); tombstone_manager - .create(vec![Key::compare_and_swap("bar"), Key::new("foo")]) + .create(vec![b"bar".to_vec(), b"foo".to_vec()]) .await .unwrap(); tombstone_manager @@ -541,4 +346,216 @@ mod tests { .unwrap(); assert!(kv_backend.is_empty()); } + + #[tokio::test] + async fn test_move_values() { + let kv_backend = Arc::new(MemoryKvBackend::default()); + let tombstone_manager = TombstoneManager::new(kv_backend.clone()); + let kvs = HashMap::from([ + (b"bar".to_vec(), b"baz".to_vec()), + (b"foo".to_vec(), b"hi".to_vec()), + (b"baz".to_vec(), b"hello".to_vec()), + ]); + for (key, value) in &kvs { + kv_backend + .put( + PutRequest::new() + .with_key(key.clone()) + .with_value(value.clone()), + ) + .await + .unwrap(); + } + let move_values = kvs + .iter() + .map(|(key, value)| MoveValue { + key: key.clone(), + dest_key: to_tombstone(key), + value: value.clone(), + }) + .collect::>(); + let (keys, dest_keys): (Vec<_>, Vec<_>) = move_values + .clone() + .into_iter() + .map(|kv| (kv.key, kv.dest_key)) + .unzip(); + tombstone_manager + .move_values(keys.clone(), dest_keys.clone()) + .await + .unwrap(); + check_moved_values(kv_backend.clone(), &move_values).await; + // Moves again + tombstone_manager + .move_values(keys.clone(), dest_keys.clone()) + .await + .unwrap(); + check_moved_values(kv_backend.clone(), &move_values).await; + } + + #[tokio::test] + async fn test_move_values_with_non_exists_values() { + let kv_backend = Arc::new(MemoryKvBackend::default()); + let tombstone_manager = TombstoneManager::new(kv_backend.clone()); + let kvs = HashMap::from([ + (b"bar".to_vec(), b"baz".to_vec()), + (b"foo".to_vec(), b"hi".to_vec()), + (b"baz".to_vec(), b"hello".to_vec()), + ]); + for (key, value) in &kvs { + kv_backend + .put( + PutRequest::new() + .with_key(key.clone()) + .with_value(value.clone()), + ) + .await + .unwrap(); + } + let move_values = kvs + .iter() + .map(|(key, value)| MoveValue { + key: key.clone(), + dest_key: to_tombstone(key), + value: value.clone(), + }) + .collect::>(); + let (mut keys, mut dest_keys): (Vec<_>, Vec<_>) = move_values + .clone() + .into_iter() + .map(|kv| (kv.key, kv.dest_key)) + .unzip(); + keys.push(b"non-exists".to_vec()); + dest_keys.push(b"hi/non-exists".to_vec()); + tombstone_manager + .move_values(keys.clone(), dest_keys.clone()) + .await + .unwrap(); + check_moved_values(kv_backend.clone(), &move_values).await; + // Moves again + tombstone_manager + .move_values(keys.clone(), dest_keys.clone()) + .await + .unwrap(); + check_moved_values(kv_backend.clone(), &move_values).await; + } + + #[tokio::test] + async fn test_move_values_changed() { + let kv_backend = Arc::new(MemoryKvBackend::default()); + let tombstone_manager = TombstoneManager::new(kv_backend.clone()); + let kvs = HashMap::from([ + (b"bar".to_vec(), b"baz".to_vec()), + (b"foo".to_vec(), b"hi".to_vec()), + (b"baz".to_vec(), b"hello".to_vec()), + ]); + for (key, value) in &kvs { + kv_backend + .put( + PutRequest::new() + .with_key(key.clone()) + .with_value(value.clone()), + ) + .await + .unwrap(); + } + + kv_backend + .put(PutRequest::new().with_key("baz").with_value("changed")) + .await + .unwrap(); + + let move_values = kvs + .iter() + .map(|(key, value)| MoveValue { + key: key.clone(), + dest_key: to_tombstone(key), + value: value.clone(), + }) + .collect::>(); + let (keys, dest_keys): (Vec<_>, Vec<_>) = move_values + .clone() + .into_iter() + .map(|kv| (kv.key, kv.dest_key)) + .unzip(); + tombstone_manager + .move_values(keys, dest_keys) + .await + .unwrap(); + } + + #[tokio::test] + async fn test_move_values_overwrite_dest_values() { + let kv_backend = Arc::new(MemoryKvBackend::default()); + let tombstone_manager = TombstoneManager::new(kv_backend.clone()); + let kvs = HashMap::from([ + (b"bar".to_vec(), b"baz".to_vec()), + (b"foo".to_vec(), b"hi".to_vec()), + (b"baz".to_vec(), b"hello".to_vec()), + ]); + for (key, value) in &kvs { + kv_backend + .put( + PutRequest::new() + .with_key(key.clone()) + .with_value(value.clone()), + ) + .await + .unwrap(); + } + + // Prepares + let move_values = kvs + .iter() + .map(|(key, value)| MoveValue { + key: key.clone(), + dest_key: to_tombstone(key), + value: value.clone(), + }) + .collect::>(); + let (keys, dest_keys): (Vec<_>, Vec<_>) = move_values + .clone() + .into_iter() + .map(|kv| (kv.key, kv.dest_key)) + .unzip(); + tombstone_manager + .move_values(keys, dest_keys) + .await + .unwrap(); + check_moved_values(kv_backend.clone(), &move_values).await; + + // Overwrites existing dest keys. + let kvs = HashMap::from([ + (b"bar".to_vec(), b"new baz".to_vec()), + (b"foo".to_vec(), b"new hi".to_vec()), + (b"baz".to_vec(), b"new baz".to_vec()), + ]); + for (key, value) in &kvs { + kv_backend + .put( + PutRequest::new() + .with_key(key.clone()) + .with_value(value.clone()), + ) + .await + .unwrap(); + } + let move_values = kvs + .iter() + .map(|(key, value)| MoveValue { + key: key.clone(), + dest_key: to_tombstone(key), + value: value.clone(), + }) + .collect::>(); + let (keys, dest_keys): (Vec<_>, Vec<_>) = move_values + .clone() + .into_iter() + .map(|kv| (kv.key, kv.dest_key)) + .unzip(); + tombstone_manager + .move_values(keys, dest_keys) + .await + .unwrap(); + check_moved_values(kv_backend.clone(), &move_values).await; + } } diff --git a/src/common/meta/src/key/txn_helper.rs b/src/common/meta/src/key/txn_helper.rs index b06c8acee9..f4571ba528 100644 --- a/src/common/meta/src/key/txn_helper.rs +++ b/src/common/meta/src/key/txn_helper.rs @@ -21,21 +21,9 @@ use crate::kv_backend::txn::{Compare, CompareOp, Txn, TxnOp, TxnOpResponse}; use crate::rpc::KeyValue; /// The response set of [TxnOpResponse::ResponseGet] -pub(crate) struct TxnOpGetResponseSet(Vec); +pub struct TxnOpGetResponseSet(Vec); impl TxnOpGetResponseSet { - /// Returns a [TxnOp] to retrieve the value corresponding `key` and - /// a filter to consume corresponding [KeyValue] from [TxnOpGetResponseSet]. - pub(crate) fn build_get_op>>( - key: T, - ) -> ( - TxnOp, - impl FnMut(&'_ mut TxnOpGetResponseSet) -> Option>, - ) { - let key = key.into(); - (TxnOp::Get(key.clone()), TxnOpGetResponseSet::filter(key)) - } - /// Returns a filter to consume a [KeyValue] where the key equals `key`. pub(crate) fn filter(key: Vec) -> impl FnMut(&mut TxnOpGetResponseSet) -> Option> { move |set| { @@ -80,30 +68,6 @@ impl From<&mut Vec> for TxnOpGetResponseSet { } } -// TODO(weny): using `TxnOpGetResponseSet`. -pub(crate) fn build_txn_response_decoder_fn( - raw_key: Vec, -) -> impl FnOnce(&Vec) -> Result>> -where - T: Serialize + DeserializeOwned + TableMetaValue, -{ - move |txn_res: &Vec| { - txn_res - .iter() - .filter_map(|resp| { - if let TxnOpResponse::ResponseGet(r) = resp { - Some(r) - } else { - None - } - }) - .flat_map(|r| &r.kvs) - .find(|kv| kv.key == raw_key) - .map(|kv| DeserializedValueWithBytes::from_inner_slice(&kv.value)) - .transpose() - } -} - pub(crate) fn build_put_if_absent_txn(key: Vec, value: Vec) -> Txn { Txn::new() .when(vec![Compare::with_not_exist_value( diff --git a/src/common/meta/src/rpc/store.rs b/src/common/meta/src/rpc/store.rs index f2b43a9c3a..f763d6b443 100644 --- a/src/common/meta/src/rpc/store.rs +++ b/src/common/meta/src/rpc/store.rs @@ -97,7 +97,6 @@ impl Display for RangeRequest { } impl RangeRequest { - #[inline] pub fn new() -> Self { Self { key: vec![], @@ -114,7 +113,6 @@ impl RangeRequest { /// key is the first key for the range, If range_end is not given, the /// request only looks up key. - #[inline] pub fn with_key(mut self, key: impl Into>) -> Self { self.key = key.into(); self @@ -129,7 +127,6 @@ impl RangeRequest { /// then the range request gets all keys prefixed with key. /// If both key and range_end are '\0', then the range request returns all /// keys. - #[inline] pub fn with_range(mut self, key: impl Into>, range_end: impl Into>) -> Self { self.key = key.into(); self.range_end = range_end.into(); @@ -138,7 +135,6 @@ impl RangeRequest { /// Gets all keys prefixed with key. /// range_end is the key plus one (e.g., "aa"+1 == "ab", "a\xff"+1 == "b"), - #[inline] pub fn with_prefix(mut self, key: impl Into>) -> Self { self.key = key.into(); self.range_end = util::get_prefix_end_key(&self.key); @@ -147,14 +143,12 @@ impl RangeRequest { /// limit is a limit on the number of keys returned for the request. When /// limit is set to 0, it is treated as no limit. - #[inline] pub fn with_limit(mut self, limit: i64) -> Self { self.limit = limit; self } /// keys_only when set returns only the keys and not the values. - #[inline] pub fn with_keys_only(mut self) -> Self { self.keys_only = true; self @@ -204,7 +198,6 @@ impl RangeResponse { } } - #[inline] pub fn take_kvs(&mut self) -> Vec { self.kvs.drain(..).collect() } @@ -244,7 +237,6 @@ impl From for PutRequest { } impl PutRequest { - #[inline] pub fn new() -> Self { Self { key: vec![], @@ -254,7 +246,6 @@ impl PutRequest { } /// key is the key, in bytes, to put into the key-value store. - #[inline] pub fn with_key(mut self, key: impl Into>) -> Self { self.key = key.into(); self @@ -262,7 +253,6 @@ impl PutRequest { /// value is the value, in bytes, to associate with the key in the /// key-value store. - #[inline] pub fn with_value(mut self, value: impl Into>) -> Self { self.value = value.into(); self @@ -270,7 +260,6 @@ impl PutRequest { /// If prev_kv is set, gets the previous key-value pair before changing it. /// The previous key-value pair will be returned in the put response. - #[inline] pub fn with_prev_kv(mut self) -> Self { self.prev_kv = true; self @@ -330,18 +319,15 @@ impl Default for BatchGetRequest { } impl BatchGetRequest { - #[inline] pub fn new() -> Self { Self { keys: vec![] } } - #[inline] pub fn with_keys(mut self, keys: Vec>) -> Self { self.keys = keys; self } - #[inline] pub fn add_key(mut self, key: impl Into>) -> Self { self.keys.push(key.into()); self @@ -416,7 +402,6 @@ impl From for BatchPutRequest { } impl BatchPutRequest { - #[inline] pub fn new() -> Self { Self { kvs: vec![], @@ -424,7 +409,6 @@ impl BatchPutRequest { } } - #[inline] pub fn add_kv(mut self, key: impl Into>, value: impl Into>) -> Self { self.kvs.push(KeyValue { key: key.into(), @@ -435,7 +419,6 @@ impl BatchPutRequest { /// If prev_kv is set, gets the previous key-value pair before changing it. /// The previous key-value pair will be returned in the put response. - #[inline] pub fn with_prev_kv(mut self) -> Self { self.prev_kv = true; self @@ -467,7 +450,6 @@ impl BatchPutResponse { } } - #[inline] pub fn take_prev_kvs(&mut self) -> Vec { self.prev_kvs.drain(..).collect() } @@ -501,7 +483,6 @@ impl From for BatchDeleteRequest { } impl BatchDeleteRequest { - #[inline] pub fn new() -> Self { Self { keys: vec![], @@ -509,7 +490,12 @@ impl BatchDeleteRequest { } } - #[inline] + /// Sets `keys`. + pub fn with_keys(mut self, keys: Vec>) -> Self { + self.keys = keys; + self + } + pub fn add_key(mut self, key: impl Into>) -> Self { self.keys.push(key.into()); self @@ -517,7 +503,6 @@ impl BatchDeleteRequest { /// If prev_kv is set, gets the previous key-value pair before deleting it. /// The previous key-value pair will be returned in the batch delete response. - #[inline] pub fn with_prev_kv(mut self) -> Self { self.prev_kv = true; self @@ -582,7 +567,6 @@ impl From for CompareAndPutRequest { } impl CompareAndPutRequest { - #[inline] pub fn new() -> Self { Self { key: vec![], @@ -592,14 +576,12 @@ impl CompareAndPutRequest { } /// key is the key, in bytes, to put into the key-value store. - #[inline] pub fn with_key(mut self, key: impl Into>) -> Self { self.key = key.into(); self } /// expect is the previous value, in bytes - #[inline] pub fn with_expect(mut self, expect: impl Into>) -> Self { self.expect = expect.into(); self @@ -607,7 +589,6 @@ impl CompareAndPutRequest { /// value is the value, in bytes, to associate with the key in the /// key-value store. - #[inline] pub fn with_value(mut self, value: impl Into>) -> Self { self.value = value.into(); self @@ -649,12 +630,10 @@ impl CompareAndPutResponse { } } - #[inline] pub fn is_success(&self) -> bool { self.success } - #[inline] pub fn take_prev_kv(&mut self) -> Option { self.prev_kv.take() } @@ -703,7 +682,6 @@ impl From for DeleteRangeRequest { } impl DeleteRangeRequest { - #[inline] pub fn new() -> Self { Self { key: vec![], @@ -719,7 +697,6 @@ impl DeleteRangeRequest { /// key is the first key to delete in the range. If range_end is not given, /// the range is defined to contain only the key argument. - #[inline] pub fn with_key(mut self, key: impl Into>) -> Self { self.key = key.into(); self @@ -735,7 +712,6 @@ impl DeleteRangeRequest { /// the keys with the prefix (the given key). /// If range_end is '\0', the range is all keys greater than or equal to the /// key argument. - #[inline] pub fn with_range(mut self, key: impl Into>, range_end: impl Into>) -> Self { self.key = key.into(); self.range_end = range_end.into(); @@ -744,7 +720,6 @@ impl DeleteRangeRequest { /// Deletes all keys prefixed with key. /// range_end is one bit larger than the given key. - #[inline] pub fn with_prefix(mut self, key: impl Into>) -> Self { self.key = key.into(); self.range_end = util::get_prefix_end_key(&self.key); @@ -753,7 +728,6 @@ impl DeleteRangeRequest { /// If prev_kv is set, gets the previous key-value pairs before deleting it. /// The previous key-value pairs will be returned in the delete response. - #[inline] pub fn with_prev_kv(mut self) -> Self { self.prev_kv = true; self @@ -788,12 +762,10 @@ impl DeleteRangeResponse { } } - #[inline] pub fn deleted(&self) -> i64 { self.deleted } - #[inline] pub fn take_prev_kvs(&mut self) -> Vec { self.prev_kvs.drain(..).collect() } diff --git a/src/common/meta/src/sequence.rs b/src/common/meta/src/sequence.rs index 9e2469332a..b1acc961d4 100644 --- a/src/common/meta/src/sequence.rs +++ b/src/common/meta/src/sequence.rs @@ -172,9 +172,7 @@ impl Inner { if !res.success { if let Some(kv) = res.prev_kv { - expect = kv.value.clone(); - - let v: [u8; 8] = match kv.value.try_into() { + let v: [u8; 8] = match kv.value.clone().try_into() { Ok(a) => a, Err(v) => { return error::UnexpectedSequenceValueSnafu { @@ -184,13 +182,12 @@ impl Inner { } }; let v = u64::from_le_bytes(v); - // If the existed value is smaller than the initial, we should start from the initial. start = v.max(self.initial); + expect = kv.value; } else { - expect = vec![]; - start = self.initial; + expect = vec![]; } continue; } diff --git a/src/common/meta/src/state_store.rs b/src/common/meta/src/state_store.rs index eb3a91de17..1cf1ea8649 100644 --- a/src/common/meta/src/state_store.rs +++ b/src/common/meta/src/state_store.rs @@ -22,6 +22,7 @@ use common_procedure::store::util::multiple_value_stream; use common_procedure::Result as ProcedureResult; use futures::future::try_join_all; use futures::StreamExt; +use itertools::Itertools; use snafu::ResultExt; use crate::error::Result; @@ -79,17 +80,21 @@ fn decode_kv(kv: KeyValue) -> Result<(String, Vec)> { Ok((key, value)) } -enum SplitValue<'a> { - Single(&'a [u8]), - Multiple(Vec<&'a [u8]>), +enum SplitValue { + Single(Vec), + Multiple(Vec>), } -fn split_value(value: &[u8], max_value_size: Option) -> SplitValue<'_> { +fn split_value(value: Vec, max_value_size: Option) -> SplitValue { if let Some(max_value_size) = max_value_size { if value.len() <= max_value_size { SplitValue::Single(value) } else { - SplitValue::Multiple(value.chunks(max_value_size).collect::>()) + let mut values = vec![]; + for chunk in value.into_iter().chunks(max_value_size).into_iter() { + values.push(chunk.collect()); + } + SplitValue::Multiple(values) } } else { SplitValue::Single(value) @@ -99,10 +104,10 @@ fn split_value(value: &[u8], max_value_size: Option) -> SplitValue<'_> { #[async_trait] impl StateStore for KvStateStore { async fn put(&self, key: &str, value: Vec) -> ProcedureResult<()> { - let split = split_value(&value, self.max_value_size); + let split = split_value(value, self.max_value_size); let key = with_prefix(key); match split { - SplitValue::Single(_) => { + SplitValue::Single(value) => { self.kv_backend .put( PutRequest::new() diff --git a/src/common/meta/src/test_util.rs b/src/common/meta/src/test_util.rs index f1d2f33f11..4ed396509a 100644 --- a/src/common/meta/src/test_util.rs +++ b/src/common/meta/src/test_util.rs @@ -14,14 +14,13 @@ use std::sync::Arc; +use api::region::RegionResponse; use api::v1::region::{QueryRequest, RegionRequest}; pub use common_base::AffectedRows; use common_recordbatch::SendableRecordBatchStream; use crate::cache_invalidator::DummyCacheInvalidator; -use crate::datanode_manager::{ - Datanode, DatanodeManager, DatanodeManagerRef, DatanodeRef, HandleResponse, -}; +use crate::datanode_manager::{Datanode, DatanodeManager, DatanodeManagerRef, DatanodeRef}; use crate::ddl::table_meta::TableMetadataAllocator; use crate::ddl::DdlContext; use crate::error::Result; @@ -35,7 +34,7 @@ use crate::wal_options_allocator::WalOptionsAllocator; #[async_trait::async_trait] pub trait MockDatanodeHandler: Sync + Send + Clone { - async fn handle(&self, peer: &Peer, request: RegionRequest) -> Result; + async fn handle(&self, peer: &Peer, request: RegionRequest) -> Result; async fn handle_query( &self, @@ -65,7 +64,7 @@ struct MockDatanode { #[async_trait::async_trait] impl Datanode for MockDatanode { - async fn handle(&self, request: RegionRequest) -> Result { + async fn handle(&self, request: RegionRequest) -> Result { self.handler.handle(&self.peer, request).await } diff --git a/src/common/meta/src/wal_options_allocator.rs b/src/common/meta/src/wal_options_allocator.rs index 202b2958ba..ac55127894 100644 --- a/src/common/meta/src/wal_options_allocator.rs +++ b/src/common/meta/src/wal_options_allocator.rs @@ -163,7 +163,7 @@ mod tests { let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef; let mut topic_manager = KafkaTopicManager::new(config.clone(), kv_backend); // Replaces the default topic pool with the constructed topics. - topic_manager.topic_pool = topics.clone(); + topic_manager.topic_pool.clone_from(&topics); // Replaces the default selector with a round-robin selector without shuffled. topic_manager.topic_selector = Arc::new(RoundRobinTopicSelector::default()); diff --git a/src/common/meta/src/wal_options_allocator/kafka/topic_manager.rs b/src/common/meta/src/wal_options_allocator/kafka/topic_manager.rs index ea2f89554b..ab64a6fa0f 100644 --- a/src/common/meta/src/wal_options_allocator/kafka/topic_manager.rs +++ b/src/common/meta/src/wal_options_allocator/kafka/topic_manager.rs @@ -291,7 +291,7 @@ mod tests { let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef; let mut manager = TopicManager::new(config.clone(), kv_backend); // Replaces the default topic pool with the constructed topics. - manager.topic_pool = topics.clone(); + manager.topic_pool.clone_from(&topics); // Replaces the default selector with a round-robin selector without shuffled. manager.topic_selector = Arc::new(RoundRobinTopicSelector::default()); manager.start().await.unwrap(); diff --git a/src/common/procedure/src/local/rwlock.rs b/src/common/procedure/src/local/rwlock.rs index a170132036..538914e395 100644 --- a/src/common/procedure/src/local/rwlock.rs +++ b/src/common/procedure/src/local/rwlock.rs @@ -19,19 +19,19 @@ use std::sync::{Arc, Mutex}; use tokio::sync::{OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock}; pub enum OwnedKeyRwLockGuard { - Read(OwnedRwLockReadGuard<()>), - Write(OwnedRwLockWriteGuard<()>), + Read { _guard: OwnedRwLockReadGuard<()> }, + Write { _guard: OwnedRwLockWriteGuard<()> }, } impl From> for OwnedKeyRwLockGuard { fn from(guard: OwnedRwLockReadGuard<()>) -> Self { - OwnedKeyRwLockGuard::Read(guard) + OwnedKeyRwLockGuard::Read { _guard: guard } } } impl From> for OwnedKeyRwLockGuard { fn from(guard: OwnedRwLockWriteGuard<()>) -> Self { - OwnedKeyRwLockGuard::Write(guard) + OwnedKeyRwLockGuard::Write { _guard: guard } } } diff --git a/src/common/query/src/columnar_value.rs b/src/common/query/src/columnar_value.rs index 4e79d7819f..419bc5ed3a 100644 --- a/src/common/query/src/columnar_value.rs +++ b/src/common/query/src/columnar_value.rs @@ -17,7 +17,7 @@ use datatypes::prelude::ConcreteDataType; use datatypes::vectors::{Helper, VectorRef}; use snafu::ResultExt; -use crate::error::{self, IntoVectorSnafu, Result}; +use crate::error::{self, GeneralDataFusionSnafu, IntoVectorSnafu, Result}; use crate::prelude::ScalarValue; /// Represents the result from an expression @@ -43,7 +43,9 @@ impl ColumnarValue { Ok(match self { ColumnarValue::Vector(v) => v, ColumnarValue::Scalar(s) => { - let v = s.to_array_of_size(num_rows); + let v = s + .to_array_of_size(num_rows) + .context(GeneralDataFusionSnafu)?; let data_type = v.data_type().clone(); Helper::try_into_vector(v).context(IntoVectorSnafu { data_type })? } diff --git a/src/common/query/src/lib.rs b/src/common/query/src/lib.rs index cb39d37fb4..ca81ad9e41 100644 --- a/src/common/query/src/lib.rs +++ b/src/common/query/src/lib.rs @@ -94,8 +94,8 @@ impl Debug for OutputData { OutputData::RecordBatches(recordbatches) => { write!(f, "OutputData::RecordBatches({recordbatches:?})") } - OutputData::Stream(_) => { - write!(f, "OutputData::Stream()") + OutputData::Stream(s) => { + write!(f, "OutputData::Stream(<{}>)", s.name()) } } } diff --git a/src/common/query/src/logical_plan.rs b/src/common/query/src/logical_plan.rs index ac20c74b5b..ab5dedc14f 100644 --- a/src/common/query/src/logical_plan.rs +++ b/src/common/query/src/logical_plan.rs @@ -72,6 +72,7 @@ pub fn create_aggregate_function( mod tests { use std::sync::Arc; + use datafusion_common::DFSchema; use datafusion_expr::{ ColumnarValue as DfColumnarValue, ScalarUDF as DfScalarUDF, TypeSignature as DfTypeSignature, @@ -135,15 +136,17 @@ mod tests { // test into_df_udf let df_udf: DfScalarUDF = udf.into(); - assert_eq!("and", df_udf.name); + assert_eq!("and", df_udf.name()); let types = vec![DataType::Boolean, DataType::Boolean]; assert!( - matches!(&df_udf.signature.type_signature, DfTypeSignature::Exact(ts) if ts.clone() == types) + matches!(&df_udf.signature().type_signature, DfTypeSignature::Exact(ts) if ts.clone() == types) ); assert_eq!( - Arc::new(DataType::Boolean), - (df_udf.return_type)(&[]).unwrap() + DataType::Boolean, + df_udf + .return_type_from_exprs(&[], &DFSchema::empty(), &[]) + .unwrap() ); let args = vec![ @@ -152,7 +155,7 @@ mod tests { ]; // call the function - let result = (df_udf.fun)(&args).unwrap(); + let result = (df_udf.fun())(&args).unwrap(); match result { DfColumnarValue::Array(arr) => { diff --git a/src/common/query/src/logical_plan/accumulator.rs b/src/common/query/src/logical_plan/accumulator.rs index f0c272ada1..32f1b4587c 100644 --- a/src/common/query/src/logical_plan/accumulator.rs +++ b/src/common/query/src/logical_plan/accumulator.rs @@ -126,7 +126,7 @@ impl DfAccumulatorAdaptor { } impl DfAccumulator for DfAccumulatorAdaptor { - fn state(&self) -> DfResult> { + fn state(&mut self) -> DfResult> { let state_values = self.accumulator.state()?; let state_types = self.creator.state_types()?; if state_values.len() != state_types.len() { @@ -161,7 +161,7 @@ impl DfAccumulator for DfAccumulatorAdaptor { Ok(()) } - fn evaluate(&self) -> DfResult { + fn evaluate(&mut self) -> DfResult { let value = self.accumulator.evaluate()?; let output_type = self.creator.output_type()?; let scalar_value = value diff --git a/src/common/query/src/logical_plan/expr.rs b/src/common/query/src/logical_plan/expr.rs index e5abddc4d2..c63de87f61 100644 --- a/src/common/query/src/logical_plan/expr.rs +++ b/src/common/query/src/logical_plan/expr.rs @@ -44,9 +44,7 @@ pub fn build_filter_from_timestamp( ts_col_name: &str, time_range: Option<&TimestampRange>, ) -> Option { - let Some(time_range) = time_range else { - return None; - }; + let time_range = time_range?; let ts_col_expr = DfExpr::Column(Column { relation: None, name: ts_col_name.to_string(), @@ -94,10 +92,10 @@ mod tests { #[test] fn test_from_df_expr() { - let df_expr = DfExpr::Wildcard; + let df_expr = DfExpr::Wildcard { qualifier: None }; let expr: Expr = df_expr.into(); - assert_eq!(DfExpr::Wildcard, *expr.df_expr()); + assert_eq!(DfExpr::Wildcard { qualifier: None }, *expr.df_expr()); } } diff --git a/src/common/query/src/logical_plan/udaf.rs b/src/common/query/src/logical_plan/udaf.rs index b96de5b888..b9a11cfbfb 100644 --- a/src/common/query/src/logical_plan/udaf.rs +++ b/src/common/query/src/logical_plan/udaf.rs @@ -16,15 +16,18 @@ //! //! Modified from DataFusion. +use std::any::Any; use std::fmt::{self, Debug, Formatter}; use std::sync::Arc; +use datafusion::arrow::datatypes::Field; +use datafusion_common::Result; +use datafusion_expr::function::AccumulatorArgs; use datafusion_expr::{ - AccumulatorFactoryFunction, AggregateUDF as DfAggregateUdf, - StateTypeFunction as DfStateTypeFunction, + Accumulator, AccumulatorFactoryFunction, AggregateUDF as DfAggregateUdf, AggregateUDFImpl, }; use datatypes::arrow::datatypes::DataType as ArrowDataType; -use datatypes::prelude::*; +use datatypes::data_type::DataType; use crate::function::{ to_df_return_type, AccumulatorFunctionImpl, ReturnTypeFunction, StateTypeFunction, @@ -90,13 +93,72 @@ impl AggregateFunction { impl From for DfAggregateUdf { fn from(udaf: AggregateFunction) -> Self { - DfAggregateUdf::new( - &udaf.name, - &udaf.signature.into(), - &to_df_return_type(udaf.return_type), - &to_df_accumulator_func(udaf.accumulator, udaf.creator.clone()), - &to_df_state_type(udaf.state_type), - ) + struct DfUdafAdapter { + name: String, + signature: datafusion_expr::Signature, + return_type_func: datafusion_expr::ReturnTypeFunction, + accumulator: AccumulatorFactoryFunction, + creator: AggregateFunctionCreatorRef, + } + + impl Debug for DfUdafAdapter { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("DfUdafAdapter") + .field("name", &self.name) + .field("signature", &self.signature) + .finish() + } + } + + impl AggregateUDFImpl for DfUdafAdapter { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + &self.name + } + + fn signature(&self) -> &datafusion_expr::Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[ArrowDataType]) -> Result { + (self.return_type_func)(arg_types).map(|x| x.as_ref().clone()) + } + + fn accumulator(&self, acc_args: AccumulatorArgs) -> Result> { + (self.accumulator)(acc_args) + } + + fn state_fields( + &self, + name: &str, + _value_type: ArrowDataType, + _ordering_fields: Vec, + ) -> Result> { + self.creator + .state_types() + .map(|x| { + (0..x.len()) + .zip(x) + .map(|(i, t)| { + Field::new(format!("{}_{}", name, i), t.as_arrow_type(), true) + }) + .collect::>() + }) + .map_err(|e| e.into()) + } + } + + DfUdafAdapter { + name: udaf.name, + signature: udaf.signature.into(), + return_type_func: to_df_return_type(udaf.return_type), + accumulator: to_df_accumulator_func(udaf.accumulator, udaf.creator.clone()), + creator: udaf.creator, + } + .into() } } @@ -110,19 +172,3 @@ fn to_df_accumulator_func( Ok(Box::new(DfAccumulatorAdaptor::new(accumulator, creator)) as _) }) } - -fn to_df_state_type(func: StateTypeFunction) -> DfStateTypeFunction { - let df_func = move |data_type: &ArrowDataType| { - // DataFusion DataType -> ConcreteDataType - let concrete_data_type = ConcreteDataType::from_arrow_type(data_type); - - // evaluate ConcreteDataType - let eval_result = (func)(&concrete_data_type); - - // ConcreteDataType -> DataFusion DataType - eval_result - .map(|ts| Arc::new(ts.iter().map(|t| t.as_arrow_type()).collect())) - .map_err(|e| e.into()) - }; - Arc::new(df_func) -} diff --git a/src/common/query/src/logical_plan/udf.rs b/src/common/query/src/logical_plan/udf.rs index 31d3561745..df5cec762c 100644 --- a/src/common/query/src/logical_plan/udf.rs +++ b/src/common/query/src/logical_plan/udf.rs @@ -70,6 +70,8 @@ impl ScalarUdf { impl From for DfScalarUDF { fn from(udf: ScalarUdf) -> Self { + // TODO(LFC): remove deprecated + #[allow(deprecated)] DfScalarUDF::new( &udf.name, &udf.signature.into(), diff --git a/src/common/query/src/physical_plan.rs b/src/common/query/src/physical_plan.rs index ddfb1bfb6c..919d104450 100644 --- a/src/common/query/src/physical_plan.rs +++ b/src/common/query/src/physical_plan.rs @@ -21,10 +21,9 @@ use common_recordbatch::{DfSendableRecordBatchStream, SendableRecordBatchStream} use datafusion::arrow::datatypes::SchemaRef as DfSchemaRef; use datafusion::error::Result as DfResult; pub use datafusion::execution::context::{SessionContext, TaskContext}; -use datafusion::physical_plan::expressions::PhysicalSortExpr; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; pub use datafusion::physical_plan::Partitioning; -use datafusion::physical_plan::{DisplayAs, DisplayFormatType, Statistics}; +use datafusion::physical_plan::{DisplayAs, DisplayFormatType, PlanProperties}; use datatypes::schema::SchemaRef; use snafu::ResultExt; @@ -47,13 +46,9 @@ pub trait PhysicalPlan: Debug + Send + Sync { /// Get the schema for this physical plan fn schema(&self) -> SchemaRef; - /// Specifies the output partitioning scheme of this plan - fn output_partitioning(&self) -> Partitioning; - - /// returns `Some(keys)` that describes how the output was sorted. - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { - None - } + /// Return properties of the output of the [PhysicalPlan], such as output + /// ordering(s), partitioning information etc. + fn properties(&self) -> &PlanProperties; /// Get a list of child physical plans that provide the input for this plan. The returned list /// will be empty for leaf nodes, will contain a single value for unary nodes, or two @@ -107,8 +102,8 @@ impl PhysicalPlan for PhysicalPlanAdapter { self.schema.clone() } - fn output_partitioning(&self) -> Partitioning { - self.df_plan.output_partitioning() + fn properties(&self) -> &PlanProperties { + self.df_plan.properties() } fn children(&self) -> Vec { @@ -170,14 +165,6 @@ impl DfPhysicalPlan for DfPhysicalPlanAdapter { self.0.schema().arrow_schema().clone() } - fn output_partitioning(&self) -> Partitioning { - self.0.output_partitioning() - } - - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { - self.0.output_ordering() - } - fn children(&self) -> Vec> { self.0 .children() @@ -213,13 +200,13 @@ impl DfPhysicalPlan for DfPhysicalPlanAdapter { Ok(Box::pin(DfRecordBatchStreamAdapter::new(stream))) } - fn statistics(&self) -> Statistics { - Statistics::default() - } - fn metrics(&self) -> Option { self.0.metrics() } + + fn properties(&self) -> &PlanProperties { + self.0.properties() + } } impl DisplayAs for DfPhysicalPlanAdapter { @@ -232,10 +219,12 @@ impl DisplayAs for DfPhysicalPlanAdapter { mod test { use async_trait::async_trait; use common_recordbatch::{RecordBatch, RecordBatches}; + use datafusion::arrow::datatypes::SchemaRef as DfSchemaRef; use datafusion::datasource::{DefaultTableSource, TableProvider as DfTableProvider, TableType}; use datafusion::execution::context::{SessionContext, SessionState}; - use datafusion::physical_plan::collect; + use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::empty::EmptyExec; + use datafusion::physical_plan::{collect, ExecutionMode}; use datafusion_expr::logical_plan::builder::LogicalPlanBuilder; use datafusion_expr::{Expr, TableSource}; use datatypes::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; @@ -272,10 +261,13 @@ mod test { _filters: &[Expr], _limit: Option, ) -> DfResult> { - let schema = Schema::try_from(self.schema()).unwrap(); - let my_plan = Arc::new(MyExecutionPlan { - schema: Arc::new(schema), - }); + let schema = Arc::new(Schema::try_from(self.schema()).unwrap()); + let properties = PlanProperties::new( + EquivalenceProperties::new(schema.arrow_schema().clone()), + Partitioning::UnknownPartitioning(1), + ExecutionMode::Bounded, + ); + let my_plan = Arc::new(MyExecutionPlan { schema, properties }); let df_plan = DfPhysicalPlanAdapter(my_plan); Ok(Arc::new(df_plan)) } @@ -289,9 +281,10 @@ mod test { } } - #[derive(Debug)] + #[derive(Debug, Clone)] struct MyExecutionPlan { schema: SchemaRef, + properties: PlanProperties, } impl PhysicalPlan for MyExecutionPlan { @@ -303,8 +296,8 @@ mod test { self.schema.clone() } - fn output_partitioning(&self) -> Partitioning { - Partitioning::UnknownPartitioning(1) + fn properties(&self) -> &PlanProperties { + &self.properties } fn children(&self) -> Vec { @@ -312,7 +305,7 @@ mod test { } fn with_new_children(&self, _children: Vec) -> Result { - unimplemented!() + Ok(Arc::new(self.clone())) } fn execute( @@ -381,7 +374,7 @@ mod test { let plan = PhysicalPlanAdapter::new( Arc::new(Schema::try_from(df_schema.clone()).unwrap()), - Arc::new(EmptyExec::new(true, df_schema.clone())), + Arc::new(EmptyExec::new(df_schema.clone())), ); let _ = plan.df_plan.as_any().downcast_ref::().unwrap(); diff --git a/src/common/query/src/signature.rs b/src/common/query/src/signature.rs index 9e92a10e17..a234990bf6 100644 --- a/src/common/query/src/signature.rs +++ b/src/common/query/src/signature.rs @@ -31,6 +31,8 @@ pub enum TypeSignature { // A function such as `array` is `VariadicEqual` // The first argument decides the type used for coercion VariadicEqual, + /// One or more arguments with arbitrary types + VariadicAny, /// fixed number of arguments of an arbitrary but equal type out of a list of valid types // A function of one argument of f64 is `Uniform(1, vec![ConcreteDataType::Float64])` // A function of one argument of f64 or f32 is `Uniform(1, vec![ConcreteDataType::Float32, ConcreteDataType::Float64])` @@ -79,6 +81,15 @@ impl Signature { volatility, } } + + /// variadic_any - Creates a variadic signature that represents an arbitrary number of arguments of any type. + pub fn variadic_any(volatility: Volatility) -> Self { + Self { + type_signature: TypeSignature::VariadicAny, + volatility, + } + } + /// uniform - Creates a function with a fixed number of arguments of the same type, which must be from valid_types. pub fn uniform( arg_count: usize, @@ -131,6 +142,7 @@ impl From for DfTypeSignature { TypeSignature::OneOf(ts) => { DfTypeSignature::OneOf(ts.into_iter().map(Into::into).collect()) } + TypeSignature::VariadicAny => DfTypeSignature::VariadicAny, } } } diff --git a/src/common/recordbatch/Cargo.toml b/src/common/recordbatch/Cargo.toml index 12494cdcf5..d82b445c8e 100644 --- a/src/common/recordbatch/Cargo.toml +++ b/src/common/recordbatch/Cargo.toml @@ -11,6 +11,7 @@ workspace = true arc-swap = "1.6" common-error.workspace = true common-macro.workspace = true +common-telemetry.workspace = true datafusion.workspace = true datafusion-common.workspace = true datatypes.workspace = true diff --git a/src/common/recordbatch/src/adapter.rs b/src/common/recordbatch/src/adapter.rs index 12f5ecfdc9..5f5cc45abb 100644 --- a/src/common/recordbatch/src/adapter.rs +++ b/src/common/recordbatch/src/adapter.rs @@ -103,7 +103,7 @@ where "Trying to cast a RecordBatch into an incompatible schema. RecordBatch: {}, Target: {}", projected_column.schema(), projected_schema, - )))); + )), None)); } let mut columns = Vec::with_capacity(projected_schema.fields.len()); @@ -218,6 +218,10 @@ impl RecordBatchStreamAdapter { } impl RecordBatchStream for RecordBatchStreamAdapter { + fn name(&self) -> &str { + "RecordBatchStreamAdapter" + } + fn schema(&self) -> SchemaRef { self.schema.clone() } diff --git a/src/common/recordbatch/src/error.rs b/src/common/recordbatch/src/error.rs index 42a2754bb2..b22c0f488d 100644 --- a/src/common/recordbatch/src/error.rs +++ b/src/common/recordbatch/src/error.rs @@ -18,6 +18,7 @@ use std::any::Any; use common_error::ext::{BoxedError, ErrorExt}; use common_error::status_code::StatusCode; use common_macro::stack_trace_debug; +use datafusion_common::ScalarValue; use datatypes::prelude::ConcreteDataType; use snafu::{Location, Snafu}; @@ -69,8 +70,9 @@ pub enum Error { location: Location, }, - #[snafu(display("Failed to init Recordbatch stream"))] - InitRecordbatchStream { + #[snafu(display("Failed to convert {v:?} to Arrow scalar"))] + ToArrowScalar { + v: ScalarValue, #[snafu(source)] error: datafusion_common::DataFusionError, location: Location, @@ -128,7 +130,7 @@ impl ErrorExt for Error { | Error::CreateRecordBatches { .. } | Error::PollStream { .. } | Error::Format { .. } - | Error::InitRecordbatchStream { .. } + | Error::ToArrowScalar { .. } | Error::ColumnNotExists { .. } | Error::ProjectArrowRecordBatch { .. } | Error::ArrowCompute { .. } => StatusCode::Internal, diff --git a/src/common/recordbatch/src/filter.rs b/src/common/recordbatch/src/filter.rs index 3175ace37e..7a5e361138 100644 --- a/src/common/recordbatch/src/filter.rs +++ b/src/common/recordbatch/src/filter.rs @@ -22,7 +22,7 @@ use datafusion_common::ScalarValue; use datatypes::vectors::VectorRef; use snafu::ResultExt; -use crate::error::{ArrowComputeSnafu, Result, UnsupportedOperationSnafu}; +use crate::error::{ArrowComputeSnafu, Result, ToArrowScalarSnafu, UnsupportedOperationSnafu}; /// An inplace expr evaluator for simple filter. Only support /// - `col` `op` `literal` @@ -69,9 +69,10 @@ impl SimpleFilterEvaluator { _ => return None, }; + let literal = rhs.to_scalar().ok()?; Some(Self { column_name: lhs.name.clone(), - literal: rhs.clone().to_scalar(), + literal, op, }) } @@ -85,7 +86,10 @@ impl SimpleFilterEvaluator { } pub fn evaluate_scalar(&self, input: &ScalarValue) -> Result { - let result = self.evaluate_datum(&input.to_scalar())?; + let input = input + .to_scalar() + .with_context(|_| ToArrowScalarSnafu { v: input.clone() })?; + let result = self.evaluate_datum(&input)?; Ok(result.value(0)) } diff --git a/src/common/recordbatch/src/lib.rs b/src/common/recordbatch/src/lib.rs index f062a6474f..ff3135d9e4 100644 --- a/src/common/recordbatch/src/lib.rs +++ b/src/common/recordbatch/src/lib.rs @@ -37,6 +37,10 @@ pub use recordbatch::RecordBatch; use snafu::{ensure, ResultExt}; pub trait RecordBatchStream: Stream> { + fn name(&self) -> &str { + "RecordBatchStream" + } + fn schema(&self) -> SchemaRef; fn output_ordering(&self) -> Option<&[OrderOption]>; @@ -243,6 +247,10 @@ impl RecordBatchStreamWrapper { impl> + Unpin> RecordBatchStream for RecordBatchStreamWrapper { + fn name(&self) -> &str { + "RecordBatchStreamWrapper" + } + fn schema(&self) -> SchemaRef { self.schema.clone() } diff --git a/src/common/substrait/Cargo.toml b/src/common/substrait/Cargo.toml index 9ac4fc150f..a2fb0e2725 100644 --- a/src/common/substrait/Cargo.toml +++ b/src/common/substrait/Cargo.toml @@ -12,7 +12,9 @@ async-trait.workspace = true bytes.workspace = true catalog.workspace = true common-error.workspace = true +common-function.workspace = true common-macro.workspace = true +common-telemetry.workspace = true datafusion.workspace = true datafusion-common.workspace = true datafusion-expr.workspace = true @@ -20,6 +22,7 @@ datafusion-substrait.workspace = true datatypes.workspace = true promql.workspace = true prost.workspace = true +session.workspace = true snafu.workspace = true [dependencies.substrait_proto] diff --git a/src/common/substrait/src/df_substrait.rs b/src/common/substrait/src/df_substrait.rs index c4e1db9a56..cfc2736829 100644 --- a/src/common/substrait/src/df_substrait.rs +++ b/src/common/substrait/src/df_substrait.rs @@ -16,18 +16,24 @@ use std::sync::Arc; use async_trait::async_trait; use bytes::{Buf, Bytes, BytesMut}; -use datafusion::catalog::CatalogList; +use common_function::function_registry::FUNCTION_REGISTRY; +use common_function::scalars::udf::create_udf; +use datafusion::catalog::CatalogProviderList; use datafusion::execution::context::SessionState; use datafusion::execution::runtime_env::RuntimeEnv; +use datafusion::execution::FunctionRegistry; use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion_expr::LogicalPlan; use datafusion_substrait::logical_plan::consumer::from_substrait_plan; use datafusion_substrait::logical_plan::producer::to_substrait_plan; +use datafusion_substrait::substrait::proto::Plan; use prost::Message; +use session::context::QueryContextRef; use snafu::ResultExt; -use substrait_proto::proto::Plan; -use crate::error::{DecodeDfPlanSnafu, DecodeRelSnafu, EncodeDfPlanSnafu, EncodeRelSnafu, Error}; +use crate::error::{ + DFInternalSnafu, DecodeDfPlanSnafu, DecodeRelSnafu, EncodeDfPlanSnafu, EncodeRelSnafu, Error, +}; use crate::extension_serializer::ExtensionSerializer; use crate::SubstraitPlan; @@ -42,17 +48,20 @@ impl SubstraitPlan for DFLogicalSubstraitConvertor { async fn decode( &self, message: B, - catalog_list: Arc, - catalog: &str, - schema: &str, + catalog_list: Arc, + mut state: SessionState, + query_ctx: QueryContextRef, ) -> Result { - let state_config = SessionConfig::new().with_default_catalog_and_schema(catalog, schema); - let state = SessionState::new_with_config_rt(state_config, Arc::new(RuntimeEnv::default())) - .with_serializer_registry(Arc::new(ExtensionSerializer)); + // substrait decoder will look up the UDFs in SessionState, so we need to register them + for func in FUNCTION_REGISTRY.functions() { + let udf = Arc::new(create_udf(func, query_ctx.clone(), Default::default()).into()); + state.register_udf(udf).context(DFInternalSnafu)?; + } + let mut context = SessionContext::new_with_state(state); context.register_catalog_list(catalog_list); let plan = Plan::decode(message).context(DecodeRelSnafu)?; - let df_plan = from_substrait_plan(&mut context, &plan) + let df_plan = from_substrait_plan(&context, &plan) .await .context(DecodeDfPlanSnafu)?; Ok(df_plan) diff --git a/src/common/substrait/src/extension_serializer.rs b/src/common/substrait/src/extension_serializer.rs index 813c525843..89944db508 100644 --- a/src/common/substrait/src/extension_serializer.rs +++ b/src/common/substrait/src/extension_serializer.rs @@ -19,7 +19,7 @@ use datafusion::execution::registry::SerializerRegistry; use datafusion_common::DataFusionError; use datafusion_expr::UserDefinedLogicalNode; use promql::extension_plan::{ - EmptyMetric, InstantManipulate, RangeManipulate, SeriesDivide, SeriesNormalize, + EmptyMetric, InstantManipulate, RangeManipulate, ScalarCalculate, SeriesDivide, SeriesNormalize, }; pub struct ExtensionSerializer; @@ -50,6 +50,13 @@ impl SerializerRegistry for ExtensionSerializer { .expect("Failed to downcast to RangeManipulate"); Ok(range_manipulate.serialize()) } + name if name == ScalarCalculate::name() => { + let scalar_calculate = node + .as_any() + .downcast_ref::() + .expect("Failed to downcast to ScalarCalculate"); + Ok(scalar_calculate.serialize()) + } name if name == SeriesDivide::name() => { let series_divide = node .as_any() @@ -92,6 +99,10 @@ impl SerializerRegistry for ExtensionSerializer { let series_divide = SeriesDivide::deserialize(bytes)?; Ok(Arc::new(series_divide)) } + name if name == ScalarCalculate::name() => { + let scalar_calculate = ScalarCalculate::deserialize(bytes)?; + Ok(Arc::new(scalar_calculate)) + } name if name == EmptyMetric::name() => Err(DataFusionError::Substrait( "EmptyMetric should not be deserialized".to_string(), )), diff --git a/src/common/substrait/src/lib.rs b/src/common/substrait/src/lib.rs index 51f8119dbb..ca7e28e8d1 100644 --- a/src/common/substrait/src/lib.rs +++ b/src/common/substrait/src/lib.rs @@ -21,7 +21,9 @@ use std::sync::Arc; use async_trait::async_trait; use bytes::{Buf, Bytes}; -use datafusion::catalog::CatalogList; +use datafusion::catalog::CatalogProviderList; +use datafusion::execution::context::SessionState; +use session::context::QueryContextRef; pub use substrait_proto; pub use crate::df_substrait::DFLogicalSubstraitConvertor; @@ -35,9 +37,9 @@ pub trait SubstraitPlan { async fn decode( &self, message: B, - catalog_list: Arc, - catalog: &str, - schema: &str, + catalog_list: Arc, + state: SessionState, + query_ctx: QueryContextRef, ) -> Result; fn encode(&self, plan: &Self::Plan) -> Result; diff --git a/src/common/time/src/datetime.rs b/src/common/time/src/datetime.rs index d12b87f7f1..f1980a38d1 100644 --- a/src/common/time/src/datetime.rs +++ b/src/common/time/src/datetime.rs @@ -35,11 +35,11 @@ pub struct DateTime(i64); impl Display for DateTime { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - if let Some(abs_time) = NaiveDateTime::from_timestamp_millis(self.0) { + if let Some(abs_time) = chrono::DateTime::from_timestamp_millis(self.0) { write!( f, "{}", - format_utc_datetime(&abs_time, DATETIME_FORMAT_WITH_TZ) + format_utc_datetime(&abs_time.naive_utc(), DATETIME_FORMAT_WITH_TZ) ) } else { write!(f, "DateTime({})", self.0) @@ -55,7 +55,7 @@ impl From for serde_json::Value { impl From for DateTime { fn from(value: NaiveDateTime) -> Self { - DateTime::from(value.timestamp_millis()) + DateTime::from(value.and_utc().timestamp_millis()) } } @@ -87,13 +87,15 @@ impl DateTime { pub fn from_str(s: &str, timezone: Option<&Timezone>) -> Result { let s = s.trim(); let timestamp_millis = if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(s) { - dt.naive_utc().timestamp_millis() + dt.naive_utc().and_utc().timestamp_millis() } else if let Ok(d) = NaiveDateTime::parse_from_str(s, DATETIME_FORMAT) { match datetime_to_utc(&d, get_timezone(timezone)) { LocalResult::None => { return InvalidDateStrSnafu { raw: s }.fail(); } - LocalResult::Single(utc) | LocalResult::Ambiguous(utc, _) => utc.timestamp_millis(), + LocalResult::Single(t) | LocalResult::Ambiguous(t, _) => { + t.and_utc().timestamp_millis() + } } } else if let Ok(v) = chrono::DateTime::parse_from_str(s, DATETIME_FORMAT_WITH_TZ) { v.timestamp_millis() @@ -116,7 +118,7 @@ impl DateTime { /// Convert to [NaiveDateTime]. pub fn to_chrono_datetime(&self) -> Option { - NaiveDateTime::from_timestamp_millis(self.0) + chrono::DateTime::from_timestamp_millis(self.0).map(|x| x.naive_utc()) } /// Format DateTime for given format and timezone. diff --git a/src/common/time/src/timestamp.rs b/src/common/time/src/timestamp.rs index 7010ff8293..a20a1925d0 100644 --- a/src/common/time/src/timestamp.rs +++ b/src/common/time/src/timestamp.rs @@ -357,7 +357,7 @@ impl Timestamp { pub fn to_chrono_datetime(&self) -> Option { let (sec, nsec) = self.split(); - NaiveDateTime::from_timestamp_opt(sec, nsec) + chrono::DateTime::from_timestamp(sec, nsec).map(|x| x.naive_utc()) } pub fn to_chrono_datetime_with_timezone(&self, tz: Option<&Timezone>) -> Option { @@ -380,8 +380,8 @@ impl Timestamp { } pub fn from_chrono_datetime(ndt: NaiveDateTime) -> Option { - let sec = ndt.timestamp(); - let nsec = ndt.timestamp_subsec_nanos(); + let sec = ndt.and_utc().timestamp(); + let nsec = ndt.and_utc().timestamp_subsec_nanos(); Timestamp::from_splits(sec, nsec) } @@ -1063,9 +1063,9 @@ mod tests { let _ = Timestamp::new(i64::MAX, TimeUnit::Nanosecond).split(); let _ = Timestamp::new(i64::MIN, TimeUnit::Nanosecond).split(); let (sec, nsec) = Timestamp::new(i64::MIN, TimeUnit::Nanosecond).split(); - let time = NaiveDateTime::from_timestamp_opt(sec, nsec).unwrap(); - assert_eq!(sec, time.timestamp()); - assert_eq!(nsec, time.timestamp_subsec_nanos()); + let time = DateTime::from_timestamp(sec, nsec).unwrap().naive_utc(); + assert_eq!(sec, time.and_utc().timestamp()); + assert_eq!(nsec, time.and_utc().timestamp_subsec_nanos()); } #[test] @@ -1159,12 +1159,12 @@ mod tests { #[test] fn test_subtract_timestamp() { assert_eq!( - Some(chrono::Duration::milliseconds(42)), + chrono::Duration::try_milliseconds(42), Timestamp::new_millisecond(100).sub(&Timestamp::new_millisecond(58)) ); assert_eq!( - Some(chrono::Duration::milliseconds(-42)), + chrono::Duration::try_milliseconds(-42), Timestamp::new_millisecond(58).sub(&Timestamp::new_millisecond(100)) ); } @@ -1286,8 +1286,8 @@ mod tests { #[test] fn test_from_naive_date_time() { - let naive_date_time_min = NaiveDateTime::MIN; - let naive_date_time_max = NaiveDateTime::MAX; + let naive_date_time_min = NaiveDateTime::MIN.and_utc(); + let naive_date_time_max = NaiveDateTime::MAX.and_utc(); let min_sec = Timestamp::new_second(naive_date_time_min.timestamp()); let max_sec = Timestamp::new_second(naive_date_time_max.timestamp()); diff --git a/src/common/time/src/timezone.rs b/src/common/time/src/timezone.rs index 8d98b072a9..b1fefceb21 100644 --- a/src/common/time/src/timezone.rs +++ b/src/common/time/src/timezone.rs @@ -15,7 +15,7 @@ use std::fmt::Display; use std::str::FromStr; -use chrono::{FixedOffset, NaiveDateTime, TimeZone}; +use chrono::{FixedOffset, TimeZone}; use chrono_tz::{OffsetComponents, Tz}; use once_cell::sync::OnceCell; use snafu::{OptionExt, ResultExt}; @@ -114,7 +114,9 @@ impl Timezone { match self { Self::Offset(offset) => offset.local_minus_utc().into(), Self::Named(tz) => { - let datetime = NaiveDateTime::from_timestamp_opt(0, 0).unwrap(); + let datetime = chrono::DateTime::from_timestamp(0, 0) + .map(|x| x.naive_utc()) + .expect("invalid timestamp"); let datetime = tz.from_utc_datetime(&datetime); let utc_offset = datetime.offset().base_utc_offset(); let dst_offset = datetime.offset().dst_offset(); diff --git a/src/common/time/src/util.rs b/src/common/time/src/util.rs index 6ce824764a..19fe3bc911 100644 --- a/src/common/time/src/util.rs +++ b/src/common/time/src/util.rs @@ -69,7 +69,10 @@ pub fn current_time_rfc3339() -> String { /// Returns the yesterday time in rfc3339 format. pub fn yesterday_rfc3339() -> String { let now = chrono::Utc::now(); - let day_before = now - chrono::Duration::days(1); + let day_before = now + - chrono::Duration::try_days(1).unwrap_or_else(|| { + panic!("now time ('{now}') is too early to calculate the day before") + }); day_before.to_rfc3339() } diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs index d9dc267b3c..1af0fda462 100644 --- a/src/datanode/src/region_server.rs +++ b/src/datanode/src/region_server.rs @@ -18,14 +18,14 @@ use std::fmt::Debug; use std::ops::Deref; use std::sync::{Arc, Mutex, RwLock}; -use api::v1::region::{region_request, QueryRequest, RegionResponse}; +use api::region::RegionResponse; +use api::v1::region::{region_request, QueryRequest, RegionResponse as RegionResponseV1}; use api::v1::{ResponseHeader, Status}; use arrow_flight::{FlightData, Ticket}; use async_trait::async_trait; use bytes::Bytes; use common_error::ext::BoxedError; use common_error::status_code::StatusCode; -use common_meta::datanode_manager::HandleResponse; use common_query::logical_plan::Expr; use common_query::physical_plan::DfPhysicalPlanAdapter; use common_query::{DfPhysicalPlan, OutputData}; @@ -36,7 +36,7 @@ use common_telemetry::tracing_context::{FutureExt, TracingContext}; use common_telemetry::{info, warn}; use dashmap::DashMap; use datafusion::catalog::schema::SchemaProvider; -use datafusion::catalog::{CatalogList, CatalogProvider}; +use datafusion::catalog::{CatalogProvider, CatalogProviderList}; use datafusion::datasource::TableProvider; use datafusion::error::Result as DfResult; use datafusion::execution::context::SessionState; @@ -129,7 +129,7 @@ impl RegionServer { &self, region_id: RegionId, request: RegionRequest, - ) -> Result { + ) -> Result { self.inner.handle_request(region_id, request).await } @@ -218,7 +218,7 @@ impl RegionServer { #[async_trait] impl RegionServerHandler for RegionServer { - async fn handle(&self, request: region_request::Body) -> ServerResult { + async fn handle(&self, request: region_request::Body) -> ServerResult { let is_parallel = matches!( request, region_request::Body::Inserts(_) | region_request::Body::Deletes(_) @@ -276,7 +276,7 @@ impl RegionServerHandler for RegionServer { extension.extend(result.extension); } - Ok(RegionResponse { + Ok(RegionResponseV1 { header: Some(ResponseHeader { status: Some(Status { status_code: StatusCode::Success as _, @@ -465,7 +465,7 @@ impl RegionServerInner { &self, region_id: RegionId, request: RegionRequest, - ) -> Result { + ) -> Result { let request_type = request.request_type(); let _timer = crate::metrics::HANDLE_REGION_REQUEST_ELAPSED .with_label_values(&[request_type]) @@ -490,7 +490,7 @@ impl RegionServerInner { let engine = match self.get_engine(region_id, ®ion_change)? { CurrentEngine::Engine(engine) => engine, - CurrentEngine::EarlyReturn(rows) => return Ok(HandleResponse::new(rows)), + CurrentEngine::EarlyReturn(rows) => return Ok(RegionResponse::new(rows)), }; // Sets corresponding region status to registering/deregistering before the operation. @@ -505,7 +505,7 @@ impl RegionServerInner { // Sets corresponding region status to ready. self.set_region_status_ready(region_id, engine, region_change) .await?; - Ok(HandleResponse { + Ok(RegionResponse { affected_rows: result.affected_rows, extension: result.extension, }) @@ -643,10 +643,15 @@ impl RegionServerInner { .await?; let catalog_list = Arc::new(DummyCatalogList::with_table_provider(table_provider)); - + let query_engine_ctx = self.query_engine.engine_context(ctx.clone()); // decode substrait plan to logical plan and execute it let logical_plan = DFLogicalSubstraitConvertor - .decode(Bytes::from(plan), catalog_list, "", "") + .decode( + Bytes::from(plan), + catalog_list, + query_engine_ctx.state().clone(), + ctx.clone(), + ) .await .context(DecodeLogicalPlanSnafu)?; @@ -728,7 +733,7 @@ impl DummyCatalogList { } } -impl CatalogList for DummyCatalogList { +impl CatalogProviderList for DummyCatalogList { fn as_any(&self) -> &dyn Any { self } @@ -786,8 +791,8 @@ impl SchemaProvider for DummySchemaProvider { vec![] } - async fn table(&self, _name: &str) -> Option> { - Some(self.table.clone()) + async fn table(&self, _name: &str) -> DfResult>> { + Ok(Some(self.table.clone())) } fn table_exist(&self, _name: &str) -> bool { @@ -827,7 +832,10 @@ impl TableProvider for DummyTableProvider { limit: Option, ) -> DfResult> { let mut request = self.scan_request.lock().unwrap().clone(); - request.projection = projection.cloned(); + request.projection = match projection { + Some(x) if !x.is_empty() => Some(x.clone()), + _ => None, + }; request.filters = filters.iter().map(|e| Expr::from(e.clone())).collect(); request.limit = limit; diff --git a/src/datanode/src/tests.rs b/src/datanode/src/tests.rs index cffa382865..84244a4679 100644 --- a/src/datanode/src/tests.rs +++ b/src/datanode/src/tests.rs @@ -16,6 +16,7 @@ use std::any::Any; use std::sync::Arc; use std::time::Duration; +use api::region::RegionResponse; use async_trait::async_trait; use common_error::ext::BoxedError; use common_function::function::FunctionRef; @@ -31,7 +32,7 @@ use query::query_engine::DescribeResult; use query::{QueryEngine, QueryEngineContext}; use session::context::QueryContextRef; use store_api::metadata::RegionMetadataRef; -use store_api::region_engine::{RegionEngine, RegionHandleResult, RegionRole, SetReadonlyResponse}; +use store_api::region_engine::{RegionEngine, RegionRole, SetReadonlyResponse}; use store_api::region_request::{AffectedRows, RegionRequest}; use store_api::storage::{RegionId, ScanRequest}; use table::TableRef; @@ -166,18 +167,18 @@ impl RegionEngine for MockRegionEngine { &self, region_id: RegionId, request: RegionRequest, - ) -> Result { + ) -> Result { if let Some(delay) = self.handle_request_delay { tokio::time::sleep(delay).await; } if let Some(mock_fn) = &self.handle_request_mock_fn { return mock_fn(region_id, request) .map_err(BoxedError::new) - .map(RegionHandleResult::new); + .map(RegionResponse::new); }; let _ = self.sender.send((region_id, request)).await; - Ok(RegionHandleResult::new(0)) + Ok(RegionResponse::new(0)) } async fn handle_query( diff --git a/src/datatypes/src/error.rs b/src/datatypes/src/error.rs index 316b50e327..df56be87b4 100644 --- a/src/datatypes/src/error.rs +++ b/src/datatypes/src/error.rs @@ -139,6 +139,13 @@ pub enum Error { error: arrow::error::ArrowError, location: Location, }, + + #[snafu(display("Failed to convert Arrow array to scalars"))] + ConvertArrowArrayToScalars { + #[snafu(source)] + error: datafusion_common::DataFusionError, + location: Location, + }, } impl ErrorExt for Error { diff --git a/src/datatypes/src/scalars.rs b/src/datatypes/src/scalars.rs index 60e6da2183..272d91eebf 100644 --- a/src/datatypes/src/scalars.rs +++ b/src/datatypes/src/scalars.rs @@ -437,10 +437,8 @@ mod tests { #[test] fn test_list_value_scalar() { - let list_value = ListValue::new( - Some(Box::new(vec![Value::Int32(123)])), - ConcreteDataType::int32_datatype(), - ); + let list_value = + ListValue::new(vec![Value::Int32(123)], ConcreteDataType::int32_datatype()); let list_ref = ListValueRef::Ref { val: &list_value }; assert_eq!(list_ref, list_value.as_scalar_ref()); assert_eq!(list_value, list_ref.to_owned_scalar()); diff --git a/src/datatypes/src/types/list_type.rs b/src/datatypes/src/types/list_type.rs index 18115837f5..91f4c8654f 100644 --- a/src/datatypes/src/types/list_type.rs +++ b/src/datatypes/src/types/list_type.rs @@ -61,7 +61,7 @@ impl DataType for ListType { } fn default_value(&self) -> Value { - Value::List(ListValue::new(None, *self.item_type.clone())) + Value::List(ListValue::new(vec![], *self.item_type.clone())) } fn as_arrow_type(&self) -> ArrowDataType { @@ -95,7 +95,7 @@ mod tests { assert_eq!("List", t.name()); assert_eq!(LogicalTypeId::List, t.logical_type_id()); assert_eq!( - Value::List(ListValue::new(None, ConcreteDataType::boolean_datatype())), + Value::List(ListValue::new(vec![], ConcreteDataType::boolean_datatype())), t.default_value() ); assert_eq!( diff --git a/src/datatypes/src/value.rs b/src/datatypes/src/value.rs index 1483b82adc..01c1983f32 100644 --- a/src/datatypes/src/value.rs +++ b/src/datatypes/src/value.rs @@ -17,6 +17,7 @@ use std::fmt::{Display, Formatter}; use std::sync::Arc; use arrow::datatypes::{DataType as ArrowDataType, Field}; +use arrow_array::{Array, ListArray}; use common_base::bytes::{Bytes, StringBytes}; use common_decimal::Decimal128; use common_telemetry::logging; @@ -31,8 +32,7 @@ pub use ordered_float::OrderedFloat; use serde::{Deserialize, Serialize}; use snafu::{ensure, ResultExt}; -use crate::error; -use crate::error::{Error, Result, TryFromValueSnafu}; +use crate::error::{self, ConvertArrowArrayToScalarsSnafu, Error, Result, TryFromValueSnafu}; use crate::prelude::*; use crate::type_id::LogicalTypeId; use crate::types::{IntervalType, ListType}; @@ -110,9 +110,8 @@ impl Display for Value { Value::Interval(v) => write!(f, "{}", v.to_iso8601_string()), Value::Duration(d) => write!(f, "{d}"), Value::List(v) => { - let default = Box::>::default(); - let items = v.items().as_ref().unwrap_or(&default); - let items = items + let items = v + .items() .iter() .map(|i| i.to_string()) .collect::>() @@ -424,9 +423,10 @@ pub fn to_null_scalar_value(output_type: &ConcreteDataType) -> Result ScalarValue::IntervalDayTime(None), IntervalType::MonthDayNano(_) => ScalarValue::IntervalMonthDayNano(None), }, - ConcreteDataType::List(_) => { - ScalarValue::List(None, Arc::new(new_item_field(output_type.as_arrow_type()))) - } + ConcreteDataType::List(_) => ScalarValue::List(Arc::new(ListArray::new_null( + Arc::new(new_item_field(output_type.as_arrow_type())), + 0, + ))), ConcreteDataType::Dictionary(dict) => ScalarValue::Dictionary( Box::new(dict.key_type().as_arrow_type()), Box::new(to_null_scalar_value(dict.value_type())?), @@ -715,9 +715,7 @@ impl TryFrom for serde_json::Value { /// List value. #[derive(Debug, Clone, PartialEq, Hash, Serialize, Deserialize)] pub struct ListValue { - /// List of nested Values (boxed to reduce size_of(Value)) - #[allow(clippy::box_collection)] - items: Option>>, + items: Vec, /// Inner values datatype, to distinguish empty lists of different datatypes. /// Restricted by DataFusion, cannot use null datatype for empty list. datatype: ConcreteDataType, @@ -726,11 +724,11 @@ pub struct ListValue { impl Eq for ListValue {} impl ListValue { - pub fn new(items: Option>>, datatype: ConcreteDataType) -> Self { + pub fn new(items: Vec, datatype: ConcreteDataType) -> Self { Self { items, datatype } } - pub fn items(&self) -> &Option>> { + pub fn items(&self) -> &[Value] { &self.items } @@ -739,38 +737,30 @@ impl ListValue { } fn try_to_scalar_value(&self, output_type: &ListType) -> Result { - let vs = if let Some(items) = self.items() { - Some( - items - .iter() - .map(|v| v.try_to_scalar_value(output_type.item_type())) - .collect::>>()?, - ) - } else { - None - }; - - Ok(ScalarValue::List( - vs, - Arc::new(new_item_field(output_type.item_type().as_arrow_type())), - )) + let vs = self + .items + .iter() + .map(|v| v.try_to_scalar_value(output_type.item_type())) + .collect::>>()?; + Ok(ScalarValue::List(ScalarValue::new_list( + &vs, + &self.datatype.as_arrow_type(), + ))) } /// use 'the first item size' * 'length of items' to estimate the size. /// it could be inaccurate. fn estimated_size(&self) -> usize { - if let Some(items) = &self.items { - if let Some(item) = items.first() { - return item.as_value_ref().data_size() * items.len(); - } - } - 0 + self.items + .first() + .map(|x| x.as_value_ref().data_size() * self.items.len()) + .unwrap_or(0) } } impl Default for ListValue { fn default() -> ListValue { - ListValue::new(None, ConcreteDataType::null_datatype()) + ListValue::new(vec![], ConcreteDataType::null_datatype()) } } @@ -824,17 +814,14 @@ impl TryFrom for Value { ScalarValue::Binary(b) | ScalarValue::LargeBinary(b) | ScalarValue::FixedSizeBinary(_, b) => Value::from(b.map(Bytes::from)), - ScalarValue::List(vs, field) | ScalarValue::Fixedsizelist(vs, field, _) => { - let items = if let Some(vs) = vs { - let vs = vs - .into_iter() - .map(ScalarValue::try_into) - .collect::>()?; - Some(Box::new(vs)) - } else { - None - }; - let datatype = ConcreteDataType::try_from(field.data_type())?; + ScalarValue::List(array) => { + let datatype = ConcreteDataType::try_from(array.data_type())?; + let items = ScalarValue::convert_array_to_scalar_vec(array.as_ref()) + .context(ConvertArrowArrayToScalarsSnafu)? + .into_iter() + .flatten() + .map(|x| x.try_into()) + .collect::>>()?; Value::List(ListValue::new(items, datatype)) } ScalarValue::Date32(d) => d.map(|x| Value::Date(Date::new(x))).unwrap_or(Value::Null), @@ -891,8 +878,11 @@ impl TryFrom for Value { .map(|v| Value::Decimal128(Decimal128::new(v, p, s))) .unwrap_or(Value::Null), ScalarValue::Decimal256(_, _, _) - | ScalarValue::Struct(_, _) - | ScalarValue::Dictionary(_, _) => { + | ScalarValue::Struct(_) + | ScalarValue::FixedSizeList(_) + | ScalarValue::LargeList(_) + | ScalarValue::Dictionary(_, _) + | ScalarValue::Union(_, _, _) => { return error::UnsupportedArrowTypeSnafu { arrow_type: v.data_type(), } @@ -1382,19 +1372,22 @@ mod tests { assert_eq!( Value::List(ListValue::new( - Some(Box::new(vec![Value::Int32(1), Value::Null])), - ConcreteDataType::int32_datatype() + vec![Value::Int32(1), Value::Null], + ConcreteDataType::list_datatype(ConcreteDataType::int32_datatype()) )), - ScalarValue::new_list( - Some(vec![ScalarValue::Int32(Some(1)), ScalarValue::Int32(None)]), - ArrowDataType::Int32, - ) + ScalarValue::List(ScalarValue::new_list( + &[ScalarValue::Int32(Some(1)), ScalarValue::Int32(None)], + &ArrowDataType::Int32, + )) .try_into() .unwrap() ); assert_eq!( - Value::List(ListValue::new(None, ConcreteDataType::uint32_datatype())), - ScalarValue::new_list(None, ArrowDataType::UInt32) + Value::List(ListValue::new( + vec![], + ConcreteDataType::list_datatype(ConcreteDataType::uint32_datatype()) + )), + ScalarValue::List(ScalarValue::new_list(&[], &ArrowDataType::UInt32)) .try_into() .unwrap() ); @@ -1664,7 +1657,7 @@ mod tests { check_type_and_value( &ConcreteDataType::list_datatype(ConcreteDataType::int32_datatype()), &Value::List(ListValue::new( - Some(Box::new(vec![Value::Int32(10)])), + vec![Value::Int32(10)], ConcreteDataType::int32_datatype(), )), ); @@ -1829,7 +1822,7 @@ mod tests { assert_eq!( json_value, to_json(Value::List(ListValue { - items: Some(Box::new(vec![Value::Int32(123)])), + items: vec![Value::Int32(123)], datatype: ConcreteDataType::int32_datatype(), })) ); @@ -1897,7 +1890,7 @@ mod tests { check_as_value_ref!(DateTime, DateTime::new(1034)); let list = ListValue { - items: None, + items: vec![], datatype: ConcreteDataType::int32_datatype(), }; assert_eq!( @@ -1935,7 +1928,7 @@ mod tests { check_as_correct!(Time::new_second(12), Time, as_time); check_as_correct!(Duration::new_second(12), Duration, as_duration); let list = ListValue { - items: None, + items: vec![], datatype: ConcreteDataType::int32_datatype(), }; check_as_correct!(ListValueRef::Ref { val: &list }, List, as_list); @@ -1991,7 +1984,7 @@ mod tests { ); assert_eq!( Value::List(ListValue::new( - Some(Box::new(vec![Value::Int8(1), Value::Int8(2)])), + vec![Value::Int8(1), Value::Int8(2)], ConcreteDataType::int8_datatype(), )) .to_string(), @@ -1999,7 +1992,7 @@ mod tests { ); assert_eq!( Value::List(ListValue::new( - Some(Box::default()), + vec![], ConcreteDataType::timestamp_second_datatype(), )) .to_string(), @@ -2007,7 +2000,7 @@ mod tests { ); assert_eq!( Value::List(ListValue::new( - Some(Box::default()), + vec![], ConcreteDataType::timestamp_millisecond_datatype(), )) .to_string(), @@ -2015,7 +2008,7 @@ mod tests { ); assert_eq!( Value::List(ListValue::new( - Some(Box::default()), + vec![], ConcreteDataType::timestamp_microsecond_datatype(), )) .to_string(), @@ -2023,7 +2016,7 @@ mod tests { ); assert_eq!( Value::List(ListValue::new( - Some(Box::default()), + vec![], ConcreteDataType::timestamp_nanosecond_datatype(), )) .to_string(), @@ -2253,19 +2246,29 @@ mod tests { #[test] fn test_list_value_to_scalar_value() { - let items = Some(Box::new(vec![Value::Int32(-1), Value::Null])); + let items = vec![Value::Int32(-1), Value::Null]; let list = Value::List(ListValue::new(items, ConcreteDataType::int32_datatype())); let df_list = list .try_to_scalar_value(&ConcreteDataType::list_datatype( ConcreteDataType::int32_datatype(), )) .unwrap(); - assert!(matches!(df_list, ScalarValue::List(_, _))); + assert!(matches!(df_list, ScalarValue::List(_))); match df_list { - ScalarValue::List(vs, field) => { - assert_eq!(ArrowDataType::Int32, *field.data_type()); + ScalarValue::List(vs) => { + assert_eq!( + ArrowDataType::List(Arc::new(Field::new_list_field( + ArrowDataType::Int32, + true + ))), + *vs.data_type() + ); - let vs = vs.unwrap(); + let vs = ScalarValue::convert_array_to_scalar_vec(vs.as_ref()) + .unwrap() + .into_iter() + .flatten() + .collect::>(); assert_eq!( vs, vec![ScalarValue::Int32(Some(-1)), ScalarValue::Int32(None)] @@ -2367,10 +2370,10 @@ mod tests { check_value_ref_size_eq( &ValueRef::List(ListValueRef::Ref { val: &ListValue { - items: Some(Box::new(vec![ + items: vec![ Value::String("hello world".into()), Value::String("greptimedb".into()), - ])), + ], datatype: ConcreteDataType::string_datatype(), }, }), @@ -2387,7 +2390,6 @@ mod tests { for vec_opt in &data { if let Some(vec) = vec_opt { let values = vec.iter().map(|v| Value::from(*v)).collect(); - let values = Some(Box::new(values)); let list_value = ListValue::new(values, ConcreteDataType::int32_datatype()); builder.push(Some(ListValueRef::Ref { val: &list_value })); diff --git a/src/datatypes/src/vectors/constant.rs b/src/datatypes/src/vectors/constant.rs index 2d731ee993..66587cf1d7 100644 --- a/src/datatypes/src/vectors/constant.rs +++ b/src/datatypes/src/vectors/constant.rs @@ -223,7 +223,7 @@ mod tests { assert_eq!(10, c.len()); assert!(c.validity().is_all_valid()); assert!(!c.only_null()); - assert_eq!(64, c.memory_size()); + assert_eq!(4, c.memory_size()); for i in 0..10 { assert!(!c.is_null(i)); diff --git a/src/datatypes/src/vectors/helper.rs b/src/datatypes/src/vectors/helper.rs index ef7dab842f..21c37ec077 100644 --- a/src/datatypes/src/vectors/helper.rs +++ b/src/datatypes/src/vectors/helper.rs @@ -26,9 +26,9 @@ use datafusion_common::ScalarValue; use snafu::{OptionExt, ResultExt}; use crate::data_type::ConcreteDataType; -use crate::error::{self, Result}; +use crate::error::{self, ConvertArrowArrayToScalarsSnafu, Result}; use crate::scalars::{Scalar, ScalarVectorBuilder}; -use crate::value::{ListValue, ListValueRef}; +use crate::value::{ListValue, ListValueRef, Value}; use crate::vectors::{ BinaryVector, BooleanVector, ConstantVector, DateTimeVector, DateVector, Decimal128Vector, DurationMicrosecondVector, DurationMillisecondVector, DurationNanosecondVector, @@ -160,19 +160,18 @@ impl Helper { | ScalarValue::FixedSizeBinary(_, v) => { ConstantVector::new(Arc::new(BinaryVector::from(vec![v])), length) } - ScalarValue::List(v, field) | ScalarValue::Fixedsizelist(v, field, _) => { - let item_type = ConcreteDataType::try_from(field.data_type())?; + ScalarValue::List(array) => { + let item_type = ConcreteDataType::try_from(&array.value_type())?; let mut builder = ListVectorBuilder::with_type_capacity(item_type.clone(), 1); - if let Some(values) = v { - let values = values - .into_iter() - .map(ScalarValue::try_into) - .collect::>()?; - let list_value = ListValue::new(Some(Box::new(values)), item_type); - builder.push(Some(ListValueRef::Ref { val: &list_value })); - } else { - builder.push(None); - } + let values = ScalarValue::convert_array_to_scalar_vec(array.as_ref()) + .context(ConvertArrowArrayToScalarsSnafu)? + .into_iter() + .flatten() + .map(ScalarValue::try_into) + .collect::>>()?; + builder.push(Some(ListValueRef::Ref { + val: &ListValue::new(values, item_type), + })); let list_vector = builder.to_vector(); ConstantVector::new(list_vector, length) } @@ -236,8 +235,11 @@ impl Helper { ConstantVector::new(Arc::new(vector), length) } ScalarValue::Decimal256(_, _, _) - | ScalarValue::Struct(_, _) - | ScalarValue::Dictionary(_, _) => { + | ScalarValue::Struct(_) + | ScalarValue::FixedSizeList(_) + | ScalarValue::LargeList(_) + | ScalarValue::Dictionary(_, _) + | ScalarValue::Union(_, _, _) => { return error::ConversionSnafu { from: format!("Unsupported scalar value: {value}"), } @@ -351,7 +353,11 @@ impl Helper { | ArrowDataType::Dictionary(_, _) | ArrowDataType::Decimal256(_, _) | ArrowDataType::Map(_, _) - | ArrowDataType::RunEndEncoded(_, _) => { + | ArrowDataType::RunEndEncoded(_, _) + | ArrowDataType::BinaryView + | ArrowDataType::Utf8View + | ArrowDataType::ListView(_) + | ArrowDataType::LargeListView(_) => { return error::UnsupportedArrowTypeSnafu { arrow_type: array.as_ref().data_type().clone(), } @@ -396,7 +402,7 @@ mod tests { TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array, }; - use arrow::datatypes::{Field, Int32Type}; + use arrow::datatypes::Int32Type; use arrow_array::DictionaryArray; use common_decimal::Decimal128; use common_time::time::Time; @@ -486,13 +492,10 @@ mod tests { #[test] fn test_try_from_list_value() { - let value = ScalarValue::List( - Some(vec![ - ScalarValue::Int32(Some(1)), - ScalarValue::Int32(Some(2)), - ]), - Arc::new(Field::new("item", ArrowDataType::Int32, true)), - ); + let value = ScalarValue::List(ScalarValue::new_list( + &[ScalarValue::Int32(Some(1)), ScalarValue::Int32(Some(2))], + &ArrowDataType::Int32, + )); let vector = Helper::try_from_scalar_value(value, 3).unwrap(); assert_eq!( ConcreteDataType::list_datatype(ConcreteDataType::int32_datatype()), @@ -501,8 +504,8 @@ mod tests { assert_eq!(3, vector.len()); for i in 0..vector.len() { let v = vector.get(i); - let items = v.as_list().unwrap().unwrap().items().as_ref().unwrap(); - assert_eq!(vec![Value::Int32(1), Value::Int32(2)], **items); + let items = v.as_list().unwrap().unwrap().items(); + assert_eq!(vec![Value::Int32(1), Value::Int32(2)], items); } } diff --git a/src/datatypes/src/vectors/list.rs b/src/datatypes/src/vectors/list.rs index aaf88bbd98..99399edcf4 100644 --- a/src/datatypes/src/vectors/list.rs +++ b/src/datatypes/src/vectors/list.rs @@ -114,10 +114,7 @@ impl Vector for ListVector { let values = (0..vector.len()) .map(|i| vector.get(i)) .collect::>(); - Value::List(ListValue::new( - Some(Box::new(values)), - self.item_type.clone(), - )) + Value::List(ListValue::new(values, self.item_type.clone())) } fn get_ref(&self, index: usize) -> ValueRef { @@ -248,11 +245,8 @@ impl ListVectorBuilder { } fn push_list_value(&mut self, list_value: &ListValue) -> Result<()> { - if let Some(items) = list_value.items() { - for item in &**items { - self.values_builder - .try_push_value_ref(item.as_value_ref())?; - } + for v in list_value.items() { + self.values_builder.try_push_value_ref(v.as_value_ref())?; } self.finish_list(true); @@ -496,7 +490,6 @@ pub mod tests { for vec_opt in data { if let Some(vec) = vec_opt { let values = vec.iter().map(|v| Value::from(*v)).collect(); - let values = Some(Box::new(values)); let list_value = ListValue::new(values, ConcreteDataType::int32_datatype()); builder.push(Some(ListValueRef::Ref { val: &list_value })); @@ -576,11 +569,7 @@ pub mod tests { assert_eq!( Value::List(ListValue::new( - Some(Box::new(vec![ - Value::Int32(1), - Value::Int32(2), - Value::Int32(3) - ])), + vec![Value::Int32(1), Value::Int32(2), Value::Int32(3)], ConcreteDataType::int32_datatype() )), list_vector.get(0) @@ -599,11 +588,7 @@ pub mod tests { assert_eq!(Value::Null, list_vector.get(1)); assert_eq!( Value::List(ListValue::new( - Some(Box::new(vec![ - Value::Int32(4), - Value::Null, - Value::Int32(6) - ])), + vec![Value::Int32(4), Value::Null, Value::Int32(6)], ConcreteDataType::int32_datatype() )), list_vector.get(2) @@ -680,11 +665,7 @@ pub mod tests { ListType::new(ConcreteDataType::int32_datatype()).create_mutable_vector(3); builder.push_value_ref(ValueRef::List(ListValueRef::Ref { val: &ListValue::new( - Some(Box::new(vec![ - Value::Int32(4), - Value::Null, - Value::Int32(6), - ])), + vec![Value::Int32(4), Value::Null, Value::Int32(6)], ConcreteDataType::int32_datatype(), ), })); @@ -717,11 +698,7 @@ pub mod tests { builder.push(None); builder.push(Some(ListValueRef::Ref { val: &ListValue::new( - Some(Box::new(vec![ - Value::Int32(4), - Value::Null, - Value::Int32(6), - ])), + vec![Value::Int32(4), Value::Null, Value::Int32(6)], ConcreteDataType::int32_datatype(), ), })); @@ -772,11 +749,7 @@ pub mod tests { builder.push(None); builder.push(Some(ListValueRef::Ref { val: &ListValue::new( - Some(Box::new(vec![ - Value::Int32(4), - Value::Null, - Value::Int32(6), - ])), + vec![Value::Int32(4), Value::Null, Value::Int32(6)], ConcreteDataType::int32_datatype(), ), })); diff --git a/src/datatypes/src/vectors/primitive.rs b/src/datatypes/src/vectors/primitive.rs index 7e7e3dd50b..fc0007a4b6 100644 --- a/src/datatypes/src/vectors/primitive.rs +++ b/src/datatypes/src/vectors/primitive.rs @@ -531,9 +531,9 @@ mod tests { #[test] fn test_memory_size() { let v = Int32Vector::from_slice((0..5).collect::>()); - assert_eq!(64, v.memory_size()); + assert_eq!(20, v.memory_size()); let v = Int64Vector::from(vec![Some(0i64), Some(1i64), Some(2i64), None, None]); - assert_eq!(128, v.memory_size()); + assert_eq!(144, v.memory_size()); } #[test] diff --git a/src/file-engine/Cargo.toml b/src/file-engine/Cargo.toml index 3ce83b6b79..f9cd1113f5 100644 --- a/src/file-engine/Cargo.toml +++ b/src/file-engine/Cargo.toml @@ -12,7 +12,7 @@ test = ["common-test-util"] workspace = true [dependencies] -api = { workspace = true, optional = true } +api.workspace = true async-trait = "0.1" common-catalog.workspace = true common-datasource.workspace = true @@ -25,6 +25,7 @@ common-telemetry.workspace = true common-test-util = { workspace = true, optional = true } common-time.workspace = true datafusion.workspace = true +datafusion-expr.workspace = true datatypes.workspace = true futures.workspace = true object-store.workspace = true diff --git a/src/file-engine/src/engine.rs b/src/file-engine/src/engine.rs index 0b9192b5c9..fa4f2c5a3f 100644 --- a/src/file-engine/src/engine.rs +++ b/src/file-engine/src/engine.rs @@ -16,6 +16,7 @@ use std::any::Any; use std::collections::HashMap; use std::sync::{Arc, RwLock}; +use api::region::RegionResponse; use async_trait::async_trait; use common_catalog::consts::FILE_ENGINE; use common_error::ext::BoxedError; @@ -24,7 +25,7 @@ use common_telemetry::{error, info}; use object_store::ObjectStore; use snafu::{ensure, OptionExt}; use store_api::metadata::RegionMetadataRef; -use store_api::region_engine::{RegionEngine, RegionHandleResult, RegionRole, SetReadonlyResponse}; +use store_api::region_engine::{RegionEngine, RegionRole, SetReadonlyResponse}; use store_api::region_request::{ AffectedRows, RegionCloseRequest, RegionCreateRequest, RegionDropRequest, RegionOpenRequest, RegionRequest, @@ -60,7 +61,7 @@ impl RegionEngine for FileRegionEngine { &self, region_id: RegionId, request: RegionRequest, - ) -> Result { + ) -> Result { self.inner .handle_request(region_id, request) .await @@ -154,7 +155,7 @@ impl EngineInner { &self, region_id: RegionId, request: RegionRequest, - ) -> EngineResult { + ) -> EngineResult { let result = match request { RegionRequest::Create(req) => self.handle_create(region_id, req).await, RegionRequest::Drop(req) => self.handle_drop(region_id, req).await, @@ -165,7 +166,7 @@ impl EngineInner { } .fail(), }; - result.map(RegionHandleResult::new) + result.map(RegionResponse::new) } async fn stop(&self) -> EngineResult<()> { diff --git a/src/file-engine/src/query/file_stream.rs b/src/file-engine/src/query/file_stream.rs index 105785979e..f3afd1fbc5 100644 --- a/src/file-engine/src/query/file_stream.rs +++ b/src/file-engine/src/query/file_stream.rs @@ -23,15 +23,16 @@ use common_query::prelude::Expr; use common_query::DfPhysicalPlan; use common_recordbatch::adapter::RecordBatchStreamAdapter; use common_recordbatch::SendableRecordBatchStream; -use datafusion::common::ToDFSchema; +use datafusion::common::{Statistics, ToDFSchema}; +use datafusion::config::TableParquetOptions; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::object_store::ObjectStoreUrl; use datafusion::datasource::physical_plan::{FileOpener, FileScanConfig, FileStream, ParquetExec}; -use datafusion::optimizer::utils::conjunction; use datafusion::physical_expr::create_physical_expr; use datafusion::physical_expr::execution_props::ExecutionProps; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion::prelude::SessionContext; +use datafusion_expr::utils::conjunction; use datatypes::arrow::datatypes::Schema as ArrowSchema; use datatypes::schema::SchemaRef; use object_store::ObjectStore; @@ -101,6 +102,7 @@ fn build_record_batch_stream( projection: Option<&Vec>, limit: Option, ) -> Result { + let statistics = Statistics::new_unknown(file_schema.as_ref()); let stream = FileStream::new( &FileScanConfig { object_store_url: ObjectStoreUrl::parse("empty://").unwrap(), // won't be used @@ -109,12 +111,11 @@ fn build_record_batch_stream( .iter() .map(|filename| PartitionedFile::new(filename.to_string(), 0)) .collect::>()], - statistics: Default::default(), + statistics, projection: projection.cloned(), limit, table_partition_cols: vec![], output_ordering: vec![], - infinite_source: false, }, 0, // partition: hard-code opener, @@ -173,12 +174,11 @@ fn new_parquet_stream_with_exec_plan( .iter() .map(|filename| PartitionedFile::new(filename.to_string(), 0)) .collect::>()], - statistics: Default::default(), + statistics: Statistics::new_unknown(file_schema.as_ref()), projection: projection.cloned(), limit: *limit, table_partition_cols: vec![], output_ordering: vec![], - infinite_source: false, }; // build predicate filter @@ -192,7 +192,7 @@ fn new_parquet_stream_with_exec_plan( .to_dfschema_ref() .context(error::ParquetScanPlanSnafu)?; - let filters = create_physical_expr(&expr, &df_schema, &file_schema, &ExecutionProps::new()) + let filters = create_physical_expr(&expr, &df_schema, &ExecutionProps::new()) .context(error::ParquetScanPlanSnafu)?; Some(filters) } else { @@ -201,7 +201,7 @@ fn new_parquet_stream_with_exec_plan( // TODO(ruihang): get this from upper layer let task_ctx = SessionContext::default().task_ctx(); - let parquet_exec = ParquetExec::new(scan_config, filters, None) + let parquet_exec = ParquetExec::new(scan_config, filters, None, TableParquetOptions::default()) .with_parquet_file_reader_factory(Arc::new(DefaultParquetFileReaderFactory::new( store.clone(), ))); diff --git a/src/flow/clippy.toml b/src/flow/clippy.toml index 5a9ebd2a5b..87e7f07bc8 100644 --- a/src/flow/clippy.toml +++ b/src/flow/clippy.toml @@ -1,3 +1,5 @@ # Whether to only check for missing documentation in items visible within the current crate. For example, pub(crate) items. (default: false) # This is a config for clippy::missing_docs_in_private_items missing-docs-in-crate-items = true + +too-many-lines-threshold = 500 diff --git a/src/flow/src/transform/expr.rs b/src/flow/src/transform/expr.rs index 3f65c4b607..c8bff7da5c 100644 --- a/src/flow/src/transform/expr.rs +++ b/src/flow/src/transform/expr.rs @@ -33,6 +33,28 @@ use crate::repr::{ColumnType, RelationType}; use crate::transform::literal::{from_substrait_literal, from_substrait_type}; use crate::transform::FunctionExtensions; +// TODO: found proper place for this +/// ref to `arrow_schema::datatype` for type name +fn typename_to_cdt(name: &str) -> CDT { + match name { + "Int8" => CDT::int8_datatype(), + "Int16" => CDT::int16_datatype(), + "Int32" => CDT::int32_datatype(), + "Int64" => CDT::int64_datatype(), + "UInt8" => CDT::uint8_datatype(), + "UInt16" => CDT::uint16_datatype(), + "UInt32" => CDT::uint32_datatype(), + "UInt64" => CDT::uint64_datatype(), + "Float32" => CDT::float32_datatype(), + "Float64" => CDT::float64_datatype(), + "Boolean" => CDT::boolean_datatype(), + "String" => CDT::string_datatype(), + "Date" => CDT::date_datatype(), + "Timestamp" => CDT::timestamp_second_datatype(), + _ => CDT::null_datatype(), + } +} + impl TypedExpr { /// Convert ScalarFunction into Flow's ScalarExpr pub fn from_substrait_scalar_func( @@ -87,6 +109,21 @@ impl TypedExpr { Ok(TypedExpr::new(arg.call_unary(func), ret_type)) } + 2 if fn_name == "arrow_cast" => { + let cast_to = arg_exprs[1] + .clone() + .as_literal() + .and_then(|lit| lit.as_string()) + .with_context(|| InvalidQuerySnafu { + reason: "array_cast's second argument must be a literal string", + })?; + let cast_to = typename_to_cdt(&cast_to); + let func = UnaryFunc::Cast(cast_to); + let arg = arg_exprs[0].clone(); + let ret_type = ColumnType::new_nullable(func.signature().output.clone()); + + Ok(TypedExpr::new(arg.call_unary(func), ret_type)) + } // because variadic function can also have 2 arguments, we need to check if it's a variadic function first 2 if VariadicFunc::from_str_and_types(fn_name, &arg_types).is_err() => { let (func, signature) = diff --git a/src/frontend/src/heartbeat/handler/tests.rs b/src/frontend/src/heartbeat/handler/tests.rs index 1b6885ddb7..c5b5b4ecde 100644 --- a/src/frontend/src/heartbeat/handler/tests.rs +++ b/src/frontend/src/heartbeat/handler/tests.rs @@ -22,6 +22,7 @@ use common_meta::heartbeat::handler::{ }; use common_meta::heartbeat::mailbox::{HeartbeatMailbox, MessageMeta}; use common_meta::instruction::{CacheIdent, Instruction}; +use common_meta::key::schema_name::{SchemaName, SchemaNameKey}; use common_meta::key::table_info::TableInfoKey; use common_meta::key::TableMetaKey; use partition::manager::TableRouteCacheInvalidator; @@ -53,6 +54,27 @@ impl TableRouteCacheInvalidator for MockTableRouteCacheInvalidator { } } +pub fn test_message_meta(id: u64, subject: &str, to: &str, from: &str) -> MessageMeta { + MessageMeta { + id, + subject: subject.to_string(), + to: to.to_string(), + from: from.to_string(), + } +} + +async fn handle_instruction( + executor: Arc, + mailbox: Arc, + instruction: Instruction, +) { + let response = HeartbeatResponse::default(); + let mut ctx: HeartbeatResponseHandlerContext = + HeartbeatResponseHandlerContext::new(mailbox, response); + ctx.incoming_message = Some((test_message_meta(1, "hi", "foo", "bar"), instruction)); + executor.handle(ctx).await.unwrap(); +} + #[tokio::test] async fn test_invalidate_table_cache_handler() { let table_id = 1; @@ -92,23 +114,45 @@ async fn test_invalidate_table_cache_handler() { .await; } -pub fn test_message_meta(id: u64, subject: &str, to: &str, from: &str) -> MessageMeta { - MessageMeta { - id, - subject: subject.to_string(), - to: to.to_string(), - from: from.to_string(), - } -} +#[tokio::test] +async fn test_invalidate_schema_key_handler() { + let (catalog, schema) = ("foo", "bar"); + let schema_key = SchemaNameKey { catalog, schema }; + let inner = HashMap::from([(schema_key.as_raw_key(), 1)]); + let backend = Arc::new(MockKvCacheInvalidator { + inner: Mutex::new(inner), + }); -async fn handle_instruction( - executor: Arc, - mailbox: Arc, - instruction: Instruction, -) { - let response = HeartbeatResponse::default(); - let mut ctx: HeartbeatResponseHandlerContext = - HeartbeatResponseHandlerContext::new(mailbox, response); - ctx.incoming_message = Some((test_message_meta(1, "hi", "foo", "bar"), instruction)); - executor.handle(ctx).await.unwrap(); + let executor = Arc::new(HandlerGroupExecutor::new(vec![Arc::new( + InvalidateTableCacheHandler::new(backend.clone()), + )])); + + let (tx, _) = mpsc::channel(8); + let mailbox = Arc::new(HeartbeatMailbox::new(tx)); + + // removes a valid key + let valid_key = SchemaName { + catalog_name: catalog.to_string(), + schema_name: schema.to_string(), + }; + handle_instruction( + executor.clone(), + mailbox.clone(), + Instruction::InvalidateCaches(vec![CacheIdent::SchemaName(valid_key.clone())]), + ) + .await; + + assert!(!backend + .inner + .lock() + .unwrap() + .contains_key(&schema_key.as_raw_key())); + + // removes a invalid key + handle_instruction( + executor, + mailbox, + Instruction::InvalidateCaches(vec![CacheIdent::SchemaName(valid_key)]), + ) + .await; } diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs index d1f3df5aea..69a6cc2a12 100644 --- a/src/frontend/src/instance.rs +++ b/src/frontend/src/instance.rs @@ -491,11 +491,20 @@ pub fn check_permission( // database ops won't be checked Statement::CreateDatabase(_) | Statement::ShowDatabases(_) | Statement::DropDatabase(_) => { } - // show create table and alter are not supported yet - Statement::ShowCreateTable(_) | Statement::CreateExternalTable(_) | Statement::Alter(_) => { + + Statement::ShowCreateTable(stmt) => { + validate_param(&stmt.table_name, query_ctx)?; + } + Statement::CreateExternalTable(stmt) => { + validate_param(&stmt.name, query_ctx)?; + } + Statement::Alter(stmt) => { + validate_param(stmt.table_name(), query_ctx)?; } // set/show variable now only alter/show variable in session Statement::SetVariables(_) | Statement::ShowVariables(_) => {} + // show charset and show collation won't be checked + Statement::ShowCharset(_) | Statement::ShowCollation(_) => {} Statement::Insert(insert) => { validate_param(insert.table_name(), query_ctx)?; diff --git a/src/frontend/src/instance/standalone.rs b/src/frontend/src/instance/standalone.rs index 615e14f5d9..4001c4e59d 100644 --- a/src/frontend/src/instance/standalone.rs +++ b/src/frontend/src/instance/standalone.rs @@ -14,11 +14,12 @@ use std::sync::Arc; -use api::v1::region::{QueryRequest, RegionRequest, RegionResponse}; +use api::region::RegionResponse; +use api::v1::region::{QueryRequest, RegionRequest, RegionResponse as RegionResponseV1}; use async_trait::async_trait; use client::region::check_response_header; use common_error::ext::BoxedError; -use common_meta::datanode_manager::{Datanode, DatanodeManager, DatanodeRef, HandleResponse}; +use common_meta::datanode_manager::{Datanode, DatanodeManager, DatanodeRef}; use common_meta::error::{self as meta_error, Result as MetaResult}; use common_meta::peer::Peer; use common_recordbatch::SendableRecordBatchStream; @@ -49,7 +50,7 @@ impl RegionInvoker { Arc::new(Self { region_server }) } - async fn handle_inner(&self, request: RegionRequest) -> Result { + async fn handle_inner(&self, request: RegionRequest) -> Result { let body = request.body.with_context(|| InvalidRegionRequestSnafu { reason: "body not found", })?; @@ -63,7 +64,7 @@ impl RegionInvoker { #[async_trait] impl Datanode for RegionInvoker { - async fn handle(&self, request: RegionRequest) -> MetaResult { + async fn handle(&self, request: RegionRequest) -> MetaResult { let span = request .header .as_ref() @@ -79,7 +80,7 @@ impl Datanode for RegionInvoker { check_response_header(&response.header) .map_err(BoxedError::new) .context(meta_error::ExternalSnafu)?; - Ok(HandleResponse::from_region_response(response)) + Ok(RegionResponse::from_region_response(response)) } async fn handle_query(&self, request: QueryRequest) -> MetaResult { diff --git a/src/index/src/inverted_index/format/writer/single.rs b/src/index/src/inverted_index/format/writer/single.rs index c652c76390..07d10b3880 100644 --- a/src/index/src/inverted_index/format/writer/single.rs +++ b/src/index/src/inverted_index/format/writer/single.rs @@ -120,7 +120,7 @@ where // update min/max, assume values are appended in lexicographic order if stats.distinct_count == 1 { - stats.min_value = value.clone(); + stats.min_value.clone_from(&value); } stats.max_value = value; } diff --git a/src/meta-client/src/client.rs b/src/meta-client/src/client.rs index ddb169ebc5..cb8341b3a8 100644 --- a/src/meta-client/src/client.rs +++ b/src/meta-client/src/client.rs @@ -510,9 +510,6 @@ impl MetaClient { #[cfg(test)] mod tests { use api::v1::meta::{HeartbeatRequest, Peer}; - use meta_srv::metasrv::SelectorContext; - use meta_srv::selector::{Namespace, Selector, SelectorOptions}; - use meta_srv::Result as MetaResult; use super::*; use crate::{error, mocks}; @@ -662,36 +659,6 @@ mod tests { }); } - struct MockSelector; - - #[async_trait::async_trait] - impl Selector for MockSelector { - type Context = SelectorContext; - type Output = Vec; - - async fn select( - &self, - _ns: Namespace, - _ctx: &Self::Context, - _opts: SelectorOptions, - ) -> MetaResult { - Ok(vec![ - Peer { - id: 0, - addr: "peer0".to_string(), - }, - Peer { - id: 1, - addr: "peer1".to_string(), - }, - Peer { - id: 2, - addr: "peer2".to_string(), - }, - ]) - } - } - #[tokio::test] async fn test_range_get() { let tc = new_client("test_range_get").await; diff --git a/src/meta-srv/src/metasrv/builder.rs b/src/meta-srv/src/metasrv/builder.rs index 003d8ba5c6..bd4ab0394d 100644 --- a/src/meta-srv/src/metasrv/builder.rs +++ b/src/meta-srv/src/metasrv/builder.rs @@ -65,7 +65,7 @@ use crate::procedure::region_migration::DefaultContextFactory; use crate::pubsub::PublishRef; use crate::selector::lease_based::LeaseBasedSelector; use crate::service::mailbox::MailboxRef; -use crate::service::store::cached_kv::{CheckLeader, LeaderCachedKvBackend}; +use crate::service::store::cached_kv::LeaderCachedKvBackend; use crate::state::State; use crate::table_meta_alloc::MetasrvPeerAllocator; @@ -436,13 +436,3 @@ impl Default for MetasrvBuilder { Self::new() } } - -struct CheckLeaderByElection(Option); - -impl CheckLeader for CheckLeaderByElection { - fn check(&self) -> bool { - self.0 - .as_ref() - .map_or(false, |election| election.is_leader()) - } -} diff --git a/src/meta-srv/src/selector/load_based.rs b/src/meta-srv/src/selector/load_based.rs index 5f89a5cfeb..de9a293446 100644 --- a/src/meta-srv/src/selector/load_based.rs +++ b/src/meta-srv/src/selector/load_based.rs @@ -205,11 +205,9 @@ mod tests { let alive_stat_kvs = filter_out_expired_datanode(stat_kvs, &lease_kvs); assert_eq!(1, alive_stat_kvs.len()); - assert!(alive_stat_kvs - .get(&StatKey { - cluster_id: 1, - node_id: 1 - }) - .is_some()); + assert!(alive_stat_kvs.contains_key(&StatKey { + cluster_id: 1, + node_id: 1 + })); } } diff --git a/src/meta-srv/src/service/admin.rs b/src/meta-srv/src/service/admin.rs index 6e38fc22fc..3cd1a22938 100644 --- a/src/meta-srv/src/service/admin.rs +++ b/src/meta-srv/src/service/admin.rs @@ -29,7 +29,7 @@ use std::task::{Context, Poll}; use tonic::body::BoxBody; use tonic::codegen::{empty_body, http, BoxFuture, Service}; -use tonic::transport::NamedService; +use tonic::server::NamedService; use crate::metasrv::Metasrv; diff --git a/src/meta-srv/src/service/store/cached_kv.rs b/src/meta-srv/src/service/store/cached_kv.rs index df898164d1..8772c47a1d 100644 --- a/src/meta-srv/src/service/store/cached_kv.rs +++ b/src/meta-srv/src/service/store/cached_kv.rs @@ -263,7 +263,7 @@ impl KvBackend for LeaderCachedKvBackend { .collect::>(); metrics::METRIC_META_KV_CACHE_HIT - .with_label_values(&[&"batch_get"]) + .with_label_values(&["batch_get"]) .inc_by(hit_keys.len() as u64); let missed_keys = req @@ -273,7 +273,7 @@ impl KvBackend for LeaderCachedKvBackend { .cloned() .collect::>(); metrics::METRIC_META_KV_CACHE_MISS - .with_label_values(&[&"batch_get"]) + .with_label_values(&["batch_get"]) .inc_by(missed_keys.len() as u64); let remote_req = BatchGetRequest { keys: missed_keys }; diff --git a/src/metric-engine/src/engine.rs b/src/metric-engine/src/engine.rs index 41f14fd766..1cf36f5661 100644 --- a/src/metric-engine/src/engine.rs +++ b/src/metric-engine/src/engine.rs @@ -27,6 +27,7 @@ use std::any::Any; use std::collections::HashMap; use std::sync::{Arc, RwLock}; +use api::region::RegionResponse; use async_trait::async_trait; use common_error::ext::{BoxedError, ErrorExt}; use common_error::status_code::StatusCode; @@ -34,7 +35,7 @@ use common_recordbatch::SendableRecordBatchStream; use mito2::engine::MitoEngine; use store_api::metadata::RegionMetadataRef; use store_api::metric_engine_consts::METRIC_ENGINE_NAME; -use store_api::region_engine::{RegionEngine, RegionHandleResult, RegionRole, SetReadonlyResponse}; +use store_api::region_engine::{RegionEngine, RegionRole, SetReadonlyResponse}; use store_api::region_request::RegionRequest; use store_api::storage::{RegionId, ScanRequest}; @@ -122,7 +123,7 @@ impl RegionEngine for MetricEngine { &self, region_id: RegionId, request: RegionRequest, - ) -> Result { + ) -> Result { let mut extension_return_value = HashMap::new(); let result = match request { @@ -148,12 +149,10 @@ impl RegionEngine for MetricEngine { RegionRequest::Catchup(_) => Ok(0), }; - result - .map_err(BoxedError::new) - .map(|rows| RegionHandleResult { - affected_rows: rows, - extension: extension_return_value, - }) + result.map_err(BoxedError::new).map(|rows| RegionResponse { + affected_rows: rows, + extension: extension_return_value, + }) } /// Handles substrait query and return a stream of record batches diff --git a/src/metric-engine/src/engine/create.rs b/src/metric-engine/src/engine/create.rs index 0ff5bb75f8..a7e3c5c364 100644 --- a/src/metric-engine/src/engine/create.rs +++ b/src/metric-engine/src/engine/create.rs @@ -381,6 +381,10 @@ impl MetricEngineInner { // concat region dir let metadata_region_dir = join_dir(&request.region_dir, METADATA_REGION_SUBDIR); + // remove TTL option + let mut options = request.options.clone(); + options.remove("ttl"); + RegionCreateRequest { engine: MITO_ENGINE_NAME.to_string(), column_metadatas: vec![ @@ -389,7 +393,7 @@ impl MetricEngineInner { value_column_metadata, ], primary_key: vec![METADATA_SCHEMA_KEY_COLUMN_INDEX as _], - options: request.options.clone(), + options, region_dir: metadata_region_dir, } } @@ -553,12 +557,12 @@ mod test { let mut options = HashMap::new(); options.insert(PHYSICAL_TABLE_METADATA_KEY.to_string(), "value".to_string()); - request.options = options.clone(); + request.options.clone_from(&options); let result = MetricEngineInner::verify_region_create_request(&request); assert!(result.is_ok()); options.insert(LOGICAL_TABLE_METADATA_KEY.to_string(), "value".to_string()); - request.options = options.clone(); + request.options.clone_from(&options); let result = MetricEngineInner::verify_region_create_request(&request); assert!(result.is_err()); @@ -569,7 +573,10 @@ mod test { } #[tokio::test] - async fn test_create_request_for_data_region() { + async fn test_create_request_for_physical_regions() { + // original request + let mut ttl_options = HashMap::new(); + ttl_options.insert("ttl".to_string(), "60m".to_string()); let request = RegionCreateRequest { engine: METRIC_ENGINE_NAME.to_string(), column_metadatas: vec![ @@ -593,15 +600,17 @@ mod test { }, ], primary_key: vec![0], - options: HashMap::new(), + options: ttl_options, region_dir: "/test_dir".to_string(), }; + // set up let env = TestEnv::new().await; let engine = MetricEngine::new(env.mito()); let engine_inner = engine.inner; - let data_region_request = engine_inner.create_request_for_data_region(&request); + // check create data region request + let data_region_request = engine_inner.create_request_for_data_region(&request); assert_eq!( data_region_request.region_dir, "/test_dir/data/".to_string() @@ -611,5 +620,14 @@ mod test { data_region_request.primary_key, vec![ReservedColumnId::table_id(), ReservedColumnId::tsid(), 1] ); + assert!(data_region_request.options.contains_key("ttl")); + + // check create metadata region request + let metadata_region_request = engine_inner.create_request_for_metadata_region(&request); + assert_eq!( + metadata_region_request.region_dir, + "/test_dir/metadata/".to_string() + ); + assert!(!metadata_region_request.options.contains_key("ttl")); } } diff --git a/src/mito2/src/cache/cache_size.rs b/src/mito2/src/cache/cache_size.rs index 8ecd2d5e99..2b63441e64 100644 --- a/src/mito2/src/cache/cache_size.rs +++ b/src/mito2/src/cache/cache_size.rs @@ -137,6 +137,6 @@ mod tests { fn test_parquet_meta_size() { let metadata = parquet_meta(); - assert_eq!(948, parquet_meta_size(&metadata)); + assert_eq!(964, parquet_meta_size(&metadata)); } } diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs index 955aa9c7ee..71954678ad 100644 --- a/src/mito2/src/engine.rs +++ b/src/mito2/src/engine.rs @@ -53,6 +53,7 @@ use std::any::Any; use std::sync::Arc; use std::time::Instant; +use api::region::RegionResponse; use async_trait::async_trait; use common_error::ext::BoxedError; use common_recordbatch::SendableRecordBatchStream; @@ -61,7 +62,7 @@ use object_store::manager::ObjectStoreManagerRef; use snafu::{ensure, OptionExt, ResultExt}; use store_api::logstore::LogStore; use store_api::metadata::RegionMetadataRef; -use store_api::region_engine::{RegionEngine, RegionHandleResult, RegionRole, SetReadonlyResponse}; +use store_api::region_engine::{RegionEngine, RegionRole, SetReadonlyResponse}; use store_api::region_request::{AffectedRows, RegionRequest}; use store_api::storage::{RegionId, ScanRequest}; use tokio::sync::oneshot; @@ -299,7 +300,7 @@ impl RegionEngine for MitoEngine { &self, region_id: RegionId, request: RegionRequest, - ) -> Result { + ) -> Result { let _timer = HANDLE_REQUEST_ELAPSED .with_label_values(&[request.request_type()]) .start_timer(); @@ -307,7 +308,7 @@ impl RegionEngine for MitoEngine { self.inner .handle_request(region_id, request) .await - .map(RegionHandleResult::new) + .map(RegionResponse::new) .map_err(BoxedError::new) } diff --git a/src/mito2/src/engine/basic_test.rs b/src/mito2/src/engine/basic_test.rs index dfbf22c4b0..64c13778e3 100644 --- a/src/mito2/src/engine/basic_test.rs +++ b/src/mito2/src/engine/basic_test.rs @@ -550,7 +550,7 @@ async fn test_region_usage() { flush_region(&engine, region_id, None).await; let region_stat = region.region_usage().await; - assert_eq!(region_stat.sst_usage, 2962); + assert_eq!(region_stat.sst_usage, 3010); // region total usage // Some memtables may share items. diff --git a/src/mito2/src/error.rs b/src/mito2/src/error.rs index 03d0e60715..201717fc7e 100644 --- a/src/mito2/src/error.rs +++ b/src/mito2/src/error.rs @@ -685,7 +685,9 @@ impl ErrorExt for Error { CleanDir { .. } => StatusCode::Unexpected, InvalidConfig { .. } => StatusCode::InvalidArguments, StaleLogEntry { .. } => StatusCode::Unexpected, + FilterRecordBatch { source, .. } => source.status_code(), + Upload { .. } => StatusCode::StorageUnavailable, BiError { .. } => StatusCode::Internal, EncodeMemtable { .. } | ReadDataPart { .. } => StatusCode::Internal, diff --git a/src/mito2/src/flush.rs b/src/mito2/src/flush.rs index f3890cafda..bdf27bdafd 100644 --- a/src/mito2/src/flush.rs +++ b/src/mito2/src/flush.rs @@ -497,9 +497,7 @@ impl FlushScheduler { &mut self, region_id: RegionId, ) -> Option<(Vec, Vec)> { - let Some(flush_status) = self.region_status.get_mut(®ion_id) else { - return None; - }; + let flush_status = self.region_status.get_mut(®ion_id)?; // This region doesn't have running flush job. flush_status.flushing = false; diff --git a/src/mito2/src/memtable/partition_tree/tree.rs b/src/mito2/src/memtable/partition_tree/tree.rs index ca0b478c87..3f3dd236c4 100644 --- a/src/mito2/src/memtable/partition_tree/tree.rs +++ b/src/mito2/src/memtable/partition_tree/tree.rs @@ -211,8 +211,9 @@ impl PartitionTree { }; let filters = predicate - .map(|p| { - p.exprs() + .map(|predicate| { + predicate + .exprs() .iter() .filter_map(|f| SimpleFilterEvaluator::try_new(f.df_expr())) .collect::>() diff --git a/src/mito2/src/memtable/time_series.rs b/src/mito2/src/memtable/time_series.rs index 3991504e51..fe93882681 100644 --- a/src/mito2/src/memtable/time_series.rs +++ b/src/mito2/src/memtable/time_series.rs @@ -238,7 +238,9 @@ impl Memtable for TimeSeriesMemtable { .collect() }; - let iter = self.series_set.iter_series(projection, filters, self.dedup); + let iter = self + .series_set + .iter_series(projection, filters, self.dedup)?; Ok(Box::new(iter)) } @@ -348,7 +350,7 @@ impl SeriesSet { projection: HashSet, predicate: Option, dedup: bool, - ) -> Iter { + ) -> Result { let primary_key_schema = primary_key_schema(&self.region_metadata); let primary_key_datatypes = self .region_metadata @@ -356,7 +358,7 @@ impl SeriesSet { .map(|pk| pk.column_schema.data_type.clone()) .collect(); - Iter::new( + Iter::try_new( self.region_metadata.clone(), self.series.clone(), projection, @@ -417,7 +419,7 @@ struct Iter { impl Iter { #[allow(clippy::too_many_arguments)] - pub(crate) fn new( + pub(crate) fn try_new( metadata: RegionMetadataRef, series: Arc, projection: HashSet, @@ -426,27 +428,28 @@ impl Iter { pk_datatypes: Vec, codec: Arc, dedup: bool, - ) -> Self { - let simple_filters = predicate - .map(|p| { - p.exprs() + ) -> Result { + let predicate = predicate + .map(|predicate| { + predicate + .exprs() .iter() .filter_map(|f| SimpleFilterEvaluator::try_new(f.df_expr())) .collect::>() }) .unwrap_or_default(); - Self { + Ok(Self { metadata, series, projection, last_key: None, - predicate: simple_filters, + predicate, pk_schema, pk_datatypes, codec, dedup, metrics: Metrics::default(), - } + }) } } diff --git a/src/mito2/src/region/options.rs b/src/mito2/src/region/options.rs index a56976874b..5a2df0d8cc 100644 --- a/src/mito2/src/region/options.rs +++ b/src/mito2/src/region/options.rs @@ -440,7 +440,7 @@ mod tests { // No need to add compatible tests for RegionOptions since the above tests already check for compatibility. #[test] fn test_with_any_wal_options() { - let all_wal_options = vec![ + let all_wal_options = [ WalOptions::RaftEngine, WalOptions::Kafka(KafkaWalOptions { topic: "test_topic".to_string(), diff --git a/src/mito2/src/sst/parquet/reader.rs b/src/mito2/src/sst/parquet/reader.rs index 60aa0afa54..71996eba5b 100644 --- a/src/mito2/src/sst/parquet/reader.rs +++ b/src/mito2/src/sst/parquet/reader.rs @@ -180,11 +180,12 @@ impl ParquetReaderBuilder { metrics.build_cost = start.elapsed(); - let predicate = if let Some(p) = &self.predicate { - p.exprs() + let predicate = if let Some(predicate) = &self.predicate { + predicate + .exprs() .iter() .filter_map(|expr| SimpleFilterEvaluator::try_new(expr.df_expr())) - .collect() + .collect::>() } else { vec![] }; diff --git a/src/mito2/src/sst/parquet/row_selection.rs b/src/mito2/src/sst/parquet/row_selection.rs index 93accf11ac..5ab961c223 100644 --- a/src/mito2/src/sst/parquet/row_selection.rs +++ b/src/mito2/src/sst/parquet/row_selection.rs @@ -82,7 +82,7 @@ mod tests { #[test] fn test_non_contiguous_ranges() { - let ranges = vec![1..3, 5..8]; + let ranges = [1..3, 5..8]; let (selection, skipped) = row_selection_from_row_ranges(ranges.iter().cloned(), 10); let expected = RowSelection::from(vec![ RowSelector::skip(1), @@ -96,7 +96,7 @@ mod tests { #[test] fn test_empty_range() { - let ranges = vec![]; + let ranges = []; let (selection, skipped) = row_selection_from_row_ranges(ranges.iter().cloned(), 10); let expected = RowSelection::from(vec![]); assert_eq!(selection, expected); @@ -105,7 +105,7 @@ mod tests { #[test] fn test_adjacent_ranges() { - let ranges = vec![1..2, 2..3]; + let ranges = [1..2, 2..3]; let (selection, skipped) = row_selection_from_row_ranges(ranges.iter().cloned(), 10); let expected = RowSelection::from(vec![RowSelector::skip(1), RowSelector::select(2)]); assert_eq!(selection, expected); @@ -114,7 +114,7 @@ mod tests { #[test] fn test_large_gap_between_ranges() { - let ranges = vec![1..2, 100..101]; + let ranges = [1..2, 100..101]; let (selection, skipped) = row_selection_from_row_ranges(ranges.iter().cloned(), 10240); let expected = RowSelection::from(vec![ RowSelector::skip(1), diff --git a/src/mito2/src/sst/parquet/stats.rs b/src/mito2/src/sst/parquet/stats.rs index a17e8ace45..43e0e3cfb9 100644 --- a/src/mito2/src/sst/parquet/stats.rs +++ b/src/mito2/src/sst/parquet/stats.rs @@ -18,8 +18,8 @@ use std::borrow::Borrow; use std::collections::HashSet; use datafusion::physical_optimizer::pruning::PruningStatistics; -use datafusion_common::Column; -use datatypes::arrow::array::ArrayRef; +use datafusion_common::{Column, ScalarValue}; +use datatypes::arrow::array::{ArrayRef, BooleanArray}; use parquet::file::metadata::RowGroupMetaData; use store_api::storage::ColumnId; @@ -81,4 +81,14 @@ impl<'a, T: Borrow> PruningStatistics for RowGroupPruningStats let column_id = self.column_id_to_prune(&column.name)?; self.read_format.null_counts(self.row_groups, column_id) } + + fn row_counts(&self, _column: &Column) -> Option { + // TODO(LFC): Impl it. + None + } + + fn contained(&self, _column: &Column, _values: &HashSet) -> Option { + // TODO(LFC): Impl it. + None + } } diff --git a/src/operator/src/delete.rs b/src/operator/src/delete.rs index 60e44cb7c6..46f235123c 100644 --- a/src/operator/src/delete.rs +++ b/src/operator/src/delete.rs @@ -124,6 +124,7 @@ impl Deleter { let request_factory = RegionRequestFactory::new(RegionRequestHeader { tracing_context: TracingContext::from_current_span().to_w3c(), dbname: ctx.get_db_string(), + ..Default::default() }); let tasks = self diff --git a/src/operator/src/expr_factory.rs b/src/operator/src/expr_factory.rs index 0975f9b17d..42d8ee3187 100644 --- a/src/operator/src/expr_factory.rs +++ b/src/operator/src/expr_factory.rs @@ -224,10 +224,15 @@ fn find_primary_keys( let columns_pk = columns .iter() .filter_map(|x| { - if x.options - .iter() - .any(|o| matches!(o.option, ColumnOption::Unique { is_primary: true })) - { + if x.options.iter().any(|o| { + matches!( + o.option, + ColumnOption::Unique { + is_primary: true, + .. + } + ) + }) { Some(x.name.value.clone()) } else { None @@ -249,6 +254,7 @@ fn find_primary_keys( name: _, columns, is_primary: true, + .. } => Some(columns.iter().map(|ident| ident.value.clone())), _ => None, }) @@ -276,6 +282,7 @@ pub fn find_time_index(constraints: &[TableConstraint]) -> Result { name: Some(name), columns, is_primary: false, + .. } => { if name.value == TIME_INDEX { Some(columns.iter().map(|ident| &ident.value)) diff --git a/src/operator/src/insert.rs b/src/operator/src/insert.rs index 5460fe9a59..61d2d2cfcf 100644 --- a/src/operator/src/insert.rs +++ b/src/operator/src/insert.rs @@ -196,6 +196,7 @@ impl Inserter { let request_factory = RegionRequestFactory::new(RegionRequestHeader { tracing_context: TracingContext::from_current_span().to_w3c(), dbname: ctx.get_db_string(), + ..Default::default() }); let tasks = self diff --git a/src/operator/src/request.rs b/src/operator/src/request.rs index 3aba4dfd70..7cfd5a8a1a 100644 --- a/src/operator/src/request.rs +++ b/src/operator/src/request.rs @@ -162,6 +162,7 @@ impl Requester { let request_factory = RegionRequestFactory::new(RegionRequestHeader { tracing_context: TracingContext::from_current_span().to_w3c(), dbname: db_string.unwrap_or_else(|| ctx.get_db_string()), + ..Default::default() }); let tasks = requests.into_iter().map(|req_body| { diff --git a/src/operator/src/statement.rs b/src/operator/src/statement.rs index 17ef9f81fc..c947db82b8 100644 --- a/src/operator/src/statement.rs +++ b/src/operator/src/statement.rs @@ -118,6 +118,10 @@ impl StatementExecutor { Statement::ShowTables(stmt) => self.show_tables(stmt, query_ctx).await, + Statement::ShowCollation(kind) => self.show_collation(kind, query_ctx).await, + + Statement::ShowCharset(kind) => self.show_charset(kind, query_ctx).await, + Statement::Copy(sql::statements::copy::Copy::CopyTable(stmt)) => { let req = to_copy_table_request(stmt, query_ctx.clone())?; match req.direction { diff --git a/src/operator/src/statement/copy_table_from.rs b/src/operator/src/statement/copy_table_from.rs index c8c1ae3688..378c3af317 100644 --- a/src/operator/src/statement/copy_table_from.rs +++ b/src/operator/src/statement/copy_table_from.rs @@ -15,7 +15,6 @@ use std::collections::HashMap; use std::future::Future; use std::sync::Arc; -use std::usize; use client::{Output, OutputData, OutputMeta}; use common_base::readable_size::ReadableSize; @@ -36,6 +35,7 @@ use datafusion::datasource::physical_plan::{FileOpener, FileScanConfig, FileStre use datafusion::parquet::arrow::arrow_reader::ArrowReaderMetadata; use datafusion::parquet::arrow::ParquetRecordBatchStreamBuilder; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; +use datafusion_common::Statistics; use datatypes::arrow::compute::can_cast_types; use datatypes::arrow::datatypes::{Schema, SchemaRef}; use datatypes::vectors::Helper; @@ -184,17 +184,17 @@ impl StatementExecutor { filename: &str, file_schema: SchemaRef, ) -> Result { + let statistics = Statistics::new_unknown(file_schema.as_ref()); let stream = FileStream::new( &FileScanConfig { object_store_url: ObjectStoreUrl::parse("empty://").unwrap(), // won't be used file_schema, file_groups: vec![vec![PartitionedFile::new(filename.to_string(), 10)]], - statistics: Default::default(), + statistics, projection: None, limit: None, table_partition_cols: vec![], output_ordering: vec![], - infinite_source: false, }, 0, opener, diff --git a/src/operator/src/statement/copy_table_to.rs b/src/operator/src/statement/copy_table_to.rs index 2a4d4a0ca7..282bb03979 100644 --- a/src/operator/src/statement/copy_table_to.rs +++ b/src/operator/src/statement/copy_table_to.rs @@ -118,7 +118,7 @@ impl StatementExecutor { let table_source = Arc::new(DefaultTableSource::new(table_provider)); let mut builder = LogicalPlanBuilder::scan_with_filters( - df_table_ref.to_owned_reference(), + df_table_ref, table_source, None, filters.clone(), diff --git a/src/operator/src/statement/show.rs b/src/operator/src/statement/show.rs index 694ba1951e..7ba8bc999f 100644 --- a/src/operator/src/statement/show.rs +++ b/src/operator/src/statement/show.rs @@ -21,7 +21,9 @@ use session::context::QueryContextRef; use snafu::ResultExt; use sql::ast::Ident; use sql::statements::create::Partitions; -use sql::statements::show::{ShowColumns, ShowDatabases, ShowIndex, ShowTables, ShowVariables}; +use sql::statements::show::{ + ShowColumns, ShowDatabases, ShowIndex, ShowKind, ShowTables, ShowVariables, +}; use table::TableRef; use crate::error::{self, ExecuteStatementSnafu, Result}; @@ -97,6 +99,24 @@ impl StatementExecutor { pub fn show_variable(&self, stmt: ShowVariables, query_ctx: QueryContextRef) -> Result { query::sql::show_variable(stmt, query_ctx).context(error::ExecuteStatementSnafu) } + + #[tracing::instrument(skip_all)] + pub async fn show_collation( + &self, + kind: ShowKind, + query_ctx: QueryContextRef, + ) -> Result { + query::sql::show_collations(kind, &self.query_engine, &self.catalog_manager, query_ctx) + .await + .context(error::ExecuteStatementSnafu) + } + + #[tracing::instrument(skip_all)] + pub async fn show_charset(&self, kind: ShowKind, query_ctx: QueryContextRef) -> Result { + query::sql::show_charsets(kind, &self.query_engine, &self.catalog_manager, query_ctx) + .await + .context(error::ExecuteStatementSnafu) + } } pub(crate) fn create_partitions_stmt(partitions: Vec) -> Result> { diff --git a/src/operator/src/statement/tql.rs b/src/operator/src/statement/tql.rs index 1654005d49..96bb341cf1 100644 --- a/src/operator/src/statement/tql.rs +++ b/src/operator/src/statement/tql.rs @@ -39,7 +39,9 @@ impl StatementExecutor { end: eval.end, step: eval.step, query: eval.query, - lookback: eval.lookback.unwrap_or(DEFAULT_LOOKBACK_STRING.to_string()), + lookback: eval + .lookback + .unwrap_or_else(|| DEFAULT_LOOKBACK_STRING.to_string()), }; QueryLanguageParser::parse_promql(&promql, &query_ctx).context(ParseQuerySnafu)? } @@ -48,7 +50,7 @@ impl StatementExecutor { query: explain.query, lookback: explain .lookback - .unwrap_or(DEFAULT_LOOKBACK_STRING.to_string()), + .unwrap_or_else(|| DEFAULT_LOOKBACK_STRING.to_string()), ..PromQuery::default() }; let explain_node_name = if explain.is_verbose { @@ -71,7 +73,7 @@ impl StatementExecutor { query: analyze.query, lookback: analyze .lookback - .unwrap_or(DEFAULT_LOOKBACK_STRING.to_string()), + .unwrap_or_else(|| DEFAULT_LOOKBACK_STRING.to_string()), }; let analyze_node_name = if analyze.is_verbose { ANALYZE_VERBOSE_NODE_NAME diff --git a/src/promql/Cargo.toml b/src/promql/Cargo.toml index 9e7bc01ae8..893b8b24c3 100644 --- a/src/promql/Cargo.toml +++ b/src/promql/Cargo.toml @@ -20,9 +20,12 @@ common-query.workspace = true common-recordbatch.workspace = true common-telemetry.workspace = true datafusion.workspace = true +datafusion-expr.workspace = true +datafusion-functions.workspace = true datatypes.workspace = true futures = "0.3" greptime-proto.workspace = true +itertools.workspace = true lazy_static.workspace = true prometheus.workspace = true promql-parser = "0.1.1" diff --git a/src/promql/src/extension_plan.rs b/src/promql/src/extension_plan.rs index ff2195e532..f8e32fc4dc 100644 --- a/src/promql/src/extension_plan.rs +++ b/src/promql/src/extension_plan.rs @@ -18,6 +18,7 @@ mod instant_manipulate; mod normalize; mod planner; mod range_manipulate; +mod scalar_calculate; mod series_divide; #[cfg(test)] mod test_util; @@ -30,6 +31,7 @@ pub use instant_manipulate::{InstantManipulate, InstantManipulateExec, InstantMa pub use normalize::{SeriesNormalize, SeriesNormalizeExec, SeriesNormalizeStream}; pub use planner::PromExtensionPlanner; pub use range_manipulate::{RangeManipulate, RangeManipulateExec, RangeManipulateStream}; +pub use scalar_calculate::ScalarCalculate; pub use series_divide::{SeriesDivide, SeriesDivideExec, SeriesDivideStream}; pub use union_distinct_on::{UnionDistinctOn, UnionDistinctOnExec, UnionDistinctOnStream}; diff --git a/src/promql/src/extension_plan/empty_metric.rs b/src/promql/src/extension_plan/empty_metric.rs index 74d25d8ebb..f26337b2f6 100644 --- a/src/promql/src/extension_plan/empty_metric.rs +++ b/src/promql/src/extension_plan/empty_metric.rs @@ -20,16 +20,20 @@ use std::sync::Arc; use std::task::{Context, Poll}; use datafusion::arrow::array::ArrayRef; -use datafusion::arrow::datatypes::{DataType, Schema as ArrowSchema, TimeUnit}; -use datafusion::common::{DFField, DFSchema, DFSchemaRef, Result as DataFusionResult, Statistics}; +use datafusion::arrow::datatypes::{DataType, TimeUnit}; +use datafusion::common::arrow::datatypes::Field; +use datafusion::common::stats::Precision; +use datafusion::common::{ + DFSchema, DFSchemaRef, Result as DataFusionResult, Statistics, TableReference, +}; use datafusion::error::DataFusionError; use datafusion::execution::context::{SessionState, TaskContext}; use datafusion::logical_expr::{ExprSchemable, LogicalPlan, UserDefinedLogicalNodeCore}; -use datafusion::physical_expr::{PhysicalExprRef, PhysicalSortExpr}; +use datafusion::physical_expr::{EquivalenceProperties, PhysicalExprRef}; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, RecordBatchStream, - SendableRecordBatchStream, + DisplayAs, DisplayFormatType, ExecutionMode, ExecutionPlan, Partitioning, PlanProperties, + RecordBatchStream, SendableRecordBatchStream, }; use datafusion::physical_planner::PhysicalPlanner; use datafusion::prelude::{col, lit, Expr}; @@ -66,15 +70,14 @@ impl EmptyMetric { field_column_name: String, field_expr: Option, ) -> DataFusionResult { + let qualifier = Some(TableReference::bare("")); let ts_only_schema = build_ts_only_schema(&time_index_column_name); - let mut fields = vec![ts_only_schema.field(0).clone()]; + let mut fields = vec![(qualifier.clone(), Arc::new(ts_only_schema.field(0).clone()))]; if let Some(field_expr) = &field_expr { let field_data_type = field_expr.get_type(&ts_only_schema)?; - fields.push(DFField::new( - Some(""), - &field_column_name, - field_data_type, - true, + fields.push(( + qualifier.clone(), + Arc::new(Field::new(field_column_name, field_data_type, true)), )); } let schema = Arc::new(DFSchema::new_with_metadata(fields, HashMap::new())?); @@ -102,22 +105,23 @@ impl EmptyMetric { .expr .as_ref() .map(|expr| { - physical_planner.create_physical_expr( - expr, - &self.time_index_schema, - &ArrowSchema::from(self.time_index_schema.as_ref()), - session_state, - ) + physical_planner.create_physical_expr(expr, &self.time_index_schema, session_state) }) .transpose()?; - + let result_schema: SchemaRef = Arc::new(self.result_schema.as_ref().into()); + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(result_schema.clone()), + Partitioning::UnknownPartitioning(1), + ExecutionMode::Bounded, + )); Ok(Arc::new(EmptyMetricExec { start: self.start, end: self.end, interval: self.interval, time_index_schema: Arc::new(self.time_index_schema.as_ref().into()), - result_schema: Arc::new(self.result_schema.as_ref().into()), + result_schema, expr: physical_expr, + properties, metric: ExecutionPlanMetricsSet::new(), })) } @@ -137,7 +141,11 @@ impl UserDefinedLogicalNodeCore for EmptyMetric { } fn expressions(&self) -> Vec { - vec![] + if let Some(expr) = &self.expr { + vec![expr.clone()] + } else { + vec![] + } } fn fmt_for_explain(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { @@ -148,8 +156,19 @@ impl UserDefinedLogicalNodeCore for EmptyMetric { ) } - fn from_template(&self, _expr: &[Expr], _inputs: &[LogicalPlan]) -> Self { - self.clone() + fn from_template(&self, expr: &[Expr], _inputs: &[LogicalPlan]) -> Self { + Self { + start: self.start, + end: self.end, + interval: self.interval, + expr: if !expr.is_empty() { + Some(expr[0].clone()) + } else { + None + }, + time_index_schema: self.time_index_schema.clone(), + result_schema: self.result_schema.clone(), + } } } @@ -164,7 +183,7 @@ pub struct EmptyMetricExec { /// Schema of the output record batch result_schema: SchemaRef, expr: Option, - + properties: Arc, metric: ExecutionPlanMetricsSet, } @@ -177,12 +196,8 @@ impl ExecutionPlan for EmptyMetricExec { self.result_schema.clone() } - fn output_partitioning(&self) -> Partitioning { - Partitioning::UnknownPartitioning(1) - } - - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { - None + fn properties(&self) -> &PlanProperties { + self.properties.as_ref() } fn maintains_input_order(&self) -> Vec { @@ -222,16 +237,15 @@ impl ExecutionPlan for EmptyMetricExec { Some(self.metric.clone_inner()) } - fn statistics(&self) -> Statistics { + fn statistics(&self) -> DataFusionResult { let estimated_row_num = (self.end - self.start) as f64 / self.interval as f64; let total_byte_size = estimated_row_num * std::mem::size_of::() as f64; - Statistics { - num_rows: Some(estimated_row_num.floor() as _), - total_byte_size: Some(total_byte_size.floor() as _), - column_statistics: None, - is_exact: true, - } + Ok(Statistics { + num_rows: Precision::Inexact(estimated_row_num.floor() as _), + total_byte_size: Precision::Inexact(total_byte_size.floor() as _), + column_statistics: Statistics::unknown_column(&self.schema()), + }) } } @@ -285,21 +299,21 @@ impl Stream for EmptyMetricStream { let num_rows = time_array.len(); let input_record_batch = RecordBatch::try_new(self.time_index_schema.clone(), vec![time_array.clone()]) - .map_err(DataFusionError::ArrowError)?; + .map_err(|e| DataFusionError::ArrowError(e, None))?; let mut result_arrays: Vec = vec![time_array]; // evaluate the field expr and get the result if let Some(field_expr) = &self.expr { result_arrays.push( field_expr - .evaluate(&input_record_batch)? - .into_array(num_rows), + .evaluate(&input_record_batch) + .and_then(|x| x.into_array(num_rows))?, ); } // assemble the output record batch let batch = RecordBatch::try_new(self.result_schema.clone(), result_arrays) - .map_err(DataFusionError::ArrowError); + .map_err(|e| DataFusionError::ArrowError(e, None)); Poll::Ready(Some(batch)) } else { @@ -311,14 +325,17 @@ impl Stream for EmptyMetricStream { /// Build a schema that only contains **millisecond** timestamp column fn build_ts_only_schema(column_name: &str) -> DFSchema { - let ts_field = DFField::new( - Some(""), + let ts_field = Field::new( column_name, DataType::Timestamp(TimeUnit::Millisecond, None), false, ); // safety: should not fail (UT covers this) - DFSchema::new_with_metadata(vec![ts_field], HashMap::new()).unwrap() + DFSchema::new_with_metadata( + vec![(Some(TableReference::bare("")), Arc::new(ts_field))], + HashMap::new(), + ) + .unwrap() } // Convert timestamp column to UNIX epoch second: diff --git a/src/promql/src/extension_plan/histogram_fold.rs b/src/promql/src/extension_plan/histogram_fold.rs index c5fabd4a0f..13315a22a0 100644 --- a/src/promql/src/extension_plan/histogram_fold.rs +++ b/src/promql/src/extension_plan/histogram_fold.rs @@ -24,16 +24,17 @@ use datafusion::arrow::array::AsArray; use datafusion::arrow::compute::{self, concat_batches, SortOptions}; use datafusion::arrow::datatypes::{DataType, Float64Type, SchemaRef}; use datafusion::arrow::record_batch::RecordBatch; -use datafusion::common::{DFSchema, DFSchemaRef}; +use datafusion::common::stats::Precision; +use datafusion::common::{ColumnStatistics, DFSchema, DFSchemaRef, Statistics}; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::execution::TaskContext; use datafusion::logical_expr::{LogicalPlan, UserDefinedLogicalNodeCore}; -use datafusion::physical_expr::{PhysicalSortExpr, PhysicalSortRequirement}; +use datafusion::physical_expr::{EquivalenceProperties, PhysicalSortRequirement}; use datafusion::physical_plan::expressions::{CastExpr as PhyCast, Column as PhyColumn}; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, Partitioning, PhysicalExpr, - RecordBatchStream, SendableRecordBatchStream, Statistics, + PlanProperties, RecordBatchStream, SendableRecordBatchStream, }; use datafusion::prelude::{Column, Expr}; use datatypes::prelude::{ConcreteDataType, DataType as GtDataType}; @@ -141,16 +142,13 @@ impl HistogramFold { ) -> DataFusionResult<()> { let check_column = |col| { if !input_schema.has_column_with_unqualified_name(col) { - return Err(DataFusionError::SchemaError( + Err(DataFusionError::SchemaError( datafusion::common::SchemaError::FieldNotFound { field: Box::new(Column::new(None::, col)), - valid_fields: input_schema - .fields() - .iter() - .map(|f| f.qualified_column()) - .collect(), + valid_fields: input_schema.columns(), }, - )); + Box::new(None), + )) } else { Ok(()) } @@ -166,25 +164,29 @@ impl HistogramFold { // safety: those fields are checked in `check_schema()` let le_column_index = input_schema .index_of_column_by_name(None, &self.le_column) - .unwrap() .unwrap(); let field_column_index = input_schema .index_of_column_by_name(None, &self.field_column) - .unwrap() .unwrap(); let ts_column_index = input_schema .index_of_column_by_name(None, &self.ts_column) - .unwrap() .unwrap(); + let output_schema: SchemaRef = Arc::new(self.output_schema.as_ref().into()); + let properties = PlanProperties::new( + EquivalenceProperties::new(output_schema.clone()), + Partitioning::UnknownPartitioning(1), + exec_input.properties().execution_mode(), + ); Arc::new(HistogramFoldExec { le_column_index, field_column_index, ts_column_index, input: exec_input, quantile: self.quantile.into(), - output_schema: Arc::new(self.output_schema.as_ref().into()), + output_schema, metric: ExecutionPlanMetricsSet::new(), + properties, }) } @@ -195,15 +197,16 @@ impl HistogramFold { input_schema: &DFSchemaRef, le_column: &str, ) -> DataFusionResult { - let mut fields = input_schema.fields().clone(); + let fields = input_schema.fields(); // safety: those fields are checked in `check_schema()` - let le_column_idx = input_schema - .index_of_column_by_name(None, le_column)? - .unwrap(); - fields.remove(le_column_idx); - + let mut new_fields = Vec::with_capacity(fields.len() - 1); + for f in fields { + if f.name() != le_column { + new_fields.push((None, f.clone())); + } + } Ok(Arc::new(DFSchema::new_with_metadata( - fields, + new_fields, HashMap::new(), )?)) } @@ -220,6 +223,7 @@ pub struct HistogramFoldExec { ts_column_index: usize, quantile: f64, metric: ExecutionPlanMetricsSet, + properties: PlanProperties, } impl ExecutionPlan for HistogramFoldExec { @@ -227,16 +231,8 @@ impl ExecutionPlan for HistogramFoldExec { self } - fn schema(&self) -> SchemaRef { - self.output_schema.clone() - } - - fn output_partitioning(&self) -> Partitioning { - self.input.output_partitioning() - } - - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { - self.input.output_ordering() + fn properties(&self) -> &PlanProperties { + &self.properties } fn required_input_ordering(&self) -> Vec>> { @@ -276,8 +272,7 @@ impl ExecutionPlan for HistogramFoldExec { } fn required_input_distribution(&self) -> Vec { - // partition on all tag columns, i.e., non-le, non-ts and non-field columns - vec![Distribution::HashPartitioned(self.tag_col_exprs())] + vec![Distribution::SinglePartition; self.children().len()] } fn maintains_input_order(&self) -> Vec { @@ -302,6 +297,7 @@ impl ExecutionPlan for HistogramFoldExec { quantile: self.quantile, output_schema: self.output_schema.clone(), field_column_index: self.field_column_index, + properties: self.properties.clone(), })) } @@ -343,13 +339,16 @@ impl ExecutionPlan for HistogramFoldExec { Some(self.metric.clone_inner()) } - fn statistics(&self) -> Statistics { - Statistics { - num_rows: None, - total_byte_size: None, - column_statistics: None, - is_exact: false, - } + fn statistics(&self) -> DataFusionResult { + Ok(Statistics { + num_rows: Precision::Absent, + total_byte_size: Precision::Absent, + column_statistics: vec![ + ColumnStatistics::new_unknown(); + // plus one more for the removed column by function `convert_schema` + self.schema().all_fields().len() + 1 + ], + }) } } @@ -594,7 +593,7 @@ impl HistogramFoldStream { self.output_buffered_rows = 0; RecordBatch::try_new(self.output_schema.clone(), columns) .map(Some) - .map_err(DataFusionError::ArrowError) + .map_err(|e| DataFusionError::ArrowError(e, None)) } /// Find the first `+Inf` which indicates the end of the bucket group @@ -695,6 +694,7 @@ mod test { use datafusion::arrow::datatypes::{Field, Schema}; use datafusion::common::ToDFSchema; use datafusion::physical_plan::memory::MemoryExec; + use datafusion::physical_plan::ExecutionMode; use datafusion::prelude::SessionContext; use datatypes::arrow_array::StringArray; @@ -759,7 +759,7 @@ mod test { #[tokio::test] async fn fold_overall() { let memory_exec = Arc::new(prepare_test_data()); - let output_schema = Arc::new( + let output_schema: SchemaRef = Arc::new( (*HistogramFold::convert_schema( &Arc::new(memory_exec.schema().to_dfschema().unwrap()), "le", @@ -769,6 +769,11 @@ mod test { .clone() .into(), ); + let properties = PlanProperties::new( + EquivalenceProperties::new(output_schema.clone()), + Partitioning::UnknownPartitioning(1), + ExecutionMode::Bounded, + ); let fold_exec = Arc::new(HistogramFoldExec { le_column_index: 1, field_column_index: 2, @@ -777,6 +782,7 @@ mod test { input: memory_exec, output_schema, metric: ExecutionPlanMetricsSet::new(), + properties, }); let session_context = SessionContext::default(); diff --git a/src/promql/src/extension_plan/instant_manipulate.rs b/src/promql/src/extension_plan/instant_manipulate.rs index e65592bb37..03e2c373ee 100644 --- a/src/promql/src/extension_plan/instant_manipulate.rs +++ b/src/promql/src/extension_plan/instant_manipulate.rs @@ -21,14 +21,14 @@ use std::task::{Context, Poll}; use datafusion::arrow::array::{Array, Float64Array, TimestampMillisecondArray, UInt64Array}; use datafusion::arrow::datatypes::SchemaRef; use datafusion::arrow::record_batch::RecordBatch; -use datafusion::common::{DFSchema, DFSchemaRef}; +use datafusion::common::stats::Precision; +use datafusion::common::{ColumnStatistics, DFSchema, DFSchemaRef}; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::execution::context::TaskContext; use datafusion::logical_expr::{EmptyRelation, Expr, LogicalPlan, UserDefinedLogicalNodeCore}; -use datafusion::physical_expr::PhysicalSortExpr; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, RecordBatchStream, + DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, PlanProperties, RecordBatchStream, SendableRecordBatchStream, Statistics, }; use datatypes::arrow::compute; @@ -194,16 +194,17 @@ impl ExecutionPlan for InstantManipulateExec { self.input.schema() } - fn output_partitioning(&self) -> Partitioning { - self.input.output_partitioning() + fn properties(&self) -> &PlanProperties { + self.input.properties() } - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { - self.input.output_ordering() + fn required_input_distribution(&self) -> Vec { + self.input.required_input_distribution() } + // Prevent reordering of input fn maintains_input_order(&self) -> Vec { - vec![true; self.children().len()] + vec![false; self.children().len()] } fn children(&self) -> Vec> { @@ -262,23 +263,28 @@ impl ExecutionPlan for InstantManipulateExec { Some(self.metric.clone_inner()) } - fn statistics(&self) -> Statistics { - let input_stats = self.input.statistics(); + fn statistics(&self) -> DataFusionResult { + let input_stats = self.input.statistics()?; let estimated_row_num = (self.end - self.start) as f64 / self.interval as f64; let estimated_total_bytes = input_stats .total_byte_size - .zip(input_stats.num_rows) - .map(|(size, rows)| (size as f64 / rows as f64) * estimated_row_num) - .map(|size| size.floor() as _); + .get_value() + .zip(input_stats.num_rows.get_value()) + .map(|(size, rows)| { + Precision::Inexact(((*size as f64 / *rows as f64) * estimated_row_num).floor() as _) + }) + .unwrap_or(Precision::Absent); - Statistics { - num_rows: Some(estimated_row_num.floor() as _), + Ok(Statistics { + num_rows: Precision::Inexact(estimated_row_num.floor() as _), total_byte_size: estimated_total_bytes, // TODO(ruihang): support this column statistics - column_statistics: None, - is_exact: false, - } + column_statistics: vec![ + ColumnStatistics::new_unknown(); + self.schema().all_fields().len() + ], + }) } } @@ -438,7 +444,7 @@ impl InstantManipulateStream { arrays[self.time_index] = Arc::new(TimestampMillisecondArray::from(aligned_ts)); let result = RecordBatch::try_new(record_batch.schema(), arrays) - .map_err(DataFusionError::ArrowError)?; + .map_err(|e| DataFusionError::ArrowError(e, None))?; Ok(result) } } diff --git a/src/promql/src/extension_plan/normalize.rs b/src/promql/src/extension_plan/normalize.rs index 968e8a6d37..957c55fade 100644 --- a/src/promql/src/extension_plan/normalize.rs +++ b/src/promql/src/extension_plan/normalize.rs @@ -23,10 +23,9 @@ use datafusion::common::{DFSchema, DFSchemaRef, Result as DataFusionResult, Stat use datafusion::error::DataFusionError; use datafusion::execution::context::TaskContext; use datafusion::logical_expr::{EmptyRelation, Expr, LogicalPlan, UserDefinedLogicalNodeCore}; -use datafusion::physical_expr::PhysicalSortExpr; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, Partitioning, RecordBatchStream, + DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, PlanProperties, RecordBatchStream, SendableRecordBatchStream, }; use datatypes::arrow::array::TimestampMillisecondArray; @@ -170,12 +169,8 @@ impl ExecutionPlan for SeriesNormalizeExec { vec![Distribution::SinglePartition] } - fn output_partitioning(&self) -> Partitioning { - self.input.output_partitioning() - } - - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { - self.input.output_ordering() + fn properties(&self) -> &PlanProperties { + self.input.properties() } fn children(&self) -> Vec> { @@ -223,7 +218,7 @@ impl ExecutionPlan for SeriesNormalizeExec { Some(self.metric.clone_inner()) } - fn statistics(&self) -> Statistics { + fn statistics(&self) -> DataFusionResult { self.input.statistics() } } @@ -299,7 +294,7 @@ impl SeriesNormalizeStream { } let result = compute::filter_record_batch(&ordered_batch, &BooleanArray::from(filter)) - .map_err(DataFusionError::ArrowError)?; + .map_err(|e| DataFusionError::ArrowError(e, None))?; Ok(result) } } diff --git a/src/promql/src/extension_plan/planner.rs b/src/promql/src/extension_plan/planner.rs index 80cd565bd2..10b6efd438 100644 --- a/src/promql/src/extension_plan/planner.rs +++ b/src/promql/src/extension_plan/planner.rs @@ -21,7 +21,7 @@ use datafusion::logical_expr::{LogicalPlan, UserDefinedLogicalNode}; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_planner::{ExtensionPlanner, PhysicalPlanner}; -use super::{HistogramFold, UnionDistinctOn}; +use super::{HistogramFold, ScalarCalculate, UnionDistinctOn}; use crate::extension_plan::{ EmptyMetric, InstantManipulate, RangeManipulate, SeriesDivide, SeriesNormalize, }; @@ -48,6 +48,8 @@ impl ExtensionPlanner for PromExtensionPlanner { Ok(Some(node.to_execution_plan(physical_inputs[0].clone()))) } else if let Some(node) = node.as_any().downcast_ref::() { Ok(Some(node.to_execution_plan(session_state, planner)?)) + } else if let Some(node) = node.as_any().downcast_ref::() { + Ok(Some(node.to_execution_plan(physical_inputs[0].clone())?)) } else if let Some(node) = node.as_any().downcast_ref::() { Ok(Some(node.to_execution_plan(physical_inputs[0].clone()))) } else if let Some(node) = node.as_any().downcast_ref::() { diff --git a/src/promql/src/extension_plan/range_manipulate.rs b/src/promql/src/extension_plan/range_manipulate.rs index e6e93d8b50..49002dabfa 100644 --- a/src/promql/src/extension_plan/range_manipulate.rs +++ b/src/promql/src/extension_plan/range_manipulate.rs @@ -23,14 +23,15 @@ use datafusion::arrow::compute; use datafusion::arrow::datatypes::{Field, SchemaRef}; use datafusion::arrow::error::ArrowError; use datafusion::arrow::record_batch::RecordBatch; -use datafusion::common::{DFField, DFSchema, DFSchemaRef}; +use datafusion::common::stats::Precision; +use datafusion::common::{DFSchema, DFSchemaRef}; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::execution::context::TaskContext; use datafusion::logical_expr::{EmptyRelation, Expr, LogicalPlan, UserDefinedLogicalNodeCore}; -use datafusion::physical_expr::PhysicalSortExpr; +use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, RecordBatchStream, + DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, PlanProperties, RecordBatchStream, SendableRecordBatchStream, Statistics, }; use datafusion::sql::TableReference; @@ -110,44 +111,55 @@ impl RangeManipulate { time_index: &str, field_columns: &[String], ) -> DataFusionResult { - let mut columns = input_schema.fields().clone(); + let columns = input_schema.fields(); + let mut new_columns = Vec::with_capacity(columns.len() + 1); + for i in 0..columns.len() { + let x = input_schema.qualified_field(i); + new_columns.push((x.0.cloned(), Arc::new(x.1.clone()))); + } // process time index column // the raw timestamp field is preserved. And a new timestamp_range field is appended to the last. - let Some(ts_col_index) = input_schema.index_of_column_by_name(None, time_index)? else { + let Some(ts_col_index) = input_schema.index_of_column_by_name(None, time_index) else { return Err(datafusion::common::field_not_found( None::, time_index, input_schema.as_ref(), )); }; - let ts_col_field = columns[ts_col_index].field(); + let ts_col_field = &columns[ts_col_index]; let timestamp_range_field = Field::new( Self::build_timestamp_range_name(time_index), RangeArray::convert_field(ts_col_field).data_type().clone(), ts_col_field.is_nullable(), ); - columns.push(DFField::from(timestamp_range_field)); + new_columns.push((None, Arc::new(timestamp_range_field))); // process value columns for name in field_columns { - let Some(index) = input_schema.index_of_column_by_name(None, name)? else { + let Some(index) = input_schema.index_of_column_by_name(None, name) else { return Err(datafusion::common::field_not_found( None::, name, input_schema.as_ref(), )); }; - columns[index] = DFField::from(RangeArray::convert_field(columns[index].field())); + new_columns[index] = (None, Arc::new(RangeArray::convert_field(&columns[index]))); } Ok(Arc::new(DFSchema::new_with_metadata( - columns, + new_columns, HashMap::new(), )?)) } pub fn to_execution_plan(&self, exec_input: Arc) -> Arc { + let output_schema: SchemaRef = SchemaRef::new(self.output_schema.as_ref().into()); + let properties = PlanProperties::new( + EquivalenceProperties::new(output_schema.clone()), + exec_input.properties().partitioning.clone(), + exec_input.properties().execution_mode, + ); Arc::new(RangeManipulateExec { start: self.start, end: self.end, @@ -157,8 +169,9 @@ impl RangeManipulate { time_range_column: self.range_timestamp_name(), field_columns: self.field_columns.clone(), input: exec_input, - output_schema: SchemaRef::new(self.output_schema.as_ref().into()), + output_schema, metric: ExecutionPlanMetricsSet::new(), + properties, }) } @@ -247,6 +260,7 @@ pub struct RangeManipulateExec { input: Arc, output_schema: SchemaRef, metric: ExecutionPlanMetricsSet, + properties: PlanProperties, } impl ExecutionPlan for RangeManipulateExec { @@ -258,12 +272,8 @@ impl ExecutionPlan for RangeManipulateExec { self.output_schema.clone() } - fn output_partitioning(&self) -> Partitioning { - self.input.output_partitioning() - } - - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { - self.input.output_ordering() + fn properties(&self) -> &PlanProperties { + &self.properties } fn maintains_input_order(&self) -> Vec { @@ -274,6 +284,10 @@ impl ExecutionPlan for RangeManipulateExec { vec![self.input.clone()] } + fn required_input_distribution(&self) -> Vec { + vec![Distribution::SinglePartition] + } + fn with_new_children( self: Arc, children: Vec>, @@ -290,6 +304,7 @@ impl ExecutionPlan for RangeManipulateExec { output_schema: self.output_schema.clone(), input: children[0].clone(), metric: self.metric.clone(), + properties: self.properties.clone(), })) } @@ -333,23 +348,25 @@ impl ExecutionPlan for RangeManipulateExec { Some(self.metric.clone_inner()) } - fn statistics(&self) -> Statistics { - let input_stats = self.input.statistics(); + fn statistics(&self) -> DataFusionResult { + let input_stats = self.input.statistics()?; let estimated_row_num = (self.end - self.start) as f64 / self.interval as f64; let estimated_total_bytes = input_stats .total_byte_size - .zip(input_stats.num_rows) - .map(|(size, rows)| (size as f64 / rows as f64) * estimated_row_num) - .map(|size| size.floor() as _); + .get_value() + .zip(input_stats.num_rows.get_value()) + .map(|(size, rows)| { + Precision::Inexact(((*size as f64 / *rows as f64) * estimated_row_num).floor() as _) + }) + .unwrap_or_default(); - Statistics { - num_rows: Some(estimated_row_num.floor() as _), + Ok(Statistics { + num_rows: Precision::Inexact(estimated_row_num as _), total_byte_size: estimated_total_bytes, // TODO(ruihang): support this column statistics - column_statistics: None, - is_exact: false, - } + column_statistics: Statistics::unknown_column(&self.schema()), + }) } } @@ -452,7 +469,7 @@ impl RangeManipulateStream { RecordBatch::try_new(self.output_schema.clone(), new_columns) .map(Some) - .map_err(DataFusionError::ArrowError) + .map_err(|e| DataFusionError::ArrowError(e, None)) } fn calculate_range(&self, input: &RecordBatch) -> (ArrayRef, Vec<(u32, u32)>) { @@ -500,7 +517,9 @@ mod test { ArrowPrimitiveType, DataType, Field, Int64Type, Schema, TimestampMillisecondType, }; use datafusion::common::ToDFSchema; + use datafusion::physical_expr::Partitioning; use datafusion::physical_plan::memory::MemoryExec; + use datafusion::physical_plan::ExecutionMode; use datafusion::prelude::SessionContext; use datatypes::arrow::array::TimestampMillisecondArray; @@ -556,6 +575,11 @@ mod test { .as_ref() .into(), ); + let properties = PlanProperties::new( + EquivalenceProperties::new(manipulate_output_schema.clone()), + Partitioning::UnknownPartitioning(1), + ExecutionMode::Bounded, + ); let normalize_exec = Arc::new(RangeManipulateExec { start, end, @@ -567,6 +591,7 @@ mod test { time_index_column: time_index, input: memory_exec, metric: ExecutionPlanMetricsSet::new(), + properties, }); let session_context = SessionContext::default(); let result = datafusion::physical_plan::collect(normalize_exec, session_context.task_ctx()) diff --git a/src/promql/src/extension_plan/scalar_calculate.rs b/src/promql/src/extension_plan/scalar_calculate.rs new file mode 100644 index 0000000000..19dfd8283c --- /dev/null +++ b/src/promql/src/extension_plan/scalar_calculate.rs @@ -0,0 +1,602 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::any::Any; +use std::collections::HashMap; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use datafusion::common::stats::Precision; +use datafusion::common::{DFSchema, DFSchemaRef, Result as DataFusionResult, Statistics}; +use datafusion::error::DataFusionError; +use datafusion::execution::context::TaskContext; +use datafusion::logical_expr::{EmptyRelation, LogicalPlan, UserDefinedLogicalNodeCore}; +use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, Partitioning, PlanProperties, + RecordBatchStream, SendableRecordBatchStream, +}; +use datafusion::prelude::Expr; +use datafusion::sql::TableReference; +use datatypes::arrow::array::{Array, Float64Array, StringArray, TimestampMillisecondArray}; +use datatypes::arrow::compute::{cast_with_options, concat_batches, CastOptions}; +use datatypes::arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; +use datatypes::arrow::record_batch::RecordBatch; +use futures::{ready, Stream, StreamExt}; +use greptime_proto::substrait_extension as pb; +use prost::Message; +use snafu::ResultExt; + +use super::Millisecond; +use crate::error::{ColumnNotFoundSnafu, DataFusionPlanningSnafu, DeserializeSnafu, Result}; + +/// `ScalarCalculate` is the custom logical plan to calculate +/// [`scalar`](https://prometheus.io/docs/prometheus/latest/querying/functions/#scalar) +/// in PromQL, return NaN when have multiple time series. +/// return the time series as scalar value when only have one time series. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ScalarCalculate { + start: Millisecond, + end: Millisecond, + interval: Millisecond, + + time_index: String, + tag_columns: Vec, + field_column: String, + input: LogicalPlan, + output_schema: DFSchemaRef, +} + +impl ScalarCalculate { + /// create a new `ScalarCalculate` plan + #[allow(clippy::too_many_arguments)] + pub fn new( + start: Millisecond, + end: Millisecond, + interval: Millisecond, + input: LogicalPlan, + time_index: &str, + tag_colunms: &[String], + field_column: &str, + table_name: Option<&str>, + ) -> Result { + let input_schema = input.schema(); + let Ok(ts_field) = input_schema + .field_with_unqualified_name(time_index) + .cloned() + else { + return ColumnNotFoundSnafu { col: time_index }.fail(); + }; + let val_field = Field::new(format!("scalar({})", field_column), DataType::Float64, true); + let qualifier = table_name.map(TableReference::bare); + let schema = DFSchema::new_with_metadata( + vec![ + (qualifier.clone(), Arc::new(ts_field)), + (qualifier, Arc::new(val_field)), + ], + input_schema.metadata().clone(), + ) + .context(DataFusionPlanningSnafu)?; + + Ok(Self { + start, + end, + interval, + time_index: time_index.to_string(), + tag_columns: tag_colunms.to_vec(), + field_column: field_column.to_string(), + input, + output_schema: Arc::new(schema), + }) + } + + /// The name of this custom plan + pub const fn name() -> &'static str { + "ScalarCalculate" + } + + /// Create a new execution plan from ScalarCalculate + pub fn to_execution_plan( + &self, + exec_input: Arc, + ) -> DataFusionResult> { + let fields: Vec<_> = self + .output_schema + .fields() + .iter() + .map(|field| Field::new(field.name(), field.data_type().clone(), field.is_nullable())) + .collect(); + let input_schema = exec_input.schema(); + let ts_index = input_schema + .index_of(&self.time_index) + .map_err(|e| DataFusionError::ArrowError(e, None))?; + let val_index = input_schema + .index_of(&self.field_column) + .map_err(|e| DataFusionError::ArrowError(e, None))?; + let schema = Arc::new(Schema::new(fields)); + let properties = PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + Partitioning::UnknownPartitioning(1), + exec_input.properties().execution_mode, + ); + Ok(Arc::new(ScalarCalculateExec { + start: self.start, + end: self.end, + interval: self.interval, + schema, + input: exec_input, + project_index: (ts_index, val_index), + tag_columns: self.tag_columns.clone(), + metric: ExecutionPlanMetricsSet::new(), + properties, + })) + } + + pub fn serialize(&self) -> Vec { + pb::ScalarCalculate { + start: self.start, + end: self.end, + interval: self.interval, + time_index: self.time_index.clone(), + tag_columns: self.tag_columns.clone(), + field_column: self.field_column.clone(), + } + .encode_to_vec() + } + + pub fn deserialize(bytes: &[u8]) -> Result { + let pb_scalar_calculate = pb::ScalarCalculate::decode(bytes).context(DeserializeSnafu)?; + let placeholder_plan = LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: Arc::new(DFSchema::empty()), + }); + // TODO(Taylor-lagrange): Supports timestamps of different precisions + let ts_field = Field::new( + &pb_scalar_calculate.time_index, + DataType::Timestamp(TimeUnit::Millisecond, None), + true, + ); + let val_field = Field::new( + format!("scalar({})", pb_scalar_calculate.field_column), + DataType::Float64, + true, + ); + // TODO(Taylor-lagrange): missing tablename in pb + let schema = DFSchema::new_with_metadata( + vec![(None, Arc::new(ts_field)), (None, Arc::new(val_field))], + HashMap::new(), + ) + .context(DataFusionPlanningSnafu)?; + + Ok(Self { + start: pb_scalar_calculate.start, + end: pb_scalar_calculate.end, + interval: pb_scalar_calculate.interval, + time_index: pb_scalar_calculate.time_index, + tag_columns: pb_scalar_calculate.tag_columns, + field_column: pb_scalar_calculate.field_column, + output_schema: Arc::new(schema), + input: placeholder_plan, + }) + } +} + +impl UserDefinedLogicalNodeCore for ScalarCalculate { + fn name(&self) -> &str { + Self::name() + } + + fn inputs(&self) -> Vec<&LogicalPlan> { + vec![&self.input] + } + + fn schema(&self) -> &DFSchemaRef { + &self.output_schema + } + + fn expressions(&self) -> Vec { + vec![] + } + + fn fmt_for_explain(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "ScalarCalculate: tags={:?}", self.tag_columns) + } + + fn from_template(&self, _expr: &[Expr], inputs: &[LogicalPlan]) -> Self { + assert!(!inputs.is_empty()); + ScalarCalculate { + start: self.start, + end: self.end, + interval: self.interval, + time_index: self.time_index.clone(), + tag_columns: self.tag_columns.clone(), + field_column: self.field_column.clone(), + input: inputs[0].clone(), + output_schema: self.output_schema.clone(), + } + } +} + +#[derive(Debug, Clone)] +struct ScalarCalculateExec { + start: Millisecond, + end: Millisecond, + interval: Millisecond, + schema: SchemaRef, + project_index: (usize, usize), + input: Arc, + tag_columns: Vec, + metric: ExecutionPlanMetricsSet, + properties: PlanProperties, +} + +impl ExecutionPlan for ScalarCalculateExec { + fn as_any(&self) -> &dyn Any { + self + } + + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + fn properties(&self) -> &PlanProperties { + &self.properties + } + + fn maintains_input_order(&self) -> Vec { + vec![true; self.children().len()] + } + + fn required_input_distribution(&self) -> Vec { + vec![Distribution::SinglePartition] + } + + fn children(&self) -> Vec> { + vec![self.input.clone()] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DataFusionResult> { + Ok(Arc::new(ScalarCalculateExec { + start: self.start, + end: self.end, + interval: self.interval, + schema: self.schema.clone(), + project_index: self.project_index, + tag_columns: self.tag_columns.clone(), + input: children[0].clone(), + metric: self.metric.clone(), + properties: self.properties.clone(), + })) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> DataFusionResult { + let baseline_metric = BaselineMetrics::new(&self.metric, partition); + let input = self.input.execute(partition, context)?; + let schema = input.schema(); + let tag_indices = self + .tag_columns + .iter() + .map(|tag| { + schema + .column_with_name(tag) + .unwrap_or_else(|| panic!("tag column not found {tag}")) + .0 + }) + .collect(); + + Ok(Box::pin(ScalarCalculateStream { + start: self.start, + end: self.end, + interval: self.interval, + schema: self.schema.clone(), + project_index: self.project_index, + metric: baseline_metric, + tag_indices, + input, + have_multi_series: false, + done: false, + batch: None, + tag_value: None, + })) + } + + fn metrics(&self) -> Option { + Some(self.metric.clone_inner()) + } + + fn statistics(&self) -> DataFusionResult { + let input_stats = self.input.statistics()?; + + let estimated_row_num = (self.end - self.start) as f64 / self.interval as f64; + let estimated_total_bytes = input_stats + .total_byte_size + .get_value() + .zip(input_stats.num_rows.get_value()) + .map(|(size, rows)| { + Precision::Inexact(((*size as f64 / *rows as f64) * estimated_row_num).floor() as _) + }) + .unwrap_or_default(); + + Ok(Statistics { + num_rows: Precision::Inexact(estimated_row_num as _), + total_byte_size: estimated_total_bytes, + // TODO(ruihang): support this column statistics + column_statistics: Statistics::unknown_column(&self.schema()), + }) + } +} + +impl DisplayAs for ScalarCalculateExec { + fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match t { + DisplayFormatType::Default | DisplayFormatType::Verbose => { + write!(f, "ScalarCalculateExec: tags={:?}", self.tag_columns) + } + } + } +} + +struct ScalarCalculateStream { + start: Millisecond, + end: Millisecond, + interval: Millisecond, + schema: SchemaRef, + input: SendableRecordBatchStream, + metric: BaselineMetrics, + tag_indices: Vec, + /// with format `(ts_index, field_index)` + project_index: (usize, usize), + have_multi_series: bool, + done: bool, + batch: Option, + tag_value: Option>, +} + +impl RecordBatchStream for ScalarCalculateStream { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } +} + +impl ScalarCalculateStream { + fn update_batch(&mut self, batch: RecordBatch) -> DataFusionResult<()> { + let _timer = self.metric.elapsed_compute(); + // if have multi time series, scalar will return NaN + if self.have_multi_series { + return Ok(()); + } + // fast path: no tag columns means all data belongs to the same series. + if self.tag_indices.is_empty() { + self.append_batch(batch)?; + return Ok(()); + } + let all_same = |val: Option<&str>, array: &StringArray| -> bool { + if let Some(v) = val { + array.iter().all(|s| s == Some(v)) + } else { + array.is_empty() || array.iter().skip(1).all(|s| s == Some(array.value(0))) + } + }; + // assert the entire batch belong to the same series + let all_tag_columns_same = if let Some(tags) = &self.tag_value { + tags.iter() + .zip(self.tag_indices.iter()) + .all(|(value, index)| { + let array = batch.column(*index); + let string_array = array.as_any().downcast_ref::().unwrap(); + all_same(Some(value), string_array) + }) + } else { + let mut tag_values = Vec::with_capacity(self.tag_indices.len()); + let is_same = self.tag_indices.iter().all(|index| { + let array = batch.column(*index); + let string_array = array.as_any().downcast_ref::().unwrap(); + tag_values.push(string_array.value(0).to_string()); + all_same(None, string_array) + }); + self.tag_value = Some(tag_values); + is_same + }; + if all_tag_columns_same { + self.append_batch(batch)?; + } else { + self.have_multi_series = true; + } + Ok(()) + } + + fn append_batch(&mut self, input_batch: RecordBatch) -> DataFusionResult<()> { + let ts_column = input_batch.column(self.project_index.0).clone(); + let val_column = cast_with_options( + input_batch.column(self.project_index.1), + &DataType::Float64, + &CastOptions::default(), + )?; + let input_batch = RecordBatch::try_new(self.schema.clone(), vec![ts_column, val_column])?; + if let Some(batch) = &self.batch { + self.batch = Some(concat_batches(&self.schema, vec![batch, &input_batch])?); + } else { + self.batch = Some(input_batch); + } + Ok(()) + } +} + +impl Stream for ScalarCalculateStream { + type Item = DataFusionResult; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + loop { + if self.done { + return Poll::Ready(None); + } + match ready!(self.input.poll_next_unpin(cx)) { + Some(Ok(batch)) => { + self.update_batch(batch)?; + } + // inner had error, return to caller + Some(Err(e)) => return Poll::Ready(Some(Err(e))), + // inner is done, producing output + None => { + self.done = true; + return match self.batch.take() { + Some(batch) if !self.have_multi_series => Poll::Ready(Some(Ok(batch))), + _ => { + let time_array = (self.start..=self.end) + .step_by(self.interval as _) + .collect::>(); + let nums = time_array.len(); + let nan_batch = RecordBatch::try_new( + self.schema.clone(), + vec![ + Arc::new(TimestampMillisecondArray::from(time_array)), + Arc::new(Float64Array::from(vec![f64::NAN; nums])), + ], + )?; + Poll::Ready(Some(Ok(nan_batch))) + } + }; + } + }; + } + } +} + +#[cfg(test)] +mod test { + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::physical_plan::memory::MemoryExec; + use datafusion::physical_plan::ExecutionMode; + use datafusion::prelude::SessionContext; + use datatypes::arrow::array::{Float64Array, TimestampMillisecondArray}; + use datatypes::arrow::datatypes::TimeUnit; + + use super::*; + + fn prepare_test_data(diff_series: bool) -> MemoryExec { + let schema = Arc::new(Schema::new(vec![ + Field::new("ts", DataType::Timestamp(TimeUnit::Millisecond, None), true), + Field::new("tag1", DataType::Utf8, true), + Field::new("tag2", DataType::Utf8, true), + Field::new("val", DataType::Float64, true), + ])); + let batch_1 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(TimestampMillisecondArray::from(vec![0, 5_000])), + Arc::new(StringArray::from(vec!["foo", "foo"])), + Arc::new(StringArray::from(vec!["🥺", "🥺"])), + Arc::new(Float64Array::from(vec![1.0, 2.0])), + ], + ) + .unwrap(); + let batch_2 = if diff_series { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(TimestampMillisecondArray::from(vec![10_000, 15_000])), + Arc::new(StringArray::from(vec!["foo", "foo"])), + Arc::new(StringArray::from(vec!["🥺", "😝"])), + Arc::new(Float64Array::from(vec![3.0, 4.0])), + ], + ) + .unwrap() + } else { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(TimestampMillisecondArray::from(vec![10_000, 15_000])), + Arc::new(StringArray::from(vec!["foo", "foo"])), + Arc::new(StringArray::from(vec!["🥺", "🥺"])), + Arc::new(Float64Array::from(vec![3.0, 4.0])), + ], + ) + .unwrap() + }; + MemoryExec::try_new(&[vec![batch_1, batch_2]], schema, None).unwrap() + } + + async fn run_test(diff_series: bool, expected: &str) { + let memory_exec = Arc::new(prepare_test_data(diff_series)); + let schema = Arc::new(Schema::new(vec![ + Field::new("ts", DataType::Timestamp(TimeUnit::Millisecond, None), true), + Field::new("val", DataType::Float64, true), + ])); + let properties = PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + Partitioning::UnknownPartitioning(1), + ExecutionMode::Bounded, + ); + let scalar_exec = Arc::new(ScalarCalculateExec { + start: 0, + end: 15_000, + interval: 5000, + tag_columns: vec!["tag1".to_string(), "tag2".to_string()], + input: memory_exec, + schema, + project_index: (0, 3), + metric: ExecutionPlanMetricsSet::new(), + properties, + }); + let session_context = SessionContext::default(); + let result = datafusion::physical_plan::collect(scalar_exec, session_context.task_ctx()) + .await + .unwrap(); + let result_literal = datatypes::arrow::util::pretty::pretty_format_batches(&result) + .unwrap() + .to_string(); + assert_eq!(result_literal, expected); + } + + #[tokio::test] + async fn same_series() { + run_test( + false, + "+---------------------+-----+\ + \n| ts | val |\ + \n+---------------------+-----+\ + \n| 1970-01-01T00:00:00 | 1.0 |\ + \n| 1970-01-01T00:00:05 | 2.0 |\ + \n| 1970-01-01T00:00:10 | 3.0 |\ + \n| 1970-01-01T00:00:15 | 4.0 |\ + \n+---------------------+-----+", + ) + .await + } + + #[tokio::test] + async fn diff_series() { + run_test( + true, + "+---------------------+-----+\ + \n| ts | val |\ + \n+---------------------+-----+\ + \n| 1970-01-01T00:00:00 | NaN |\ + \n| 1970-01-01T00:00:05 | NaN |\ + \n| 1970-01-01T00:00:10 | NaN |\ + \n| 1970-01-01T00:00:15 | NaN |\ + \n+---------------------+-----+", + ) + .await + } +} diff --git a/src/promql/src/extension_plan/series_divide.rs b/src/promql/src/extension_plan/series_divide.rs index 4abf8d8788..d0524b5d53 100644 --- a/src/promql/src/extension_plan/series_divide.rs +++ b/src/promql/src/extension_plan/series_divide.rs @@ -24,12 +24,12 @@ use datafusion::common::{DFSchema, DFSchemaRef}; use datafusion::error::Result as DataFusionResult; use datafusion::execution::context::TaskContext; use datafusion::logical_expr::{EmptyRelation, Expr, LogicalPlan, UserDefinedLogicalNodeCore}; -use datafusion::physical_expr::{PhysicalSortExpr, PhysicalSortRequirement}; +use datafusion::physical_expr::PhysicalSortRequirement; use datafusion::physical_plan::expressions::Column as ColumnExpr; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, Partitioning, RecordBatchStream, - SendableRecordBatchStream, Statistics, + DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, PlanProperties, RecordBatchStream, + SendableRecordBatchStream, }; use datatypes::arrow::compute; use futures::{ready, Stream, StreamExt}; @@ -130,8 +130,8 @@ impl ExecutionPlan for SeriesDivideExec { self.input.schema() } - fn output_partitioning(&self) -> Partitioning { - Partitioning::UnknownPartitioning(1) + fn properties(&self) -> &PlanProperties { + self.input.properties() } fn required_input_distribution(&self) -> Vec { @@ -156,10 +156,6 @@ impl ExecutionPlan for SeriesDivideExec { } } - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { - self.input.output_ordering() - } - fn maintains_input_order(&self) -> Vec { vec![true; self.children().len()] } @@ -212,16 +208,6 @@ impl ExecutionPlan for SeriesDivideExec { fn metrics(&self) -> Option { Some(self.metric.clone_inner()) } - - fn statistics(&self) -> Statistics { - Statistics { - num_rows: None, - total_byte_size: None, - // TODO(ruihang): support this column statistics - column_statistics: None, - is_exact: false, - } - } } impl DisplayAs for SeriesDivideExec { diff --git a/src/promql/src/extension_plan/union_distinct_on.rs b/src/promql/src/extension_plan/union_distinct_on.rs index 20e40415c9..4624544645 100644 --- a/src/promql/src/extension_plan/union_distinct_on.rs +++ b/src/promql/src/extension_plan/union_distinct_on.rs @@ -25,11 +25,11 @@ use datafusion::common::DFSchemaRef; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::execution::context::TaskContext; use datafusion::logical_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore}; -use datafusion::physical_expr::PhysicalSortExpr; +use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ - hash_utils, DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, Partitioning, - RecordBatchStream, SendableRecordBatchStream, Statistics, + hash_utils, DisplayAs, DisplayFormatType, Distribution, ExecutionMode, ExecutionPlan, + Partitioning, PlanProperties, RecordBatchStream, SendableRecordBatchStream, }; use datatypes::arrow::compute; use futures::future::BoxFuture; @@ -91,13 +91,20 @@ impl UnionDistinctOn { left_exec: Arc, right_exec: Arc, ) -> Arc { + let output_schema: SchemaRef = Arc::new(self.output_schema.as_ref().into()); + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(output_schema.clone()), + Partitioning::UnknownPartitioning(1), + ExecutionMode::Bounded, + )); Arc::new(UnionDistinctOnExec { left: left_exec, right: right_exec, compare_keys: self.compare_keys.clone(), ts_col: self.ts_col.clone(), - output_schema: Arc::new(self.output_schema.as_ref().into()), + output_schema, metric: ExecutionPlanMetricsSet::new(), + properties, random_state: RandomState::new(), }) } @@ -151,6 +158,7 @@ pub struct UnionDistinctOnExec { ts_col: String, output_schema: SchemaRef, metric: ExecutionPlanMetricsSet, + properties: Arc, /// Shared the `RandomState` for the hashing algorithm random_state: RandomState, @@ -169,14 +177,8 @@ impl ExecutionPlan for UnionDistinctOnExec { vec![Distribution::SinglePartition, Distribution::SinglePartition] } - fn output_partitioning(&self) -> Partitioning { - Partitioning::UnknownPartitioning(1) - } - - /// [UnionDistinctOnExec] will output left first, then right. - /// So the order of the output is not maintained. - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { - None + fn properties(&self) -> &PlanProperties { + self.properties.as_ref() } fn children(&self) -> Vec> { @@ -198,6 +200,7 @@ impl ExecutionPlan for UnionDistinctOnExec { ts_col: self.ts_col.clone(), output_schema: self.output_schema.clone(), metric: self.metric.clone(), + properties: self.properties.clone(), random_state: self.random_state.clone(), })) } @@ -249,10 +252,6 @@ impl ExecutionPlan for UnionDistinctOnExec { fn metrics(&self) -> Option { Some(self.metric.clone_inner()) } - - fn statistics(&self) -> Statistics { - Statistics::default() - } } impl DisplayAs for UnionDistinctOnExec { @@ -472,7 +471,8 @@ fn interleave_batches( } // assemble new record batch - RecordBatch::try_new(schema.clone(), interleaved_arrays).map_err(DataFusionError::ArrowError) + RecordBatch::try_new(schema.clone(), interleaved_arrays) + .map_err(|e| DataFusionError::ArrowError(e, None)) } /// Utility function to take rows from a record batch. Based on [take](datafusion::arrow::compute::take) @@ -490,9 +490,10 @@ fn take_batch(batch: &RecordBatch, indices: &[usize]) -> DataFusionResult, _>>() - .map_err(DataFusionError::ArrowError)?; + .map_err(|e| DataFusionError::ArrowError(e, None))?; - let result = RecordBatch::try_new(schema, arrays).map_err(DataFusionError::ArrowError)?; + let result = + RecordBatch::try_new(schema, arrays).map_err(|e| DataFusionError::ArrowError(e, None))?; Ok(result) } diff --git a/src/promql/src/functions/extrapolate_rate.rs b/src/promql/src/functions/extrapolate_rate.rs index 7cba367145..7a57efae04 100644 --- a/src/promql/src/functions/extrapolate_rate.rs +++ b/src/promql/src/functions/extrapolate_rate.rs @@ -204,15 +204,17 @@ impl ExtrapolatedRate { } pub fn scalar_udf(range_length: i64) -> ScalarUDF { - ScalarUDF { - name: Self::name().to_string(), - signature: Signature::new( + // TODO(LFC): Use the new Datafusion UDF impl. + #[allow(deprecated)] + ScalarUDF::new( + Self::name(), + &Signature::new( TypeSignature::Exact(Self::input_type()), Volatility::Immutable, ), - return_type: Arc::new(|_| Ok(Arc::new(Self::return_type()))), - fun: Arc::new(move |input| Self::new(range_length).calc(input)), - } + &(Arc::new(|_: &_| Ok(Arc::new(Self::return_type()))) as _), + &(Arc::new(move |input: &_| Self::new(range_length).calc(input)) as _), + ) } } @@ -223,15 +225,17 @@ impl ExtrapolatedRate { } pub fn scalar_udf(range_length: i64) -> ScalarUDF { - ScalarUDF { - name: Self::name().to_string(), - signature: Signature::new( + // TODO(LFC): Use the new Datafusion UDF impl. + #[allow(deprecated)] + ScalarUDF::new( + Self::name(), + &Signature::new( TypeSignature::Exact(Self::input_type()), Volatility::Immutable, ), - return_type: Arc::new(|_| Ok(Arc::new(Self::return_type()))), - fun: Arc::new(move |input| Self::new(range_length).calc(input)), - } + &(Arc::new(|_: &_| Ok(Arc::new(Self::return_type()))) as _), + &(Arc::new(move |input: &_| Self::new(range_length).calc(input)) as _), + ) } } @@ -242,15 +246,17 @@ impl ExtrapolatedRate { } pub fn scalar_udf(range_length: i64) -> ScalarUDF { - ScalarUDF { - name: Self::name().to_string(), - signature: Signature::new( + // TODO(LFC): Use the new Datafusion UDF impl. + #[allow(deprecated)] + ScalarUDF::new( + Self::name(), + &Signature::new( TypeSignature::Exact(Self::input_type()), Volatility::Immutable, ), - return_type: Arc::new(|_| Ok(Arc::new(Self::return_type()))), - fun: Arc::new(move |input| Self::new(range_length).calc(input)), - } + &(Arc::new(|_: &_| Ok(Arc::new(Self::return_type()))) as _), + &(Arc::new(move |input: &_| Self::new(range_length).calc(input)) as _), + ) } } diff --git a/src/promql/src/functions/holt_winters.rs b/src/promql/src/functions/holt_winters.rs index 4480ccc52d..c047c1883d 100644 --- a/src/promql/src/functions/holt_winters.rs +++ b/src/promql/src/functions/holt_winters.rs @@ -68,15 +68,17 @@ impl HoltWinters { } pub fn scalar_udf(level: f64, trend: f64) -> ScalarUDF { - ScalarUDF { - name: Self::name().to_string(), - signature: Signature::new( + // TODO(LFC): Use the new Datafusion UDF impl. + #[allow(deprecated)] + ScalarUDF::new( + Self::name(), + &Signature::new( TypeSignature::Exact(Self::input_type()), Volatility::Immutable, ), - return_type: Arc::new(|_| Ok(Arc::new(Self::return_type()))), - fun: Arc::new(move |input| Self::new(level, trend).calc(input)), - } + &(Arc::new(|_: &_| Ok(Arc::new(Self::return_type()))) as _), + &(Arc::new(move |input: &_| Self::new(level, trend).calc(input)) as _), + ) } fn calc(&self, input: &[ColumnarValue]) -> Result { diff --git a/src/promql/src/functions/idelta.rs b/src/promql/src/functions/idelta.rs index cf8ad2d0d6..9a74b65fec 100644 --- a/src/promql/src/functions/idelta.rs +++ b/src/promql/src/functions/idelta.rs @@ -42,15 +42,17 @@ impl IDelta { } pub fn scalar_udf() -> ScalarUDF { - ScalarUDF { - name: Self::name().to_string(), - signature: Signature::new( + // TODO(LFC): Use the new Datafusion UDF impl. + #[allow(deprecated)] + ScalarUDF::new( + Self::name(), + &Signature::new( TypeSignature::Exact(Self::input_type()), Volatility::Immutable, ), - return_type: Arc::new(|_| Ok(Arc::new(Self::return_type()))), - fun: Arc::new(Self::calc), - } + &(Arc::new(|_: &_| Ok(Arc::new(Self::return_type()))) as _), + &(Arc::new(Self::calc) as _), + ) } // time index column and value column diff --git a/src/promql/src/functions/predict_linear.rs b/src/promql/src/functions/predict_linear.rs index 965fa28afc..c9b24a76a8 100644 --- a/src/promql/src/functions/predict_linear.rs +++ b/src/promql/src/functions/predict_linear.rs @@ -44,15 +44,17 @@ impl PredictLinear { } pub fn scalar_udf(t: i64) -> ScalarUDF { - ScalarUDF { - name: Self::name().to_string(), - signature: Signature::new( + // TODO(LFC): Use the new Datafusion UDF impl. + #[allow(deprecated)] + ScalarUDF::new( + Self::name(), + &Signature::new( TypeSignature::Exact(Self::input_type()), Volatility::Immutable, ), - return_type: Arc::new(|_| Ok(Arc::new(Self::return_type()))), - fun: Arc::new(move |input| Self::new(t).calc(input)), - } + &(Arc::new(|_: &_| Ok(Arc::new(Self::return_type()))) as _), + &(Arc::new(move |input: &_| Self::new(t).calc(input)) as _), + ) } // time index column and value column diff --git a/src/promql/src/functions/quantile.rs b/src/promql/src/functions/quantile.rs index 62ff2f2126..d055ad1227 100644 --- a/src/promql/src/functions/quantile.rs +++ b/src/promql/src/functions/quantile.rs @@ -40,15 +40,17 @@ impl QuantileOverTime { } pub fn scalar_udf(quantile: f64) -> ScalarUDF { - ScalarUDF { - name: Self::name().to_string(), - signature: Signature::new( + // TODO(LFC): Use the new Datafusion UDF impl. + #[allow(deprecated)] + ScalarUDF::new( + Self::name(), + &Signature::new( TypeSignature::Exact(Self::input_type()), Volatility::Immutable, ), - return_type: Arc::new(|_| Ok(Arc::new(Self::return_type()))), - fun: Arc::new(move |input| Self::new(quantile).calc(input)), - } + &(Arc::new(|_: &_| Ok(Arc::new(Self::return_type()))) as _), + &(Arc::new(move |input: &_| Self::new(quantile).calc(input)) as _), + ) } // time index column and value column diff --git a/src/promql/src/functions/test_util.rs b/src/promql/src/functions/test_util.rs index 5b9d4adef3..8f9558e590 100644 --- a/src/promql/src/functions/test_util.rs +++ b/src/promql/src/functions/test_util.rs @@ -32,12 +32,20 @@ pub fn simple_range_udf_runner( ColumnarValue::Array(Arc::new(input_ts.into_dict())), ColumnarValue::Array(Arc::new(input_value.into_dict())), ]; - let eval_result: Vec> = extract_array(&(range_fn.fun)(&input).unwrap()) + let eval_result: Vec> = extract_array(&(range_fn.fun())(&input).unwrap()) .unwrap() .as_any() .downcast_ref::() .unwrap() .iter() .collect(); - assert_eq!(eval_result, expected) + assert_eq!(eval_result.len(), expected.len()); + assert!(eval_result + .iter() + .zip(expected.iter()) + .all(|(x, y)| match (*x, *y) { + (Some(x), Some(y)) => (x - y).abs() < 0.0001, + (None, None) => true, + _ => false, + })); } diff --git a/src/promql/src/planner.rs b/src/promql/src/planner.rs index 6cd0e17828..0af5308838 100644 --- a/src/promql/src/planner.rs +++ b/src/promql/src/planner.rs @@ -20,20 +20,23 @@ use std::time::UNIX_EPOCH; use async_recursion::async_recursion; use catalog::table_source::DfTableSourceProvider; use common_query::prelude::GREPTIME_VALUE; -use datafusion::common::{DFSchemaRef, OwnedTableReference, Result as DfResult}; +use datafusion::common::{DFSchemaRef, Result as DfResult}; use datafusion::datasource::DefaultTableSource; -use datafusion::logical_expr::expr::{AggregateFunction, Alias, ScalarFunction, ScalarUDF}; +use datafusion::logical_expr::expr::{ + AggregateFunction, AggregateFunctionDefinition, Alias, ScalarFunction, +}; use datafusion::logical_expr::expr_rewriter::normalize_cols; use datafusion::logical_expr::{ AggregateFunction as AggregateFunctionEnum, BinaryExpr, BuiltinScalarFunction, Cast, Extension, - LogicalPlan, LogicalPlanBuilder, Operator, ScalarUDF as ScalarUdfDef, + LogicalPlan, LogicalPlanBuilder, Operator, ScalarFunctionDefinition, ScalarUDF as ScalarUdfDef, }; -use datafusion::optimizer::utils::{self, conjunction}; use datafusion::prelude as df_prelude; use datafusion::prelude::{Column, Expr as DfExpr, JoinType}; use datafusion::scalar::ScalarValue; use datafusion::sql::TableReference; +use datafusion_expr::utils::conjunction; use datatypes::arrow::datatypes::DataType as ArrowDataType; +use itertools::Itertools; use promql_parser::label::{MatchOp, Matcher, Matchers, METRIC_NAME}; use promql_parser::parser::{ token, AggregateExpr, BinModifier, BinaryExpr as PromBinaryExpr, Call, EvalStmt, @@ -54,7 +57,7 @@ use crate::error::{ }; use crate::extension_plan::{ build_special_time_expr, EmptyMetric, HistogramFold, InstantManipulate, Millisecond, - RangeManipulate, SeriesDivide, SeriesNormalize, UnionDistinctOn, + RangeManipulate, ScalarCalculate, SeriesDivide, SeriesNormalize, UnionDistinctOn, }; use crate::functions::{ AbsentOverTime, AvgOverTime, Changes, CountOverTime, Delta, Deriv, HoltWinters, IDelta, @@ -64,6 +67,8 @@ use crate::functions::{ /// `time()` function in PromQL. const SPECIAL_TIME_FUNCTION: &str = "time"; +/// `scalar()` function in PromQL. +const SCALAR_FUNCTION: &str = "scalar"; /// `histogram_quantile` function in PromQL const SPECIAL_HISTOGRAM_QUANTILE: &str = "histogram_quantile"; /// `vector` function in PromQL @@ -305,14 +310,14 @@ impl PromPlanner { let left_field_columns = self.ctx.field_columns.clone(); let mut left_table_ref = self .table_ref() - .unwrap_or_else(|_| OwnedTableReference::bare("")); + .unwrap_or_else(|_| TableReference::bare("")); let left_context = self.ctx.clone(); let right_input = self.prom_expr_to_plan(*rhs.clone()).await?; let right_field_columns = self.ctx.field_columns.clone(); let mut right_table_ref = self .table_ref() - .unwrap_or_else(|_| OwnedTableReference::bare("")); + .unwrap_or_else(|_| TableReference::bare("")); let right_context = self.ctx.clone(); // TODO(ruihang): avoid join if left and right are the same table @@ -332,9 +337,19 @@ impl PromPlanner { // normal join if left_table_ref == right_table_ref { // rename table references to avoid ambiguity - left_table_ref = OwnedTableReference::bare("lhs"); - right_table_ref = OwnedTableReference::bare("rhs"); - self.ctx.table_name = Some("lhs".to_string()); + left_table_ref = TableReference::bare("lhs"); + right_table_ref = TableReference::bare("rhs"); + // `self.ctx` have ctx in right plan, if right plan have no tag, + // we use left plan ctx as the ctx for subsequent calculations, + // to avoid case like `host + scalar(...)` + // we need preserve tag column on `host` table in subsequent projection, + // which only show in left plan ctx. + if self.ctx.tag_columns.is_empty() { + self.ctx = left_context.clone(); + self.ctx.table_name = Some("lhs".to_string()); + } else { + self.ctx.table_name = Some("rhs".to_string()); + } } let mut field_columns = left_field_columns.iter().zip(right_field_columns.iter()); @@ -343,19 +358,23 @@ impl PromPlanner { right_input, left_table_ref.clone(), right_table_ref.clone(), + // if left plan or right plan tag is empty, means case like `scalar(...) + host` or `host + scalar(...)` + // under this case we only join on time index + left_context.tag_columns.is_empty() + || right_context.tag_columns.is_empty(), )?; let join_plan_schema = join_plan.schema().clone(); let bin_expr_builder = |_: &String| { let (left_col_name, right_col_name) = field_columns.next().unwrap(); let left_col = join_plan_schema - .field_with_name(Some(&left_table_ref), left_col_name) + .qualified_field_with_name(Some(&left_table_ref), left_col_name) .context(DataFusionPlanningSnafu)? - .qualified_column(); + .into(); let right_col = join_plan_schema - .field_with_name(Some(&right_table_ref), right_col_name) + .qualified_field_with_name(Some(&right_table_ref), right_col_name) .context(DataFusionPlanningSnafu)? - .qualified_column(); + .into(); let binary_expr_builder = Self::prom_token_to_binary_expr_builder(*op)?; let mut binary_expr = binary_expr_builder( @@ -491,6 +510,7 @@ impl PromPlanner { match func.name { SPECIAL_HISTOGRAM_QUANTILE => return self.create_histogram_plan(args).await, SPECIAL_VECTOR_FUNCTION => return self.create_vector_plan(args).await, + SCALAR_FUNCTION => return self.create_scalar_plan(args).await, _ => {} } @@ -788,7 +808,7 @@ impl PromPlanner { } // change the tag columns in context - self.ctx.tag_columns = labels.labels.clone(); + self.ctx.tag_columns.clone_from(&labels.labels); // add timestamp column exprs.push(self.create_time_index_column_expr()?); @@ -859,7 +879,7 @@ impl PromPlanner { Ok(exprs) } - fn table_ref(&self) -> Result { + fn table_ref(&self) -> Result { let table_name = self .ctx .table_name @@ -868,12 +888,12 @@ impl PromPlanner { // set schema name if `__schema__` is given let table_ref = if let Some(schema_name) = &self.ctx.schema_name { - TableReference::partial(schema_name, &table_name) + TableReference::partial(schema_name.as_str(), table_name.as_str()) } else { - TableReference::bare(&table_name) + TableReference::bare(table_name.as_str()) }; - Ok(table_ref.to_owned_reference()) + Ok(table_ref) } /// Create a table scan plan and a filter plan with given filter. @@ -882,7 +902,7 @@ impl PromPlanner { /// If the filter is empty async fn create_table_scan_plan( &mut self, - table_ref: OwnedTableReference, + table_ref: TableReference, filter: Vec, ) -> Result { let provider = self @@ -1128,7 +1148,9 @@ impl PromPlanner { right: Box::new(interval_1day_lit_expr), }); let date_trunc_expr = DfExpr::ScalarFunction(ScalarFunction { - fun: BuiltinScalarFunction::DateTrunc, + func_def: ScalarFunctionDefinition::UDF( + datafusion_functions::datetime::date_trunc(), + ), args: vec![month_lit_expr, self.create_time_index_column_expr()?], }); let date_trunc_plus_interval_expr = DfExpr::BinaryExpr(BinaryExpr { @@ -1137,21 +1159,30 @@ impl PromPlanner { right: Box::new(the_1month_minus_1day_expr), }); let date_part_expr = DfExpr::ScalarFunction(ScalarFunction { - fun: BuiltinScalarFunction::DatePart, + func_def: ScalarFunctionDefinition::UDF( + datafusion_functions::datetime::date_part(), + ), args: vec![day_lit_expr, date_trunc_plus_interval_expr], }); exprs.push(date_part_expr); ScalarFunc::GeneratedExpr } - _ => ScalarFunc::DataFusionBuiltin( - BuiltinScalarFunction::from_str(func.name).map_err(|_| { - UnsupportedExprSnafu { + _ => { + if let Ok(f) = BuiltinScalarFunction::from_str(func.name) { + ScalarFunc::DataFusionBuiltin(f) + } else if let Some(f) = datafusion_functions::math::functions() + .iter() + .find(|f| f.name() == func.name) + { + ScalarFunc::DataFusionUdf(f.clone()) + } else { + return UnsupportedExprSnafu { name: func.name.to_string(), } - .build() - })?, - ), + .fail(); + } + } }; for value in &self.ctx.field_columns { @@ -1161,12 +1192,24 @@ impl PromPlanner { ScalarFunc::DataFusionBuiltin(fun) => { other_input_exprs.insert(field_column_pos, col_expr); let fn_expr = DfExpr::ScalarFunction(ScalarFunction { - fun, + func_def: ScalarFunctionDefinition::BuiltIn(fun), args: other_input_exprs.clone().into(), }); exprs.push(fn_expr); let _ = other_input_exprs.remove(field_column_pos); } + ScalarFunc::DataFusionUdf(f) => { + let args = itertools::chain!( + other_input_exprs.iter().take(field_column_pos).cloned(), + std::iter::once(col_expr), + other_input_exprs.iter().skip(field_column_pos).cloned() + ) + .collect_vec(); + exprs.push(DfExpr::ScalarFunction(ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(f), + args, + })) + } ScalarFunc::Udf(fun) => { let ts_range_expr = DfExpr::Column(Column::from_name( RangeManipulate::build_timestamp_range_name( @@ -1175,8 +1218,8 @@ impl PromPlanner { )); other_input_exprs.insert(field_column_pos, ts_range_expr); other_input_exprs.insert(field_column_pos + 1, col_expr); - let fn_expr = DfExpr::ScalarUDF(ScalarUDF { - fun: Arc::new(fun), + let fn_expr = DfExpr::ScalarFunction(ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(fun)), args: other_input_exprs.clone().into(), }); exprs.push(fn_expr); @@ -1193,8 +1236,8 @@ impl PromPlanner { other_input_exprs.insert(field_column_pos + 1, col_expr); other_input_exprs .insert(field_column_pos + 2, self.create_time_index_column_expr()?); - let fn_expr = DfExpr::ScalarUDF(ScalarUDF { - fun: Arc::new(fun), + let fn_expr = DfExpr::ScalarFunction(ScalarFunction { + func_def: ScalarFunctionDefinition::UDF(Arc::new(fun)), args: other_input_exprs.clone().into(), }); exprs.push(fn_expr); @@ -1258,7 +1301,7 @@ impl PromPlanner { exprs.push(expr); } - utils::conjunction(exprs).context(ValueNotFoundSnafu { + conjunction(exprs).context(ValueNotFoundSnafu { table: self.table_ref()?.to_quoted_string(), }) } @@ -1299,11 +1342,12 @@ impl PromPlanner { .iter() .map(|col| { DfExpr::AggregateFunction(AggregateFunction { - fun: aggr.clone(), + func_def: AggregateFunctionDefinition::BuiltIn(aggr.clone()), args: vec![DfExpr::Column(Column::from_name(col))], distinct: false, filter: None, order_by: None, + null_treatment: None, }) }) .collect(); @@ -1407,6 +1451,44 @@ impl PromPlanner { })) } + /// Create a [SCALAR_FUNCTION] plan + async fn create_scalar_plan(&mut self, args: &PromFunctionArgs) -> Result { + ensure!( + args.len() == 1, + FunctionInvalidArgumentSnafu { + fn_name: SCALAR_FUNCTION + } + ); + let input = self + .prom_expr_to_plan(args.args[0].as_ref().clone()) + .await?; + ensure!( + self.ctx.field_columns.len() == 1, + MultiFieldsNotSupportedSnafu { + operator: SCALAR_FUNCTION + }, + ); + let scalar_plan = LogicalPlan::Extension(Extension { + node: Arc::new(ScalarCalculate::new( + self.ctx.start, + self.ctx.end, + self.ctx.interval, + input, + self.ctx.time_index_column.as_ref().unwrap(), + &self.ctx.tag_columns, + &self.ctx.field_columns[0], + self.ctx.table_name.as_deref(), + )?), + }); + // scalar plan have no tag columns + self.ctx.tag_columns.clear(); + self.ctx.field_columns.clear(); + self.ctx + .field_columns + .push(scalar_plan.schema().field(1).name().clone()); + Ok(scalar_plan) + } + /// Try to build a DataFusion Literal Expression from PromQL Expr, return /// `None` if the input is not a literal expression. fn try_build_literal_expr(expr: &PromExpr) -> Option { @@ -1515,13 +1597,16 @@ impl PromPlanner { token::T_LTE => Ok(Box::new(|lhs, rhs| Ok(lhs.lt_eq(rhs)))), token::T_POW => Ok(Box::new(|lhs, rhs| { Ok(DfExpr::ScalarFunction(ScalarFunction { - fun: BuiltinScalarFunction::Power, + func_def: ScalarFunctionDefinition::UDF(datafusion_functions::math::power()), args: vec![lhs, rhs], })) })), token::T_ATAN2 => Ok(Box::new(|lhs, rhs| { Ok(DfExpr::ScalarFunction(ScalarFunction { - fun: BuiltinScalarFunction::Atan2, + // func_def: ScalarFunctionDefinition::BuiltIn(BuiltinScalarFunction::Atan2), + func_def: datafusion_expr::ScalarFunctionDefinition::UDF( + datafusion_functions::math::atan2(), + ), args: vec![lhs, rhs], })) })), @@ -1553,19 +1638,24 @@ impl PromPlanner { } /// Build a inner join on time index column and tag columns to concat two logical plans. + /// When `only_join_time_index == true` we only join on the time index, because these two plan may not have the same tag columns fn join_on_non_field_columns( &self, left: LogicalPlan, right: LogicalPlan, - left_table_ref: OwnedTableReference, - right_table_ref: OwnedTableReference, + left_table_ref: TableReference, + right_table_ref: TableReference, + only_join_time_index: bool, ) -> Result { - let mut tag_columns = self - .ctx - .tag_columns - .iter() - .map(Column::from_name) - .collect::>(); + let mut tag_columns = if only_join_time_index { + vec![] + } else { + self.ctx + .tag_columns + .iter() + .map(Column::from_name) + .collect::>() + }; // push time index column if it exist if let Some(time_index_column) = &self.ctx.time_index_column { @@ -1778,8 +1868,8 @@ impl PromPlanner { .difference(&right_tag_cols_set) .cloned() .collect::>(); - let left_qualifier = left.schema().field(0).qualifier().cloned(); - let right_qualifier = right.schema().field(0).qualifier().cloned(); + let left_qualifier = left.schema().qualified_field(0).0.cloned(); + let right_qualifier = right.schema().qualified_field(0).0.cloned(); let left_qualifier_string = left_qualifier .as_ref() .map(|l| l.to_string()) @@ -1953,7 +2043,7 @@ impl PromPlanner { let field_columns_iter = result_field_columns .into_iter() .zip(self.ctx.field_columns.iter()) - .map(|(expr, name)| Ok(DfExpr::Alias(Alias::new(expr, name.to_string())))); + .map(|(expr, name)| Ok(DfExpr::Alias(Alias::new(expr, None::, name)))); // chain non-field columns (unchanged) and field columns (applied computation then alias) let project_fields = non_field_columns_iter @@ -2007,7 +2097,7 @@ impl PromPlanner { })?, ); let fn_expr = DfExpr::ScalarFunction(ScalarFunction { - fun: BuiltinScalarFunction::DatePart, + func_def: ScalarFunctionDefinition::UDF(datafusion_functions::datetime::date_part()), args: vec![lit_expr, input_expr], }); Ok(fn_expr) @@ -2023,6 +2113,8 @@ struct FunctionArgs { #[derive(Debug, Clone)] enum ScalarFunc { DataFusionBuiltin(BuiltinScalarFunction), + /// The UDF that is defined by Datafusion itself. + DataFusionUdf(Arc), Udf(ScalarUdfDef), // todo(ruihang): maybe merge with Udf later /// UDF that require extra information like range length to be evaluated. @@ -2436,7 +2528,7 @@ mod test { #[tokio::test] async fn aggregate_stdvar() { - do_aggregate_expr_plan("stdvar", "VARIANCE_POP").await; + do_aggregate_expr_plan("stdvar", "VAR_POP").await; } #[tokio::test] @@ -2511,7 +2603,7 @@ mod test { .unwrap(); let expected = String::from( - "Projection: lhs.tag_0, lhs.timestamp, lhs.field_0 + rhs.field_0 AS lhs.field_0 + rhs.field_0 [tag_0:Utf8, timestamp:Timestamp(Millisecond, None), lhs.field_0 + rhs.field_0:Float64;N]\ + "Projection: rhs.tag_0, rhs.timestamp, lhs.field_0 + rhs.field_0 AS lhs.field_0 + rhs.field_0 [tag_0:Utf8, timestamp:Timestamp(Millisecond, None), lhs.field_0 + rhs.field_0:Float64;N]\ \n Inner Join: lhs.tag_0 = rhs.tag_0, lhs.timestamp = rhs.timestamp [tag_0:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, tag_0:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N]\ \n SubqueryAlias: lhs [tag_0:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N]\ \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N]\ diff --git a/src/query/src/datafusion.rs b/src/query/src/datafusion.rs index b6f0e9543f..f646d6b90c 100644 --- a/src/query/src/datafusion.rs +++ b/src/query/src/datafusion.rs @@ -161,9 +161,9 @@ impl DatafusionQueryEngine { } #[tracing::instrument(skip_all)] - async fn delete<'a>( + async fn delete( &self, - table_name: &ResolvedTableReference<'a>, + table_name: &ResolvedTableReference, table: &TableRef, column_vectors: HashMap, query_ctx: QueryContextRef, @@ -205,9 +205,9 @@ impl DatafusionQueryEngine { } #[tracing::instrument(skip_all)] - async fn insert<'a>( + async fn insert( &self, - table_name: &ResolvedTableReference<'a>, + table_name: &ResolvedTableReference, column_vectors: HashMap, query_ctx: QueryContextRef, ) -> Result { @@ -226,7 +226,7 @@ impl DatafusionQueryEngine { .context(TableMutationSnafu) } - async fn find_table(&self, table_name: &ResolvedTableReference<'_>) -> Result { + async fn find_table(&self, table_name: &ResolvedTableReference) -> Result { let catalog_name = table_name.catalog.as_ref(); let schema_name = table_name.schema.as_ref(); let table_name = table_name.table.as_ref(); @@ -309,7 +309,9 @@ impl QueryEngine for DatafusionQueryEngine { } fn engine_context(&self, query_ctx: QueryContextRef) -> QueryEngineContext { - QueryEngineContext::new(self.state.session_state(), query_ctx) + let mut state = self.state.session_state(); + state.config_mut().set_extension(query_ctx.clone()); + QueryEngineContext::new(state, query_ctx) } } @@ -435,7 +437,7 @@ impl QueryExecutor for DatafusionQueryEngine { let exec_timer = metrics::EXEC_PLAN_ELAPSED.start_timer(); let task_ctx = ctx.build_task_ctx(); - match plan.output_partitioning().partition_count() { + match plan.properties().output_partitioning().partition_count() { 0 => Ok(Box::pin(EmptyRecordBatchStream::new(plan.schema()))), 1 => { let stream = plan @@ -453,7 +455,7 @@ impl QueryExecutor for DatafusionQueryEngine { // merge into a single partition let plan = CoalescePartitionsExec::new(df_plan.clone()); // CoalescePartitionsExec must produce a single partition - assert_eq!(1, plan.output_partitioning().partition_count()); + assert_eq!(1, plan.properties().output_partitioning().partition_count()); let df_stream = plan .execute(0, task_ctx) .context(error::DatafusionSnafu) @@ -475,7 +477,6 @@ impl QueryExecutor for DatafusionQueryEngine { #[cfg(test)] mod tests { - use std::borrow::Cow::Borrowed; use std::sync::Arc; use catalog::RegisterTableRequest; @@ -576,9 +577,9 @@ mod tests { .unwrap(); let table = engine .find_table(&ResolvedTableReference { - catalog: Borrowed("greptime"), - schema: Borrowed("public"), - table: Borrowed("numbers"), + catalog: "greptime".into(), + schema: "public".into(), + table: "numbers".into(), }) .await .unwrap(); diff --git a/src/query/src/datafusion/planner.rs b/src/query/src/datafusion/planner.rs index 67ec4c8a3c..1ff9770a56 100644 --- a/src/query/src/datafusion/planner.rs +++ b/src/query/src/datafusion/planner.rs @@ -23,13 +23,13 @@ use common_query::logical_plan::create_aggregate_function; use datafusion::catalog::TableReference; use datafusion::error::Result as DfResult; use datafusion::execution::context::SessionState; -use datafusion::physical_plan::udaf::AggregateUDF; use datafusion::physical_plan::udf::ScalarUDF; use datafusion::sql::planner::ContextProvider; +use datafusion::variable::VarType; use datafusion_common::config::ConfigOptions; -use datafusion_common::{DataFusionError, OwnedTableReference}; -use datafusion_expr::{TableSource, WindowUDF}; -use datafusion_physical_expr::var_provider::{is_system_variables, VarType}; +use datafusion_common::DataFusionError; +use datafusion_expr::var_provider::is_system_variables; +use datafusion_expr::{AggregateUDF, TableSource, WindowUDF}; use datafusion_sql::parser::Statement as DfStatement; use session::context::QueryContextRef; use snafu::ResultExt; @@ -79,7 +79,7 @@ impl DfContextProviderAdapter { } async fn resolve_tables( - table_names: Vec, + table_names: Vec, table_provider: &mut DfTableSourceProvider, ) -> Result>> { let mut tables = HashMap::with_capacity(table_names.len()); @@ -102,7 +102,7 @@ async fn resolve_tables( } impl ContextProvider for DfContextProviderAdapter { - fn get_table_provider(&self, name: TableReference) -> DfResult> { + fn get_table_source(&self, name: TableReference) -> DfResult> { let table_ref = self.table_provider.resolve_table_ref(name)?; self.tables .get(&table_ref.to_string()) @@ -159,4 +159,19 @@ impl ContextProvider for DfContextProviderAdapter { fn options(&self) -> &ConfigOptions { self.session_state.config_options() } + + fn udfs_names(&self) -> Vec { + // TODO(LFC): Impl it. + vec![] + } + + fn udafs_names(&self) -> Vec { + // TODO(LFC): Impl it. + vec![] + } + + fn udwfs_names(&self) -> Vec { + // TODO(LFC): Impl it. + vec![] + } } diff --git a/src/query/src/dist_plan/analyzer.rs b/src/query/src/dist_plan/analyzer.rs index 10c2c2e9e2..870b926339 100644 --- a/src/query/src/dist_plan/analyzer.rs +++ b/src/query/src/dist_plan/analyzer.rs @@ -17,7 +17,7 @@ use std::sync::Arc; use datafusion::datasource::DefaultTableSource; use datafusion::error::Result as DfResult; use datafusion_common::config::ConfigOptions; -use datafusion_common::tree_node::{RewriteRecursion, Transformed, TreeNode, TreeNodeRewriter}; +use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRewriter}; use datafusion_expr::expr::{Exists, InSubquery}; use datafusion_expr::{col, Expr, LogicalPlan, LogicalPlanBuilder, Subquery}; use datafusion_optimizer::analyzer::AnalyzerRule; @@ -47,12 +47,12 @@ impl AnalyzerRule for DistPlannerAnalyzer { // preprocess the input plan let optimizer_context = OptimizerContext::new(); let plan = SimplifyExpressions::new() - .try_optimize(&plan, &optimizer_context)? - .unwrap_or(plan); + .rewrite(plan, &optimizer_context)? + .data; let plan = plan.transform(&Self::inspect_plan_with_subquery)?; let mut rewriter = PlanRewriter::default(); - let result = plan.rewrite(&mut rewriter)?; + let result = plan.data.rewrite(&mut rewriter)?.data; Ok(result) } @@ -63,35 +63,40 @@ impl DistPlannerAnalyzer { let exprs = plan .expressions() .into_iter() - .map(|e| e.transform(&Self::transform_subquery)) + .map(|e| e.transform(&Self::transform_subquery).map(|x| x.data)) .collect::>>()?; let inputs = plan.inputs().into_iter().cloned().collect::>(); - Ok(Transformed::Yes(plan.with_new_exprs(exprs, &inputs)?)) + Ok(Transformed::yes(plan.with_new_exprs(exprs, inputs)?)) } fn transform_subquery(expr: Expr) -> DfResult> { match expr { - Expr::Exists(exists) => Ok(Transformed::Yes(Expr::Exists(Exists { + Expr::Exists(exists) => Ok(Transformed::yes(Expr::Exists(Exists { subquery: Self::handle_subquery(exists.subquery)?, negated: exists.negated, }))), - Expr::InSubquery(in_subquery) => Ok(Transformed::Yes(Expr::InSubquery(InSubquery { + Expr::InSubquery(in_subquery) => Ok(Transformed::yes(Expr::InSubquery(InSubquery { expr: in_subquery.expr, subquery: Self::handle_subquery(in_subquery.subquery)?, negated: in_subquery.negated, }))), - Expr::ScalarSubquery(scalar_subquery) => Ok(Transformed::Yes(Expr::ScalarSubquery( + Expr::ScalarSubquery(scalar_subquery) => Ok(Transformed::yes(Expr::ScalarSubquery( Self::handle_subquery(scalar_subquery)?, ))), - _ => Ok(Transformed::No(expr)), + _ => Ok(Transformed::no(expr)), } } fn handle_subquery(subquery: Subquery) -> DfResult { let mut rewriter = PlanRewriter::default(); - let mut rewrote_subquery = subquery.subquery.as_ref().clone().rewrite(&mut rewriter)?; + let mut rewrote_subquery = subquery + .subquery + .as_ref() + .clone() + .rewrite(&mut rewriter)? + .data; // Workaround. DF doesn't support the first plan in subquery to be an Extension if matches!(rewrote_subquery, LogicalPlan::Extension(_)) { let output_schema = rewrote_subquery.schema().clone(); @@ -232,35 +237,34 @@ impl PlanRewriter { } impl TreeNodeRewriter for PlanRewriter { - type N = LogicalPlan; + type Node = LogicalPlan; /// descend - fn pre_visit<'a>(&'a mut self, node: &'a Self::N) -> DfResult { + fn f_down<'a>(&mut self, node: Self::Node) -> DfResult> { self.level += 1; self.stack.push((node.clone(), self.level)); // decendening will clear the stage self.stage.clear(); self.set_unexpanded(); self.partition_cols = None; - - Ok(RewriteRecursion::Continue) + Ok(Transformed::no(node)) } /// ascend /// /// Besure to call `pop_stack` before returning - fn mutate(&mut self, node: Self::N) -> DfResult { + fn f_up(&mut self, node: Self::Node) -> DfResult> { // only expand once on each ascending if self.is_expanded() { self.pop_stack(); - return Ok(node); + return Ok(Transformed::no(node)); } // only expand when the leaf is table scan if node.inputs().is_empty() && !matches!(node, LogicalPlan::TableScan(_)) { self.set_expanded(); self.pop_stack(); - return Ok(node); + return Ok(Transformed::no(node)); } self.maybe_set_partitions(&node); @@ -270,12 +274,12 @@ impl TreeNodeRewriter for PlanRewriter { let mut node = MergeScanLogicalPlan::new(node, false).into_logical_plan(); // expand stages for new_stage in self.stage.drain(..) { - node = new_stage.with_new_inputs(&[node])? + node = new_stage.with_new_exprs(node.expressions(), vec![node.clone()])? } self.set_expanded(); self.pop_stack(); - return Ok(node); + return Ok(Transformed::yes(node)); }; // TODO(ruihang): avoid this clone @@ -285,16 +289,16 @@ impl TreeNodeRewriter for PlanRewriter { let mut node = MergeScanLogicalPlan::new(node, false).into_logical_plan(); // expand stages for new_stage in self.stage.drain(..) { - node = new_stage.with_new_inputs(&[node])? + node = new_stage.with_new_exprs(node.expressions(), vec![node.clone()])? } self.set_expanded(); self.pop_stack(); - return Ok(node); + return Ok(Transformed::yes(node)); } self.pop_stack(); - Ok(node) + Ok(Transformed::no(node)) } } diff --git a/src/query/src/dist_plan/commutativity.rs b/src/query/src/dist_plan/commutativity.rs index 5bb125e1ba..f59dc57268 100644 --- a/src/query/src/dist_plan/commutativity.rs +++ b/src/query/src/dist_plan/commutativity.rs @@ -107,6 +107,7 @@ impl Categorizer { LogicalPlan::Dml(_) => Commutativity::Unsupported, LogicalPlan::Ddl(_) => Commutativity::Unsupported, LogicalPlan::Copy(_) => Commutativity::Unsupported, + LogicalPlan::RecursiveQuery(_) => Commutativity::Unsupported, } } @@ -142,8 +143,7 @@ impl Categorizer { | Expr::Between(_) | Expr::Sort(_) | Expr::Exists(_) - | Expr::ScalarFunction(_) - | Expr::ScalarUDF(_) => Commutativity::Commutative, + | Expr::ScalarFunction(_) => Commutativity::Commutative, Expr::Like(_) | Expr::SimilarTo(_) @@ -155,14 +155,13 @@ impl Categorizer { | Expr::TryCast(_) | Expr::AggregateFunction(_) | Expr::WindowFunction(_) - | Expr::AggregateUDF(_) | Expr::InList(_) | Expr::InSubquery(_) | Expr::ScalarSubquery(_) - | Expr::Wildcard => Commutativity::Unimplemented, + | Expr::Wildcard { .. } => Commutativity::Unimplemented, Expr::Alias(_) - | Expr::QualifiedWildcard { .. } + | Expr::Unnest(_) | Expr::GroupingSet(_) | Expr::Placeholder(_) | Expr::OuterReferenceColumn(_, _) => Commutativity::Unimplemented, diff --git a/src/query/src/dist_plan/merge_scan.rs b/src/query/src/dist_plan/merge_scan.rs index ed31ecb0c8..1f294a836b 100644 --- a/src/query/src/dist_plan/merge_scan.rs +++ b/src/query/src/dist_plan/merge_scan.rs @@ -33,15 +33,18 @@ use common_telemetry::tracing_context::TracingContext; use datafusion::physical_plan::metrics::{ Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, MetricsSet, Time, }; -use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning}; -use datafusion_common::{Result, Statistics}; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionMode, ExecutionPlan, Partitioning, PlanProperties, +}; +use datafusion_common::Result; use datafusion_expr::{Extension, LogicalPlan, UserDefinedLogicalNodeCore}; -use datafusion_physical_expr::PhysicalSortExpr; +use datafusion_physical_expr::EquivalenceProperties; use datatypes::schema::{Schema, SchemaRef}; use futures_util::StreamExt; use greptime_proto::v1::region::{QueryRequest, RegionRequestHeader}; use meter_core::data::ReadItem; use meter_macros::read_meter; +use session::context::QueryContextRef; use snafu::ResultExt; use store_api::storage::RegionId; use tokio::time::Instant; @@ -123,6 +126,8 @@ pub struct MergeScanExec { arrow_schema: ArrowSchemaRef, region_query_handler: RegionQueryHandlerRef, metric: ExecutionPlanMetricsSet, + properties: PlanProperties, + query_ctx: QueryContextRef, } impl std::fmt::Debug for MergeScanExec { @@ -142,8 +147,14 @@ impl MergeScanExec { substrait_plan: Bytes, arrow_schema: &ArrowSchema, region_query_handler: RegionQueryHandlerRef, + query_ctx: QueryContextRef, ) -> Result { let arrow_schema_without_metadata = Self::arrow_schema_without_metadata(arrow_schema); + let properties = PlanProperties::new( + EquivalenceProperties::new(arrow_schema_without_metadata.clone()), + Partitioning::UnknownPartitioning(1), + ExecutionMode::Bounded, + ); let schema_without_metadata = Self::arrow_schema_to_schema(arrow_schema_without_metadata.clone())?; Ok(Self { @@ -154,6 +165,8 @@ impl MergeScanExec { arrow_schema: arrow_schema_without_metadata, region_query_handler, metric: ExecutionPlanMetricsSet::new(), + properties, + query_ctx, }) } @@ -166,6 +179,7 @@ impl MergeScanExec { let dbname = context.task_id().unwrap_or_default(); let tracing_context = TracingContext::from_json(context.session_id().as_str()); + let tz = self.query_ctx.timezone().to_string(); let stream = Box::pin(stream!({ MERGE_SCAN_REGIONS.observe(regions.len() as f64); @@ -178,6 +192,7 @@ impl MergeScanExec { header: Some(RegionRequestHeader { tracing_context: tracing_context.to_w3c(), dbname: dbname.clone(), + timezone: tz.clone(), }), region_id: region_id.into(), plan: substrait_plan.clone(), @@ -266,12 +281,8 @@ impl ExecutionPlan for MergeScanExec { self.arrow_schema.clone() } - fn output_partitioning(&self) -> Partitioning { - Partitioning::UnknownPartitioning(1) - } - - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { - None + fn properties(&self) -> &PlanProperties { + &self.properties } fn children(&self) -> Vec> { @@ -297,10 +308,6 @@ impl ExecutionPlan for MergeScanExec { ))) } - fn statistics(&self) -> Statistics { - Statistics::default() - } - fn metrics(&self) -> Option { Some(self.metric.clone_inner()) } diff --git a/src/query/src/dist_plan/planner.rs b/src/query/src/dist_plan/planner.rs index c3e0cca94e..1d29fe7aba 100644 --- a/src/query/src/dist_plan/planner.rs +++ b/src/query/src/dist_plan/planner.rs @@ -25,10 +25,11 @@ use datafusion::datasource::DefaultTableSource; use datafusion::execution::context::SessionState; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_planner::{ExtensionPlanner, PhysicalPlanner}; -use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeVisitor, VisitRecursion}; +use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion, TreeNodeVisitor}; use datafusion_common::TableReference; use datafusion_expr::{LogicalPlan, UserDefinedLogicalNode}; use datafusion_optimizer::analyzer::Analyzer; +use session::context::QueryContext; use snafu::{OptionExt, ResultExt}; use store_api::storage::RegionId; use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan}; @@ -103,12 +104,18 @@ impl ExtensionPlanner for DistExtensionPlanner { .encode(&amended_plan) .context(error::EncodeSubstraitLogicalPlanSnafu)? .into(); + + let query_ctx = session_state + .config() + .get_extension() + .unwrap_or_else(QueryContext::arc); let merge_scan_plan = MergeScanExec::new( table_name, regions, substrait_plan, &schema, self.region_query_handler.clone(), + query_ctx, )?; Ok(Some(Arc::new(merge_scan_plan) as _)) } @@ -125,6 +132,7 @@ impl DistExtensionPlanner { /// Apply the fully resolved table name to the TableScan plan fn plan_with_full_table_name(plan: LogicalPlan, name: &TableName) -> Result { plan.transform(&|plan| TableNameRewriter::rewrite_table_name(plan, name)) + .map(|x| x.data) } async fn get_regions(&self, table_name: &TableName) -> Result> { @@ -163,9 +171,9 @@ struct TableNameExtractor { } impl TreeNodeVisitor for TableNameExtractor { - type N = LogicalPlan; + type Node = LogicalPlan; - fn pre_visit(&mut self, node: &Self::N) -> Result { + fn f_down(&mut self, node: &Self::Node) -> Result { match node { LogicalPlan::TableScan(scan) => { if let Some(source) = scan.source.as_any().downcast_ref::() { @@ -182,7 +190,7 @@ impl TreeNodeVisitor for TableNameExtractor { info.name.clone(), )); } - return Ok(VisitRecursion::Stop); + return Ok(TreeNodeRecursion::Stop); } } match &scan.table_name { @@ -192,32 +200,32 @@ impl TreeNodeVisitor for TableNameExtractor { table, } => { self.table_name = Some(TableName::new( - catalog.clone(), - schema.clone(), - table.clone(), + catalog.to_string(), + schema.to_string(), + table.to_string(), )); - Ok(VisitRecursion::Stop) + Ok(TreeNodeRecursion::Stop) } // TODO(ruihang): Maybe the following two cases should not be valid TableReference::Partial { schema, table } => { self.table_name = Some(TableName::new( DEFAULT_CATALOG_NAME.to_string(), - schema.clone(), - table.clone(), + schema.to_string(), + table.to_string(), )); - Ok(VisitRecursion::Stop) + Ok(TreeNodeRecursion::Stop) } TableReference::Bare { table } => { self.table_name = Some(TableName::new( DEFAULT_CATALOG_NAME.to_string(), DEFAULT_SCHEMA_NAME.to_string(), - table.clone(), + table.to_string(), )); - Ok(VisitRecursion::Stop) + Ok(TreeNodeRecursion::Stop) } } } - _ => Ok(VisitRecursion::Continue), + _ => Ok(TreeNodeRecursion::Continue), } } } @@ -236,9 +244,9 @@ impl TableNameRewriter { name.schema_name.clone(), name.table_name.clone(), ); - Transformed::Yes(LogicalPlan::TableScan(table_scan)) + Transformed::yes(LogicalPlan::TableScan(table_scan)) } - _ => Transformed::No(plan), + _ => Transformed::no(plan), }) } } diff --git a/src/query/src/metrics.rs b/src/query/src/metrics.rs index ac9397eec3..d091f9c91b 100644 --- a/src/query/src/metrics.rs +++ b/src/query/src/metrics.rs @@ -76,6 +76,10 @@ impl OnDone { } impl RecordBatchStream for OnDone { + fn name(&self) -> &str { + self.stream.name() + } + fn schema(&self) -> SchemaRef { self.stream.schema() } diff --git a/src/query/src/optimizer.rs b/src/query/src/optimizer.rs index cfce77f039..fc99f8be12 100644 --- a/src/query/src/optimizer.rs +++ b/src/query/src/optimizer.rs @@ -29,9 +29,6 @@ pub trait ExtensionAnalyzerRule { ctx: &QueryEngineContext, config: &ConfigOptions, ) -> Result; - - /// A human readable name for this analyzer rule - fn name(&self) -> &str; } pub mod order_hint; diff --git a/src/query/src/optimizer/order_hint.rs b/src/query/src/optimizer/order_hint.rs index b9a34df0a8..6c027568a2 100644 --- a/src/query/src/optimizer/order_hint.rs +++ b/src/query/src/optimizer/order_hint.rs @@ -15,7 +15,7 @@ use arrow_schema::SortOptions; use common_recordbatch::OrderOption; use datafusion::datasource::DefaultTableSource; -use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeVisitor, VisitRecursion}; +use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion, TreeNodeVisitor}; use datafusion_common::Result as DataFusionResult; use datafusion_expr::expr::Sort; use datafusion_expr::{Expr, LogicalPlan}; @@ -48,6 +48,7 @@ impl OrderHintRule { if let Some(order_expr) = visitor.order_expr.take() { plan.clone() .transform_down(&|plan| Self::set_ordering_hint(plan, &order_expr)) + .map(|x| x.data) } else { Ok(plan.clone()) } @@ -74,7 +75,7 @@ impl OrderHintRule { for sort in order_expr { let name = match sort.expr.try_into_col() { Ok(col) => col.name, - Err(_) => return Ok(Transformed::No(plan)), + Err(_) => return Ok(Transformed::no(plan)), }; opts.push(OrderOption { name, @@ -89,12 +90,12 @@ impl OrderHintRule { } } if transformed { - Ok(Transformed::Yes(plan)) + Ok(Transformed::yes(plan)) } else { - Ok(Transformed::No(plan)) + Ok(Transformed::no(plan)) } } - _ => Ok(Transformed::No(plan)), + _ => Ok(Transformed::no(plan)), } } } @@ -106,9 +107,9 @@ struct OrderHintVisitor { } impl TreeNodeVisitor for OrderHintVisitor { - type N = LogicalPlan; + type Node = LogicalPlan; - fn pre_visit(&mut self, node: &Self::N) -> DataFusionResult { + fn f_down(&mut self, node: &Self::Node) -> DataFusionResult { if let LogicalPlan::Sort(sort) = node { let mut exprs = vec![]; for expr in &sort.expr { @@ -118,7 +119,7 @@ impl TreeNodeVisitor for OrderHintVisitor { } self.order_expr = Some(exprs); } - Ok(VisitRecursion::Continue) + Ok(TreeNodeRecursion::Continue) } } diff --git a/src/query/src/optimizer/string_normalization.rs b/src/query/src/optimizer/string_normalization.rs index 1dfcc4d518..56fb36c3df 100644 --- a/src/query/src/optimizer/string_normalization.rs +++ b/src/query/src/optimizer/string_normalization.rs @@ -32,10 +32,11 @@ impl AnalyzerRule for StringNormalizationRule { let expr = plan .expressions() .into_iter() - .map(|e| e.rewrite(&mut converter)) + .map(|e| e.rewrite(&mut converter).map(|x| x.data)) .collect::>>()?; - plan.with_new_exprs(expr, &inputs).map(Transformed::Yes) + plan.with_new_exprs(expr, inputs).map(Transformed::yes) }) + .map(|x| x.data) } fn name(&self) -> &str { @@ -46,12 +47,12 @@ impl AnalyzerRule for StringNormalizationRule { struct StringNormalizationConverter; impl TreeNodeRewriter for StringNormalizationConverter { - type N = Expr; + type Node = Expr; /// remove extra whitespaces from the String value when /// there is a CAST from a String to Timestamp. /// Otherwise - no modifications applied - fn mutate(&mut self, expr: Expr) -> Result { + fn f_up(&mut self, expr: Expr) -> Result> { let new_expr = match expr { Expr::Cast(Cast { expr, data_type }) => { let expr = match data_type { @@ -71,7 +72,7 @@ impl TreeNodeRewriter for StringNormalizationConverter { } expr => expr, }; - Ok(new_expr) + Ok(Transformed::yes(new_expr)) } } diff --git a/src/query/src/optimizer/type_conversion.rs b/src/query/src/optimizer/type_conversion.rs index aa1fc73d4c..87d0dc29e8 100644 --- a/src/query/src/optimizer/type_conversion.rs +++ b/src/query/src/optimizer/type_conversion.rs @@ -48,8 +48,8 @@ impl ExtensionAnalyzerRule for TypeConversionRule { schema: filter.input.schema().clone(), query_ctx: ctx.query_ctx(), }; - let rewritten = filter.predicate.clone().rewrite(&mut converter)?; - Ok(Transformed::Yes(LogicalPlan::Filter(Filter::try_new( + let rewritten = filter.predicate.clone().rewrite(&mut converter)?.data; + Ok(Transformed::yes(LogicalPlan::Filter(Filter::try_new( rewritten, filter.input, )?))) @@ -68,9 +68,9 @@ impl ExtensionAnalyzerRule for TypeConversionRule { }; let rewrite_filters = filters .into_iter() - .map(|e| e.rewrite(&mut converter)) + .map(|e| e.rewrite(&mut converter).map(|x| x.data)) .collect::>>()?; - Ok(Transformed::Yes(LogicalPlan::TableScan(TableScan { + Ok(Transformed::yes(LogicalPlan::TableScan(TableScan { table_name: table_name.clone(), source: source.clone(), projection, @@ -100,10 +100,10 @@ impl ExtensionAnalyzerRule for TypeConversionRule { let expr = plan .expressions() .into_iter() - .map(|e| e.rewrite(&mut converter)) + .map(|e| e.rewrite(&mut converter).map(|x| x.data)) .collect::>>()?; - plan.with_new_exprs(expr, &inputs).map(Transformed::Yes) + plan.with_new_exprs(expr, inputs).map(Transformed::yes) } LogicalPlan::Subquery { .. } @@ -116,12 +116,10 @@ impl ExtensionAnalyzerRule for TypeConversionRule { | LogicalPlan::Unnest(_) | LogicalPlan::Statement(_) | LogicalPlan::Ddl(_) - | LogicalPlan::Copy(_) => Ok(Transformed::No(plan)), + | LogicalPlan::Copy(_) + | LogicalPlan::RecursiveQuery(_) => Ok(Transformed::no(plan)), }) - } - - fn name(&self) -> &str { - "TypeConversionRule" + .map(|x| x.data) } } @@ -155,9 +153,9 @@ impl TypeConverter { _ => Ok(ScalarValue::Boolean(None)), }, (target_type, value) => { - let value_arr = value.to_array(); - let arr = - compute::cast(&value_arr, target_type).map_err(DataFusionError::ArrowError)?; + let value_arr = value.to_array()?; + let arr = compute::cast(&value_arr, target_type) + .map_err(|e| DataFusionError::ArrowError(e, None))?; ScalarValue::try_from_array( &arr, @@ -207,9 +205,9 @@ impl TypeConverter { } impl TreeNodeRewriter for TypeConverter { - type N = Expr; + type Node = Expr; - fn mutate(&mut self, expr: Expr) -> Result { + fn f_up(&mut self, expr: Expr) -> Result> { let new_expr = match expr { Expr::BinaryExpr(BinaryExpr { left, op, right }) => match op { Operator::Eq @@ -275,7 +273,7 @@ impl TreeNodeRewriter for TypeConverter { }, expr => expr, }; - Ok(new_expr) + Ok(Transformed::yes(new_expr)) } } @@ -310,7 +308,9 @@ mod tests { use std::sync::Arc; use datafusion::logical_expr::expr::AggregateFunction as AggrExpr; - use datafusion_common::{Column, DFField, DFSchema}; + use datafusion_common::arrow::datatypes::Field; + use datafusion_common::{Column, DFSchema}; + use datafusion_expr::expr::AggregateFunctionDefinition; use datafusion_expr::{AggregateFunction, LogicalPlanBuilder}; use datafusion_sql::TableReference; use session::context::QueryContext; @@ -390,11 +390,13 @@ mod tests { let schema = Arc::new( DFSchema::new_with_metadata( - vec![DFField::new( + vec![( None::, - "ts", - DataType::Timestamp(ArrowTimeUnit::Millisecond, None), - true, + Arc::new(Field::new( + "ts", + DataType::Timestamp(ArrowTimeUnit::Millisecond, None), + true, + )), )], HashMap::new(), ) @@ -411,12 +413,13 @@ mod tests { None ))), converter - .mutate( + .f_up( Expr::Column(Column::from_name("ts")).gt(Expr::Literal(ScalarValue::Utf8( Some("2020-09-08T05:42:29+08:00".to_string()), ))) ) .unwrap() + .data ); } @@ -425,11 +428,9 @@ mod tests { let col_name = "is_valid"; let schema = Arc::new( DFSchema::new_with_metadata( - vec![DFField::new( + vec![( None::, - col_name, - DataType::Boolean, - false, + Arc::new(Field::new(col_name, DataType::Boolean, false)), )], HashMap::new(), ) @@ -444,11 +445,12 @@ mod tests { Expr::Column(Column::from_name(col_name)) .eq(Expr::Literal(ScalarValue::Boolean(Some(true)))), converter - .mutate( + .f_up( Expr::Column(Column::from_name(col_name)) .eq(Expr::Literal(ScalarValue::Utf8(Some("true".to_string())))) ) .unwrap() + .data ); } @@ -475,11 +477,12 @@ mod tests { .aggregate( Vec::::new(), vec![Expr::AggregateFunction(AggrExpr { - fun: AggregateFunction::Count, + func_def: AggregateFunctionDefinition::BuiltIn(AggregateFunction::Count), args: vec![Expr::Column(Column::from_name("column1"))], distinct: false, filter: None, order_by: None, + null_treatment: None, })], ) .unwrap() diff --git a/src/query/src/parser.rs b/src/query/src/parser.rs index 4da25d3649..859ae924e7 100644 --- a/src/query/src/parser.rs +++ b/src/query/src/parser.rs @@ -305,9 +305,9 @@ mod test { sort_by: [], \ having: None, \ named_window: [], \ - qualify: None \ - }), order_by: [], limit: None, offset: None, fetch: None, locks: [] } }))"); - + qualify: None, \ + value_table_mode: None \ + }), order_by: [], limit: None, limit_by: [], offset: None, fetch: None, locks: [], for_clause: None } }))"); assert_eq!(format!("{stmt:?}"), expected); } diff --git a/src/query/src/plan.rs b/src/query/src/plan.rs index e81d00a845..34495dee98 100644 --- a/src/query/src/plan.rs +++ b/src/query/src/plan.rs @@ -16,6 +16,7 @@ use std::collections::HashMap; use std::fmt::{Debug, Display}; use common_query::prelude::ScalarValue; +use datafusion_common::ParamValues; use datafusion_expr::LogicalPlan as DfLogicalPlan; use datatypes::data_type::ConcreteDataType; use datatypes::schema::Schema; @@ -82,7 +83,7 @@ impl LogicalPlan { let LogicalPlan::DfPlan(plan) = self; plan.clone() - .replace_params_with_values(values) + .replace_params_with_values(&ParamValues::List(values.to_vec())) .context(DataFusionSnafu) .map(LogicalPlan::DfPlan) } diff --git a/src/query/src/query_engine/state.rs b/src/query/src/query_engine/state.rs index 942ac7c18f..a56439efba 100644 --- a/src/query/src/query_engine/state.rs +++ b/src/query/src/query_engine/state.rs @@ -26,7 +26,6 @@ use common_function::state::FunctionState; use common_query::physical_plan::SessionContext; use common_query::prelude::ScalarUdf; use common_telemetry::warn; -use datafusion::catalog::MemoryCatalogList; use datafusion::dataframe::DataFrame; use datafusion::error::Result as DfResult; use datafusion::execution::context::{QueryPlanner, SessionConfig, SessionState}; @@ -101,18 +100,14 @@ impl QueryEngineState { let mut optimizer = Optimizer::new(); optimizer.rules.push(Arc::new(OrderHintRule)); - let session_state = SessionState::new_with_config_rt_and_catalog_list( - session_config, - runtime_env, - Arc::new(MemoryCatalogList::default()), // pass a dummy catalog list - ) - .with_serializer_registry(Arc::new(ExtensionSerializer)) - .with_analyzer_rules(analyzer.rules) - .with_query_planner(Arc::new(DfQueryPlanner::new( - catalog_list.clone(), - region_query_handler, - ))) - .with_optimizer_rules(optimizer.rules); + let session_state = SessionState::new_with_config_rt(session_config, runtime_env) + .with_serializer_registry(Arc::new(ExtensionSerializer)) + .with_analyzer_rules(analyzer.rules) + .with_query_planner(Arc::new(DfQueryPlanner::new( + catalog_list.clone(), + region_query_handler, + ))) + .with_optimizer_rules(optimizer.rules); let df_context = SessionContext::new_with_state(session_state); diff --git a/src/query/src/range_select/plan.rs b/src/query/src/range_select/plan.rs index bff30d4789..d31097efc0 100644 --- a/src/query/src/range_select/plan.rs +++ b/src/query/src/range_select/plan.rs @@ -33,12 +33,14 @@ use datafusion::execution::context::SessionState; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::udaf::create_aggregate_expr as create_aggr_udf_expr; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, RecordBatchStream, + DisplayAs, DisplayFormatType, ExecutionMode, ExecutionPlan, PlanProperties, RecordBatchStream, SendableRecordBatchStream, }; use datafusion::physical_planner::create_physical_sort_expr; +use datafusion_common::hash_utils::create_hashes; use datafusion_common::utils::{get_arrayref_at_indices, get_row_at_idx}; -use datafusion_common::{DFField, DFSchema, DFSchemaRef, DataFusionError, ScalarValue}; +use datafusion_common::{DFSchema, DFSchemaRef, DataFusionError, ScalarValue}; +use datafusion_expr::expr::AggregateFunctionDefinition; use datafusion_expr::utils::{exprlist_to_fields, COUNT_STAR_EXPANSION}; use datafusion_expr::{ lit, Accumulator, AggregateFunction, Expr, ExprSchemable, LogicalPlan, @@ -46,9 +48,9 @@ use datafusion_expr::{ }; use datafusion_physical_expr::aggregate::utils::down_cast_any_ref; use datafusion_physical_expr::expressions::create_aggregate_expr as create_aggr_expr; -use datafusion_physical_expr::hash_utils::create_hashes; use datafusion_physical_expr::{ - create_physical_expr, AggregateExpr, Distribution, PhysicalExpr, PhysicalSortExpr, + create_physical_expr, AggregateExpr, Distribution, EquivalenceProperties, Partitioning, + PhysicalExpr, PhysicalSortExpr, }; use datatypes::arrow::array::{ Array, ArrayRef, TimestampMillisecondArray, TimestampMillisecondBuilder, UInt32Builder, @@ -181,7 +183,7 @@ impl Accumulator for RangeFirstListValueAcc { Ok(()) } - fn evaluate(&self) -> DataFusionResult { + fn evaluate(&mut self) -> DataFusionResult { Ok(self.data.clone().unwrap_or(ScalarValue::Null)) } @@ -189,7 +191,7 @@ impl Accumulator for RangeFirstListValueAcc { std::mem::size_of_val(self) } - fn state(&self) -> DataFusionResult> { + fn state(&mut self) -> DataFusionResult> { unreachable!("Accumulator::state will not be used in range query") } @@ -466,12 +468,13 @@ impl RangeSelect { fill, .. }| { - Ok(DFField::new_unqualified( + let field = Field::new( name, data_type.clone(), // Only when data fill with Const option, the data can't be null !matches!(fill, Some(Fill::Const(..))), - )) + ); + Ok((None, Arc::new(field))) }, ) .collect::>>() @@ -480,11 +483,10 @@ impl RangeSelect { let ts_field = time_index .to_field(input.schema().as_ref()) .context(DataFusionSnafu)?; - let time_index_name = ts_field.name().clone(); + let time_index_name = ts_field.1.name().clone(); fields.push(ts_field); // add by - let by_fields = - exprlist_to_fields(by.iter().collect::>(), &input).context(DataFusionSnafu)?; + let by_fields = exprlist_to_fields(&by, &input).context(DataFusionSnafu)?; fields.extend(by_fields.clone()); let schema_before_project = Arc::new( DFSchema::new_with_metadata(fields, input.schema().metadata().clone()) @@ -502,7 +504,6 @@ impl RangeSelect { if let Expr::Column(column) = project_expr { schema_before_project .index_of_column_by_name(column.relation.as_ref(), &column.name) - .unwrap_or(None) .ok_or(()) } else { Err(()) @@ -513,7 +514,10 @@ impl RangeSelect { let schema = if let Some(project) = &schema_project { let project_field = project .iter() - .map(|i| schema_before_project.fields()[*i].clone()) + .map(|i| { + let f = schema_before_project.qualified_field(*i); + (f.0.cloned(), Arc::new(f.1.clone())) + }) .collect(); Arc::new( DFSchema::new_with_metadata(project_field, input.schema().metadata().clone()) @@ -555,6 +559,8 @@ impl UserDefinedLogicalNodeCore for RangeSelect { self.range_expr .iter() .map(|expr| expr.expr.clone()) + .chain([self.time_expr.clone()]) + .chain(self.by.clone()) .collect() } @@ -578,18 +584,32 @@ impl UserDefinedLogicalNodeCore for RangeSelect { ) } - fn from_template(&self, _exprs: &[Expr], inputs: &[LogicalPlan]) -> Self { + fn from_template(&self, exprs: &[Expr], inputs: &[LogicalPlan]) -> Self { assert!(!inputs.is_empty()); - + assert!(exprs.len() == self.range_expr.len() + self.by.len() + 1); + let range_expr = exprs + .iter() + .zip(self.range_expr.iter()) + .map(|(e, range)| RangeFn { + name: range.name.clone(), + data_type: range.data_type.clone(), + expr: e.clone(), + range: range.range, + fill: range.fill.clone(), + need_cast: range.need_cast, + }) + .collect(); + let time_expr = exprs[self.range_expr.len()].clone(); + let by = exprs[self.range_expr.len() + 1..].to_vec(); Self { align: self.align, align_to: self.align_to, - range_expr: self.range_expr.clone(), + range_expr, input: Arc::new(inputs[0].clone()), time_index: self.time_index.clone(), - time_expr: self.time_expr.clone(), + time_expr, schema: self.schema.clone(), - by: self.by.clone(), + by, by_schema: self.by_schema.clone(), schema_project: self.schema_project.clone(), schema_before_project: self.schema_before_project.clone(), @@ -603,7 +623,6 @@ impl RangeSelect { is_count_aggr: bool, exprs: &[Expr], df_schema: &Arc, - schema: &Schema, session_state: &SessionState, ) -> DfResult>> { exprs @@ -614,13 +633,12 @@ impl RangeSelect { // At this time, aggregate plan has been replaced by a custom range plan, // so `CountWildcardRule` has not been applied. // We manually modify it when creating the physical plan. - Expr::Wildcard if is_count_aggr => create_physical_expr( + Expr::Wildcard { .. } if is_count_aggr => create_physical_expr( &lit(COUNT_STAR_EXPANSION), - df_schema, - schema, + df_schema.as_ref(), session_state.execution_props(), ), - _ => create_physical_expr(e, df_schema, schema, session_state.execution_props()), + _ => create_physical_expr(e, df_schema.as_ref(), session_state.execution_props()), }) .collect::>>() } @@ -650,10 +668,25 @@ impl RangeSelect { .iter() .map(|range_fn| { let expr = match &range_fn.expr { - Expr::AggregateFunction(aggr) - if aggr.fun == AggregateFunction::FirstValue - || aggr.fun == AggregateFunction::LastValue => - { + Expr::AggregateFunction( + aggr @ datafusion_expr::expr::AggregateFunction { + func_def: + AggregateFunctionDefinition::BuiltIn(AggregateFunction::FirstValue), + .. + }, + ) + | Expr::AggregateFunction( + aggr @ datafusion_expr::expr::AggregateFunction { + func_def: + AggregateFunctionDefinition::BuiltIn(AggregateFunction::LastValue), + .. + }, + ) => { + let is_last_value_func = matches!( + aggr.func_def, + AggregateFunctionDefinition::BuiltIn(AggregateFunction::LastValue) + ); + // Because we only need to find the first_value/last_value, // the complexity of sorting the entire batch is O(nlogn). // We can sort the batch with limit 1. @@ -665,13 +698,12 @@ impl RangeSelect { .map(|x| { create_physical_sort_expr( x, - input_dfschema, - &input_schema, + input_dfschema.as_ref(), session_state.execution_props(), ) .map(|expr| { // reverse the last_value sort - if aggr.fun == AggregateFunction::LastValue { + if is_last_value_func { PhysicalSortExpr { expr: expr.expr, options: SortOptions { @@ -689,14 +721,13 @@ impl RangeSelect { // if user not assign order by, time index is needed as default ordering let time_index = create_physical_expr( &self.time_expr, - input_dfschema, - &input_schema, + input_dfschema.as_ref(), session_state.execution_props(), )?; vec![PhysicalSortExpr { expr: time_index, options: SortOptions { - descending: aggr.fun == AggregateFunction::LastValue, + descending: is_last_value_func, nulls_first: false, }, }] @@ -705,7 +736,6 @@ impl RangeSelect { false, &aggr.args, input_dfschema, - &input_schema, session_state, )?; // first_value/last_value has only one param. @@ -723,8 +753,7 @@ impl RangeSelect { .map(|x| { create_physical_sort_expr( x, - input_dfschema, - &input_schema, + input_dfschema.as_ref(), session_state.execution_props(), ) }) @@ -732,36 +761,39 @@ impl RangeSelect { } else { vec![] }; - let expr = create_aggr_expr( - &aggr.fun, - false, - &self.create_physical_expr_list( - aggr.fun == AggregateFunction::Count, - &aggr.args, - input_dfschema, - &input_schema, - session_state, - )?, - &order_by, - &input_schema, - range_fn.expr.display_name()?, + + let input_phy_exprs = self.create_physical_expr_list( + matches!( + aggr.func_def, + AggregateFunctionDefinition::BuiltIn(AggregateFunction::Count,) + ), + &aggr.args, + input_dfschema, + session_state, )?; - Ok(expr) - } - Expr::AggregateUDF(aggr_udf) => { - let expr = create_aggr_udf_expr( - &aggr_udf.fun, - &self.create_physical_expr_list( + match &aggr.func_def { + AggregateFunctionDefinition::BuiltIn(fun) => create_aggr_expr( + fun, false, - &aggr_udf.args, - input_dfschema, + &input_phy_exprs, + &order_by, &input_schema, - session_state, - )?, - &input_schema, - range_fn.expr.display_name()?, - )?; - Ok(expr) + range_fn.expr.display_name()?, + false, + ), + AggregateFunctionDefinition::UDF(fun) => create_aggr_udf_expr( + fun, + &input_phy_exprs, + &[], + &[], + &input_schema, + range_fn.expr.display_name()?, + false, + ), + f => Err(DataFusionError::NotImplemented(format!( + "Range function from {f:?}" + ))), + } } _ => Err(DataFusionError::Plan(format!( "Unexpected Expr:{} in RangeSelect", @@ -788,24 +820,25 @@ impl RangeSelect { } else { schema_before_project.clone() }; + let by = self.create_physical_expr_list(false, &self.by, input_dfschema, session_state)?; + let cache = PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + Partitioning::UnknownPartitioning(1), + ExecutionMode::Bounded, + ); Ok(Arc::new(RangeSelectExec { input: exec_input, range_exec, align: self.align.as_millis() as Millisecond, align_to: self.align_to, - by: self.create_physical_expr_list( - false, - &self.by, - input_dfschema, - &input_schema, - session_state, - )?, + by, time_index: self.time_index.clone(), schema, by_schema: Arc::new(Schema::new(by_fields)), metric: ExecutionPlanMetricsSet::new(), schema_before_project, schema_project: self.schema_project.clone(), + cache, })) } } @@ -848,6 +881,7 @@ pub struct RangeSelectExec { metric: ExecutionPlanMetricsSet, schema_project: Option>, schema_before_project: SchemaRef, + cache: PlanProperties, } impl DisplayAs for RangeSelectExec { @@ -882,16 +916,12 @@ impl ExecutionPlan for RangeSelectExec { self.schema.clone() } - fn output_partitioning(&self) -> Partitioning { - Partitioning::UnknownPartitioning(1) - } - fn required_input_distribution(&self) -> Vec { vec![Distribution::SinglePartition] } - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { - self.input.output_ordering() + fn properties(&self) -> &PlanProperties { + &self.cache } fn children(&self) -> Vec> { @@ -915,6 +945,7 @@ impl ExecutionPlan for RangeSelectExec { metric: self.metric.clone(), schema_before_project: self.schema_before_project.clone(), schema_project: self.schema_project.clone(), + cache: self.cache.clone(), })) } @@ -963,8 +994,8 @@ impl ExecutionPlan for RangeSelectExec { Some(self.metric.clone_inner()) } - fn statistics(&self) -> Statistics { - self.input.statistics() + fn statistics(&self) -> DataFusionResult { + Ok(Statistics::new_unknown(self.schema.as_ref())) } } @@ -1054,7 +1085,7 @@ impl RangeSelectStream { .iter() .map(|expr| { let value = expr.evaluate(batch)?; - Ok(value.into_array(batch.num_rows())) + value.into_array(batch.num_rows()) }) .collect::>>() } @@ -1168,7 +1199,7 @@ impl RangeSelectStream { for SeriesState { row, align_ts_accumulator, - } in self.series_map.values() + } in self.series_map.values_mut() { // skip empty time series if align_ts_accumulator.is_empty() { @@ -1184,8 +1215,8 @@ impl RangeSelectStream { align_ts_accumulator.keys().copied().collect::>() }; for ts in &align_ts { - if let Some(slot) = align_ts_accumulator.get(ts) { - for (column, acc) in all_scalar.iter_mut().zip(slot.iter()) { + if let Some(slot) = align_ts_accumulator.get_mut(ts) { + for (column, acc) in all_scalar.iter_mut().zip(slot.iter_mut()) { column.push(acc.evaluate()?); } } else { @@ -1415,6 +1446,11 @@ mod test { Field::new(TIME_INDEX_COLUMN, TimestampMillisecondType::DATA_TYPE, true), Field::new("host", DataType::Utf8, true), ])); + let cache = PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + Partitioning::UnknownPartitioning(1), + ExecutionMode::Bounded, + ); let range_select_exec = Arc::new(RangeSelectExec { input: memory_exec, range_exec: vec![ @@ -1450,6 +1486,7 @@ mod test { schema_project: None, by_schema: Arc::new(Schema::new(vec![Field::new("host", DataType::Utf8, true)])), metric: ExecutionPlanMetricsSet::new(), + cache, }); let sort_exec = SortExec::new( vec![ diff --git a/src/query/src/range_select/plan_rewrite.rs b/src/query/src/range_select/plan_rewrite.rs index bab81279e7..4035f20a63 100644 --- a/src/query/src/range_select/plan_rewrite.rs +++ b/src/query/src/range_select/plan_rewrite.rs @@ -25,9 +25,8 @@ use common_time::{Interval, Timestamp, Timezone}; use datafusion::datasource::DefaultTableSource; use datafusion::prelude::Column; use datafusion::scalar::ScalarValue; -use datafusion_common::tree_node::{TreeNode, TreeNodeRewriter, VisitRecursion}; +use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion, TreeNodeRewriter}; use datafusion_common::{DFSchema, DataFusionError, Result as DFResult}; -use datafusion_expr::expr::ScalarUDF; use datafusion_expr::{ Aggregate, Analyze, Explain, Expr, ExprSchemable, Extension, LogicalPlan, LogicalPlanBuilder, Projection, @@ -163,11 +162,7 @@ fn parse_expr_list(args: &[Expr], start: usize, len: usize) -> DFResult args[i].clone(), other => { return Err(dispose_parse_error(*other)); @@ -195,11 +190,11 @@ macro_rules! inconsistent_check { } impl<'a> TreeNodeRewriter for RangeExprRewriter<'a> { - type N = Expr; + type Node = Expr; - fn mutate(&mut self, node: Expr) -> DFResult { - if let Expr::ScalarUDF(func) = &node { - if func.fun.name == "range_fn" { + fn f_down(&mut self, node: Expr) -> DFResult> { + if let Expr::ScalarFunction(func) = &node { + if func.name() == "range_fn" { // `range_fn(func, range, fill, byc, [byv], align, to)` // `[byv]` are variadic arguments, byc indicate the length of arguments let range_expr = self.get_range_expr(&func.args, 0)?; @@ -246,10 +241,10 @@ impl<'a> TreeNodeRewriter for RangeExprRewriter<'a> { }; let alias = Expr::Column(Column::from_name(range_fn.name.clone())); self.range_fn.insert(range_fn); - return Ok(alias); + return Ok(Transformed::yes(alias)); } } - Ok(node) + Ok(Transformed::no(node)) } } @@ -317,7 +312,7 @@ impl RangePlanRewriter { }; let new_expr = expr .iter() - .map(|expr| expr.clone().rewrite(&mut range_rewriter)) + .map(|expr| expr.clone().rewrite(&mut range_rewriter).map(|x| x.data)) .collect::>>() .context(DataFusionSnafu)?; if range_rewriter.by.is_empty() { @@ -385,7 +380,7 @@ impl RangePlanRewriter { .context(DataFusionSnafu)? .build() } - _ => plan.with_new_inputs(&inputs), + _ => plan.with_new_exprs(plan.expressions(), inputs), } .context(DataFusionSnafu)?; Ok(Some(plan)) @@ -401,10 +396,11 @@ impl RangePlanRewriter { /// If the user does not explicitly use the `by` keyword to indicate time series, /// `[row_columns]` will be use as default time series async fn get_index_by(&mut self, schema: &Arc) -> Result<(Expr, Vec)> { - let mut time_index_expr = Expr::Wildcard; + let mut time_index_expr = Expr::Wildcard { qualifier: None }; let mut default_by = vec![]; - for field in schema.fields() { - if let Some(table_ref) = field.qualifier() { + for i in 0..schema.fields().len() { + let (qualifier, _) = schema.qualified_field(i); + if let Some(table_ref) = qualifier { let table = self .table_provider .resolve_table(table_ref.clone()) @@ -446,7 +442,7 @@ impl RangePlanRewriter { } } } - if time_index_expr == Expr::Wildcard { + if matches!(time_index_expr, Expr::Wildcard { .. }) { TimeIndexNotFoundSnafu { table: schema.to_string(), } @@ -461,13 +457,13 @@ fn have_range_in_exprs(exprs: &[Expr]) -> bool { exprs.iter().any(|expr| { let mut find_range = false; let _ = expr.apply(&mut |expr| { - if let Expr::ScalarUDF(ScalarUDF { fun, .. }) = expr { - if fun.name == "range_fn" { + Ok(match expr { + Expr::ScalarFunction(func) if func.name() == "range_fn" => { find_range = true; - return Ok(VisitRecursion::Stop); + TreeNodeRecursion::Stop } - } - Ok(VisitRecursion::Continue) + _ => TreeNodeRecursion::Continue, + }) }); find_range }) @@ -581,8 +577,8 @@ mod test { let query = r#"SELECT (covar(field_0 + field_1, field_1)/4) RANGE '5m' FROM test ALIGN '1h';"#; let expected = String::from( - "Projection: COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m / Int64(4) [COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m / Int64(4):Float64;N]\ - \n RangeSelect: range_exprs=[COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1, test.tag_2, test.tag_3, test.tag_4], time_index=timestamp [COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8]\ + "Projection: COVAR(test.field_0 + test.field_1,test.field_1) RANGE 5m / Int64(4) [COVAR(test.field_0 + test.field_1,test.field_1) RANGE 5m / Int64(4):Float64;N]\ + \n RangeSelect: range_exprs=[COVAR(test.field_0 + test.field_1,test.field_1) RANGE 5m], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1, test.tag_2, test.tag_3, test.tag_4], time_index=timestamp [COVAR(test.field_0 + test.field_1,test.field_1) RANGE 5m:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8]\ \n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]" ); query_plan_compare(query, expected).await; @@ -662,7 +658,7 @@ mod test { async fn complex_range_expr() { let query = r#"SELECT gcd(CAST(max(field_0 + 1) Range '5m' FILL NULL AS Int64), CAST(tag_0 AS Int64)) + round(max(field_2+1) Range '6m' FILL NULL + 1) + max(field_2+3) Range '10m' FILL NULL * CAST(tag_1 AS Float64) + 1 FROM test ALIGN '1h' by (tag_0, tag_1);"#; let expected = String::from( - "Projection: gcd(CAST(MAX(test.field_0 + Int64(1)) RANGE 5m FILL NULL AS Int64), CAST(test.tag_0 AS Int64)) + round(MAX(test.field_2 + Int64(1)) RANGE 6m FILL NULL + Int64(1)) + MAX(test.field_2 + Int64(3)) RANGE 10m FILL NULL * CAST(test.tag_1 AS Float64) + Int64(1) [gcd(MAX(test.field_0 + Int64(1)) RANGE 5m FILL NULL,test.tag_0) + round(MAX(test.field_2 + Int64(1)) RANGE 6m FILL NULL + Int64(1)) + MAX(test.field_2 + Int64(3)) RANGE 10m FILL NULL * test.tag_1 + Int64(1):Float64;N]\ + "Projection: gcd(arrow_cast(MAX(test.field_0 + Int64(1)) RANGE 5m FILL NULL, Utf8(\"Int64\")), arrow_cast(test.tag_0, Utf8(\"Int64\"))) + round(MAX(test.field_2 + Int64(1)) RANGE 6m FILL NULL + Int64(1)) + MAX(test.field_2 + Int64(3)) RANGE 10m FILL NULL * arrow_cast(test.tag_1, Utf8(\"Float64\")) + Int64(1) [gcd(arrow_cast(MAX(test.field_0 + Int64(1)) RANGE 5m FILL NULL,Utf8(\"Int64\")),arrow_cast(test.tag_0,Utf8(\"Int64\"))) + round(MAX(test.field_2 + Int64(1)) RANGE 6m FILL NULL + Int64(1)) + MAX(test.field_2 + Int64(3)) RANGE 10m FILL NULL * arrow_cast(test.tag_1,Utf8(\"Float64\")) + Int64(1):Float64;N]\ \n RangeSelect: range_exprs=[MAX(test.field_0 + Int64(1)) RANGE 5m FILL NULL, MAX(test.field_2 + Int64(1)) RANGE 6m FILL NULL, MAX(test.field_2 + Int64(3)) RANGE 10m FILL NULL], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1], time_index=timestamp [MAX(test.field_0 + Int64(1)) RANGE 5m FILL NULL:Float64;N, MAX(test.field_2 + Int64(1)) RANGE 6m FILL NULL:Float64;N, MAX(test.field_2 + Int64(3)) RANGE 10m FILL NULL:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\ \n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]" ); @@ -673,7 +669,7 @@ mod test { async fn range_linear_on_integer() { let query = r#"SELECT min(CAST(field_0 AS Int64) + CAST(field_1 AS Int64)) RANGE '5m' FILL LINEAR FROM test ALIGN '1h' by (tag_0,tag_1);"#; let expected = String::from( - "RangeSelect: range_exprs=[MIN(test.field_0 + test.field_1) RANGE 5m FILL LINEAR], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1], time_index=timestamp [MIN(test.field_0 + test.field_1) RANGE 5m FILL LINEAR:Float64;N]\ + "RangeSelect: range_exprs=[MIN(arrow_cast(test.field_0,Utf8(\"Int64\")) + arrow_cast(test.field_1,Utf8(\"Int64\"))) RANGE 5m FILL LINEAR], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1], time_index=timestamp [MIN(arrow_cast(test.field_0,Utf8(\"Int64\")) + arrow_cast(test.field_1,Utf8(\"Int64\"))) RANGE 5m FILL LINEAR:Float64;N]\ \n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]" ); query_plan_compare(query, expected).await; diff --git a/src/query/src/sql.rs b/src/query/src/sql.rs index 645bac9f31..5aee6cc4ae 100644 --- a/src/query/src/sql.rs +++ b/src/query/src/sql.rs @@ -18,7 +18,8 @@ use std::collections::HashMap; use std::sync::Arc; use catalog::information_schema::{ - columns, key_column_usage, schemata, tables, COLUMNS, KEY_COLUMN_USAGE, SCHEMATA, TABLES, + columns, key_column_usage, schemata, tables, CHARACTER_SETS, COLLATIONS, COLUMNS, + KEY_COLUMN_USAGE, SCHEMATA, TABLES, }; use catalog::CatalogManagerRef; use common_catalog::consts::{ @@ -378,7 +379,7 @@ pub async fn show_index( lit("").alias(INDEX_COMMENT_COLUMN), lit(YES_STR).alias(INDEX_VISIBLE_COLUMN), null().alias(INDEX_EXPRESSION_COLUMN), - Expr::Wildcard, + Expr::Wildcard { qualifier: None }, ]; let projects = vec![ @@ -469,6 +470,76 @@ pub async fn show_tables( .await } +/// Execute `SHOW COLLATION` statement and returns the `Output` if success. +pub async fn show_collations( + kind: ShowKind, + query_engine: &QueryEngineRef, + catalog_manager: &CatalogManagerRef, + query_ctx: QueryContextRef, +) -> Result { + // Refer to https://dev.mysql.com/doc/refman/8.0/en/show-collation.html + let projects = vec![ + ("collation_name", "Collation"), + ("character_set_name", "Charset"), + ("id", "Id"), + ("is_default", "Default"), + ("is_compiled", "Compiled"), + ("sortlen", "Sortlen"), + ]; + + let filters = vec![]; + let like_field = Some("collation_name"); + let sort = vec![]; + + query_from_information_schema_table( + query_engine, + catalog_manager, + query_ctx, + COLLATIONS, + vec![], + projects, + filters, + like_field, + sort, + kind, + ) + .await +} + +/// Execute `SHOW CHARSET` statement and returns the `Output` if success. +pub async fn show_charsets( + kind: ShowKind, + query_engine: &QueryEngineRef, + catalog_manager: &CatalogManagerRef, + query_ctx: QueryContextRef, +) -> Result { + // Refer to https://dev.mysql.com/doc/refman/8.0/en/show-character-set.html + let projects = vec![ + ("character_set_name", "Charset"), + ("description", "Description"), + ("default_collate_name", "Default collation"), + ("maxlen", "Maxlen"), + ]; + + let filters = vec![]; + let like_field = Some("character_set_name"); + let sort = vec![]; + + query_from_information_schema_table( + query_engine, + catalog_manager, + query_ctx, + CHARACTER_SETS, + vec![], + projects, + filters, + like_field, + sort, + kind, + ) + .await +} + pub fn show_variable(stmt: ShowVariables, query_ctx: QueryContextRef) -> Result { let variable = stmt.variable.to_string().to_uppercase(); let value = match variable.as_str() { diff --git a/src/query/src/sql/show_create_table.rs b/src/query/src/sql/show_create_table.rs index 97d8aba4fb..2004590aa5 100644 --- a/src/query/src/sql/show_create_table.rs +++ b/src/query/src/sql/show_create_table.rs @@ -46,7 +46,7 @@ fn string_value(s: impl Into) -> SqlValue { fn sql_option(name: &str, value: SqlValue) -> SqlOption { SqlOption { name: name.into(), - value, + value: Expr::Value(value), } } @@ -141,6 +141,7 @@ fn create_table_constraints( name: Some(TIME_INDEX.into()), columns: vec![Ident::with_quote(quote_style, column_name)], is_primary: false, + characteristics: None, }); } if !table_meta.primary_key_indices.is_empty() { @@ -152,6 +153,7 @@ fn create_table_constraints( name: None, columns, is_primary: true, + characteristics: None, }); } diff --git a/src/query/src/tests/time_range_filter_test.rs b/src/query/src/tests/time_range_filter_test.rs index b47ecce99f..9c19b1f9b1 100644 --- a/src/query/src/tests/time_range_filter_test.rs +++ b/src/query/src/tests/time_range_filter_test.rs @@ -58,7 +58,7 @@ struct DataSourceWrapper { impl DataSource for DataSourceWrapper { fn get_stream(&self, request: ScanRequest) -> Result { - *self.filter.write().unwrap() = request.filters.clone(); + self.filter.write().unwrap().clone_from(&request.filters); self.inner.get_stream(request) } } diff --git a/src/script/Cargo.toml b/src/script/Cargo.toml index 50dca7ccbb..4602a334c9 100644 --- a/src/script/Cargo.toml +++ b/src/script/Cargo.toml @@ -11,6 +11,7 @@ python = [ "dep:datafusion", "dep:datafusion-common", "dep:datafusion-expr", + "dep:datafusion-functions", "dep:datafusion-physical-expr", "dep:rustpython-vm", "dep:rustpython-parser", @@ -45,6 +46,7 @@ crossbeam-utils = "0.8.14" datafusion = { workspace = true, optional = true } datafusion-common = { workspace = true, optional = true } datafusion-expr = { workspace = true, optional = true } +datafusion-functions = { workspace = true, optional = true } datafusion-physical-expr = { workspace = true, optional = true } datatypes.workspace = true futures.workspace = true @@ -54,7 +56,7 @@ paste = { workspace = true, optional = true } prometheus.workspace = true query.workspace = true # TODO(discord9): This is a forked and tweaked version of RustPython, please update it to newest original RustPython After RustPython support GC -pyo3 = { version = "0.19", optional = true, features = ["abi3", "abi3-py37"] } +pyo3 = { version = "0.20", optional = true, features = ["abi3", "abi3-py37"] } rustpython-codegen = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" } rustpython-compiler = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" } rustpython-compiler-core = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" } diff --git a/src/script/src/python/engine.rs b/src/script/src/python/engine.rs index 5ebbd202ab..58e54d4b8e 100644 --- a/src/script/src/python/engine.rs +++ b/src/script/src/python/engine.rs @@ -126,6 +126,10 @@ impl Function for PyUDF { } fn signature(&self) -> common_query::prelude::Signature { + if self.copr.arg_types.is_empty() { + return Signature::any(0, Volatility::Volatile); + } + // try our best to get a type signature let mut arg_types = Vec::with_capacity(self.copr.arg_types.len()); let mut know_all_types = true; diff --git a/src/script/src/python/ffi_types/copr.rs b/src/script/src/python/ffi_types/copr.rs index 1af3f416f3..704b0db77d 100644 --- a/src/script/src/python/ffi_types/copr.rs +++ b/src/script/src/python/ffi_types/copr.rs @@ -21,8 +21,10 @@ use std::sync::{Arc, Weak}; use common_query::OutputData; use common_recordbatch::{RecordBatch, RecordBatches}; +use datafusion_common::ScalarValue; use datatypes::arrow::compute; use datatypes::data_type::{ConcreteDataType, DataType}; +use datatypes::prelude::Value; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; use datatypes::vectors::{Helper, VectorRef}; // use crate::python::builtins::greptime_builtin; @@ -42,7 +44,9 @@ use vm::{pyclass as rspyclass, PyObjectRef, PyPayload, PyResult, VirtualMachine} use super::py_recordbatch::PyRecordBatch; use crate::engine::EvalContext; -use crate::python::error::{ensure, ArrowSnafu, OtherSnafu, Result, TypeCastSnafu}; +use crate::python::error::{ + ensure, ArrowSnafu, DataFusionSnafu, OtherSnafu, Result, TypeCastSnafu, +}; use crate::python::ffi_types::PyVector; #[cfg(feature = "pyo3_backend")] use crate::python::pyo3::pyo3_exec_parsed; @@ -179,6 +183,25 @@ impl Coprocessor { /// check if real types and annotation types(if have) is the same, if not try cast columns to annotated type pub(crate) fn check_and_cast_type(&self, cols: &mut [VectorRef]) -> Result<()> { + for col in cols.iter_mut() { + if let ConcreteDataType::List(x) = col.data_type() { + let values = + ScalarValue::convert_array_to_scalar_vec(col.to_arrow_array().as_ref()) + .context(DataFusionSnafu)? + .into_iter() + .flatten() + .map(Value::try_from) + .collect::, _>>() + .context(TypeCastSnafu)?; + + let mut builder = x.item_type().create_mutable_vector(values.len()); + for v in values.iter() { + builder.push_value_ref(v.as_value_ref()); + } + *col = builder.to_vector(); + } + } + let return_types = &self.return_types; // allow ignore Return Type Annotation if return_types.is_empty() { diff --git a/src/script/src/python/ffi_types/pair_tests.rs b/src/script/src/python/ffi_types/pair_tests.rs index 37e3c76994..781bae9b9e 100644 --- a/src/script/src/python/ffi_types/pair_tests.rs +++ b/src/script/src/python/ffi_types/pair_tests.rs @@ -209,6 +209,7 @@ fn eval_pyo3(case: CodeBlockTestCase) { let res_vec = locals .get_item("ret") .unwrap() + .unwrap() .extract::() .map_err(|e| { dbg!(&case.script); diff --git a/src/script/src/python/ffi_types/vector.rs b/src/script/src/python/ffi_types/vector.rs index 1f75efdecf..bbede552cf 100644 --- a/src/script/src/python/ffi_types/vector.rs +++ b/src/script/src/python/ffi_types/vector.rs @@ -525,17 +525,12 @@ pub fn val_to_pyobj(val: value::Value, vm: &VirtualMachine) -> PyResult { // FIXME(dennis): lose the timestamp unit here Value::Timestamp(v) => vm.ctx.new_int(v.value()).into(), value::Value::List(list) => { - let list = list.items().as_ref(); - match list { - Some(list) => { - let list: Vec<_> = list - .iter() - .map(|v| val_to_pyobj(v.clone(), vm)) - .collect::>()?; - vm.ctx.new_list(list).into() - } - None => vm.ctx.new_list(Vec::new()).into(), - } + let list: Vec<_> = list + .items() + .iter() + .map(|v| val_to_pyobj(v.clone(), vm)) + .collect::>()?; + vm.ctx.new_list(list).into() } #[allow(unreachable_patterns)] _ => return Err(vm.new_type_error(format!("Convert from {val:?} is not supported yet"))), diff --git a/src/script/src/python/pyo3/builtins.rs b/src/script/src/python/pyo3/builtins.rs index c4c10cc0e4..bc8f23107a 100644 --- a/src/script/src/python/pyo3/builtins.rs +++ b/src/script/src/python/pyo3/builtins.rs @@ -19,7 +19,7 @@ use common_function::function_registry::FUNCTION_REGISTRY; use datafusion::arrow::array::{ArrayRef, NullArray}; use datafusion::physical_plan::expressions; use datafusion_expr::ColumnarValue; -use datafusion_physical_expr::{math_expressions, AggregateExpr}; +use datafusion_physical_expr::AggregateExpr; use datatypes::vectors::VectorRef; use pyo3::exceptions::{PyKeyError, PyValueError}; use pyo3::prelude::*; @@ -133,7 +133,7 @@ fn get_globals(py: Python) -> PyResult<&PyDict> { fn dataframe(py: Python) -> PyResult { let globals = get_globals(py)?; let df = globals - .get_item("__dataframe__") + .get_item("__dataframe__")? .ok_or_else(|| PyKeyError::new_err("No __dataframe__ variable is found"))? .extract::()?; Ok(df) @@ -144,7 +144,7 @@ fn dataframe(py: Python) -> PyResult { pub(crate) fn query_engine(py: Python) -> PyResult { let globals = get_globals(py)?; let query = globals - .get_item("__query__") + .get_item("__query__")? .ok_or_else(|| PyKeyError::new_err("No __query__ variable is found"))? .extract::()?; Ok(query) @@ -237,7 +237,9 @@ macro_rules! bind_call_unary_math_function { fn $DF_FUNC(py: Python<'_>, val: PyObject) -> PyResult { let args = &[all_to_f64(try_into_columnar_value(py, val)?).map_err(PyValueError::new_err)?]; - let res = math_expressions::$DF_FUNC(args).map_err(|e| PyValueError::new_err(format!("{e:?}")))?; + let res = datafusion_functions::math::$DF_FUNC() + .invoke(args) + .map_err(|e| PyValueError::new_err(format!("{e:?}")))?; columnar_value_to_py_any(py, res) } )* @@ -293,18 +295,19 @@ fn random(py: Python<'_>, len: usize) -> PyResult { // more info at: https://doc.rust-lang.org/reference/procedural-macros.html#procedural-macro-hygiene let arg = NullArray::new(len); let args = &[ColumnarValue::Array(std::sync::Arc::new(arg) as _)]; - let res = - math_expressions::random(args).map_err(|e| PyValueError::new_err(format!("{e:?}")))?; - + let res = datafusion_functions::math::random() + .invoke(args) + .map_err(|e| PyValueError::new_err(format!("{e:?}")))?; columnar_value_to_py_any(py, res) } #[pyfunction] fn round(py: Python<'_>, val: PyObject) -> PyResult { let value = try_into_columnar_value(py, val)?; - let array = value.into_array(1); - let result = - math_expressions::round(&[array]).map_err(|e| PyValueError::new_err(format!("{e:?}")))?; + let result = datafusion_functions::math::round() + .invoke(&[value]) + .and_then(|x| x.into_array(1)) + .map_err(|e| PyValueError::new_err(format!("{e:?}")))?; columnar_value_to_py_any(py, ColumnarValue::Array(result)) } @@ -368,7 +371,19 @@ fn approx_percentile_cont(py: Python<'_>, values: &PyVector, percent: f64) -> Py ) } -bind_aggr_expr!(array_agg, ArrayAgg,[v0], v0, expr0=>0); +#[pyfunction] +fn array_agg(py: Python<'_>, v: &PyVector) -> PyResult { + eval_df_aggr_expr( + py, + expressions::ArrayAgg::new( + Arc::new(expressions::Column::new("expr0", 0)) as _, + "ArrayAgg", + v.arrow_data_type(), + true, + ), + &[v.to_arrow_array()], + ) +} bind_aggr_expr!(avg, Avg,[v0], v0, expr0=>0); diff --git a/src/script/src/python/pyo3/copr_impl.rs b/src/script/src/python/pyo3/copr_impl.rs index 50c4fd3d23..2dbd574827 100644 --- a/src/script/src/python/pyo3/copr_impl.rs +++ b/src/script/src/python/pyo3/copr_impl.rs @@ -144,7 +144,9 @@ coprocessor = copr // could generate a call in python code and use Python::run to run it, just like in RustPython // Expect either: a PyVector Or a List/Tuple of PyVector py.run(&script, Some(globals), Some(locals))?; - let result = locals.get_item("_return_from_coprocessor").ok_or_else(|| PyValueError::new_err("Can't find return value of coprocessor function"))?; + let result = locals.get_item("_return_from_coprocessor")?.ok_or_else(|| + PyValueError::new_err(format!("cannot find the return value of script '{script}'")) + )?; let col_len = rb.as_ref().map(|rb| rb.num_rows()).unwrap_or(1); py_any_to_vec(result, col_len) diff --git a/src/script/src/python/pyo3/utils.rs b/src/script/src/python/pyo3/utils.rs index 04146a6a6f..9738b8ca9e 100644 --- a/src/script/src/python/pyo3/utils.rs +++ b/src/script/src/python/pyo3/utils.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::{Arc, Mutex}; +use std::sync::Mutex; use arrow::pyarrow::PyArrowException; use common_telemetry::info; @@ -27,7 +27,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::{PyBool, PyFloat, PyInt, PyList, PyTuple}; -use crate::python::ffi_types::utils::{collect_diff_types_string, new_item_field}; +use crate::python::ffi_types::utils::collect_diff_types_string; use crate::python::ffi_types::PyVector; use crate::python::metric; use crate::python::pyo3::builtins::greptime_builtins; @@ -75,9 +75,10 @@ pub fn val_to_py_any(py: Python<'_>, val: Value) -> PyResult { Value::DateTime(val) => val.val().to_object(py), Value::Timestamp(val) => val.value().to_object(py), Value::List(val) => { - let list = val.items().clone().unwrap_or(Default::default()); - let list = list - .into_iter() + let list = val + .items() + .iter() + .cloned() .map(|v| val_to_py_any(py, v)) .collect::>>()?; list.to_object(py) @@ -211,9 +212,13 @@ pub fn scalar_value_to_py_any(py: Python<'_>, val: ScalarValue) -> PyResult Ok(py.None()), $(ScalarValue::$scalar_ty(Some(v)) => Ok(v.to_object(py)),)* - ScalarValue::List(Some(col), _) => { + ScalarValue::List(array) => { + let col = ScalarValue::convert_array_to_scalar_vec(array.as_ref()).map_err(|e| + PyValueError::new_err(format!("{e}")) + )?; let list:Vec = col .into_iter() + .flatten() .map(|v| scalar_value_to_py_any(py, v)) .collect::>()?; let list = PyList::new(py, list); @@ -281,8 +286,7 @@ pub fn try_into_columnar_value(py: Python<'_>, obj: PyObject) -> PyResult, obj: PyObject) -> PyResult Stri .unwrap_or_else(|| "Nothing".to_string()) } -fn new_item_field(data_type: ArrowDataType) -> Field { - Field::new("item", data_type, false) -} - /// try to turn a Python Object into a PyVector or a scalar that can be use for calculate /// /// supported scalar are(leftside is python data type, right side is rust type): @@ -119,8 +113,7 @@ pub fn try_into_columnar_value(obj: PyObjectRef, vm: &VirtualMachine) -> PyResul if ret.is_empty() { // TODO(dennis): empty list, we set type as null. return Ok(DFColValue::Scalar(ScalarValue::List( - None, - Arc::new(new_item_field(ArrowDataType::Null)), + ScalarValue::new_list(&[], &ArrowDataType::Null), ))); } @@ -132,8 +125,7 @@ pub fn try_into_columnar_value(obj: PyObjectRef, vm: &VirtualMachine) -> PyResul ))); } Ok(DFColValue::Scalar(ScalarValue::List( - Some(ret), - Arc::new(new_item_field(ty)), + ScalarValue::new_list(&ret, &ty), ))) } else { Err(vm.new_type_error(format!( @@ -176,9 +168,11 @@ fn scalar_val_try_into_py_obj(val: ScalarValue, vm: &VirtualMachine) -> PyResult ScalarValue::Float64(Some(v)) => Ok(PyFloat::from(v).into_pyobject(vm)), ScalarValue::Int64(Some(v)) => Ok(PyInt::from(v).into_pyobject(vm)), ScalarValue::UInt64(Some(v)) => Ok(PyInt::from(v).into_pyobject(vm)), - ScalarValue::List(Some(col), _) => { - let list = col + ScalarValue::List(list) => { + let list = ScalarValue::convert_array_to_scalar_vec(list.as_ref()) + .map_err(|e| from_df_err(e, vm))? .into_iter() + .flatten() .map(|v| scalar_val_try_into_py_obj(v, vm)) .collect::>()?; let list = vm.ctx.new_list(list); @@ -228,9 +222,10 @@ macro_rules! bind_call_unary_math_function { ($DF_FUNC: ident, $vm: ident $(,$ARG: ident)*) => { fn inner_fn($($ARG: PyObjectRef,)* vm: &VirtualMachine) -> PyResult { let args = &[$(all_to_f64(try_into_columnar_value($ARG, vm)?, vm)?,)*]; - let res = math_expressions::$DF_FUNC(args).map_err(|err| from_df_err(err, vm))?; - let ret = try_into_py_obj(res, vm)?; - Ok(ret) + datafusion_functions::math::$DF_FUNC() + .invoke(args) + .map_err(|e| from_df_err(e, vm)) + .and_then(|x| try_into_py_obj(x, vm)) } return inner_fn($($ARG,)* $vm); }; @@ -295,7 +290,6 @@ pub(crate) mod greptime_builtin { use datafusion::dataframe::DataFrame as DfDataFrame; use datafusion::physical_plan::expressions; use datafusion_expr::{ColumnarValue as DFColValue, Expr as DfExpr}; - use datafusion_physical_expr::math_expressions; use datatypes::arrow::array::{ArrayRef, Int64Array, NullArray}; use datatypes::arrow::error::ArrowError; use datatypes::arrow::{self, compute}; @@ -548,8 +542,10 @@ pub(crate) mod greptime_builtin { #[pyfunction] fn round(val: PyObjectRef, vm: &VirtualMachine) -> PyResult { let value = try_into_columnar_value(val, vm)?; - let array = value.into_array(1); - let result = math_expressions::round(&[array]).map_err(|e| from_df_err(e, vm))?; + let result = datafusion_functions::math::round() + .invoke(&[value]) + .and_then(|x| x.into_array(1)) + .map_err(|e| from_df_err(e, vm))?; try_into_py_obj(DFColValue::Array(result), vm) } @@ -604,7 +600,9 @@ pub(crate) mod greptime_builtin { // more info at: https://doc.rust-lang.org/reference/procedural-macros.html#procedural-macro-hygiene let arg = NullArray::new(len); let args = &[DFColValue::Array(std::sync::Arc::new(arg) as _)]; - let res = math_expressions::random(args).map_err(|err| from_df_err(err, vm))?; + let res = datafusion_functions::math::random() + .invoke(args) + .map_err(|err| from_df_err(err, vm))?; let ret = try_into_py_obj(res, vm)?; Ok(ret) } @@ -673,13 +671,16 @@ pub(crate) mod greptime_builtin { /// effectively equals to `list(vector)` #[pyfunction] fn array_agg(values: PyVectorRef, vm: &VirtualMachine) -> PyResult { - bind_aggr_fn!( - ArrayAgg, - vm, + eval_aggr_fn( + expressions::ArrayAgg::new( + Arc::new(expressions::Column::new("expr0", 0)) as _, + "ArrayAgg", + values.arrow_data_type(), + false, + ), &[values.to_arrow_array()], - values.arrow_data_type(), - expr0 - ); + vm, + ) } /// directly port from datafusion's `avg` function diff --git a/src/script/src/python/rspython/builtins/test.rs b/src/script/src/python/rspython/builtins/test.rs index 0245cd3e9d..3a5ea2115b 100644 --- a/src/script/src/python/rspython/builtins/test.rs +++ b/src/script/src/python/rspython/builtins/test.rs @@ -18,6 +18,7 @@ use std::io::Read; use std::path::Path; use std::sync::Arc; +use arrow::array::Array; use common_telemetry::{error, info}; use datatypes::arrow::array::{Float64Array, Int64Array}; use datatypes::arrow::compute; @@ -68,18 +69,18 @@ fn convert_scalar_to_py_obj_and_back() { } else { panic!("Convert errors, expect 1") } - let col = DFColValue::Scalar(ScalarValue::List( - Some(vec![ - ScalarValue::Int64(Some(1)), - ScalarValue::Int64(Some(2)), - ]), - Arc::new(Field::new("item", ArrowDataType::Int64, false)), - )); + let col = DFColValue::Scalar(ScalarValue::List(ScalarValue::new_list( + &[ScalarValue::Int64(Some(1)), ScalarValue::Int64(Some(2))], + &ArrowDataType::Int64, + ))); let to = try_into_py_obj(col, vm).unwrap(); let back = try_into_columnar_value(to, vm).unwrap(); - if let DFColValue::Scalar(ScalarValue::List(Some(list), field)) = back { - assert_eq!(list.len(), 2); - assert_eq!(*field.data_type(), ArrowDataType::Int64); + if let DFColValue::Scalar(ScalarValue::List(list)) = back { + assert_eq!(list.len(), 1); + assert_eq!( + list.data_type(), + &ArrowDataType::List(Arc::new(Field::new_list_field(ArrowDataType::Int64, true))) + ); } let list: Vec = vec![vm.ctx.new_int(1).into(), vm.ctx.new_int(2).into()]; let nested_list: Vec = diff --git a/src/script/src/python/rspython/builtins/testcases.ron b/src/script/src/python/rspython/builtins/testcases.ron index 8e637c6ed8..b5882048e8 100644 --- a/src/script/src/python/rspython/builtins/testcases.ron +++ b/src/script/src/python/rspython/builtins/testcases.ron @@ -707,7 +707,7 @@ from greptime import * sin(num)"#, expect: Ok(( ty: Float64, - value: Float(0.8414709848078965) + value: FloatVec([0.8414709848078965]) )) ), TestCase( @@ -722,7 +722,7 @@ from greptime import * sin(num)"#, expect: Ok(( ty: Float64, - value: Float(0.8414709848078965) + value: FloatVec([0.8414709848078965]) )) ), TestCase( @@ -732,7 +732,7 @@ from greptime import * sin(True)"#, expect: Ok(( ty: Float64, - value: Float(0.8414709848078965) + value: FloatVec([0.8414709848078965]) )) ), TestCase( @@ -747,7 +747,7 @@ from greptime import * sin(num)"#, expect: Ok(( ty: Float64, - value: Float(0.0) + value: FloatVec([0.0]) )) ), // test if string returns error correctly diff --git a/src/script/src/python/rspython/copr_impl.rs b/src/script/src/python/rspython/copr_impl.rs index 89f12f6678..1d5cb4e3bd 100644 --- a/src/script/src/python/rspython/copr_impl.rs +++ b/src/script/src/python/rspython/copr_impl.rs @@ -38,7 +38,7 @@ use crate::python::rspython::dataframe_impl::data_frame::set_dataframe_in_scope; use crate::python::rspython::dataframe_impl::init_data_frame; use crate::python::rspython::utils::{format_py_error, is_instance, py_obj_to_vec}; -thread_local!(static INTERPRETER: RefCell>> = RefCell::new(None)); +thread_local!(static INTERPRETER: RefCell>> = const { RefCell::new(None) }); /// Using `RustPython` to run a parsed `Coprocessor` struct as input to execute python code pub(crate) fn rspy_exec_parsed( diff --git a/src/script/src/python/rspython/utils.rs b/src/script/src/python/rspython/utils.rs index e24b3005c3..ccaf5454fc 100644 --- a/src/script/src/python/rspython/utils.rs +++ b/src/script/src/python/rspython/utils.rs @@ -14,12 +14,13 @@ use std::sync::Arc; +use arrow::array::ArrayRef; use datafusion_common::ScalarValue; use datafusion_expr::ColumnarValue as DFColValue; use datatypes::prelude::ScalarVector; use datatypes::value::Value; use datatypes::vectors::{ - BooleanVector, Float64Vector, Helper, Int64Vector, NullVector, StringVector, VectorRef, + BooleanVector, Float64Vector, Helper, Int64Vector, StringVector, VectorRef, }; use rustpython_vm::builtins::{PyBaseExceptionRef, PyBool, PyFloat, PyInt, PyList, PyStr}; use rustpython_vm::object::PyObjectPayload; @@ -134,15 +135,9 @@ pub fn py_obj_to_vec( try_into_columnar_value(obj.clone(), vm).map_err(|e| format_py_error(e, vm))?; match columnar_value { - DFColValue::Scalar(ScalarValue::List(scalars, _datatype)) => match scalars { - Some(scalars) => { - let array = - ScalarValue::iter_to_array(scalars).context(error::DataFusionSnafu)?; - - Helper::try_into_vector(array).context(error::TypeCastSnafu) - } - None => Ok(Arc::new(NullVector::new(0))), - }, + DFColValue::Scalar(ScalarValue::List(array)) => { + Helper::try_into_vector(array as ArrayRef).context(error::TypeCastSnafu) + } _ => unreachable!(), } } else { diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml index b965a3dc3f..ef8a2f751d 100644 --- a/src/servers/Cargo.toml +++ b/src/servers/Cargo.toml @@ -99,7 +99,7 @@ tokio.workspace = true tokio-rustls = "0.25" tokio-stream = { workspace = true, features = ["net"] } tonic.workspace = true -tonic-reflection = "0.10" +tonic-reflection = "0.11" tower = { version = "0.4", features = ["full"] } tower-http = { version = "0.4", features = ["full"] } urlencoding = "2.1" diff --git a/src/servers/src/grpc/authorize.rs b/src/servers/src/grpc/authorize.rs index ae003640ea..4b0c986912 100644 --- a/src/servers/src/grpc/authorize.rs +++ b/src/servers/src/grpc/authorize.rs @@ -20,7 +20,7 @@ use auth::UserProviderRef; use hyper::Body; use session::context::QueryContext; use tonic::body::BoxBody; -use tonic::transport::NamedService; +use tonic::server::NamedService; use tower::{Layer, Service}; use crate::http::authorize::{extract_catalog_and_schema, extract_username_and_password}; diff --git a/src/servers/src/grpc/flight.rs b/src/servers/src/grpc/flight.rs index 9ed2ed85d3..cd1d2a4bd0 100644 --- a/src/servers/src/grpc/flight.rs +++ b/src/servers/src/grpc/flight.rs @@ -21,7 +21,7 @@ use api::v1::GreptimeRequest; use arrow_flight::flight_service_server::FlightService; use arrow_flight::{ Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightInfo, - HandshakeRequest, HandshakeResponse, PutResult, SchemaResult, Ticket, + HandshakeRequest, HandshakeResponse, PollInfo, PutResult, SchemaResult, Ticket, }; use async_trait::async_trait; use common_grpc::flight::{FlightEncoder, FlightMessage}; @@ -96,6 +96,13 @@ impl FlightService for FlightCraftWrapper { Err(Status::unimplemented("Not yet implemented")) } + async fn poll_flight_info( + &self, + _: Request, + ) -> TonicResult> { + Err(Status::unimplemented("Not yet implemented")) + } + async fn get_schema( &self, _: Request, diff --git a/src/servers/src/http/handler.rs b/src/servers/src/http/handler.rs index bf91f11a9a..8a16065df2 100644 --- a/src/servers/src/http/handler.rs +++ b/src/servers/src/http/handler.rs @@ -218,7 +218,7 @@ impl From for PromQuery { step: query.step, lookback: query .lookback - .unwrap_or(DEFAULT_LOOKBACK_STRING.to_string()), + .unwrap_or_else(|| DEFAULT_LOOKBACK_STRING.to_string()), } } } diff --git a/src/servers/src/http/prometheus.rs b/src/servers/src/http/prometheus.rs index 0eebf43367..84574e6b3f 100644 --- a/src/servers/src/http/prometheus.rs +++ b/src/servers/src/http/prometheus.rs @@ -182,7 +182,7 @@ pub async fn instant_query( lookback: params .lookback .or(form_params.lookback) - .unwrap_or(DEFAULT_LOOKBACK_STRING.to_string()), + .unwrap_or_else(|| DEFAULT_LOOKBACK_STRING.to_string()), }; let result = handler.do_query(&prom_query, query_ctx).await; @@ -225,7 +225,7 @@ pub async fn range_query( lookback: params .lookback .or(form_params.lookback) - .unwrap_or(DEFAULT_LOOKBACK_STRING.to_string()), + .unwrap_or_else(|| DEFAULT_LOOKBACK_STRING.to_string()), }; let result = handler.do_query(&prom_query, query_ctx).await; @@ -324,7 +324,7 @@ pub async fn labels_query( let lookback = params .lookback .or(form_params.lookback) - .unwrap_or(DEFAULT_LOOKBACK_STRING.to_string()); + .unwrap_or_else(|| DEFAULT_LOOKBACK_STRING.to_string()); let mut labels = HashSet::new(); let _ = labels.insert(METRIC_NAME.to_string()); @@ -607,7 +607,7 @@ pub async fn label_values_query( let end = params.end.unwrap_or_else(current_time_rfc3339); let lookback = params .lookback - .unwrap_or(DEFAULT_LOOKBACK_STRING.to_string()); + .unwrap_or_else(|| DEFAULT_LOOKBACK_STRING.to_string()); let mut label_values = HashSet::new(); @@ -752,7 +752,7 @@ pub async fn series_query( let lookback = params .lookback .or(form_params.lookback) - .unwrap_or(DEFAULT_LOOKBACK_STRING.to_string()); + .unwrap_or_else(|| DEFAULT_LOOKBACK_STRING.to_string()); let mut series = Vec::new(); let mut merge_map = HashMap::new(); diff --git a/src/servers/src/mysql/federated.rs b/src/servers/src/mysql/federated.rs index d9f16bd654..13f8a63bb5 100644 --- a/src/servers/src/mysql/federated.rs +++ b/src/servers/src/mysql/federated.rs @@ -35,8 +35,6 @@ static MYSQL_CONN_JAVA_PATTERN: Lazy = Lazy::new(|| Regex::new("(?i)^(/\\* mysql-connector-j(.*))").unwrap()); static SHOW_LOWER_CASE_PATTERN: Lazy = Lazy::new(|| Regex::new("(?i)^(SHOW VARIABLES LIKE 'lower_case_table_names'(.*))").unwrap()); -static SHOW_COLLATION_PATTERN: Lazy = - Lazy::new(|| Regex::new("(?i)^(show collation where(.*))").unwrap()); static SHOW_VARIABLES_LIKE_PATTERN: Lazy = Lazy::new(|| Regex::new("(?i)^(SHOW VARIABLES( LIKE (.*))?)").unwrap()); @@ -70,9 +68,6 @@ static OTHER_NOT_SUPPORTED_STMT: Lazy = Lazy::new(|| { "(?i)^(SET @@(.*))", "(?i)^(SET PROFILING(.*))", - "(?i)^(SHOW COLLATION)", - "(?i)^(SHOW CHARSET)", - // mysqlclient. "(?i)^(SELECT \\$\\$)", @@ -96,8 +91,6 @@ static OTHER_NOT_SUPPORTED_STMT: Lazy = Lazy::new(|| { "(?i)^(SHOW WARNINGS)", "(?i)^(/\\* ApplicationName=(.*)SHOW WARNINGS)", "(?i)^(/\\* ApplicationName=(.*)SHOW PLUGINS)", - "(?i)^(/\\* ApplicationName=(.*)SHOW COLLATION)", - "(?i)^(/\\* ApplicationName=(.*)SHOW CHARSET)", "(?i)^(/\\* ApplicationName=(.*)SHOW ENGINES)", "(?i)^(/\\* ApplicationName=(.*)SELECT @@(.*))", "(?i)^(/\\* ApplicationName=(.*)SHOW @@(.*))", @@ -248,8 +241,7 @@ fn check_show_variables(query: &str) -> Option { Some(show_variables("sql_mode", "ONLY_FULL_GROUP_BY STRICT_TRANS_TABLES NO_ZERO_IN_DATE NO_ZERO_DATE ERROR_FOR_DIVISION_BY_ZERO NO_ENGINE_SUBSTITUTION")) } else if SHOW_LOWER_CASE_PATTERN.is_match(query) { Some(show_variables("lower_case_table_names", "0")) - } else if SHOW_COLLATION_PATTERN.is_match(query) || SHOW_VARIABLES_LIKE_PATTERN.is_match(query) - { + } else if SHOW_VARIABLES_LIKE_PATTERN.is_match(query) { Some(show_variables("", "")) } else { None @@ -379,12 +371,6 @@ mod test { +------------------------+-------+"; test(query, expected); - let query = "show collation"; - let expected = "\ -++ -++"; // empty - test(query, expected); - let query = "SELECT TIMEDIFF(NOW(), UTC_TIMESTAMP())"; let expected = "\ +----------------------------------+ diff --git a/src/servers/src/mysql/helper.rs b/src/servers/src/mysql/helper.rs index 6109ec706d..c5d509016d 100644 --- a/src/servers/src/mysql/helper.rs +++ b/src/servers/src/mysql/helper.rs @@ -179,6 +179,7 @@ pub fn convert_value(param: &ParamValue, t: &ConcreteDataType) -> Result(idx, &client_type)?; match server_type { ConcreteDataType::Timestamp(unit) => match *unit { - TimestampType::Second(_) => { - ScalarValue::TimestampSecond(data.map(|ts| ts.timestamp()), None) - } + TimestampType::Second(_) => ScalarValue::TimestampSecond( + data.map(|ts| ts.and_utc().timestamp()), + None, + ), TimestampType::Millisecond(_) => ScalarValue::TimestampMillisecond( - data.map(|ts| ts.timestamp_millis()), + data.map(|ts| ts.and_utc().timestamp_millis()), None, ), TimestampType::Microsecond(_) => ScalarValue::TimestampMicrosecond( - data.map(|ts| ts.timestamp_micros()), + data.map(|ts| ts.and_utc().timestamp_micros()), None, ), TimestampType::Nanosecond(_) => ScalarValue::TimestampNanosecond( - data.map(|ts| ts.timestamp_micros()), + data.map(|ts| ts.and_utc().timestamp_micros()), None, ), }, ConcreteDataType::DateTime(_) => { - ScalarValue::Date64(data.map(|d| d.timestamp_millis())) + ScalarValue::Date64(data.map(|d| d.and_utc().timestamp_millis())) } _ => { return Err(invalid_parameter_error( @@ -814,10 +815,7 @@ mod test { let err = encode_value( &query_context, - &Value::List(ListValue::new( - Some(Box::default()), - ConcreteDataType::int16_datatype(), - )), + &Value::List(ListValue::new(vec![], ConcreteDataType::int16_datatype())), &mut builder, ) .unwrap_err(); diff --git a/src/servers/src/prom_store.rs b/src/servers/src/prom_store.rs index 23c7e45c38..515a28b44b 100644 --- a/src/servers/src/prom_store.rs +++ b/src/servers/src/prom_store.rs @@ -103,11 +103,11 @@ pub fn query_to_plan(dataframe: DataFrame, q: &Query) -> Result { } // Case sensitive regexp match MatcherType::Re => { - conditions.push(regexp_match(vec![col(name), lit(value)]).is_not_null()); + conditions.push(regexp_match(col(name), lit(value)).is_not_null()); } // Case sensitive regexp not match MatcherType::Nre => { - conditions.push(regexp_match(vec![col(name), lit(value)]).is_null()); + conditions.push(regexp_match(col(name), lit(value)).is_null()); } } } diff --git a/src/session/src/context.rs b/src/session/src/context.rs index 25091e1cf5..d2c060919e 100644 --- a/src/session/src/context.rs +++ b/src/session/src/context.rs @@ -22,7 +22,7 @@ use arc_swap::ArcSwap; use auth::UserInfoRef; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_catalog::{build_db_string, parse_catalog_and_schema_from_db_string}; -use common_time::timezone::get_timezone; +use common_time::timezone::{get_timezone, parse_timezone}; use common_time::Timezone; use derive_builder::Builder; use sql::dialect::{Dialect, GreptimeDbDialect, MySqlDialect, PostgreSqlDialect}; @@ -89,8 +89,7 @@ impl From<&RegionRequestHeader> for QueryContext { current_catalog: catalog.to_string(), current_schema: schema.to_string(), current_user: Default::default(), - // for request send to datanode, all timestamp have converted to UTC, so timezone is not important - timezone: ArcSwap::new(Arc::new(get_timezone(None).clone())), + timezone: ArcSwap::new(Arc::new(parse_timezone(Some(&value.timezone)))), sql_dialect: Arc::new(GreptimeDbDialect {}), extension: Default::default(), configuration_parameter: Default::default(), diff --git a/src/sql/src/parser.rs b/src/sql/src/parser.rs index 9010106d85..e873343bbd 100644 --- a/src/sql/src/parser.rs +++ b/src/sql/src/parser.rs @@ -83,14 +83,14 @@ impl<'a> ParserContext<'a> { } pub(crate) fn intern_parse_table_name(&mut self) -> Result { - let raw_table_name = self - .parser - .parse_object_name() - .context(error::UnexpectedSnafu { - sql: self.sql, - expected: "a table name", - actual: self.parser.peek_token().to_string(), - })?; + let raw_table_name = + self.parser + .parse_object_name(false) + .context(error::UnexpectedSnafu { + sql: self.sql, + expected: "a table name", + actual: self.parser.peek_token().to_string(), + })?; Ok(Self::canonicalize_object_name(raw_table_name)) } @@ -100,7 +100,7 @@ impl<'a> ParserContext<'a> { .try_with_sql(sql) .context(SyntaxSnafu)?; - let function_name = parser.parse_identifier().context(SyntaxSnafu)?; + let function_name = parser.parse_identifier(false).context(SyntaxSnafu)?; parser .parse_function(ObjectName(vec![function_name])) .context(SyntaxSnafu) @@ -222,6 +222,22 @@ impl<'a> ParserContext<'a> { .collect(), ) } + + /// Simply a shortcut for sqlparser's same name method `parse_object_name`, + /// but with constant argument "false". + /// Because the argument is always "false" for us (it's introduced by BigQuery), + /// we don't want to write it again and again. + pub(crate) fn parse_object_name(&mut self) -> std::result::Result { + self.parser.parse_object_name(false) + } + + /// Simply a shortcut for sqlparser's same name method `parse_identifier`, + /// but with constant argument "false". + /// Because the argument is always "false" for us (it's introduced by BigQuery), + /// we don't want to write it again and again. + pub(crate) fn parse_identifier(&mut self) -> std::result::Result { + self.parser.parse_identifier(false) + } } #[cfg(test)] diff --git a/src/sql/src/parsers/alter_parser.rs b/src/sql/src/parsers/alter_parser.rs index 483c432636..687604e370 100644 --- a/src/sql/src/parsers/alter_parser.rs +++ b/src/sql/src/parsers/alter_parser.rs @@ -33,7 +33,7 @@ impl<'a> ParserContext<'a> { let parser = &mut self.parser; parser.expect_keywords(&[Keyword::ALTER, Keyword::TABLE])?; - let raw_table_name = parser.parse_object_name()?; + let raw_table_name = parser.parse_object_name(false)?; let table_name = Self::canonicalize_object_name(raw_table_name); let alter_operation = if parser.parse_keyword(Keyword::ADD) { @@ -48,7 +48,7 @@ impl<'a> ParserContext<'a> { } else if let Token::Word(word) = parser.peek_token().token { if word.value.to_ascii_uppercase() == "AFTER" { let _ = parser.next_token(); - let name = Self::canonicalize_identifier(parser.parse_identifier()?); + let name = Self::canonicalize_identifier(self.parse_identifier()?); Some(AddColumnLocation::After { column_name: name.value, }) @@ -65,7 +65,7 @@ impl<'a> ParserContext<'a> { } } else if parser.parse_keyword(Keyword::DROP) { if parser.parse_keyword(Keyword::COLUMN) { - let name = Self::canonicalize_identifier(self.parser.parse_identifier()?); + let name = Self::canonicalize_identifier(self.parse_identifier()?); AlterTableOperation::DropColumn { name } } else { return Err(ParserError::ParserError(format!( @@ -74,7 +74,7 @@ impl<'a> ParserContext<'a> { ))); } } else if parser.parse_keyword(Keyword::RENAME) { - let new_table_name_obj_raw = parser.parse_object_name()?; + let new_table_name_obj_raw = self.parse_object_name()?; let new_table_name_obj = Self::canonicalize_object_name(new_table_name_obj_raw); let new_table_name = match &new_table_name_obj.0[..] { [table] => table.value.clone(), @@ -128,7 +128,7 @@ mod tests { location, } => { assert_eq!("tagk_i", column_def.name.value); - assert_eq!(DataType::String, column_def.data_type); + assert_eq!(DataType::String(None), column_def.data_type); assert!(column_def .options .iter() @@ -164,7 +164,7 @@ mod tests { location, } => { assert_eq!("tagk_i", column_def.name.value); - assert_eq!(DataType::String, column_def.data_type); + assert_eq!(DataType::String(None), column_def.data_type); assert!(column_def .options .iter() @@ -200,7 +200,7 @@ mod tests { location, } => { assert_eq!("tagk_i", column_def.name.value); - assert_eq!(DataType::String, column_def.data_type); + assert_eq!(DataType::String(None), column_def.data_type); assert!(column_def .options .iter() diff --git a/src/sql/src/parsers/copy_parser.rs b/src/sql/src/parsers/copy_parser.rs index bd365e51ef..be4b187644 100644 --- a/src/sql/src/parsers/copy_parser.rs +++ b/src/sql/src/parsers/copy_parser.rs @@ -47,14 +47,13 @@ impl<'a> ParserContext<'a> { } fn parser_copy_database(&mut self) -> Result { - let database_name = - self.parser - .parse_object_name() - .with_context(|_| error::UnexpectedSnafu { - sql: self.sql, - expected: "a database name", - actual: self.peek_token_as_string(), - })?; + let database_name = self + .parse_object_name() + .with_context(|_| error::UnexpectedSnafu { + sql: self.sql, + expected: "a database name", + actual: self.peek_token_as_string(), + })?; let req = if self.parser.parse_keyword(Keyword::TO) { let (with, connection, location) = self.parse_copy_parameters()?; @@ -82,14 +81,13 @@ impl<'a> ParserContext<'a> { } fn parse_copy_table(&mut self) -> Result { - let raw_table_name = - self.parser - .parse_object_name() - .with_context(|_| error::UnexpectedSnafu { - sql: self.sql, - expected: "a table name", - actual: self.peek_token_as_string(), - })?; + let raw_table_name = self + .parse_object_name() + .with_context(|_| error::UnexpectedSnafu { + sql: self.sql, + expected: "a table name", + actual: self.peek_token_as_string(), + })?; let table_name = Self::canonicalize_object_name(raw_table_name); if self.parser.parse_keyword(Keyword::TO) { diff --git a/src/sql/src/parsers/create_parser.rs b/src/sql/src/parsers/create_parser.rs index ec61a20d53..08764aaa6e 100644 --- a/src/sql/src/parsers/create_parser.rs +++ b/src/sql/src/parsers/create_parser.rs @@ -111,14 +111,11 @@ impl<'a> ParserContext<'a> { self.parser .parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]); - let database_name = self - .parser - .parse_object_name() - .context(error::UnexpectedSnafu { - sql: self.sql, - expected: "a database name", - actual: self.peek_token_as_string(), - })?; + let database_name = self.parse_object_name().context(error::UnexpectedSnafu { + sql: self.sql, + expected: "a database name", + actual: self.peek_token_as_string(), + })?; let database_name = Self::canonicalize_object_name(database_name); Ok(Statement::CreateDatabase(CreateDatabase { name: database_name, @@ -319,6 +316,7 @@ impl<'a> ParserContext<'a> { quote_style: None, }], is_primary: false, + characteristics: None, }; constraints.push(constraint); } @@ -367,7 +365,7 @@ impl<'a> ParserContext<'a> { pub fn parse_column_def(&mut self) -> std::result::Result { let parser = &mut self.parser; - let name = parser.parse_identifier()?; + let name = parser.parse_identifier(false)?; if name.quote_style.is_none() && // "ALL_KEYWORDS" are sorted. ALL_KEYWORDS.binary_search(&name.value.to_uppercase().as_str()).is_ok() @@ -380,14 +378,14 @@ impl<'a> ParserContext<'a> { let data_type = parser.parse_data_type()?; let collation = if parser.parse_keyword(Keyword::COLLATE) { - Some(parser.parse_object_name()?) + Some(parser.parse_object_name(false)?) } else { None }; let mut options = vec![]; loop { if parser.parse_keyword(Keyword::CONSTRAINT) { - let name = Some(parser.parse_identifier()?); + let name = Some(parser.parse_identifier(false)?); if let Some(option) = Self::parse_optional_column_option(parser)? { options.push(ColumnOptionDef { name, option }); } else { @@ -415,7 +413,7 @@ impl<'a> ParserContext<'a> { ) -> std::result::Result, ParserError> { if parser.parse_keywords(&[Keyword::CHARACTER, Keyword::SET]) { Ok(Some(ColumnOption::CharacterSet( - parser.parse_object_name()?, + parser.parse_object_name(false)?, ))) } else if parser.parse_keywords(&[Keyword::NOT, Keyword::NULL]) { Ok(Some(ColumnOption::NotNull)) @@ -432,9 +430,15 @@ impl<'a> ParserContext<'a> { } else if parser.parse_keyword(Keyword::DEFAULT) { Ok(Some(ColumnOption::Default(parser.parse_expr()?))) } else if parser.parse_keywords(&[Keyword::PRIMARY, Keyword::KEY]) { - Ok(Some(ColumnOption::Unique { is_primary: true })) + Ok(Some(ColumnOption::Unique { + is_primary: true, + characteristics: None, + })) } else if parser.parse_keyword(Keyword::UNIQUE) { - Ok(Some(ColumnOption::Unique { is_primary: false })) + Ok(Some(ColumnOption::Unique { + is_primary: false, + characteristics: None, + })) } else if parser.parse_keywords(&[Keyword::TIME, Keyword::INDEX]) { // Use a DialectSpecific option for time index Ok(Some(ColumnOption::DialectSpecific(vec![ @@ -456,7 +460,7 @@ impl<'a> ParserContext<'a> { fn parse_optional_table_constraint(&mut self) -> Result> { let name = if self.parser.parse_keyword(Keyword::CONSTRAINT) { - let raw_name = self.parser.parse_identifier().context(error::SyntaxSnafu)?; + let raw_name = self.parse_identifier().context(SyntaxSnafu)?; Some(Self::canonicalize_identifier(raw_name)) } else { None @@ -485,6 +489,7 @@ impl<'a> ParserContext<'a> { name, columns, is_primary: true, + characteristics: None, })) } TokenWithLocation { @@ -524,6 +529,7 @@ impl<'a> ParserContext<'a> { }), columns, is_primary: false, + characteristics: None, })) } unexpected => { @@ -568,6 +574,7 @@ fn validate_time_index(columns: &[ColumnDef], constraints: &[TableConstraint]) - name: Some(ident), columns, is_primary: false, + .. } = c { if ident.value == TIME_INDEX { @@ -857,7 +864,7 @@ mod tests { assert_column_def(&columns[0], "host", "STRING"); assert_column_def(&columns[1], "ts", "TIMESTAMP"); assert_column_def(&columns[2], "cpu", "FLOAT"); - assert_column_def(&columns[3], "memory", "DOUBLE"); + assert_column_def(&columns[3], "memory", "FLOAT64"); let constraints = &c.constraints; assert_matches!( @@ -1108,6 +1115,7 @@ ENGINE=mito"; name, columns, is_primary, + .. } => { assert_eq!(name.unwrap().to_string(), "__time_index"); assert_eq!(columns.len(), 1); @@ -1314,6 +1322,7 @@ ENGINE=mito"; name, columns, is_primary, + .. } => { assert_eq!(name.unwrap().to_string(), "__time_index"); assert_eq!(columns.len(), 1); @@ -1422,7 +1431,7 @@ ENGINE=mito"; assert_column_def(&columns[0], "host", "STRING"); assert_column_def(&columns[1], "ts", "TIMESTAMP"); assert_column_def(&columns[2], "cpu", "FLOAT"); - assert_column_def(&columns[3], "memory", "DOUBLE"); + assert_column_def(&columns[3], "memory", "FLOAT64"); let constraints = &c.constraints; assert_matches!( diff --git a/src/sql/src/parsers/describe_parser.rs b/src/sql/src/parsers/describe_parser.rs index 7ba1ffc003..9525b3edb2 100644 --- a/src/sql/src/parsers/describe_parser.rs +++ b/src/sql/src/parsers/describe_parser.rs @@ -31,8 +31,7 @@ impl<'a> ParserContext<'a> { fn parse_describe_table(&mut self) -> Result { let raw_table_idents = - self.parser - .parse_object_name() + self.parse_object_name() .with_context(|_| error::UnexpectedSnafu { sql: self.sql, expected: "a table name", diff --git a/src/sql/src/parsers/drop_parser.rs b/src/sql/src/parsers/drop_parser.rs index d5d872ee16..8cc62189d8 100644 --- a/src/sql/src/parsers/drop_parser.rs +++ b/src/sql/src/parsers/drop_parser.rs @@ -40,8 +40,7 @@ impl<'a> ParserContext<'a> { let if_exists = self.parser.parse_keywords(&[Keyword::IF, Keyword::EXISTS]); let raw_table_ident = - self.parser - .parse_object_name() + self.parse_object_name() .with_context(|_| error::UnexpectedSnafu { sql: self.sql, expected: "a table name", @@ -62,14 +61,13 @@ impl<'a> ParserContext<'a> { let _ = self.parser.next_token(); let if_exists = self.parser.parse_keywords(&[Keyword::IF, Keyword::EXISTS]); - let database_name = - self.parser - .parse_object_name() - .with_context(|_| error::UnexpectedSnafu { - sql: self.sql, - expected: "a database name", - actual: self.peek_token_as_string(), - })?; + let database_name = self + .parse_object_name() + .with_context(|_| error::UnexpectedSnafu { + sql: self.sql, + expected: "a database name", + actual: self.peek_token_as_string(), + })?; let database_name = Self::canonicalize_object_name(database_name); Ok(Statement::DropDatabase(DropDatabase::new( diff --git a/src/sql/src/parsers/explain_parser.rs b/src/sql/src/parsers/explain_parser.rs index 906bd792ec..57349220fd 100644 --- a/src/sql/src/parsers/explain_parser.rs +++ b/src/sql/src/parsers/explain_parser.rs @@ -13,6 +13,7 @@ // limitations under the License. use snafu::ResultExt; +use sqlparser::ast::DescribeAlias; use crate::error::{self, Result}; use crate::parser::ParserContext; @@ -22,14 +23,14 @@ use crate::statements::statement::Statement; /// EXPLAIN statement parser implementation impl<'a> ParserContext<'a> { pub(crate) fn parse_explain(&mut self) -> Result { - let explain_statement = - self.parser - .parse_explain(false) - .with_context(|_| error::UnexpectedSnafu { - sql: self.sql, - expected: "a query statement", - actual: self.peek_token_as_string(), - })?; + let explain_statement = self + .parser + .parse_explain(DescribeAlias::Explain) + .with_context(|_| error::UnexpectedSnafu { + sql: self.sql, + expected: "a query statement", + actual: self.peek_token_as_string(), + })?; Ok(Statement::Explain(Explain::try_from(explain_statement)?)) } @@ -80,6 +81,7 @@ mod tests { having: None, qualify: None, named_window: vec![], + value_table_mode: None, }; let sp_statement = SpStatement::Query(Box::new(SpQuery { @@ -87,13 +89,15 @@ mod tests { body: Box::new(sqlparser::ast::SetExpr::Select(Box::new(select))), order_by: vec![], limit: None, + limit_by: vec![], offset: None, fetch: None, locks: vec![], + for_clause: None, })); let explain = Explain::try_from(SpStatement::Explain { - describe_alias: false, + describe_alias: DescribeAlias::Explain, analyze: false, verbose: false, statement: Box::new(sp_statement), diff --git a/src/sql/src/parsers/show_parser.rs b/src/sql/src/parsers/show_parser.rs index fb04309f65..993a1c12df 100644 --- a/src/sql/src/parsers/show_parser.rs +++ b/src/sql/src/parsers/show_parser.rs @@ -31,11 +31,26 @@ impl<'a> ParserContext<'a> { if self.consume_token("DATABASES") || self.consume_token("SCHEMAS") { self.parse_show_databases() } else if self.matches_keyword(Keyword::TABLES) { - let _ = self.parser.next_token(); + self.parser.next_token(); self.parse_show_tables(false) + } else if self.matches_keyword(Keyword::CHARSET) { + self.parser.next_token(); + Ok(Statement::ShowCharset(self.parse_show_kind()?)) + } else if self.matches_keyword(Keyword::CHARACTER) { + self.parser.next_token(); + + if self.matches_keyword(Keyword::SET) { + self.parser.next_token(); + Ok(Statement::ShowCharset(self.parse_show_kind()?)) + } else { + self.unsupported(self.peek_token_as_string()) + } + } else if self.matches_keyword(Keyword::COLLATION) { + self.parser.next_token(); + Ok(Statement::ShowCollation(self.parse_show_kind()?)) } else if self.matches_keyword(Keyword::COLUMNS) || self.matches_keyword(Keyword::FIELDS) { // SHOW {COLUMNS | FIELDS} - let _ = self.parser.next_token(); + self.parser.next_token(); self.parse_show_columns(false) } else if self.consume_token("INDEX") || self.consume_token("INDEXES") @@ -59,14 +74,13 @@ impl<'a> ParserContext<'a> { self.unsupported(self.peek_token_as_string()) } } else if self.consume_token("VARIABLES") { - let variable = - self.parser - .parse_object_name() - .with_context(|_| error::UnexpectedSnafu { - sql: self.sql, - expected: "a variable name", - actual: self.peek_token_as_string(), - })?; + let variable = self + .parse_object_name() + .with_context(|_| error::UnexpectedSnafu { + sql: self.sql, + expected: "a variable name", + actual: self.peek_token_as_string(), + })?; Ok(Statement::ShowVariables(ShowVariables { variable })) } else { self.unsupported(self.peek_token_as_string()) @@ -75,14 +89,13 @@ impl<'a> ParserContext<'a> { /// Parse SHOW CREATE TABLE statement fn parse_show_create_table(&mut self) -> Result { - let raw_table_name = - self.parser - .parse_object_name() - .with_context(|_| error::UnexpectedSnafu { - sql: self.sql, - expected: "a table name", - actual: self.peek_token_as_string(), - })?; + let raw_table_name = self + .parse_object_name() + .with_context(|_| error::UnexpectedSnafu { + sql: self.sql, + expected: "a table name", + actual: self.peek_token_as_string(), + })?; let table_name = Self::canonicalize_object_name(raw_table_name); ensure!( !table_name.0.is_empty(), @@ -94,15 +107,14 @@ impl<'a> ParserContext<'a> { } fn parse_show_table_name(&mut self) -> Result { - let _ = self.parser.next_token(); - let table_name = - self.parser - .parse_object_name() - .with_context(|_| error::UnexpectedSnafu { - sql: self.sql, - expected: "a table name", - actual: self.peek_token_as_string(), - })?; + self.parser.next_token(); + let table_name = self + .parse_object_name() + .with_context(|_| error::UnexpectedSnafu { + sql: self.sql, + expected: "a table name", + actual: self.peek_token_as_string(), + })?; ensure!( table_name.0.len() == 1, @@ -118,9 +130,8 @@ impl<'a> ParserContext<'a> { } fn parse_db_name(&mut self) -> Result> { - let _ = self.parser.next_token(); + self.parser.next_token(); let db_name = self - .parser .parse_object_name() .with_context(|_| error::UnexpectedSnafu { sql: self.sql, @@ -176,34 +187,7 @@ impl<'a> ParserContext<'a> { _ => None, }; - let kind = match self.parser.peek_token().token { - Token::EOF | Token::SemiColon => ShowKind::All, - // SHOW COLUMNS [WHERE | LIKE] [EXPR] - Token::Word(w) => match w.keyword { - Keyword::LIKE => { - let _ = self.parser.next_token(); - ShowKind::Like(self.parser.parse_identifier().with_context(|_| { - error::UnexpectedSnafu { - sql: self.sql, - expected: "LIKE", - actual: self.peek_token_as_string(), - } - })?) - } - Keyword::WHERE => { - let _ = self.parser.next_token(); - ShowKind::Where(self.parser.parse_expr().with_context(|_| { - error::UnexpectedSnafu { - sql: self.sql, - expected: "some valid expression", - actual: self.peek_token_as_string(), - } - })?) - } - _ => return self.unsupported(self.peek_token_as_string()), - }, - _ => return self.unsupported(self.peek_token_as_string()), - }; + let kind = self.parse_show_kind()?; Ok(Statement::ShowColumns(ShowColumns { kind, @@ -213,6 +197,36 @@ impl<'a> ParserContext<'a> { })) } + fn parse_show_kind(&mut self) -> Result { + match self.parser.peek_token().token { + Token::EOF | Token::SemiColon => Ok(ShowKind::All), + Token::Word(w) => match w.keyword { + Keyword::LIKE => { + self.parser.next_token(); + Ok(ShowKind::Like(self.parse_identifier().with_context( + |_| error::UnexpectedSnafu { + sql: self.sql, + expected: "LIKE", + actual: self.peek_token_as_string(), + }, + )?)) + } + Keyword::WHERE => { + self.parser.next_token(); + Ok(ShowKind::Where(self.parser.parse_expr().with_context( + |_| error::UnexpectedSnafu { + sql: self.sql, + expected: "some valid expression", + actual: self.peek_token_as_string(), + }, + )?)) + } + _ => self.unsupported(self.peek_token_as_string()), + }, + _ => self.unsupported(self.peek_token_as_string()), + } + } + fn parse_show_index(&mut self) -> Result { let table = match self.parser.peek_token().token { // SHOW INDEX {in | FROM} TABLE @@ -252,7 +266,7 @@ impl<'a> ParserContext<'a> { // SHOW INDEX [WHERE] [EXPR] Token::Word(w) => match w.keyword { Keyword::WHERE => { - let _ = self.parser.next_token(); + self.parser.next_token(); ShowKind::Where(self.parser.parse_expr().with_context(|_| { error::UnexpectedSnafu { sql: self.sql, @@ -292,34 +306,7 @@ impl<'a> ParserContext<'a> { _ => None, }; - let kind = match self.parser.peek_token().token { - Token::EOF | Token::SemiColon => ShowKind::All, - // SHOW TABLES [WHERE | LIKE] [EXPR] - Token::Word(w) => match w.keyword { - Keyword::LIKE => { - let _ = self.parser.next_token(); - ShowKind::Like(self.parser.parse_identifier().with_context(|_| { - error::UnexpectedSnafu { - sql: self.sql, - expected: "LIKE", - actual: self.peek_token_as_string(), - } - })?) - } - Keyword::WHERE => { - let _ = self.parser.next_token(); - ShowKind::Where(self.parser.parse_expr().with_context(|_| { - error::UnexpectedSnafu { - sql: self.sql, - expected: "some valid expression", - actual: self.peek_token_as_string(), - } - })?) - } - _ => return self.unsupported(self.peek_token_as_string()), - }, - _ => return self.unsupported(self.peek_token_as_string()), - }; + let kind = self.parse_show_kind()?; Ok(Statement::ShowTables(ShowTables { kind, @@ -337,7 +324,7 @@ impl<'a> ParserContext<'a> { } Token::Word(w) => match w.keyword { Keyword::LIKE => Ok(Statement::ShowDatabases(ShowDatabases::new( - ShowKind::Like(self.parser.parse_identifier().with_context(|_| { + ShowKind::Like(self.parse_identifier().with_context(|_| { error::UnexpectedSnafu { sql: self.sql, expected: "LIKE", @@ -722,4 +709,66 @@ mod tests { .. }) if table == "test" && expr.to_string() == "Field = 'disk'")); } + + #[test] + fn parse_show_collation() { + let sql = "SHOW COLLATION"; + let result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); + assert!(matches!( + result.unwrap()[0], + Statement::ShowCollation(ShowKind::All) + )); + + let sql = "SHOW COLLATION WHERE Charset = 'latin1'"; + let result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); + assert!(matches!( + result.unwrap()[0], + Statement::ShowCollation(ShowKind::Where(_)) + )); + + let sql = "SHOW COLLATION LIKE 'latin1'"; + let result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); + assert!(matches!( + result.unwrap()[0], + Statement::ShowCollation(ShowKind::Like(_)) + )); + } + + #[test] + fn parse_show_charset() { + let sql = "SHOW CHARSET"; + let result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); + assert!(matches!( + result.unwrap()[0], + Statement::ShowCharset(ShowKind::All) + )); + + let sql = "SHOW CHARACTER SET"; + let result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); + assert!(matches!( + result.unwrap()[0], + Statement::ShowCharset(ShowKind::All) + )); + + let sql = "SHOW CHARSET WHERE Charset = 'latin1'"; + let result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); + assert!(matches!( + result.unwrap()[0], + Statement::ShowCharset(ShowKind::Where(_)) + )); + + let sql = "SHOW CHARACTER SET LIKE 'latin1'"; + let result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); + assert!(matches!( + result.unwrap()[0], + Statement::ShowCharset(ShowKind::Like(_)) + )); + } } diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs index a681ca1001..8ce89812f1 100644 --- a/src/sql/src/parsers/tql_parser.rs +++ b/src/sql/src/parsers/tql_parser.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use chrono::Utc; +use datafusion::execution::context::SessionState; use datafusion::optimizer::simplify_expressions::{ExprSimplifier, SimplifyContext}; use datafusion_common::config::ConfigOptions; use datafusion_common::{DFSchema, Result as DFResult, ScalarValue, TableReference}; @@ -205,7 +206,7 @@ impl<'a> ParserContext<'a> { fn parse_to_logical_expr(expr: sqlparser::ast::Expr) -> std::result::Result { let empty_df_schema = DFSchema::empty(); - SqlToRel::new(&StubContextProvider {}) + SqlToRel::new(&StubContextProvider::default()) .sql_to_expr(expr.into(), &empty_df_schema, &mut Default::default()) .context(ConvertToLogicalExpressionSnafu) } @@ -262,20 +263,29 @@ impl<'a> ParserContext<'a> { } } -#[derive(Default)] -struct StubContextProvider {} +struct StubContextProvider { + state: SessionState, +} + +impl Default for StubContextProvider { + fn default() -> Self { + Self { + state: SessionState::new_with_config_rt(Default::default(), Default::default()), + } + } +} impl ContextProvider for StubContextProvider { - fn get_table_provider(&self, _name: TableReference) -> DFResult> { + fn get_table_source(&self, _name: TableReference) -> DFResult> { unimplemented!() } - fn get_function_meta(&self, _name: &str) -> Option> { - None + fn get_function_meta(&self, name: &str) -> Option> { + self.state.scalar_functions().get(name).cloned() } - fn get_aggregate_meta(&self, _name: &str) -> Option> { - unimplemented!() + fn get_aggregate_meta(&self, name: &str) -> Option> { + self.state.aggregate_functions().get(name).cloned() } fn get_window_meta(&self, _name: &str) -> Option> { @@ -289,6 +299,18 @@ impl ContextProvider for StubContextProvider { fn options(&self) -> &ConfigOptions { unimplemented!() } + + fn udfs_names(&self) -> Vec { + self.state.scalar_functions().keys().cloned().collect() + } + + fn udafs_names(&self) -> Vec { + self.state.aggregate_functions().keys().cloned().collect() + } + + fn udwfs_names(&self) -> Vec { + self.state.window_functions().keys().cloned().collect() + } } #[cfg(test)] diff --git a/src/sql/src/parsers/truncate_parser.rs b/src/sql/src/parsers/truncate_parser.rs index d4ac0c4d14..b71c85a5e8 100644 --- a/src/sql/src/parsers/truncate_parser.rs +++ b/src/sql/src/parsers/truncate_parser.rs @@ -27,8 +27,7 @@ impl<'a> ParserContext<'a> { let _ = self.parser.parse_keyword(Keyword::TABLE); let raw_table_ident = - self.parser - .parse_object_name() + self.parse_object_name() .with_context(|_| error::UnexpectedSnafu { sql: self.sql, expected: "a table name", diff --git a/src/sql/src/statements.rs b/src/sql/src/statements.rs index d51378c098..de35b71a90 100644 --- a/src/sql/src/statements.rs +++ b/src/sql/src/statements.rs @@ -345,7 +345,7 @@ pub fn has_primary_key_option(column_def: &ColumnDef) -> bool { .options .iter() .any(|options| match options.option { - ColumnOption::Unique { is_primary } => is_primary, + ColumnOption::Unique { is_primary, .. } => is_primary, _ => false, }) } @@ -414,10 +414,15 @@ pub fn sql_column_def_to_grpc_column_def( .context(ConvertToGrpcDataTypeSnafu)? .to_parts(); - let is_primary_key = col - .options - .iter() - .any(|o| matches!(o.option, ColumnOption::Unique { is_primary: true })); + let is_primary_key = col.options.iter().any(|o| { + matches!( + o.option, + ColumnOption::Unique { + is_primary: true, + .. + } + ) + }); let semantic_type = if is_primary_key { SemanticType::Tag @@ -438,7 +443,7 @@ pub fn sql_column_def_to_grpc_column_def( pub fn sql_data_type_to_concrete_data_type(data_type: &SqlDataType) -> Result { match data_type { - SqlDataType::BigInt(_) => Ok(ConcreteDataType::int64_datatype()), + SqlDataType::BigInt(_) | SqlDataType::Int64 => Ok(ConcreteDataType::int64_datatype()), SqlDataType::UnsignedBigInt(_) => Ok(ConcreteDataType::uint64_datatype()), SqlDataType::Int(_) | SqlDataType::Integer(_) => Ok(ConcreteDataType::int32_datatype()), SqlDataType::UnsignedInt(_) | SqlDataType::UnsignedInteger(_) => { @@ -453,9 +458,9 @@ pub fn sql_data_type_to_concrete_data_type(data_type: &SqlDataType) -> Result Ok(ConcreteDataType::string_datatype()), + | SqlDataType::String(_) => Ok(ConcreteDataType::string_datatype()), SqlDataType::Float(_) => Ok(ConcreteDataType::float32_datatype()), - SqlDataType::Double => Ok(ConcreteDataType::float64_datatype()), + SqlDataType::Double | SqlDataType::Float64 => Ok(ConcreteDataType::float64_datatype()), SqlDataType::Boolean => Ok(ConcreteDataType::boolean_datatype()), SqlDataType::Date => Ok(ConcreteDataType::date_datatype()), SqlDataType::Binary(_) @@ -502,7 +507,7 @@ pub fn concrete_data_type_to_sql_data_type(data_type: &ConcreteDataType) -> Resu ConcreteDataType::UInt16(_) => Ok(SqlDataType::UnsignedSmallInt(None)), ConcreteDataType::Int8(_) => Ok(SqlDataType::TinyInt(None)), ConcreteDataType::UInt8(_) => Ok(SqlDataType::UnsignedTinyInt(None)), - ConcreteDataType::String(_) => Ok(SqlDataType::String), + ConcreteDataType::String(_) => Ok(SqlDataType::String(None)), ConcreteDataType::Float32(_) => Ok(SqlDataType::Float(None)), ConcreteDataType::Float64(_) => Ok(SqlDataType::Double), ConcreteDataType::Boolean(_) => Ok(SqlDataType::Boolean), @@ -588,7 +593,10 @@ mod tests { ConcreteDataType::string_datatype(), ); check_type(SqlDataType::Text, ConcreteDataType::string_datatype()); - check_type(SqlDataType::String, ConcreteDataType::string_datatype()); + check_type( + SqlDataType::String(None), + ConcreteDataType::string_datatype(), + ); check_type( SqlDataType::Float(None), ConcreteDataType::float32_datatype(), @@ -966,7 +974,10 @@ mod tests { collation: None, options: vec![ColumnOptionDef { name: None, - option: ColumnOption::Unique { is_primary: true }, + option: ColumnOption::Unique { + is_primary: true, + characteristics: None, + }, }], }; @@ -1044,7 +1055,10 @@ mod tests { collation: None, options: vec![ColumnOptionDef { name: None, - option: ColumnOption::Unique { is_primary: true }, + option: ColumnOption::Unique { + is_primary: true, + characteristics: None, + }, }], }; assert!(has_primary_key_option(&column_def)); @@ -1081,7 +1095,7 @@ mod tests { let column_def = ColumnDef { name: "col2".into(), - data_type: SqlDataType::String, + data_type: SqlDataType::String(None), collation: None, options: vec![ ColumnOptionDef { diff --git a/src/sql/src/statements/explain.rs b/src/sql/src/statements/explain.rs index fe953d6618..5b3a2671f9 100644 --- a/src/sql/src/statements/explain.rs +++ b/src/sql/src/statements/explain.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::fmt::{Display, Formatter}; + use sqlparser::ast::Statement as SpStatement; use sqlparser_derive::{Visit, VisitMut}; @@ -31,8 +33,8 @@ impl TryFrom for Explain { } } -impl ToString for Explain { - fn to_string(&self) -> String { - self.inner.to_string() +impl Display for Explain { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.inner) } } diff --git a/src/sql/src/statements/insert.rs b/src/sql/src/statements/insert.rs index 81ef55734d..4eae7f1e18 100644 --- a/src/sql/src/statements/insert.rs +++ b/src/sql/src/statements/insert.rs @@ -55,10 +55,10 @@ impl Insert { match &self.inner { Statement::Insert { source: - box Query { + Some(box Query { body: box SetExpr::Values(Values { rows, .. }), .. - }, + }), .. } => sql_exprs_to_values(rows), _ => unreachable!(), @@ -71,10 +71,10 @@ impl Insert { match &self.inner { Statement::Insert { source: - box Query { + Some(box Query { body: box SetExpr::Values(Values { rows, .. }), .. - }, + }), .. } => rows.iter().all(|es| { es.iter().all(|expr| match expr { @@ -100,7 +100,8 @@ impl Insert { pub fn query_body(&self) -> Result> { Ok(match &self.inner { Statement::Insert { - source: box query, .. + source: Some(box query), + .. } => Some(query.clone().try_into()?), _ => None, }) diff --git a/src/sql/src/statements/statement.rs b/src/sql/src/statements/statement.rs index 167531168d..b5345c0d9f 100644 --- a/src/sql/src/statements/statement.rs +++ b/src/sql/src/statements/statement.rs @@ -29,7 +29,7 @@ use crate::statements::insert::Insert; use crate::statements::query::Query; use crate::statements::set_variables::SetVariables; use crate::statements::show::{ - ShowColumns, ShowCreateTable, ShowDatabases, ShowIndex, ShowTables, ShowVariables, + ShowColumns, ShowCreateTable, ShowDatabases, ShowIndex, ShowKind, ShowTables, ShowVariables, }; use crate::statements::tql::Tql; use crate::statements::truncate::TruncateTable; @@ -66,6 +66,10 @@ pub enum Statement { ShowTables(ShowTables), // SHOW COLUMNS ShowColumns(ShowColumns), + // SHOW CHARSET or SHOW CHARACTER SET + ShowCharset(ShowKind), + // SHOW COLLATION + ShowCollation(ShowKind), // SHOW INDEX ShowIndex(ShowIndex), // SHOW CREATE TABLE diff --git a/src/sql/src/statements/transform/type_alias.rs b/src/sql/src/statements/transform/type_alias.rs index 0db61538e0..353c19f68c 100644 --- a/src/sql/src/statements/transform/type_alias.rs +++ b/src/sql/src/statements/transform/type_alias.rs @@ -35,6 +35,7 @@ use crate::statements::{sql_data_type_to_concrete_data_type, TimezoneInfo}; /// - `INT32` for `int` /// - `INT64` for `bigint` /// - And `UINT8`, `UINT16` etc. for `UnsignedTinyint` etc. +/// - TinyText, MediumText, LongText for `Text`. pub(crate) struct TypeAliasTransformRule; impl TransformRule for TypeAliasTransformRule { @@ -57,28 +58,56 @@ impl TransformRule for TypeAliasTransformRule { } fn visit_expr(&self, expr: &mut Expr) -> ControlFlow<()> { + fn cast_expr_to_arrow_cast_func(expr: Expr, cast_type: String) -> Function { + Function { + name: ObjectName(vec![Ident::new("arrow_cast")]), + args: vec![ + FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)), + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( + Value::SingleQuotedString(cast_type), + ))), + ], + filter: None, + null_treatment: None, + over: None, + distinct: false, + special: false, + order_by: vec![], + } + } + match expr { + // In new sqlparser, the "INT64" is no longer parsed to custom datatype. + // The new "Int64" is not recognizable by Datafusion, cannot directly "CAST" to it. + // We have to replace the expr to "arrow_cast" function call here. + // Same for "FLOAT64". + Expr::Cast { + expr: cast_expr, + data_type, + .. + } if matches!(data_type, DataType::Int64 | DataType::Float64) => { + if let Some(new_type) = get_data_type_by_alias_name(&data_type.to_string()) { + if let Ok(new_type) = sql_data_type_to_concrete_data_type(&new_type) { + *expr = Expr::Function(cast_expr_to_arrow_cast_func( + (**cast_expr).clone(), + new_type.as_arrow_type().to_string(), + )); + } + } + } + // Type alias Expr::Cast { data_type: DataType::Custom(name, tokens), expr: cast_expr, + .. } if name.0.len() == 1 && tokens.is_empty() => { if let Some(new_type) = get_data_type_by_alias_name(name.0[0].value.as_str()) { - if let Ok(concrete_type) = sql_data_type_to_concrete_data_type(&new_type) { - let new_type = concrete_type.as_arrow_type(); - *expr = Expr::Function(Function { - name: ObjectName(vec![Ident::new("arrow_cast")]), - args: vec![ - FunctionArg::Unnamed(FunctionArgExpr::Expr((**cast_expr).clone())), - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( - Value::SingleQuotedString(new_type.to_string()), - ))), - ], - over: None, - distinct: false, - special: false, - order_by: vec![], - }); + if let Ok(new_type) = sql_data_type_to_concrete_data_type(&new_type) { + *expr = Expr::Function(cast_expr_to_arrow_cast_func( + (**cast_expr).clone(), + new_type.as_arrow_type().to_string(), + )); } } } @@ -88,24 +117,16 @@ impl TransformRule for TypeAliasTransformRule { Expr::Cast { data_type: DataType::Timestamp(precision, zone), expr: cast_expr, + .. } => { if let Ok(concrete_type) = sql_data_type_to_concrete_data_type(&DataType::Timestamp(*precision, *zone)) { let new_type = concrete_type.as_arrow_type(); - *expr = Expr::Function(Function { - name: ObjectName(vec![Ident::new("arrow_cast")]), - args: vec![ - FunctionArg::Unnamed(FunctionArgExpr::Expr((**cast_expr).clone())), - FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value( - Value::SingleQuotedString(new_type.to_string()), - ))), - ], - over: None, - distinct: false, - special: false, - order_by: vec![], - }); + *expr = Expr::Function(cast_expr_to_arrow_cast_func( + (**cast_expr).clone(), + new_type.to_string(), + )); } } @@ -158,6 +179,8 @@ pub fn get_data_type_by_alias_name(name: &str) -> Option { "UINT64" => Some(DataType::UnsignedBigInt(None)), "FLOAT32" => Some(DataType::Float(None)), "FLOAT64" => Some(DataType::Double), + // String type alias + "TINYTEXT" | "MEDIUMTEXT" | "LONGTEXT" => Some(DataType::Text), _ => None, } } @@ -261,6 +284,18 @@ mod tests { get_data_type_by_alias_name("Timestamp_ns"), Some(DataType::Timestamp(Some(9), TimezoneInfo::None)) ); + assert_eq!( + get_data_type_by_alias_name("TinyText"), + Some(DataType::Text) + ); + assert_eq!( + get_data_type_by_alias_name("MediumText"), + Some(DataType::Text) + ); + assert_eq!( + get_data_type_by_alias_name("LongText"), + Some(DataType::Text) + ); } fn test_timestamp_alias(alias: &str, expected: &str) { @@ -303,6 +338,9 @@ mod tests { let sql = r#" CREATE TABLE data_types ( s string, + tt tinytext, + mt mediumtext, + lt longtext, tint int8, sint int16, i int32, @@ -329,13 +367,16 @@ CREATE TABLE data_types ( Statement::CreateTable(c) => { let expected = r#"CREATE TABLE data_types ( s STRING, + tt TEXT, + mt TEXT, + lt TEXT, tint INT8, sint SMALLINT, i INT, - bint BIGINT, + bint INT64, v VARCHAR, f FLOAT, - d DOUBLE, + d FLOAT64, b BOOLEAN, vb VARBINARY, dt DATE, diff --git a/src/sql/src/util.rs b/src/sql/src/util.rs index 9a931e02cb..c9c1352252 100644 --- a/src/sql/src/util.rs +++ b/src/sql/src/util.rs @@ -17,7 +17,7 @@ use std::fmt::{Display, Formatter}; use std::sync::LazyLock; use regex::Regex; -use sqlparser::ast::{ObjectName, SqlOption, Value}; +use sqlparser::ast::{Expr, ObjectName, SqlOption, Value}; static SQL_SECRET_PATTERNS: LazyLock> = LazyLock::new(|| { vec![ @@ -47,9 +47,11 @@ pub fn format_raw_object_name(name: &ObjectName) -> String { format!("{}", Inner { name }) } -pub fn parse_option_string(value: Value) -> Option { +pub fn parse_option_string(value: Expr) -> Option { match value { - Value::SingleQuotedString(v) | Value::DoubleQuotedString(v) => Some(v), + Expr::Value(Value::SingleQuotedString(v)) | Expr::Value(Value::DoubleQuotedString(v)) => { + Some(v) + } _ => None, } } @@ -60,7 +62,9 @@ pub fn to_lowercase_options_map(opts: &[SqlOption]) -> HashMap { let mut map = HashMap::with_capacity(opts.len()); for SqlOption { name, value } in opts { let value_str = match value { - Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => s.clone(), + Expr::Value(Value::SingleQuotedString(s)) + | Expr::Value(Value::DoubleQuotedString(s)) => s.clone(), + Expr::Identifier(i) => i.value.clone(), _ => value.to_string(), }; let _ = map.insert(name.value.to_lowercase().clone(), value_str); diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs index 8ec0d4a611..7e461617aa 100644 --- a/src/store-api/src/region_engine.rs +++ b/src/store-api/src/region_engine.rs @@ -15,11 +15,11 @@ //! Region Engine's definition use std::any::Any; -use std::collections::HashMap; use std::fmt::Display; use std::sync::Arc; use api::greptime_proto::v1::meta::{GrantedRegion as PbGrantedRegion, RegionRole as PbRegionRole}; +use api::region::RegionResponse; use async_trait::async_trait; use common_error::ext::BoxedError; use common_recordbatch::SendableRecordBatchStream; @@ -27,7 +27,7 @@ use serde::{Deserialize, Serialize}; use crate::logstore::entry; use crate::metadata::RegionMetadataRef; -use crate::region_request::{AffectedRows, RegionRequest}; +use crate::region_request::RegionRequest; use crate::storage::{RegionId, ScanRequest}; /// The result of setting readonly for the region. @@ -130,7 +130,7 @@ pub trait RegionEngine: Send + Sync { &self, region_id: RegionId, request: RegionRequest, - ) -> Result; + ) -> Result; /// Handles substrait query and return a stream of record batches async fn handle_query( @@ -172,20 +172,3 @@ pub trait RegionEngine: Send + Sync { } pub type RegionEngineRef = Arc; - -// TODO: reorganize the dependence to merge this struct with the -// one in common_meta -#[derive(Debug)] -pub struct RegionHandleResult { - pub affected_rows: AffectedRows, - pub extension: HashMap>, -} - -impl RegionHandleResult { - pub fn new(affected_rows: AffectedRows) -> Self { - Self { - affected_rows, - extension: Default::default(), - } - } -} diff --git a/src/table/src/predicate.rs b/src/table/src/predicate.rs index bd5d6d2d18..aecb96f267 100644 --- a/src/table/src/predicate.rs +++ b/src/table/src/predicate.rs @@ -89,8 +89,7 @@ impl Predicate { .exprs .iter() .filter_map(|expr| { - create_physical_expr(expr.df_expr(), df_schema.as_ref(), schema, execution_props) - .ok() + create_physical_expr(expr.df_expr(), df_schema.as_ref(), execution_props).ok() }) .collect::>()) } @@ -284,7 +283,11 @@ impl<'a> TimeRangePredicateBuilder<'a> { | Operator::BitwiseShiftLeft | Operator::StringConcat | Operator::ArrowAt - | Operator::AtArrow => None, + | Operator::AtArrow + | Operator::LikeMatch + | Operator::ILikeMatch + | Operator::NotLikeMatch + | Operator::NotILikeMatch => None, } } @@ -570,6 +573,7 @@ mod tests { let file = std::fs::OpenOptions::new() .write(true) .create(true) + .truncate(true) .open(path.clone()) .unwrap(); diff --git a/src/table/src/predicate/stats.rs b/src/table/src/predicate/stats.rs index 4d707a82e9..e1cc48c320 100644 --- a/src/table/src/predicate/stats.rs +++ b/src/table/src/predicate/stats.rs @@ -12,13 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashSet; use std::sync::Arc; use datafusion::parquet::file::metadata::RowGroupMetaData; use datafusion::parquet::file::statistics::Statistics as ParquetStats; use datafusion::physical_optimizer::pruning::PruningStatistics; use datafusion_common::{Column, ScalarValue}; -use datatypes::arrow::array::{ArrayRef, UInt64Array}; +use datatypes::arrow::array::{ArrayRef, BooleanArray, UInt64Array}; use datatypes::arrow::datatypes::DataType; use paste::paste; @@ -115,4 +116,14 @@ impl<'a> PruningStatistics for RowGroupPruningStatistics<'a> { } Some(Arc::new(UInt64Array::from(values))) } + + fn row_counts(&self, _column: &Column) -> Option { + // TODO(LFC): Impl it. + None + } + + fn contained(&self, _column: &Column, _values: &HashSet) -> Option { + // TODO(LFC): Impl it. + None + } } diff --git a/src/table/src/table.rs b/src/table/src/table.rs index 27e410d27c..a0a45a0739 100644 --- a/src/table/src/table.rs +++ b/src/table/src/table.rs @@ -22,7 +22,7 @@ use store_api::data_source::DataSourceRef; use store_api::storage::ScanRequest; use crate::error::{Result, TablesRecordBatchSnafu}; -use crate::metadata::{FilterPushDownType, TableId, TableInfoRef, TableType}; +use crate::metadata::{FilterPushDownType, TableInfoRef, TableType}; pub mod adapter; mod metrics; @@ -31,13 +31,6 @@ pub mod scan; pub type TableRef = Arc; -#[async_trait::async_trait] -pub trait TableIdProvider { - async fn next_table_id(&self) -> Result; -} - -pub type TableIdProviderRef = Arc; - /// Table handle. pub struct Table { table_info: TableInfoRef, diff --git a/src/table/src/table/scan.rs b/src/table/src/table/scan.rs index 0d5b769136..4eb907063e 100644 --- a/src/table/src/table/scan.rs +++ b/src/table/src/table/scan.rs @@ -28,7 +28,8 @@ use common_telemetry::tracing::Span; use common_telemetry::tracing_context::TracingContext; use datafusion::execution::context::TaskContext; use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet}; -use datafusion_physical_expr::PhysicalSortExpr; +use datafusion::physical_plan::{ExecutionMode, PlanProperties}; +use datafusion_physical_expr::{EquivalenceProperties, PhysicalSortExpr}; use datatypes::schema::SchemaRef; use futures::{Stream, StreamExt}; use snafu::OptionExt; @@ -41,6 +42,7 @@ pub struct StreamScanAdapter { schema: SchemaRef, output_ordering: Option>, metric: ExecutionPlanMetricsSet, + properties: PlanProperties, } impl Debug for StreamScanAdapter { @@ -55,12 +57,17 @@ impl Debug for StreamScanAdapter { impl StreamScanAdapter { pub fn new(stream: SendableRecordBatchStream) -> Self { let schema = stream.schema(); - + let properties = PlanProperties::new( + EquivalenceProperties::new(schema.arrow_schema().clone()), + Partitioning::UnknownPartitioning(1), + ExecutionMode::Bounded, + ); Self { stream: Mutex::new(Some(stream)), schema, output_ordering: None, metric: ExecutionPlanMetricsSet::new(), + properties, } } @@ -79,12 +86,8 @@ impl PhysicalPlan for StreamScanAdapter { self.schema.clone() } - fn output_partitioning(&self) -> Partitioning { - Partitioning::UnknownPartitioning(1) - } - - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { - self.output_ordering.as_deref() + fn properties(&self) -> &PlanProperties { + &self.properties } fn children(&self) -> Vec { @@ -198,8 +201,15 @@ mod test { let stream = recordbatches.as_stream(); let scan = StreamScanAdapter::new(stream); - - assert_eq!(scan.schema(), schema); + let actual: SchemaRef = Arc::new( + scan.properties + .eq_properties + .schema() + .clone() + .try_into() + .unwrap(), + ); + assert_eq!(actual, schema); let stream = scan.execute(0, ctx.task_ctx()).unwrap(); let recordbatches = util::collect(stream).await.unwrap(); diff --git a/src/table/src/table_reference.rs b/src/table/src/table_reference.rs index fb1f6bc63a..3181efddd7 100644 --- a/src/table/src/table_reference.rs +++ b/src/table/src/table_reference.rs @@ -50,7 +50,7 @@ impl<'a> Display for TableReference<'a> { } } -impl<'a> From> for DfTableReference<'a> { +impl<'a> From> for DfTableReference { fn from(val: TableReference<'a>) -> Self { DfTableReference::full(val.catalog, val.schema, val.table) } diff --git a/tests-fuzz/src/executor.rs b/tests-fuzz/src/executor.rs deleted file mode 100644 index 064bd0ce34..0000000000 --- a/tests-fuzz/src/executor.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt; - -#[async_trait::async_trait] -pub(crate) trait DslExecutor { - type Error: Sync + Send + fmt::Debug; - - async fn execute(&self, input: &T) -> Result; -} diff --git a/tests-fuzz/src/generator/insert_expr.rs b/tests-fuzz/src/generator/insert_expr.rs index ec4ba4f615..f3f0dba116 100644 --- a/tests-fuzz/src/generator/insert_expr.rs +++ b/tests-fuzz/src/generator/insert_expr.rs @@ -50,7 +50,7 @@ impl Generator for InsertExprGenerator { let mut values_columns = vec![]; if omit_column_list { // If omit column list, then all columns are required in the values list - values_columns = self.table_ctx.columns.clone(); + values_columns.clone_from(&self.table_ctx.columns); } else { for column in &self.table_ctx.columns { let can_omit = column.is_nullable() || column.has_default_value(); diff --git a/tests-fuzz/src/lib.rs b/tests-fuzz/src/lib.rs index 406927d6b4..8900350224 100644 --- a/tests-fuzz/src/lib.rs +++ b/tests-fuzz/src/lib.rs @@ -12,11 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![feature(associated_type_bounds)] - pub mod context; pub mod error; -pub mod executor; pub mod fake; pub mod generator; pub mod ir; diff --git a/tests-fuzz/targets/fuzz_create_database.rs b/tests-fuzz/targets/fuzz_create_database.rs index 0c7129df70..b59ed4fe8e 100644 --- a/tests-fuzz/targets/fuzz_create_database.rs +++ b/tests-fuzz/targets/fuzz_create_database.rs @@ -17,7 +17,7 @@ use common_telemetry::info; use libfuzzer_sys::arbitrary::{Arbitrary, Unstructured}; use libfuzzer_sys::fuzz_target; -use rand::SeedableRng; +use rand::{Rng, SeedableRng}; use rand_chacha::ChaChaRng; use snafu::ResultExt; use sqlx::{MySql, Pool}; @@ -57,11 +57,13 @@ impl Arbitrary<'_> for FuzzInput { fn generate_expr(input: FuzzInput) -> Result { let mut rng = ChaChaRng::seed_from_u64(input.seed); + let if_not_exists = rng.gen_bool(0.5); let create_database_generator = CreateDatabaseExprGeneratorBuilder::default() .name_generator(Box::new(MappedGenerator::new( WordGenerator, merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map), ))) + .if_not_exists(if_not_exists) .build() .unwrap(); create_database_generator.generate(&mut rng) diff --git a/tests-fuzz/targets/fuzz_create_table.rs b/tests-fuzz/targets/fuzz_create_table.rs index 3d8bc1db9e..60bcb1628b 100644 --- a/tests-fuzz/targets/fuzz_create_table.rs +++ b/tests-fuzz/targets/fuzz_create_table.rs @@ -62,6 +62,7 @@ impl Arbitrary<'_> for FuzzInput { fn generate_expr(input: FuzzInput) -> Result { let mut rng = ChaChaRng::seed_from_u64(input.seed); let metric_engine = rng.gen_bool(0.5); + let if_not_exists = rng.gen_bool(0.5); if metric_engine { let create_table_generator = CreateTableExprGeneratorBuilder::default() .name_generator(Box::new(MappedGenerator::new( @@ -70,6 +71,7 @@ fn generate_expr(input: FuzzInput) -> Result { ))) .columns(input.columns) .engine("metric") + .if_not_exists(if_not_exists) .with_clause([("physical_metric_table".to_string(), "".to_string())]) .build() .unwrap(); @@ -82,6 +84,7 @@ fn generate_expr(input: FuzzInput) -> Result { ))) .columns(input.columns) .engine("mito") + .if_not_exists(if_not_exists) .build() .unwrap(); create_table_generator.generate(&mut rng) diff --git a/tests-integration/src/instance.rs b/tests-integration/src/instance.rs index 90c66e15e9..87a1ea476f 100644 --- a/tests-integration/src/instance.rs +++ b/tests-integration/src/instance.rs @@ -180,14 +180,14 @@ mod tests { let batches = common_recordbatch::util::collect_batches(s).await.unwrap(); let pretty_print = batches.pretty_print().unwrap(); let expected = "\ -+-------+---------------------+-------------+-----------+-----------+ -| host | ts | cpu | memory | disk_util | -+-------+---------------------+-------------+-----------+-----------+ -| 490 | 2013-12-31T16:00:00 | 0.1 | 1.0 | 9.9 | -| 550-A | 2022-12-31T16:00:00 | 1.0 | 100.0 | 9.9 | -| 550-W | 2023-12-31T16:00:00 | 10000.0 | 1000000.0 | 9.9 | -| MOSS | 2043-12-31T16:00:00 | 100000000.0 | 1.0e10 | 9.9 | -+-------+---------------------+-------------+-----------+-----------+"; ++-------+---------------------+-------------+---------------+-----------+ +| host | ts | cpu | memory | disk_util | ++-------+---------------------+-------------+---------------+-----------+ +| 490 | 2013-12-31T16:00:00 | 0.1 | 1.0 | 9.9 | +| 550-A | 2022-12-31T16:00:00 | 1.0 | 100.0 | 9.9 | +| 550-W | 2023-12-31T16:00:00 | 10000.0 | 1000000.0 | 9.9 | +| MOSS | 2043-12-31T16:00:00 | 100000000.0 | 10000000000.0 | 9.9 | ++-------+---------------------+-------------+---------------+-----------+"; assert_eq!(pretty_print, expected); } diff --git a/tests-integration/src/tests/instance_test.rs b/tests-integration/src/tests/instance_test.rs index 622bb3d861..dd6fbccb1a 100644 --- a/tests-integration/src/tests/instance_test.rs +++ b/tests-integration/src/tests/instance_test.rs @@ -411,11 +411,9 @@ async fn test_execute_insert_by_select(instance: Arc) { assert!(matches!( try_execute_sql(&instance, "insert into demo2(ts) select memory from demo1") .await - .unwrap_err(), - Error::TableOperation { - source: OperatorError::PlanStatement { .. }, - .. - } + .unwrap() + .data, + OutputData::AffectedRows(2), )); let output = execute_sql(&instance, "insert into demo2 select * from demo1") @@ -427,12 +425,14 @@ async fn test_execute_insert_by_select(instance: Arc) { .await .data; let expected = "\ -+-------+------+--------+---------------------+ -| host | cpu | memory | ts | -+-------+------+--------+---------------------+ -| host1 | 66.6 | 1024.0 | 2022-06-15T07:02:37 | -| host2 | 88.8 | 333.3 | 2022-06-15T07:02:38 | -+-------+------+--------+---------------------+"; ++-------+------+--------+-------------------------+ +| host | cpu | memory | ts | ++-------+------+--------+-------------------------+ +| | | | 1970-01-01T00:00:00.333 | +| | | | 1970-01-01T00:00:01.024 | +| host1 | 66.6 | 1024.0 | 2022-06-15T07:02:37 | +| host2 | 88.8 | 333.3 | 2022-06-15T07:02:38 | ++-------+------+--------+-------------------------+"; check_output_stream(output, expected).await; } @@ -727,7 +727,7 @@ async fn test_execute_query_external_table_parquet(instance: Arc) | 4 | 4.4 | | false | | | 1970-01-01T00:00:00 | | 5 | 6.6 | | false | 1990-01-01 | 1990-01-01T03:00:00 | 1970-01-01T00:00:00 | | 4 | 4000000.0 | | false | | | 1970-01-01T00:00:00 | -| 4 | 4.0e-6 | | false | | | 1970-01-01T00:00:00 | +| 4 | 4e-6 | | false | | | 1970-01-01T00:00:00 | +-------+-----------+----------+--------+------------+---------------------+---------------------+"; check_output_stream(output, expect).await; } @@ -1054,22 +1054,22 @@ async fn test_execute_query_external_table_json_with_schema(instance: Arc) { Duration::from_secs(60), Duration::from_secs(0), "+------------+---------------------+---------------------------------------------------------------------------------------------------------------------------------------------+\ - \n| job | ts | lhs.lhs.lhs.SUM(http_requests.value) + rhs.MIN(http_requests.value) + http_requests.MAX(http_requests.value) + rhs.AVG(http_requests.value) |\ + \n| job | ts | lhs.rhs.lhs.SUM(http_requests.value) + rhs.MIN(http_requests.value) + http_requests.MAX(http_requests.value) + rhs.AVG(http_requests.value) |\ \n+------------+---------------------+---------------------------------------------------------------------------------------------------------------------------------------------+\ \n| api-server | 1970-01-01T00:00:00 | 1750.0 |\ \n| app-server | 1970-01-01T00:00:00 | 4550.0 |\ diff --git a/tests-integration/tests/sql.rs b/tests-integration/tests/sql.rs index 0bf238b8a1..97c12f63a5 100644 --- a/tests-integration/tests/sql.rs +++ b/tests-integration/tests/sql.rs @@ -147,7 +147,7 @@ pub async fn test_mysql_crud(store_type: StorageType) { .unwrap(); for i in 0..10 { let dt: DateTime = DateTime::from_naive_utc_and_offset( - NaiveDateTime::from_timestamp_opt(60, i).unwrap(), + chrono::DateTime::from_timestamp(60, i).unwrap().naive_utc(), Utc, ); let d = NaiveDate::from_yo_opt(2015, 100).unwrap(); @@ -179,7 +179,9 @@ pub async fn test_mysql_crud(store_type: StorageType) { let expected_d = NaiveDate::from_yo_opt(2015, 100).unwrap(); assert_eq!(expected_d, d); let expected_dt: DateTime = DateTime::from_naive_utc_and_offset( - NaiveDateTime::from_timestamp_opt(60, i as u32).unwrap(), + chrono::DateTime::from_timestamp(60, i as u32) + .unwrap() + .naive_utc(), Utc, ); assert_eq!( @@ -360,7 +362,7 @@ pub async fn test_postgres_crud(store_type: StorageType) { for i in 0..10 { let d = NaiveDate::from_yo_opt(2015, 100).unwrap(); - let dt = d.and_hms_opt(0, 0, 0).unwrap().timestamp_millis(); + let dt = d.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis(); sqlx::query("insert into demo values($1, $2, $3, $4)") .bind(i) diff --git a/tests/cases/distributed/explain/join_10_tables.result b/tests/cases/distributed/explain/join_10_tables.result index 0fe8bebfb6..8b40f21fc6 100644 --- a/tests/cases/distributed/explain/join_10_tables.result +++ b/tests/cases/distributed/explain/join_10_tables.result @@ -75,6 +75,7 @@ limit 1; +-+-+ | logical_plan_| Limit: skip=0, fetch=1_| |_|_Sort: t_1.ts DESC NULLS FIRST, fetch=1_| +|_|_Projection: t_1.ts, t_1.vin, t_1.val, t_2.ts, t_2.vin, t_2.val, t_3.ts, t_3.vin, t_3.val, t_4.ts, t_4.vin, t_4.val, t_5.ts, t_5.vin, t_5.val, t_6.ts, t_6.vin, t_6.val, t_7.ts, t_7.vin, t_7.val, t_8.ts, t_8.vin, t_8.val, t_9.ts, t_9.vin, t_9.val, t_10.ts, t_10.vin, t_10.val | |_|_Inner Join: t_9.ts = t_10.ts, t_9.vin = t_10.vin_| |_|_Inner Join: t_8.ts = t_9.ts, t_8.vin = t_9.vin_| |_|_Inner Join: t_7.ts = t_8.ts, t_7.vin = t_8.vin_| diff --git a/tests/cases/distributed/explain/subqueries.result b/tests/cases/distributed/explain/subqueries.result index 5d0857b091..593e745e1d 100644 --- a/tests/cases/distributed/explain/subqueries.result +++ b/tests/cases/distributed/explain/subqueries.result @@ -13,9 +13,11 @@ EXPLAIN SELECT * FROM integers WHERE i IN ((SELECT i FROM integers)) ORDER BY i; | plan_type_| plan_| +-+-+ | logical_plan_| Sort: integers.i ASC NULLS LAST_| +|_|_Projection: integers.i, integers.j_| |_|_LeftSemi Join: integers.i = __correlated_sq_1.i_| |_|_MergeScan [is_placeholder=false]_| |_|_SubqueryAlias: __correlated_sq_1_| +|_|_Projection: integers.i_| |_|_MergeScan [is_placeholder=false]_| | physical_plan | SortPreservingMergeExec: [i@0 ASC NULLS LAST]_| |_|_SortExec: expr=[i@0 ASC NULLS LAST]_| @@ -43,6 +45,7 @@ EXPLAIN SELECT * FROM integers i1 WHERE EXISTS(SELECT i FROM integers WHERE i=i1 | plan_type_| plan_| +-+-+ | logical_plan_| Sort: i1.i ASC NULLS LAST_| +|_|_Projection: i1.i, i1.j_| |_|_LeftSemi Join: i1.i = __correlated_sq_1.i_| |_|_SubqueryAlias: i1_| |_|_MergeScan [is_placeholder=false]_| @@ -86,24 +89,24 @@ order by t.i desc; +-+-+ | logical_plan_| Sort: t.i DESC NULLS FIRST_| |_|_SubqueryAlias: t_| -|_|_Inner Join:_| -|_|_Projection:_| +|_|_CrossJoin:_| +|_|_Filter: integers.i IS NOT NULL_| +|_|_Projection: integers.i_| |_|_MergeScan [is_placeholder=false]_| -|_|_Filter: other.i IS NOT NULL_| -|_|_Projection: other.i_| +|_|_Projection:_| |_|_MergeScan [is_placeholder=false]_| | physical_plan | SortPreservingMergeExec: [i@0 DESC]_| |_|_SortExec: expr=[i@0 DESC]_| -|_|_NestedLoopJoinExec: join_type=Inner_| -|_|_RepartitionExec: partitioning=REDACTED -|_|_ProjectionExec: expr=[]_| -|_|_MergeScanExec: REDACTED +|_|_CrossJoinExec_| |_|_CoalescePartitionsExec_| |_|_CoalesceBatchesExec: target_batch_size=8192_| |_|_FilterExec: i@0 IS NOT NULL_| |_|_RepartitionExec: partitioning=REDACTED |_|_ProjectionExec: expr=[i@0 as i]_| |_|_MergeScanExec: REDACTED +|_|_RepartitionExec: partitioning=REDACTED +|_|_ProjectionExec: expr=[]_| +|_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -118,7 +121,8 @@ EXPLAIN INSERT INTO other SELECT i, 2 FROM integers WHERE i=(SELECT MAX(i) FROM | | Projection: integers.i | | | MergeScan [is_placeholder=false] | | | SubqueryAlias: __scalar_sq_1 | -| | MergeScan [is_placeholder=false] | +| | Projection: MAX(integers.i) | +| | MergeScan [is_placeholder=false] | +--------------+-------------------------------------------------------------------+ drop table other; diff --git a/tests/cases/distributed/optimizer/filter_push_down.result b/tests/cases/distributed/optimizer/filter_push_down.result index 67370b04e4..fa4dab8a9a 100644 --- a/tests/cases/distributed/optimizer/filter_push_down.result +++ b/tests/cases/distributed/optimizer/filter_push_down.result @@ -225,19 +225,47 @@ SELECT i FROM (SELECT * FROM integers i1 UNION SELECT * FROM integers i2) a WHER | 3 | +---+ --- TODO(LFC): Somehow the following SQL does not order by column 1 under new DataFusion occasionally. Should further investigate it. Comment it out temporarily. --- expected: --- +---+---+--------------+ --- | a | b | ROW_NUMBER() | --- +---+---+--------------+ --- | 1 | 1 | 1 | --- | 2 | 2 | 5 | --- | 3 | 3 | 9 | --- +---+---+--------------+ --- SELECT * FROM (SELECT i1.i AS a, i2.i AS b, row_number() OVER (ORDER BY i1.i, i2.i) FROM integers i1, integers i2 WHERE i1.i IS NOT NULL AND i2.i IS NOT NULL) a1 WHERE a=b ORDER BY 1; -SELECT * FROM (SELECT 0=1 AS cond FROM integers i1, integers i2) a1 WHERE cond ORDER BY 1; +SELECT * FROM (SELECT i1.i AS a, i2.i AS b, row_number() OVER (ORDER BY i1.i, i2.i) FROM integers i1, integers i2 WHERE i1.i IS NOT NULL AND i2.i IS NOT NULL) a1 WHERE a=b ORDER BY 1; -Error: 3001(EngineExecuteQuery), Invalid argument error: must either specify a row count or at least one column ++---+---+--------------------------------------------------------------------------------------------------------------------+ +| a | b | ROW_NUMBER() ORDER BY [i1.i ASC NULLS LAST, i2.i ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW | ++---+---+--------------------------------------------------------------------------------------------------------------------+ +| 1 | 1 | 1 | +| 2 | 2 | 5 | +| 3 | 3 | 9 | ++---+---+--------------------------------------------------------------------------------------------------------------------+ + +-- The "0=1" will be evaluated as a constant expression that is always false, and will be optimized away in the query +-- engine. In the final plan, there's no filter node. We explain it to ensure that. +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +EXPLAIN SELECT * FROM (SELECT 0=1 AS cond FROM integers i1, integers i2) a1 WHERE cond ORDER BY 1; + ++-+-+ +| plan_type_| plan_| ++-+-+ +| logical_plan_| Sort: a1.cond ASC NULLS LAST_| +|_|_SubqueryAlias: a1_| +|_|_Projection: Boolean(false) AS cond_| +|_|_CrossJoin:_| +|_|_SubqueryAlias: i1_| +|_|_Projection:_| +|_|_MergeScan [is_placeholder=false]_| +|_|_SubqueryAlias: i2_| +|_|_Projection:_| +|_|_MergeScan [is_placeholder=false]_| +| physical_plan | CoalescePartitionsExec_| +|_|_ProjectionExec: expr=[false as cond]_| +|_|_CrossJoinExec_| +|_|_ProjectionExec: expr=[]_| +|_|_MergeScanExec: REDACTED +|_|_RepartitionExec: partitioning=REDACTED +|_|_ProjectionExec: expr=[]_| +|_|_MergeScanExec: REDACTED +|_|_| ++-+-+ SELECT * FROM (SELECT 0=1 AS cond FROM integers i1, integers i2 GROUP BY 1) a1 WHERE cond ORDER BY 1; diff --git a/tests/cases/distributed/optimizer/filter_push_down.sql b/tests/cases/distributed/optimizer/filter_push_down.sql index 4870ddcb1c..c34551f25e 100644 --- a/tests/cases/distributed/optimizer/filter_push_down.sql +++ b/tests/cases/distributed/optimizer/filter_push_down.sql @@ -44,18 +44,15 @@ SELECT * FROM (SELECT DISTINCT i1.i AS a, i2.i AS b FROM integers i1, integers i SELECT i FROM (SELECT * FROM integers i1 UNION SELECT * FROM integers i2) a WHERE i=3; --- TODO(LFC): Somehow the following SQL does not order by column 1 under new DataFusion occasionally. Should further investigate it. Comment it out temporarily. --- expected: --- +---+---+--------------+ --- | a | b | ROW_NUMBER() | --- +---+---+--------------+ --- | 1 | 1 | 1 | --- | 2 | 2 | 5 | --- | 3 | 3 | 9 | --- +---+---+--------------+ --- SELECT * FROM (SELECT i1.i AS a, i2.i AS b, row_number() OVER (ORDER BY i1.i, i2.i) FROM integers i1, integers i2 WHERE i1.i IS NOT NULL AND i2.i IS NOT NULL) a1 WHERE a=b ORDER BY 1; +SELECT * FROM (SELECT i1.i AS a, i2.i AS b, row_number() OVER (ORDER BY i1.i, i2.i) FROM integers i1, integers i2 WHERE i1.i IS NOT NULL AND i2.i IS NOT NULL) a1 WHERE a=b ORDER BY 1; -SELECT * FROM (SELECT 0=1 AS cond FROM integers i1, integers i2) a1 WHERE cond ORDER BY 1; +-- The "0=1" will be evaluated as a constant expression that is always false, and will be optimized away in the query +-- engine. In the final plan, there's no filter node. We explain it to ensure that. +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +EXPLAIN SELECT * FROM (SELECT 0=1 AS cond FROM integers i1, integers i2) a1 WHERE cond ORDER BY 1; SELECT * FROM (SELECT 0=1 AS cond FROM integers i1, integers i2 GROUP BY 1) a1 WHERE cond ORDER BY 1; diff --git a/tests/cases/standalone/common/aggregate/distinct_order_by.result b/tests/cases/standalone/common/aggregate/distinct_order_by.result index 996048bed8..29028359e0 100644 --- a/tests/cases/standalone/common/aggregate/distinct_order_by.result +++ b/tests/cases/standalone/common/aggregate/distinct_order_by.result @@ -29,7 +29,7 @@ Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: For SELECT DI SELECT DISTINCT ON (1) i % 2, i FROM integers WHERE i<3 ORDER BY i; -Error: 3000(PlanQuery), Failed to plan SQL: This feature is not implemented: DISTINCT ON Exprs not supported +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: SELECT DISTINCT ON expressions must match initial ORDER BY expressions SELECT DISTINCT integers.i FROM integers ORDER BY i DESC; diff --git a/tests/cases/standalone/common/aggregate/sum.result b/tests/cases/standalone/common/aggregate/sum.result index 526fce1757..4c54f1bef8 100644 --- a/tests/cases/standalone/common/aggregate/sum.result +++ b/tests/cases/standalone/common/aggregate/sum.result @@ -24,11 +24,7 @@ SELECT SUM(-1) FROM numbers; SELECT SUM(-1) FROM numbers WHERE number=-1; -+----------------+ -| SUM(Int64(-1)) | -+----------------+ -| | -+----------------+ +Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Can't cast value -1 to type UInt32 SELECT SUM(-1) FROM numbers WHERE number>10000 limit 1000; @@ -64,11 +60,11 @@ Affected Rows: 4 SELECT sum(n) from doubles; -+----------------------+ -| SUM(doubles.n) | -+----------------------+ -| 9.007199254740992e15 | -+----------------------+ ++--------------------+ +| SUM(doubles.n) | ++--------------------+ +| 9007199254740992.0 | ++--------------------+ DROP TABLE bigints; diff --git a/tests/cases/standalone/common/cte/cte.result b/tests/cases/standalone/common/cte/cte.result index 2f143c964a..3982a3a857 100644 --- a/tests/cases/standalone/common/cte/cte.result +++ b/tests/cases/standalone/common/cte/cte.result @@ -57,7 +57,7 @@ with cte1 as (select i as j from a), cte2 as (select ref.j as k from cte1 as ref with cte1 as (select 42), cte1 as (select 42) select * FROM cte1; -Error: 3000(PlanQuery), Failed to plan SQL: sql parser error: WITH query name "cte1" specified more than once +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: WITH query name "cte1" specified more than once -- reference to CTE before its actually defined, it's not supported by datafusion with cte3 as (select ref2.j as i from cte1 as ref2), cte1 as (Select i as j from a), cte2 as (select ref.j+1 as k from cte1 as ref) select * from cte2 union all select * FROM cte3; @@ -109,7 +109,7 @@ WITH RECURSIVE cte(d) AS ( ) SELECT max(d) FROM cte; -Error: 3000(PlanQuery), Failed to plan SQL: This feature is not implemented: Recursive CTEs are not supported +Error: 3000(PlanQuery), Failed to plan SQL: This feature is not implemented: Recursive CTE is not implemented -- Nested aliases is not supported in datafusion with cte (a) as ( diff --git a/tests/cases/standalone/common/cte/cte_in_cte.result b/tests/cases/standalone/common/cte/cte_in_cte.result index b2380be7fc..3c8d6010c8 100644 --- a/tests/cases/standalone/common/cte/cte_in_cte.result +++ b/tests/cases/standalone/common/cte/cte_in_cte.result @@ -40,7 +40,7 @@ with cte1 as (with b as (Select i as j from a) select j from b), cte2 as (with c with cte1 as (select 42), cte1 as (select 42) select * FROM cte1; -Error: 3000(PlanQuery), Failed to plan SQL: sql parser error: WITH query name "cte1" specified more than once +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: WITH query name "cte1" specified more than once with cte1 as (Select i as j from a) select * from (with cte2 as (select max(j) as j from cte1) select * from cte2) f; @@ -64,7 +64,7 @@ with cte1 as (Select i as j from a) select * from cte1 where j = (with cte2 as ( -- this feature is not implemented in datafusion with cte as (Select i as j from a) select * from cte where j = (with cte as (select max(j) as j from cte) select j from cte); -Error: 3000(PlanQuery), Failed to plan SQL: sql parser error: WITH query name "cte" specified more than once +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: WITH query name "cte" specified more than once -- self-refer to non-existent cte- with cte as (select * from cte) select * from cte; diff --git a/tests/cases/standalone/common/function/arithmetic.result b/tests/cases/standalone/common/function/arithmetic.result index 563053fbce..20c612d56a 100644 --- a/tests/cases/standalone/common/function/arithmetic.result +++ b/tests/cases/standalone/common/function/arithmetic.result @@ -24,7 +24,7 @@ SELECT MOD(18.0, 4.0); SELECT MOD(18, 0); -Error: 3001(EngineExecuteQuery), DataFusion error: Divide by zero error +Error: 3001(EngineExecuteQuery), Divide by zero error SELECT POW (2, 5); @@ -76,5 +76,5 @@ SELECT CLAMP(0.5, 0, 1); SELECT CLAMP(10, 1, 0); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid function args: The second arg should be less than or equal to the third arg, have: ConstantVector([Int64(1); 1]), ConstantVector([Int64(0); 1]) +Error: 3001(EngineExecuteQuery), Invalid function args: The second arg should be less than or equal to the third arg, have: ConstantVector([Int64(1); 1]), ConstantVector([Int64(0); 1]) diff --git a/tests/cases/standalone/common/function/date.result b/tests/cases/standalone/common/function/date.result index a94cff3cd3..27b9a981ae 100644 --- a/tests/cases/standalone/common/function/date.result +++ b/tests/cases/standalone/common/function/date.result @@ -1,19 +1,19 @@ --- date_add --- SELECT date_add('2023-12-06 07:39:46.222'::TIMESTAMP_MS, INTERVAL '5 day'); -+----------------------------------------------------------------------------------------+ -| date_add(Utf8("2023-12-06 07:39:46.222"),IntervalMonthDayNano("92233720368547758080")) | -+----------------------------------------------------------------------------------------+ -| 2023-12-11T07:39:46.222 | -+----------------------------------------------------------------------------------------+ ++-----------------------------------------------------------------------------------------------------------------------------------------+ +| date_add(arrow_cast(Utf8("2023-12-06 07:39:46.222"),Utf8("Timestamp(Millisecond, None)")),IntervalMonthDayNano("92233720368547758080")) | ++-----------------------------------------------------------------------------------------------------------------------------------------+ +| 2023-12-11T07:39:46.222 | ++-----------------------------------------------------------------------------------------------------------------------------------------+ SELECT date_add('2023-12-06 07:39:46.222'::TIMESTAMP_MS, '5 day'); -+---------------------------------------------------------+ -| date_add(Utf8("2023-12-06 07:39:46.222"),Utf8("5 day")) | -+---------------------------------------------------------+ -| 2023-12-11T07:39:46.222 | -+---------------------------------------------------------+ ++----------------------------------------------------------------------------------------------------------+ +| date_add(arrow_cast(Utf8("2023-12-06 07:39:46.222"),Utf8("Timestamp(Millisecond, None)")),Utf8("5 day")) | ++----------------------------------------------------------------------------------------------------------+ +| 2023-12-11T07:39:46.222 | ++----------------------------------------------------------------------------------------------------------+ SELECT date_add('2023-12-06'::DATE, INTERVAL '3 month 5 day'); @@ -34,19 +34,19 @@ SELECT date_add('2023-12-06'::DATE, '3 month 5 day'); --- date_sub --- SELECT date_sub('2023-12-06 07:39:46.222'::TIMESTAMP_MS, INTERVAL '5 day'); -+----------------------------------------------------------------------------------------+ -| date_sub(Utf8("2023-12-06 07:39:46.222"),IntervalMonthDayNano("92233720368547758080")) | -+----------------------------------------------------------------------------------------+ -| 2023-12-01T07:39:46.222 | -+----------------------------------------------------------------------------------------+ ++-----------------------------------------------------------------------------------------------------------------------------------------+ +| date_sub(arrow_cast(Utf8("2023-12-06 07:39:46.222"),Utf8("Timestamp(Millisecond, None)")),IntervalMonthDayNano("92233720368547758080")) | ++-----------------------------------------------------------------------------------------------------------------------------------------+ +| 2023-12-01T07:39:46.222 | ++-----------------------------------------------------------------------------------------------------------------------------------------+ SELECT date_sub('2023-12-06 07:39:46.222'::TIMESTAMP_MS, '5 day'); -+---------------------------------------------------------+ -| date_sub(Utf8("2023-12-06 07:39:46.222"),Utf8("5 day")) | -+---------------------------------------------------------+ -| 2023-12-01T07:39:46.222 | -+---------------------------------------------------------+ ++----------------------------------------------------------------------------------------------------------+ +| date_sub(arrow_cast(Utf8("2023-12-06 07:39:46.222"),Utf8("Timestamp(Millisecond, None)")),Utf8("5 day")) | ++----------------------------------------------------------------------------------------------------------+ +| 2023-12-01T07:39:46.222 | ++----------------------------------------------------------------------------------------------------------+ SELECT date_sub('2023-12-06'::DATE, INTERVAL '3 month 5 day'); @@ -67,19 +67,19 @@ SELECT date_sub('2023-12-06'::DATE, '3 month 5 day'); --- date_format --- SELECT date_format('2023-12-06 07:39:46.222'::TIMESTAMP_MS, '%Y-%m-%d %H:%M:%S:%3f'); -+----------------------------------------------------------------------------+ -| date_format(Utf8("2023-12-06 07:39:46.222"),Utf8("%Y-%m-%d %H:%M:%S:%3f")) | -+----------------------------------------------------------------------------+ -| 2023-12-06 07:39:46:222 | -+----------------------------------------------------------------------------+ ++-----------------------------------------------------------------------------------------------------------------------------+ +| date_format(arrow_cast(Utf8("2023-12-06 07:39:46.222"),Utf8("Timestamp(Millisecond, None)")),Utf8("%Y-%m-%d %H:%M:%S:%3f")) | ++-----------------------------------------------------------------------------------------------------------------------------+ +| 2023-12-06 07:39:46:222 | ++-----------------------------------------------------------------------------------------------------------------------------+ SELECT date_format('2023-12-06 07:39:46.222'::TIMESTAMP_S, '%Y-%m-%d %H:%M:%S:%3f'); -+----------------------------------------------------------------------------+ -| date_format(Utf8("2023-12-06 07:39:46.222"),Utf8("%Y-%m-%d %H:%M:%S:%3f")) | -+----------------------------------------------------------------------------+ -| 2023-12-06 07:39:46:000 | -+----------------------------------------------------------------------------+ ++------------------------------------------------------------------------------------------------------------------------+ +| date_format(arrow_cast(Utf8("2023-12-06 07:39:46.222"),Utf8("Timestamp(Second, None)")),Utf8("%Y-%m-%d %H:%M:%S:%3f")) | ++------------------------------------------------------------------------------------------------------------------------+ +| 2023-12-06 07:39:46:000 | ++------------------------------------------------------------------------------------------------------------------------+ --- datetime not supported yet --- SELECT date_format('2023-12-06 07:39:46.222'::DATETIME, '%Y-%m-%d %H:%M:%S:%3f'); diff --git a/tests/cases/standalone/common/function/time.result b/tests/cases/standalone/common/function/time.result index 25f82f2a48..83589ced35 100644 --- a/tests/cases/standalone/common/function/time.result +++ b/tests/cases/standalone/common/function/time.result @@ -30,11 +30,11 @@ select to_timezone('2022-09-20T14:16:43.012345+08:00', 'Europe/Berlin'); select to_timezone('2022-09-20T14:16:43.012345+08:00'::Timestamp, 'Europe/Berlin'); -+-----------------------------------------------------------------------------+ -| to_timezone(Utf8("2022-09-20T14:16:43.012345+08:00"),Utf8("Europe/Berlin")) | -+-----------------------------------------------------------------------------+ -| 2022-09-20 08:16:43.012 | -+-----------------------------------------------------------------------------+ ++------------------------------------------------------------------------------------------------------------------------------+ +| to_timezone(arrow_cast(Utf8("2022-09-20T14:16:43.012345+08:00"),Utf8("Timestamp(Millisecond, None)")),Utf8("Europe/Berlin")) | ++------------------------------------------------------------------------------------------------------------------------------+ +| 2022-09-20 08:16:43.012 | ++------------------------------------------------------------------------------------------------------------------------------+ select to_timezone('2024-03-29T14:16:43.012345Z', 'Asia/Shanghai'); @@ -46,11 +46,11 @@ select to_timezone('2024-03-29T14:16:43.012345Z', 'Asia/Shanghai'); select to_timezone('2024-03-29T14:16:43.012345Z'::Timestamp, 'Asia/Shanghai'); -+------------------------------------------------------------------------+ -| to_timezone(Utf8("2024-03-29T14:16:43.012345Z"),Utf8("Asia/Shanghai")) | -+------------------------------------------------------------------------+ -| 2024-03-29 22:16:43.012 | -+------------------------------------------------------------------------+ ++-------------------------------------------------------------------------------------------------------------------------+ +| to_timezone(arrow_cast(Utf8("2024-03-29T14:16:43.012345Z"),Utf8("Timestamp(Millisecond, None)")),Utf8("Asia/Shanghai")) | ++-------------------------------------------------------------------------------------------------------------------------+ +| 2024-03-29 22:16:43.012 | ++-------------------------------------------------------------------------------------------------------------------------+ select to_timezone(1709992225, 'Asia/Shanghai'); @@ -62,9 +62,9 @@ select to_timezone(1709992225, 'Asia/Shanghai'); select to_timezone(1711508510000::INT64, 'Asia/Shanghai'); -+---------------------------------------------------------+ -| to_timezone(Int64(1711508510000),Utf8("Asia/Shanghai")) | -+---------------------------------------------------------+ -| 2024-03-27 11:01:50 | -+---------------------------------------------------------+ ++-----------------------------------------------------------------------------------+ +| to_timezone(arrow_cast(Int64(1711508510000),Utf8("Int64")),Utf8("Asia/Shanghai")) | ++-----------------------------------------------------------------------------------+ +| 2024-03-27 11:01:50 | ++-----------------------------------------------------------------------------------+ diff --git a/tests/cases/standalone/common/insert/insert_select.result b/tests/cases/standalone/common/insert/insert_select.result index 5bb2bb97fb..c59d1b3233 100644 --- a/tests/cases/standalone/common/insert/insert_select.result +++ b/tests/cases/standalone/common/insert/insert_select.result @@ -20,7 +20,7 @@ Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Column count insert into demo2(ts) select memory from demo1; -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Cannot automatically convert Float64 to Timestamp(Millisecond, None) +Affected Rows: 2 insert into demo2 select * from demo1; @@ -28,12 +28,14 @@ Affected Rows: 2 select * from demo2 order by ts; -+-------+------+--------+---------------------+ -| host | cpu | memory | ts | -+-------+------+--------+---------------------+ -| host1 | 66.6 | 1024.0 | 2022-06-15T07:02:37 | -| host2 | 88.8 | 333.3 | 2022-06-15T07:02:38 | -+-------+------+--------+---------------------+ ++-------+------+--------+-------------------------+ +| host | cpu | memory | ts | ++-------+------+--------+-------------------------+ +| | | | 1970-01-01T00:00:00.333 | +| | | | 1970-01-01T00:00:01.024 | +| host1 | 66.6 | 1024.0 | 2022-06-15T07:02:37 | +| host2 | 88.8 | 333.3 | 2022-06-15T07:02:38 | ++-------+------+--------+-------------------------+ drop table demo1; diff --git a/tests/cases/standalone/common/order/limit.result b/tests/cases/standalone/common/order/limit.result index 3648460758..dd9dec772f 100644 --- a/tests/cases/standalone/common/order/limit.result +++ b/tests/cases/standalone/common/order/limit.result @@ -16,27 +16,31 @@ SELECT a FROM test LIMIT 1; SELECT a FROM test LIMIT 1.25; -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: LIMIT must not be negative +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Unexpected expression in LIMIT clause SELECT a FROM test LIMIT 2-1; -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: LIMIT must not be negative ++-------------------------+ +| a | ++-------------------------+ +| 1970-01-01T00:00:00.011 | ++-------------------------+ SELECT a FROM test LIMIT a; -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: LIMIT must not be negative +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Unexpected expression in LIMIT clause SELECT a FROM test LIMIT a+1; -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: LIMIT must not be negative +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Unexpected expression in LIMIT clause SELECT a FROM test LIMIT SUM(42); -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: LIMIT must not be negative +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Unexpected expression in LIMIT clause SELECT a FROM test LIMIT row_number() OVER (); -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: LIMIT must not be negative +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Unexpected expression in LIMIT clause CREATE TABLE test2 (a STRING, ts TIMESTAMP TIME INDEX); @@ -56,7 +60,7 @@ SELECT * FROM test2 LIMIT 3; select 1 limit date '1992-01-01'; -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: LIMIT must not be negative +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Unexpected expression in LIMIT clause CREATE TABLE integers(i TIMESTAMP TIME INDEX); @@ -89,7 +93,7 @@ SELECT * FROM integers LIMIT 4; SELECT * FROM integers as int LIMIT (SELECT MIN(integers.i) FROM integers); -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: LIMIT must not be negative +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Unexpected expression in LIMIT clause SELECT * FROM integers as int OFFSET (SELECT MIN(integers.i) FROM integers); @@ -101,23 +105,23 @@ Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Unexpected ex SELECT * FROM integers as int LIMIT (SELECT max(integers.i) FROM integers where i > 5); -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: LIMIT must not be negative +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Unexpected expression in LIMIT clause SELECT * FROM integers as int LIMIT (SELECT max(integers.i) FROM integers where i > 5); -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: LIMIT must not be negative +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Unexpected expression in LIMIT clause SELECT * FROM integers as int LIMIT (SELECT NULL); -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: LIMIT must not be negative +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Unexpected expression in LIMIT clause SELECT * FROM integers as int LIMIT (SELECT -1); -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: LIMIT must not be negative +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Unexpected expression in LIMIT clause SELECT * FROM integers as int LIMIT (SELECT 'ab'); -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: LIMIT must not be negative +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Unexpected expression in LIMIT clause DROP TABLE integers; diff --git a/tests/cases/standalone/common/order/order_by.result b/tests/cases/standalone/common/order/order_by.result index 36cb023e3d..2e8014c6a2 100644 --- a/tests/cases/standalone/common/order/order_by.result +++ b/tests/cases/standalone/common/order/order_by.result @@ -210,17 +210,11 @@ SELECT a-10 AS k FROM test UNION SELECT a-10 AS l FROM test ORDER BY 1-k; +---+ -- Not compatible with duckdb, give an error in greptimedb --- TODO(LFC): Failed to meet the expected error: --- expected: --- Error: 3000(PlanQuery), Schema error: No field named 'a'. Valid fields are 'k'. SELECT a-10 AS k FROM test UNION SELECT a-10 AS l FROM test ORDER BY a-10; Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: For SELECT DISTINCT, ORDER BY expressions a must appear in select list -- Not compatible with duckdb, give an error in greptimedb --- TODO(LFC): Failed to meet the expected error: --- expected: --- Error: 3000(PlanQuery), Schema error: No field named 'a'. Valid fields are 'k'. SELECT a-10 AS k FROM test UNION SELECT a-11 AS l FROM test ORDER BY a-11; Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: For SELECT DISTINCT, ORDER BY expressions a must appear in select list diff --git a/tests/cases/standalone/common/order/order_by.sql b/tests/cases/standalone/common/order/order_by.sql index d1410bbd26..38ec572978 100644 --- a/tests/cases/standalone/common/order/order_by.sql +++ b/tests/cases/standalone/common/order/order_by.sql @@ -50,15 +50,9 @@ SELECT a-10 AS k FROM test UNION SELECT a-10 AS l FROM test ORDER BY l; SELECT a-10 AS k FROM test UNION SELECT a-10 AS l FROM test ORDER BY 1-k; -- Not compatible with duckdb, give an error in greptimedb --- TODO(LFC): Failed to meet the expected error: --- expected: --- Error: 3000(PlanQuery), Schema error: No field named 'a'. Valid fields are 'k'. SELECT a-10 AS k FROM test UNION SELECT a-10 AS l FROM test ORDER BY a-10; -- Not compatible with duckdb, give an error in greptimedb --- TODO(LFC): Failed to meet the expected error: --- expected: --- Error: 3000(PlanQuery), Schema error: No field named 'a'. Valid fields are 'k'. SELECT a-10 AS k FROM test UNION SELECT a-11 AS l FROM test ORDER BY a-11; DROP TABLE test; diff --git a/tests/cases/standalone/common/order/order_by_exceptions.result b/tests/cases/standalone/common/order/order_by_exceptions.result index 9edcddfc88..851eaacb3e 100644 --- a/tests/cases/standalone/common/order/order_by_exceptions.result +++ b/tests/cases/standalone/common/order/order_by_exceptions.result @@ -10,10 +10,15 @@ SELECT a FROM test ORDER BY 2; Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Order by column out of bounds, specified: 2, max: 1 --- Not work in greptimedb SELECT a FROM test ORDER BY 'hello', a; -Error: 3001(EngineExecuteQuery), Error during planning: Sort operation is not applicable to scalar value hello ++----+ +| a | ++----+ +| 11 | +| 12 | +| 13 | ++----+ -- Ambiguous reference in union alias, give and error in duckdb, but works in greptimedb SELECT a AS k, b FROM test UNION SELECT a, b AS k FROM test ORDER BY k; @@ -41,9 +46,6 @@ SELECT a % 2, b FROM test UNION SELECT b, a % 2 AS k ORDER BY a % 2; Error: 3000(PlanQuery), Failed to plan SQL: No field named b. -- Works duckdb, but not work in greptimedb --- TODO(LFC): Failed to meet the expected error: --- expected: --- Error: 3000(PlanQuery), Schema error: No field named 'a'. Valid fields are 'test.a % Int64(2)', 'b'. SELECT a % 2, b FROM test UNION SELECT a % 2 AS k, b FROM test ORDER BY a % 2; Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: For SELECT DISTINCT, ORDER BY expressions a must appear in select list @@ -52,9 +54,40 @@ SELECT a % 2, b FROM test UNION SELECT a % 2 AS k, b FROM test ORDER BY 3; Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Order by column out of bounds, specified: 3, max: 2 -SELECT a % 2, b FROM test UNION SELECT a % 2 AS k, b FROM test ORDER BY -1; +-- "order by -1" is generally an undefined behavior. +-- It's not supported in PostgreSQL 16, error "ORDER BY position -1 is not in select list". +-- But in Mysql 8, it can be executed, just the actual order is ignored. +-- In DataFusion, it behaves like Mysql 8. The "sort" plan node will be eliminated by the physical optimizer +-- "EnforceSorting" because it's sort key is parsed as a constant "-1". +-- We check the "explain" of the "order by -1" query to ensure that. +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (partitioning.*) REDACTED +EXPLAIN SELECT a % 2, b FROM test UNION SELECT a % 2 AS k, b FROM test ORDER BY -1; -Error: 3001(EngineExecuteQuery), Error during planning: Sort operation is not applicable to scalar value -1 ++---------------+-----------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-----------------------------------------------------------------------------------------------------------+ +| logical_plan | Sort: Int64(-1) ASC NULLS LAST | +| | Aggregate: groupBy=[[test.a % Int64(2), test.b]], aggr=[[]] | +| | Union | +| | Projection: CAST(test.a AS Int64) % Int64(2) AS test.a % Int64(2), test.b | +| | MergeScan [is_placeholder=false] | +| | Projection: CAST(test.a AS Int64) % Int64(2) AS test.a % Int64(2), test.b | +| | MergeScan [is_placeholder=false] | +| physical_plan | CoalescePartitionsExec | +| | AggregateExec: mode=FinalPartitioned, gby=[test.a % Int64(2)@0 as test.a % Int64(2), b@1 as b], aggr=[] | +| | CoalesceBatchesExec: target_batch_size=8192 | +| | RepartitionExec: REDACTED +| | AggregateExec: mode=Partial, gby=[test.a % Int64(2)@0 as test.a % Int64(2), b@1 as b], aggr=[] | +| | UnionExec | +| | ProjectionExec: expr=[CAST(a@0 AS Int64) % 2 as test.a % Int64(2), b@1 as b] | +| | RepartitionExec: REDACTED +| | MergeScanExec: REDACTED +| | ProjectionExec: expr=[CAST(a@0 AS Int64) % 2 as test.a % Int64(2), b@1 as b] | +| | RepartitionExec: REDACTED +| | MergeScanExec: REDACTED +| | | ++---------------+-----------------------------------------------------------------------------------------------------------+ SELECT a % 2, b FROM test UNION SELECT a % 2 AS k FROM test ORDER BY -1; diff --git a/tests/cases/standalone/common/order/order_by_exceptions.sql b/tests/cases/standalone/common/order/order_by_exceptions.sql index 3b12a0e7e5..2024daa290 100644 --- a/tests/cases/standalone/common/order/order_by_exceptions.sql +++ b/tests/cases/standalone/common/order/order_by_exceptions.sql @@ -4,7 +4,6 @@ INSERT INTO test VALUES (11, 22, 1), (12, 21, 2), (13, 22, 3); SELECT a FROM test ORDER BY 2; --- Not work in greptimedb SELECT a FROM test ORDER BY 'hello', a; -- Ambiguous reference in union alias, give and error in duckdb, but works in greptimedb @@ -15,14 +14,19 @@ SELECT a AS k, b FROM test UNION SELECT a AS k, b FROM test ORDER BY k; SELECT a % 2, b FROM test UNION SELECT b, a % 2 AS k ORDER BY a % 2; -- Works duckdb, but not work in greptimedb --- TODO(LFC): Failed to meet the expected error: --- expected: --- Error: 3000(PlanQuery), Schema error: No field named 'a'. Valid fields are 'test.a % Int64(2)', 'b'. SELECT a % 2, b FROM test UNION SELECT a % 2 AS k, b FROM test ORDER BY a % 2; SELECT a % 2, b FROM test UNION SELECT a % 2 AS k, b FROM test ORDER BY 3; -SELECT a % 2, b FROM test UNION SELECT a % 2 AS k, b FROM test ORDER BY -1; +-- "order by -1" is generally an undefined behavior. +-- It's not supported in PostgreSQL 16, error "ORDER BY position -1 is not in select list". +-- But in Mysql 8, it can be executed, just the actual order is ignored. +-- In DataFusion, it behaves like Mysql 8. The "sort" plan node will be eliminated by the physical optimizer +-- "EnforceSorting" because it's sort key is parsed as a constant "-1". +-- We check the "explain" of the "order by -1" query to ensure that. +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (partitioning.*) REDACTED +EXPLAIN SELECT a % 2, b FROM test UNION SELECT a % 2 AS k, b FROM test ORDER BY -1; SELECT a % 2, b FROM test UNION SELECT a % 2 AS k FROM test ORDER BY -1; diff --git a/tests/cases/standalone/common/order/order_variable_size_payload.result b/tests/cases/standalone/common/order/order_variable_size_payload.result index 050026cea7..ba4386006f 100644 --- a/tests/cases/standalone/common/order/order_variable_size_payload.result +++ b/tests/cases/standalone/common/order/order_variable_size_payload.result @@ -129,14 +129,14 @@ Affected Rows: 4 SELECT * FROM tpch_q1_agg ORDER BY l_returnflag, l_linestatus; -+--------------+--------------+---------+------------------+-----------------+-----------------------+--------------------+--------------------+---------------------+-------------+-------------------------+ -| l_returnflag | l_linestatus | sum_qty | sum_base_price | sum_disc_price | sum_charge | avg_qty | avg_price | avg_disc | count_order | t | -+--------------+--------------+---------+------------------+-----------------+-----------------------+--------------------+--------------------+---------------------+-------------+-------------------------+ -| A | F | 3774200 | 5320753880.69 | 5054096266.6828 | 5256751331.449234 | 25.537587116854997 | 36002.12382901414 | 0.05014459706340077 | 147790 | 1970-01-01T00:00:00.003 | -| N | F | 95257 | 133737795.84 | 127132372.6512 | 132286291.229445 | 25.30066401062417 | 35521.32691633466 | 0.04939442231075697 | 3765 | 1970-01-01T00:00:00.004 | -| N | O | 7459297 | 1.05122700089e10 | 9986238338.3847 | 1.0385578376585466e10 | 25.545537671232875 | 36000.9246880137 | 0.05009595890410959 | 292000 | 1970-01-01T00:00:00.001 | -| R | F | 3785523 | 5337950526.47 | 5071818532.942 | 5274405503.049367 | 25.5259438574251 | 35994.029214030925 | 0.04998927856184382 | 148301 | 1970-01-01T00:00:00.002 | -+--------------+--------------+---------+------------------+-----------------+-----------------------+--------------------+--------------------+---------------------+-------------+-------------------------+ ++--------------+--------------+---------+----------------+-----------------+--------------------+--------------------+--------------------+---------------------+-------------+-------------------------+ +| l_returnflag | l_linestatus | sum_qty | sum_base_price | sum_disc_price | sum_charge | avg_qty | avg_price | avg_disc | count_order | t | ++--------------+--------------+---------+----------------+-----------------+--------------------+--------------------+--------------------+---------------------+-------------+-------------------------+ +| A | F | 3774200 | 5320753880.69 | 5054096266.6828 | 5256751331.449234 | 25.537587116854997 | 36002.12382901414 | 0.05014459706340077 | 147790 | 1970-01-01T00:00:00.003 | +| N | F | 95257 | 133737795.84 | 127132372.6512 | 132286291.229445 | 25.30066401062417 | 35521.32691633466 | 0.04939442231075697 | 3765 | 1970-01-01T00:00:00.004 | +| N | O | 7459297 | 10512270008.9 | 9986238338.3847 | 10385578376.585466 | 25.545537671232875 | 36000.9246880137 | 0.05009595890410959 | 292000 | 1970-01-01T00:00:00.001 | +| R | F | 3785523 | 5337950526.47 | 5071818532.942 | 5274405503.049367 | 25.5259438574251 | 35994.029214030925 | 0.04998927856184382 | 148301 | 1970-01-01T00:00:00.002 | ++--------------+--------------+---------+----------------+-----------------+--------------------+--------------------+--------------------+---------------------+-------------+-------------------------+ create table test5 (i int, s varchar, t TIMESTAMP TIME INDEX); diff --git a/tests/cases/standalone/common/promql/scalar.result b/tests/cases/standalone/common/promql/scalar.result new file mode 100644 index 0000000000..0a3d5be95a --- /dev/null +++ b/tests/cases/standalone/common/promql/scalar.result @@ -0,0 +1,320 @@ +CREATE TABLE host ( + ts timestamp(3) time index, + host STRING PRIMARY KEY, + val BIGINT, +); + +Affected Rows: 0 + +INSERT INTO TABLE host VALUES + (0, 'host1', 1), + (0, 'host2', 2), + (5000, 'host1', 3), + (5000, 'host2', 4), + (10000, 'host1', 5), + (10000, 'host2', 6), + (15000, 'host1', 7), + (15000, 'host2', 8); + +Affected Rows: 8 + +-- case only have one time series, scalar return value +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host{host="host1"}); + ++---------------------+-------------+ +| ts | scalar(val) | ++---------------------+-------------+ +| 1970-01-01T00:00:00 | 1.0 | +| 1970-01-01T00:00:05 | 3.0 | +| 1970-01-01T00:00:10 | 5.0 | +| 1970-01-01T00:00:15 | 7.0 | ++---------------------+-------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host{host="host1"}) + 1; + ++---------------------+--------------------------+ +| ts | scalar(val) + Float64(1) | ++---------------------+--------------------------+ +| 1970-01-01T00:00:00 | 2.0 | +| 1970-01-01T00:00:05 | 4.0 | +| 1970-01-01T00:00:10 | 6.0 | +| 1970-01-01T00:00:15 | 8.0 | ++---------------------+--------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') 1 + scalar(host{host="host1"}); + ++---------------------+--------------------------+ +| ts | Float64(1) + scalar(val) | ++---------------------+--------------------------+ +| 1970-01-01T00:00:00 | 2.0 | +| 1970-01-01T00:00:05 | 4.0 | +| 1970-01-01T00:00:10 | 6.0 | +| 1970-01-01T00:00:15 | 8.0 | ++---------------------+--------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host{host="host1"}) + scalar(host{host="host2"}); + ++---------------------+-----------------------------------+ +| ts | lhs.scalar(val) + rhs.scalar(val) | ++---------------------+-----------------------------------+ +| 1970-01-01T00:00:00 | 3.0 | +| 1970-01-01T00:00:05 | 7.0 | +| 1970-01-01T00:00:10 | 11.0 | +| 1970-01-01T00:00:15 | 15.0 | ++---------------------+-----------------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') host{host="host1"} + scalar(host{host="host2"}); + ++-------+---------------------+---------------------------+ +| host | ts | lhs.val + rhs.scalar(val) | ++-------+---------------------+---------------------------+ +| host1 | 1970-01-01T00:00:00 | 3.0 | +| host1 | 1970-01-01T00:00:05 | 7.0 | +| host1 | 1970-01-01T00:00:10 | 11.0 | +| host1 | 1970-01-01T00:00:15 | 15.0 | ++-------+---------------------+---------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host{host="host1"}) + host{host="host2"}; + ++-------+---------------------+---------------------------+ +| host | ts | lhs.scalar(val) + rhs.val | ++-------+---------------------+---------------------------+ +| host2 | 1970-01-01T00:00:00 | 3.0 | +| host2 | 1970-01-01T00:00:05 | 7.0 | +| host2 | 1970-01-01T00:00:10 | 11.0 | +| host2 | 1970-01-01T00:00:15 | 15.0 | ++-------+---------------------+---------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') host + scalar(host{host="host2"}); + ++-------+---------------------+---------------------------+ +| host | ts | lhs.val + rhs.scalar(val) | ++-------+---------------------+---------------------------+ +| host1 | 1970-01-01T00:00:00 | 3.0 | +| host1 | 1970-01-01T00:00:05 | 7.0 | +| host1 | 1970-01-01T00:00:10 | 11.0 | +| host1 | 1970-01-01T00:00:15 | 15.0 | +| host2 | 1970-01-01T00:00:00 | 4.0 | +| host2 | 1970-01-01T00:00:05 | 8.0 | +| host2 | 1970-01-01T00:00:10 | 12.0 | +| host2 | 1970-01-01T00:00:15 | 16.0 | ++-------+---------------------+---------------------------+ + +-- SQLNESS SORT_RESULT 3 1 + +TQL EVAL (0, 15, '5s') scalar(host{host="host1"}) + host; + ++-------+---------------------+---------------------------+ +| host | ts | lhs.scalar(val) + rhs.val | ++-------+---------------------+---------------------------+ +| host1 | 1970-01-01T00:00:00 | 2.0 | +| host1 | 1970-01-01T00:00:05 | 6.0 | +| host1 | 1970-01-01T00:00:10 | 10.0 | +| host1 | 1970-01-01T00:00:15 | 14.0 | +| host2 | 1970-01-01T00:00:00 | 3.0 | +| host2 | 1970-01-01T00:00:05 | 7.0 | +| host2 | 1970-01-01T00:00:10 | 11.0 | +| host2 | 1970-01-01T00:00:15 | 15.0 | ++-------+---------------------+---------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(count(count(host) by (host))); + ++---------------------+--------------------------------+ +| ts | scalar(COUNT(COUNT(host.val))) | ++---------------------+--------------------------------+ +| 1970-01-01T00:00:00 | 2.0 | +| 1970-01-01T00:00:05 | 2.0 | +| 1970-01-01T00:00:10 | 2.0 | +| 1970-01-01T00:00:15 | 2.0 | ++---------------------+--------------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host{host="host1"} + scalar(host{host="host2"})); + ++---------------------+-----------------------------------+ +| ts | scalar(lhs.val + rhs.scalar(val)) | ++---------------------+-----------------------------------+ +| 1970-01-01T00:00:00 | 3.0 | +| 1970-01-01T00:00:05 | 7.0 | +| 1970-01-01T00:00:10 | 11.0 | +| 1970-01-01T00:00:15 | 15.0 | ++---------------------+-----------------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(scalar(host{host="host2"}) + host{host="host1"}); + ++---------------------+-----------------------------------+ +| ts | scalar(lhs.scalar(val) + rhs.val) | ++---------------------+-----------------------------------+ +| 1970-01-01T00:00:00 | 3.0 | +| 1970-01-01T00:00:05 | 7.0 | +| 1970-01-01T00:00:10 | 11.0 | +| 1970-01-01T00:00:15 | 15.0 | ++---------------------+-----------------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host + scalar(host{host="host2"})); + ++---------------------+-----------------------------------+ +| ts | scalar(lhs.val + rhs.scalar(val)) | ++---------------------+-----------------------------------+ +| 1970-01-01T00:00:00 | NaN | +| 1970-01-01T00:00:05 | NaN | +| 1970-01-01T00:00:10 | NaN | +| 1970-01-01T00:00:15 | NaN | ++---------------------+-----------------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(scalar(host{host="host2"}) + host); + ++---------------------+-----------------------------------+ +| ts | scalar(lhs.scalar(val) + rhs.val) | ++---------------------+-----------------------------------+ +| 1970-01-01T00:00:00 | NaN | +| 1970-01-01T00:00:05 | NaN | +| 1970-01-01T00:00:10 | NaN | +| 1970-01-01T00:00:15 | NaN | ++---------------------+-----------------------------------+ + +-- case have multiple time series, scalar return NaN +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host); + ++---------------------+-------------+ +| ts | scalar(val) | ++---------------------+-------------+ +| 1970-01-01T00:00:00 | NaN | +| 1970-01-01T00:00:05 | NaN | +| 1970-01-01T00:00:10 | NaN | +| 1970-01-01T00:00:15 | NaN | ++---------------------+-------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host) + 1; + ++---------------------+--------------------------+ +| ts | scalar(val) + Float64(1) | ++---------------------+--------------------------+ +| 1970-01-01T00:00:00 | NaN | +| 1970-01-01T00:00:05 | NaN | +| 1970-01-01T00:00:10 | NaN | +| 1970-01-01T00:00:15 | NaN | ++---------------------+--------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') 1 + scalar(host); + ++---------------------+--------------------------+ +| ts | Float64(1) + scalar(val) | ++---------------------+--------------------------+ +| 1970-01-01T00:00:00 | NaN | +| 1970-01-01T00:00:05 | NaN | +| 1970-01-01T00:00:10 | NaN | +| 1970-01-01T00:00:15 | NaN | ++---------------------+--------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host) + scalar(host); + ++---------------------+-----------------------------------+ +| ts | lhs.scalar(val) + rhs.scalar(val) | ++---------------------+-----------------------------------+ +| 1970-01-01T00:00:00 | NaN | +| 1970-01-01T00:00:05 | NaN | +| 1970-01-01T00:00:10 | NaN | +| 1970-01-01T00:00:15 | NaN | ++---------------------+-----------------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') host + scalar(host); + ++-------+---------------------+---------------------------+ +| host | ts | lhs.val + rhs.scalar(val) | ++-------+---------------------+---------------------------+ +| host1 | 1970-01-01T00:00:00 | NaN | +| host1 | 1970-01-01T00:00:05 | NaN | +| host1 | 1970-01-01T00:00:10 | NaN | +| host1 | 1970-01-01T00:00:15 | NaN | +| host2 | 1970-01-01T00:00:00 | NaN | +| host2 | 1970-01-01T00:00:05 | NaN | +| host2 | 1970-01-01T00:00:10 | NaN | +| host2 | 1970-01-01T00:00:15 | NaN | ++-------+---------------------+---------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host) + host; + ++-------+---------------------+---------------------------+ +| host | ts | lhs.scalar(val) + rhs.val | ++-------+---------------------+---------------------------+ +| host1 | 1970-01-01T00:00:00 | NaN | +| host1 | 1970-01-01T00:00:05 | NaN | +| host1 | 1970-01-01T00:00:10 | NaN | +| host1 | 1970-01-01T00:00:15 | NaN | +| host2 | 1970-01-01T00:00:00 | NaN | +| host2 | 1970-01-01T00:00:05 | NaN | +| host2 | 1970-01-01T00:00:10 | NaN | +| host2 | 1970-01-01T00:00:15 | NaN | ++-------+---------------------+---------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') host{host="host2"} + scalar(host); + ++-------+---------------------+---------------------------+ +| host | ts | lhs.val + rhs.scalar(val) | ++-------+---------------------+---------------------------+ +| host2 | 1970-01-01T00:00:00 | NaN | +| host2 | 1970-01-01T00:00:05 | NaN | +| host2 | 1970-01-01T00:00:10 | NaN | +| host2 | 1970-01-01T00:00:15 | NaN | ++-------+---------------------+---------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host) + host{host="host2"}; + ++-------+---------------------+---------------------------+ +| host | ts | lhs.scalar(val) + rhs.val | ++-------+---------------------+---------------------------+ +| host2 | 1970-01-01T00:00:00 | NaN | +| host2 | 1970-01-01T00:00:05 | NaN | +| host2 | 1970-01-01T00:00:10 | NaN | +| host2 | 1970-01-01T00:00:15 | NaN | ++-------+---------------------+---------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host{host="host1"} + scalar(host)); + ++---------------------+-----------------------------------+ +| ts | scalar(lhs.val + rhs.scalar(val)) | ++---------------------+-----------------------------------+ +| 1970-01-01T00:00:00 | NaN | +| 1970-01-01T00:00:05 | NaN | +| 1970-01-01T00:00:10 | NaN | +| 1970-01-01T00:00:15 | NaN | ++---------------------+-----------------------------------+ + +-- error case +TQL EVAL (0, 15, '5s') scalar(1 + scalar(host{host="host2"})); + +Error: 2000(InvalidSyntax), expected type vector in call to function 'scalar', got scalar + +TQL EVAL (0, 15, '5s') scalar(scalar(host{host="host2"}) + 1); + +Error: 2000(InvalidSyntax), expected type vector in call to function 'scalar', got scalar + +TQL EVAL (0, 15, '5s') scalar(scalar(host{host="host1"}) + scalar(host{host="host2"})); + +Error: 2000(InvalidSyntax), expected type vector in call to function 'scalar', got scalar + +Drop table host; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/promql/scalar.sql b/tests/cases/standalone/common/promql/scalar.sql new file mode 100644 index 0000000000..3c45164226 --- /dev/null +++ b/tests/cases/standalone/common/promql/scalar.sql @@ -0,0 +1,95 @@ +CREATE TABLE host ( + ts timestamp(3) time index, + host STRING PRIMARY KEY, + val BIGINT, +); + +INSERT INTO TABLE host VALUES + (0, 'host1', 1), + (0, 'host2', 2), + (5000, 'host1', 3), + (5000, 'host2', 4), + (10000, 'host1', 5), + (10000, 'host2', 6), + (15000, 'host1', 7), + (15000, 'host2', 8); + +-- case only have one time series, scalar return value + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host{host="host1"}); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host{host="host1"}) + 1; + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') 1 + scalar(host{host="host1"}); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host{host="host1"}) + scalar(host{host="host2"}); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') host{host="host1"} + scalar(host{host="host2"}); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host{host="host1"}) + host{host="host2"}; + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') host + scalar(host{host="host2"}); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host{host="host1"}) + host; + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(count(count(host) by (host))); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host{host="host1"} + scalar(host{host="host2"})); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(scalar(host{host="host2"}) + host{host="host1"}); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host + scalar(host{host="host2"})); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(scalar(host{host="host2"}) + host); + +-- case have multiple time series, scalar return NaN + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host) + 1; + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') 1 + scalar(host); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host) + scalar(host); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') host + scalar(host); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host) + host; + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') host{host="host2"} + scalar(host); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host) + host{host="host2"}; + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '5s') scalar(host{host="host1"} + scalar(host)); + +-- error case + +TQL EVAL (0, 15, '5s') scalar(1 + scalar(host{host="host2"})); + +TQL EVAL (0, 15, '5s') scalar(scalar(host{host="host2"}) + 1); + +TQL EVAL (0, 15, '5s') scalar(scalar(host{host="host1"}) + scalar(host{host="host2"})); + +Drop table host; diff --git a/tests/cases/standalone/common/range/by.result b/tests/cases/standalone/common/range/by.result index 1665eec32e..e4d9e243cd 100644 --- a/tests/cases/standalone/common/range/by.result +++ b/tests/cases/standalone/common/range/by.result @@ -52,12 +52,12 @@ SELECT ts, max(val) RANGE '5s' FROM host ALIGN '20s' BY () ORDER BY ts; SELECT ts, length(host)::INT64 + 2, max(val) RANGE '5s' FROM host ALIGN '20s' BY (length(host)::INT64 + 2) ORDER BY ts; -+---------------------+----------------------------------------+------------------------+ -| ts | character_length(host.host) + Int64(2) | MAX(host.val) RANGE 5s | -+---------------------+----------------------------------------+------------------------+ -| 1970-01-01T00:00:00 | 7 | 3 | -| 1970-01-01T00:00:20 | 7 | 5 | -+---------------------+----------------------------------------+------------------------+ ++---------------------+------------------------------------------------------------------+------------------------+ +| ts | arrow_cast(character_length(host.host),Utf8("Int64")) + Int64(2) | MAX(host.val) RANGE 5s | ++---------------------+------------------------------------------------------------------+------------------------+ +| 1970-01-01T00:00:00 | 7 | 3 | +| 1970-01-01T00:00:20 | 7 | 5 | ++---------------------+------------------------------------------------------------------+------------------------+ -- Test error -- project non-aggregation key diff --git a/tests/cases/standalone/common/range/calculate.result b/tests/cases/standalone/common/range/calculate.result index a753a9268f..97d527968b 100644 --- a/tests/cases/standalone/common/range/calculate.result +++ b/tests/cases/standalone/common/range/calculate.result @@ -23,18 +23,18 @@ Affected Rows: 10 -- Test range expr calculate SELECT ts, host, covar(val::DOUBLE, val::DOUBLE) RANGE '20s' FROM host ALIGN '10s' ORDER BY host, ts; -+---------------------+-------+-----------------------------------------+ -| ts | host | COVARIANCE(host.val,host.val) RANGE 20s | -+---------------------+-------+-----------------------------------------+ -| 1969-12-31T23:59:50 | host1 | | -| 1970-01-01T00:00:00 | host1 | 0.5 | -| 1970-01-01T00:00:10 | host1 | 0.5 | -| 1970-01-01T00:00:20 | host1 | | -| 1969-12-31T23:59:50 | host2 | | -| 1970-01-01T00:00:00 | host2 | 0.5 | -| 1970-01-01T00:00:10 | host2 | 0.5 | -| 1970-01-01T00:00:20 | host2 | | -+---------------------+-------+-----------------------------------------+ ++---------------------+-------+------------------------------------+ +| ts | host | COVAR(host.val,host.val) RANGE 20s | ++---------------------+-------+------------------------------------+ +| 1969-12-31T23:59:50 | host1 | | +| 1970-01-01T00:00:00 | host1 | 0.5 | +| 1970-01-01T00:00:10 | host1 | 0.5 | +| 1970-01-01T00:00:20 | host1 | | +| 1969-12-31T23:59:50 | host2 | | +| 1970-01-01T00:00:00 | host2 | 0.5 | +| 1970-01-01T00:00:10 | host2 | 0.5 | +| 1970-01-01T00:00:20 | host2 | | ++---------------------+-------+------------------------------------+ SELECT ts, host, 2 * min(val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; @@ -139,18 +139,18 @@ SELECT ts, host, (min(val) + max(val)) RANGE '20s' + 1.0 FROM host ALIGN '10s' O SELECT ts, host, covar(ceil(val::DOUBLE), floor(val::DOUBLE)) RANGE '20s' FROM host ALIGN '10s' ORDER BY host, ts; -+---------------------+-------+------------------------------------------------------+ -| ts | host | COVARIANCE(ceil(host.val),floor(host.val)) RANGE 20s | -+---------------------+-------+------------------------------------------------------+ -| 1969-12-31T23:59:50 | host1 | | -| 1970-01-01T00:00:00 | host1 | 0.5 | -| 1970-01-01T00:00:10 | host1 | 0.5 | -| 1970-01-01T00:00:20 | host1 | | -| 1969-12-31T23:59:50 | host2 | | -| 1970-01-01T00:00:00 | host2 | 0.5 | -| 1970-01-01T00:00:10 | host2 | 0.5 | -| 1970-01-01T00:00:20 | host2 | | -+---------------------+-------+------------------------------------------------------+ ++---------------------+-------+-------------------------------------------------+ +| ts | host | COVAR(ceil(host.val),floor(host.val)) RANGE 20s | ++---------------------+-------+-------------------------------------------------+ +| 1969-12-31T23:59:50 | host1 | | +| 1970-01-01T00:00:00 | host1 | 0.5 | +| 1970-01-01T00:00:10 | host1 | 0.5 | +| 1970-01-01T00:00:20 | host1 | | +| 1969-12-31T23:59:50 | host2 | | +| 1970-01-01T00:00:00 | host2 | 0.5 | +| 1970-01-01T00:00:10 | host2 | 0.5 | +| 1970-01-01T00:00:20 | host2 | | ++---------------------+-------+-------------------------------------------------+ SELECT ts, host, floor(cos(ceil(sin(min(val) RANGE '5s')))) FROM host ALIGN '5s' ORDER BY host, ts; @@ -171,22 +171,22 @@ SELECT ts, host, floor(cos(ceil(sin(min(val) RANGE '5s')))) FROM host ALIGN '5s' SELECT ts, host, gcd(CAST(max(floor(val::DOUBLE)) RANGE '10s' FILL PREV as INT64) * 4, max(val * 4) RANGE '10s' FILL PREV) * length(host) + 1 FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+------------------------------------------------------------------------------------------------------------------------------------------------+ -| ts | host | gcd(MAX(floor(host.val)) RANGE 10s FILL PREV * Int64(4),MAX(host.val * Int64(4)) RANGE 10s FILL PREV) * character_length(host.host) + Int64(1) | -+---------------------+-------+------------------------------------------------------------------------------------------------------------------------------------------------+ -| 1969-12-31T23:59:55 | host1 | 1 | -| 1970-01-01T00:00:00 | host1 | 1 | -| 1970-01-01T00:00:05 | host1 | 21 | -| 1970-01-01T00:00:10 | host1 | 21 | -| 1970-01-01T00:00:15 | host1 | 41 | -| 1970-01-01T00:00:20 | host1 | 41 | -| 1969-12-31T23:59:55 | host2 | 61 | -| 1970-01-01T00:00:00 | host2 | 61 | -| 1970-01-01T00:00:05 | host2 | 81 | -| 1970-01-01T00:00:10 | host2 | 81 | -| 1970-01-01T00:00:15 | host2 | 101 | -| 1970-01-01T00:00:20 | host2 | 101 | -+---------------------+-------+------------------------------------------------------------------------------------------------------------------------------------------------+ ++---------------------+-------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| ts | host | gcd(arrow_cast(MAX(floor(host.val)) RANGE 10s FILL PREV,Utf8("Int64")) * Int64(4),MAX(host.val * Int64(4)) RANGE 10s FILL PREV) * character_length(host.host) + Int64(1) | ++---------------------+-------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| 1969-12-31T23:59:55 | host1 | 1 | +| 1970-01-01T00:00:00 | host1 | 1 | +| 1970-01-01T00:00:05 | host1 | 21 | +| 1970-01-01T00:00:10 | host1 | 21 | +| 1970-01-01T00:00:15 | host1 | 41 | +| 1970-01-01T00:00:20 | host1 | 41 | +| 1969-12-31T23:59:55 | host2 | 61 | +| 1970-01-01T00:00:00 | host2 | 61 | +| 1970-01-01T00:00:05 | host2 | 81 | +| 1970-01-01T00:00:10 | host2 | 81 | +| 1970-01-01T00:00:15 | host2 | 101 | +| 1970-01-01T00:00:20 | host2 | 101 | ++---------------------+-------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ DROP TABLE host; diff --git a/tests/cases/standalone/common/range/error.result b/tests/cases/standalone/common/range/error.result index a33558589b..01beca591f 100644 --- a/tests/cases/standalone/common/range/error.result +++ b/tests/cases/standalone/common/range/error.result @@ -59,8 +59,18 @@ Error: 3000(PlanQuery), DataFusion error: Error during planning: Missing argumen -- 2.3 type mismatch SELECT covar(ceil(val), floor(val)) RANGE '20s' FROM host ALIGN '10s'; -Error: 3001(EngineExecuteQuery), Internal error: Unsupported data type Int64 for function ceil. -This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug report in our issue tracker ++-------------------------------------------------+ +| COVAR(ceil(host.val),floor(host.val)) RANGE 20s | ++-------------------------------------------------+ +| | +| 0.5 | +| 0.5 | +| | +| | +| 0.5 | +| 0.5 | +| | ++-------------------------------------------------+ -- 2.4 nest query SELECT min(max(val) RANGE '20s') RANGE '20s' FROM host ALIGN '10s'; diff --git a/tests/cases/standalone/common/show/show_charset.result b/tests/cases/standalone/common/show/show_charset.result new file mode 100644 index 0000000000..10b25864d3 --- /dev/null +++ b/tests/cases/standalone/common/show/show_charset.result @@ -0,0 +1,58 @@ +SHOW CHARACTER SET; + ++---------+---------------+-------------------+--------+ +| Charset | Description | Default collation | Maxlen | ++---------+---------------+-------------------+--------+ +| utf8 | UTF-8 Unicode | utf8_bin | 4 | ++---------+---------------+-------------------+--------+ + +SHOW CHARSET; + ++---------+---------------+-------------------+--------+ +| Charset | Description | Default collation | Maxlen | ++---------+---------------+-------------------+--------+ +| utf8 | UTF-8 Unicode | utf8_bin | 4 | ++---------+---------------+-------------------+--------+ + +SHOW CHARACTER SET LIKE 'utf8'; + ++---------+---------------+-------------------+--------+ +| Charset | Description | Default collation | Maxlen | ++---------+---------------+-------------------+--------+ +| utf8 | UTF-8 Unicode | utf8_bin | 4 | ++---------+---------------+-------------------+--------+ + +SHOW CHARACTER SET LIKE 'latin1'; + +++ +++ + +SHOW CHARSET LIKE 'utf8'; + ++---------+---------------+-------------------+--------+ +| Charset | Description | Default collation | Maxlen | ++---------+---------------+-------------------+--------+ +| utf8 | UTF-8 Unicode | utf8_bin | 4 | ++---------+---------------+-------------------+--------+ + +SHOW CHARACTER SET WHERE Charset = 'utf8'; + ++---------+---------------+-------------------+--------+ +| Charset | Description | Default collation | Maxlen | ++---------+---------------+-------------------+--------+ +| utf8 | UTF-8 Unicode | utf8_bin | 4 | ++---------+---------------+-------------------+--------+ + +SHOW CHARSET WHERE Charset = 'utf8'; + ++---------+---------------+-------------------+--------+ +| Charset | Description | Default collation | Maxlen | ++---------+---------------+-------------------+--------+ +| utf8 | UTF-8 Unicode | utf8_bin | 4 | ++---------+---------------+-------------------+--------+ + +SHOW CHARSET WHERE Charset = 'latin1'; + +++ +++ + diff --git a/tests/cases/standalone/common/show/show_charset.sql b/tests/cases/standalone/common/show/show_charset.sql new file mode 100644 index 0000000000..9b0ce0d31d --- /dev/null +++ b/tests/cases/standalone/common/show/show_charset.sql @@ -0,0 +1,15 @@ +SHOW CHARACTER SET; + +SHOW CHARSET; + +SHOW CHARACTER SET LIKE 'utf8'; + +SHOW CHARACTER SET LIKE 'latin1'; + +SHOW CHARSET LIKE 'utf8'; + +SHOW CHARACTER SET WHERE Charset = 'utf8'; + +SHOW CHARSET WHERE Charset = 'utf8'; + +SHOW CHARSET WHERE Charset = 'latin1'; diff --git a/tests/cases/standalone/common/show/show_collation.result b/tests/cases/standalone/common/show/show_collation.result new file mode 100644 index 0000000000..4dcf953059 --- /dev/null +++ b/tests/cases/standalone/common/show/show_collation.result @@ -0,0 +1,31 @@ +SHOW COLLATION; + ++-----------+---------+----+---------+----------+---------+ +| Collation | Charset | Id | Default | Compiled | Sortlen | ++-----------+---------+----+---------+----------+---------+ +| utf8_bin | utf8 | 1 | Yes | Yes | 1 | ++-----------+---------+----+---------+----------+---------+ + +SHOW COLLATION LIKE 'utf8'; + +++ +++ + +SHOW COLLATION WHERE Charset = 'utf8'; + ++-----------+---------+----+---------+----------+---------+ +| Collation | Charset | Id | Default | Compiled | Sortlen | ++-----------+---------+----+---------+----------+---------+ +| utf8_bin | utf8 | 1 | Yes | Yes | 1 | ++-----------+---------+----+---------+----------+---------+ + +SHOW COLLATION WHERE Charset = 'latin1'; + +++ +++ + +SHOW COLLATION LIKE 'latin1'; + +++ +++ + diff --git a/tests/cases/standalone/common/show/show_collation.sql b/tests/cases/standalone/common/show/show_collation.sql new file mode 100644 index 0000000000..13eb6f8d36 --- /dev/null +++ b/tests/cases/standalone/common/show/show_collation.sql @@ -0,0 +1,10 @@ +SHOW COLLATION; + +SHOW COLLATION LIKE 'utf8'; + + +SHOW COLLATION WHERE Charset = 'utf8'; + +SHOW COLLATION WHERE Charset = 'latin1'; + +SHOW COLLATION LIKE 'latin1'; diff --git a/tests/cases/standalone/common/tql-explain-analyze/analyze.result b/tests/cases/standalone/common/tql-explain-analyze/analyze.result index 9d063adcdc..fcc3d50bbd 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/analyze.result +++ b/tests/cases/standalone/common/tql-explain-analyze/analyze.result @@ -19,7 +19,6 @@ TQL ANALYZE (0, 10, '5s') test; | plan_type_| plan_| +-+-+ | Plan with Metrics | PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], REDACTED -|_|_RepartitionExec: partitioning=REDACTED |_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], REDACTED |_|_PromSeriesDivideExec: tags=["k"], REDACTED |_|_SortExec: expr=[k@2 ASC NULLS LAST], REDACTED @@ -40,7 +39,6 @@ TQL ANALYZE (0, 10, '1s', '2s') test; | plan_type_| plan_| +-+-+ | Plan with Metrics | PromInstantManipulateExec: range=[0..10000], lookback=[2000], interval=[1000], time index=[j], REDACTED -|_|_RepartitionExec: partitioning=REDACTED |_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], REDACTED |_|_PromSeriesDivideExec: tags=["k"], REDACTED |_|_SortExec: expr=[k@2 ASC NULLS LAST], REDACTED @@ -60,7 +58,6 @@ TQL ANALYZE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp | plan_type_| plan_| +-+-+ | Plan with Metrics | PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], REDACTED -|_|_RepartitionExec: partitioning=REDACTED |_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], REDACTED |_|_PromSeriesDivideExec: tags=["k"], REDACTED |_|_SortExec: expr=[k@2 ASC NULLS LAST], REDACTED @@ -82,14 +79,12 @@ TQL ANALYZE VERBOSE (0, 10, '5s') test; | plan_type_| plan_| +-+-+ | Plan with Metrics_| PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], REDACTED -|_|_RepartitionExec: partitioning=REDACTED |_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], REDACTED |_|_PromSeriesDivideExec: tags=["k"], REDACTED |_|_SortExec: expr=[k@2 ASC NULLS LAST], REDACTED |_|_MergeScanExec: REDACTED |_|_| | Plan with Full Metrics | PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j], REDACTED -|_|_RepartitionExec: partitioning=REDACTED |_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], REDACTED |_|_PromSeriesDivideExec: tags=["k"], REDACTED |_|_SortExec: expr=[k@2 ASC NULLS LAST], REDACTED diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.result b/tests/cases/standalone/common/tql-explain-analyze/explain.result index 2666a1a4d7..eb6be427d3 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/explain.result +++ b/tests/cases/standalone/common/tql-explain-analyze/explain.result @@ -20,11 +20,10 @@ TQL EXPLAIN (0, 10, '5s') test; | | PromSeriesDivide: tags=["k"] | | | MergeScan [is_placeholder=false] | | physical_plan | PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j] | -| | RepartitionExec: partitioning=REDACTED -| | PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false] | -| | PromSeriesDivideExec: tags=["k"] | -| | SortExec: expr=[k@2 ASC NULLS LAST] | -| | MergeScanExec: REDACTED +| | PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false] | +| | PromSeriesDivideExec: tags=["k"] | +| | SortExec: expr=[k@2 ASC NULLS LAST] | +| | MergeScanExec: REDACTED | | | +---------------+-----------------------------------------------------------------------------------------------+ @@ -42,11 +41,10 @@ TQL EXPLAIN (0, 10, '1s', '2s') test; | | PromSeriesDivide: tags=["k"] | | | MergeScan [is_placeholder=false] | | physical_plan | PromInstantManipulateExec: range=[0..0], lookback=[2000], interval=[300000], time index=[j] | -| | RepartitionExec: partitioning=REDACTED -| | PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false] | -| | PromSeriesDivideExec: tags=["k"] | -| | SortExec: expr=[k@2 ASC NULLS LAST] | -| | MergeScanExec: REDACTED +| | PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false] | +| | PromSeriesDivideExec: tags=["k"] | +| | SortExec: expr=[k@2 ASC NULLS LAST] | +| | MergeScanExec: REDACTED | | | +---------------+---------------------------------------------------------------------------------------------+ @@ -63,11 +61,10 @@ TQL EXPLAIN ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp | | PromSeriesDivide: tags=["k"] | | | MergeScan [is_placeholder=false] | | physical_plan | PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j] | -| | RepartitionExec: partitioning=REDACTED -| | PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false] | -| | PromSeriesDivideExec: tags=["k"] | -| | SortExec: expr=[k@2 ASC NULLS LAST] | -| | MergeScanExec: REDACTED +| | PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false] | +| | PromSeriesDivideExec: tags=["k"] | +| | SortExec: expr=[k@2 ASC NULLS LAST] | +| | MergeScanExec: REDACTED | | | +---------------+-----------------------------------------------------------------------------------------------+ @@ -86,8 +83,9 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; |_|_PromSeriesNormalize: offset=[0], time index=[j], filter NaN: [false]_| |_|_PromSeriesDivide: tags=["k"]_| |_|_Sort: test.k DESC NULLS LAST, test.j DESC NULLS LAST_| -|_|_Filter: test.j >= TimestampMillisecond(-300000, None) AND test.j <= TimestampMillisecond(300000, None) | +|_|_Filter: test.j >= TimestampMillisecond(-300000, None) AND test.j <= TimestampMillisecond(300000, None)_| |_|_TableScan: test_| +| logical_plan after apply_function_rewrites_| SAME TEXT AS ABOVE_| | logical_plan after count_wildcard_rule_| SAME TEXT AS ABOVE_| | logical_plan after StringNormalizationRule_| SAME TEXT AS ABOVE_| | logical_plan after inline_table_scan_| SAME TEXT AS ABOVE_| @@ -106,7 +104,6 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; | logical_plan after scalar_subquery_to_join_| SAME TEXT AS ABOVE_| | logical_plan after extract_equijoin_predicate_| SAME TEXT AS ABOVE_| | logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| -| logical_plan after merge_projection_| SAME TEXT AS ABOVE_| | logical_plan after rewrite_disjunctive_predicate_| SAME TEXT AS ABOVE_| | logical_plan after eliminate_duplicated_expr_| SAME TEXT AS ABOVE_| | logical_plan after eliminate_filter_| SAME TEXT AS ABOVE_| @@ -123,9 +120,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; | logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| | logical_plan after unwrap_cast_in_comparison_| SAME TEXT AS ABOVE_| | logical_plan after common_sub_expression_eliminate_| SAME TEXT AS ABOVE_| -| logical_plan after push_down_projection_| SAME TEXT AS ABOVE_| -| logical_plan after eliminate_projection_| SAME TEXT AS ABOVE_| -| logical_plan after push_down_limit_| SAME TEXT AS ABOVE_| +| logical_plan after optimize_projections_| SAME TEXT AS ABOVE_| | logical_plan after OrderHintRule_| SAME TEXT AS ABOVE_| | logical_plan_| PromInstantManipulate: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| |_|_PromSeriesNormalize: offset=[0], time index=[j], filter NaN: [false]_| @@ -136,6 +131,11 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; |_|_PromSeriesDivideExec: tags=["k"]_| |_|_MergeScanExec: REDACTED |_|_| +| initial_physical_plan_with_stats_| PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j], statistics=[Rows=Inexact(0), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]] | +|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_PromSeriesDivideExec: tags=["k"], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_MergeScanExec: REDACTED +|_|_| | physical_plan after OutputRequirements_| OutputRequirementExec_| |_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| |_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| @@ -144,25 +144,19 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; |_|_| | physical_plan after aggregate_statistics_| SAME TEXT AS ABOVE_| | physical_plan after join_selection_| SAME TEXT AS ABOVE_| -| physical_plan after EnforceDistribution_| OutputRequirementExec_| -|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| -|_|_RepartitionExec: partitioning=REDACTED -|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| -|_|_PromSeriesDivideExec: tags=["k"]_| -|_|_MergeScanExec: REDACTED -|_|_| +| physical_plan after LimitedDistinctAggregation_| SAME TEXT AS ABOVE_| +| physical_plan after EnforceDistribution_| SAME TEXT AS ABOVE_| | physical_plan after CombinePartialFinalAggregate_| SAME TEXT AS ABOVE_| | physical_plan after EnforceSorting_| OutputRequirementExec_| |_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| -|_|_RepartitionExec: partitioning=REDACTED |_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| |_|_PromSeriesDivideExec: tags=["k"]_| |_|_SortExec: expr=[k@2 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +| physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_| | physical_plan after coalesce_batches_| SAME TEXT AS ABOVE_| | physical_plan after OutputRequirements_| PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| -|_|_RepartitionExec: partitioning=REDACTED |_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| |_|_PromSeriesDivideExec: tags=["k"]_| |_|_SortExec: expr=[k@2 ASC NULLS LAST]_| @@ -170,13 +164,19 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; |_|_| | physical_plan after PipelineChecker_| SAME TEXT AS ABOVE_| | physical_plan after LimitAggregation_| SAME TEXT AS ABOVE_| +| physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_| | physical_plan_| PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| -|_|_RepartitionExec: partitioning=REDACTED |_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_| |_|_PromSeriesDivideExec: tags=["k"]_| |_|_SortExec: expr=[k@2 ASC NULLS LAST]_| |_|_MergeScanExec: REDACTED |_|_| +| physical_plan_with_stats_| PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j], statistics=[Rows=Inexact(0), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]] | +|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_PromSeriesDivideExec: tags=["k"], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_SortExec: expr=[k@2 ASC NULLS LAST], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:)]]_| +|_|_MergeScanExec: REDACTED +|_|_| +-+-+ DROP TABLE test; diff --git a/tests/cases/standalone/common/types/decimal/decimal_arithmetic.result b/tests/cases/standalone/common/types/decimal/decimal_arithmetic.result index 9be450cc7f..28a5388ca3 100644 --- a/tests/cases/standalone/common/types/decimal/decimal_arithmetic.result +++ b/tests/cases/standalone/common/types/decimal/decimal_arithmetic.result @@ -69,7 +69,7 @@ SELECT ('0.5'::DECIMAL(1,1) + 10000)::VARCHAR, -- out of range SELECT ('0.54321543215432154321543215432154321'::DECIMAL(35,35) + 10000)::VARCHAR; -Error: 3001(EngineExecuteQuery), DataFusion error: Compute error: Overflow happened on: 10000 * 100000000000000000000000000000000000 +Error: 3001(EngineExecuteQuery), Compute error: Overflow happened on: 10000 * 100000000000000000000000000000000000 -- different types SELECT '0.5'::DECIMAL(1,1) + 1::TINYINT, diff --git a/tests/cases/standalone/common/types/decimal/decimal_cast.result b/tests/cases/standalone/common/types/decimal/decimal_cast.result index 0e8de7db67..5f9bb7955e 100644 --- a/tests/cases/standalone/common/types/decimal/decimal_cast.result +++ b/tests/cases/standalone/common/types/decimal/decimal_cast.result @@ -12,19 +12,19 @@ SELECT 127::DECIMAL(3,0)::TINYINT, -127::DECIMAL(3,0)::TINYINT, -7::DECIMAL(9,1) SELECT 128::DECIMAL(3,0)::TINYINT; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: value of 128 is out of range Int8 +Error: 3001(EngineExecuteQuery), Cast error: value of 128 is out of range Int8 SELECT -128::DECIMAL(9,0)::TINYINT; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: value of 128 is out of range Int8 +Error: 3001(EngineExecuteQuery), Cast error: value of 128 is out of range Int8 SELECT 128::DECIMAL(18,0)::TINYINT; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: value of 128 is out of range Int8 +Error: 3001(EngineExecuteQuery), Cast error: value of 128 is out of range Int8 SELECT 14751947891758972421513::DECIMAL(38,0)::TINYINT; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: value of 14751947891758971486208 is out of range Int8 +Error: 3001(EngineExecuteQuery), Cast error: value of 14751947891758971486208 is out of range Int8 -- smallint SELECT 127::DECIMAL(3,0)::SMALLINT, -32767::DECIMAL(5,0)::SMALLINT, -7::DECIMAL(9,1)::SMALLINT, 27::DECIMAL(18,1)::SMALLINT, 33::DECIMAL(38,1)::SMALLINT; @@ -37,15 +37,15 @@ SELECT 127::DECIMAL(3,0)::SMALLINT, -32767::DECIMAL(5,0)::SMALLINT, -7::DECIMAL( SELECT -32768::DECIMAL(9,0)::SMALLINT; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: value of 32768 is out of range Int16 +Error: 3001(EngineExecuteQuery), Cast error: value of 32768 is out of range Int16 SELECT 32768::DECIMAL(18,0)::SMALLINT; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: value of 32768 is out of range Int16 +Error: 3001(EngineExecuteQuery), Cast error: value of 32768 is out of range Int16 SELECT 14751947891758972421513::DECIMAL(38,0)::SMALLINT; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: value of 14751947891758971486208 is out of range Int16 +Error: 3001(EngineExecuteQuery), Cast error: value of 14751947891758971486208 is out of range Int16 -- integer SELECT 127::DECIMAL(3,0)::INTEGER, -2147483647::DECIMAL(10,0)::INTEGER, -7::DECIMAL(9,1)::INTEGER, 27::DECIMAL(18,1)::INTEGER, 33::DECIMAL(38,1)::INTEGER; @@ -58,11 +58,11 @@ SELECT 127::DECIMAL(3,0)::INTEGER, -2147483647::DECIMAL(10,0)::INTEGER, -7::DECI SELECT 2147483648::DECIMAL(18,0)::INTEGER; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: value of 2147483648 is out of range Int32 +Error: 3001(EngineExecuteQuery), Cast error: value of 2147483648 is out of range Int32 SELECT 14751947891758972421513::DECIMAL(38,0)::INTEGER; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: value of 14751947891758971486208 is out of range Int32 +Error: 3001(EngineExecuteQuery), Cast error: value of 14751947891758971486208 is out of range Int32 -- bigint SELECT 127::DECIMAL(3,0)::BIGINT, -9223372036854775807::DECIMAL(19,0)::BIGINT, -7::DECIMAL(9,1)::BIGINT, 27::DECIMAL(18,1)::BIGINT, 33::DECIMAL(38,1)::BIGINT; @@ -75,7 +75,7 @@ SELECT 127::DECIMAL(3,0)::BIGINT, -9223372036854775807::DECIMAL(19,0)::BIGINT, - SELECT 14751947891758972421513::DECIMAL(38,0)::BIGINT; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: value of 14751947891758971486208 is out of range Int64 +Error: 3001(EngineExecuteQuery), Cast error: value of 14751947891758971486208 is out of range Int64 -- float SELECT 127::DECIMAL(3,0)::FLOAT, -17014118346046923173168730371588410572::DECIMAL(38,0)::FLOAT, -7::DECIMAL(9,1)::FLOAT, 27::DECIMAL(18,1)::FLOAT, 33::DECIMAL(38,1)::FLOAT; @@ -99,32 +99,32 @@ SELECT 127::DECIMAL(3,0)::DOUBLE, -17014118346046923173168730371588410572::DECIM -- tinyint SELECT 100::TINYINT::DECIMAL(18,3), 200::TINYINT::DECIMAL(3,0), (-300)::TINYINT::DECIMAL(3,0), 0::TINYINT::DECIMAL(3,3); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Can't cast value 200 to type Int8 +Error: 3001(EngineExecuteQuery), Cast error: Can't cast value 200 to type Int8 SELECT 100::TINYINT::DECIMAL(38,35), 200::TINYINT::DECIMAL(9,6); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Can't cast value 200 to type Int8 +Error: 3001(EngineExecuteQuery), Cast error: Can't cast value 200 to type Int8 -- overflow SELECT 100::TINYINT::DECIMAL(3,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 1::TINYINT::DECIMAL(3,3); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 100::TINYINT::DECIMAL(18,17); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 10000000000000000000 is too large to store in a Decimal128 of precision 18. Max is 999999999999999999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 10000000000000000000 is too large to store in a Decimal128 of precision 18. Max is 999999999999999999 SELECT 100::TINYINT::DECIMAL(9,7); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 1000000000 is too large to store in a Decimal128 of precision 9. Max is 999999999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000000000 is too large to store in a Decimal128 of precision 9. Max is 999999999 SELECT 100::TINYINT::DECIMAL(38,37); -Error: 3001(EngineExecuteQuery), DataFusion error: Compute error: Overflow happened on: 100 * 10000000000000000000000000000000000000 +Error: 3001(EngineExecuteQuery), Compute error: Overflow happened on: 100 * 10000000000000000000000000000000000000 -- smallint SELECT 100::SMALLINT::DECIMAL(18,3), 200::SMALLINT::DECIMAL(3,0), (-300)::SMALLINT::DECIMAL(3,0), 0::SMALLINT::DECIMAL(3,3); @@ -146,23 +146,23 @@ SELECT 100::SMALLINT::DECIMAL(38,35), 200::SMALLINT::DECIMAL(9,6); -- overflow SELECT 100::SMALLINT::DECIMAL(3,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 1::SMALLINT::DECIMAL(3,3); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 100::SMALLINT::DECIMAL(18,17); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 10000000000000000000 is too large to store in a Decimal128 of precision 18. Max is 999999999999999999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 10000000000000000000 is too large to store in a Decimal128 of precision 18. Max is 999999999999999999 SELECT 100::SMALLINT::DECIMAL(9,7); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 1000000000 is too large to store in a Decimal128 of precision 9. Max is 999999999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000000000 is too large to store in a Decimal128 of precision 9. Max is 999999999 SELECT 100::SMALLINT::DECIMAL(38,37); -Error: 3001(EngineExecuteQuery), DataFusion error: Compute error: Overflow happened on: 100 * 10000000000000000000000000000000000000 +Error: 3001(EngineExecuteQuery), Compute error: Overflow happened on: 100 * 10000000000000000000000000000000000000 -- integer SELECT 100::INTEGER::DECIMAL(18,3), 200::INTEGER::DECIMAL(3,0), (-300)::INTEGER::DECIMAL(3,0), 0::INTEGER::DECIMAL(3,3); @@ -184,31 +184,31 @@ SELECT 100::INTEGER::DECIMAL(38,35), 200::INTEGER::DECIMAL(9,6), 2147483647::INT -- overflow SELECT 100::INTEGER::DECIMAL(3,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 10000000::INTEGER::DECIMAL(3,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 100000000 is too large to store in a Decimal128 of precision 3. Max is 999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 100000000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT -10000000::INTEGER::DECIMAL(3,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 100000000 is too large to store in a Decimal128 of precision 3. Max is 999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 100000000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 1::INTEGER::DECIMAL(3,3); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 100::INTEGER::DECIMAL(18,17); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 10000000000000000000 is too large to store in a Decimal128 of precision 18. Max is 999999999999999999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 10000000000000000000 is too large to store in a Decimal128 of precision 18. Max is 999999999999999999 SELECT 100::INTEGER::DECIMAL(9,7); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 1000000000 is too large to store in a Decimal128 of precision 9. Max is 999999999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000000000 is too large to store in a Decimal128 of precision 9. Max is 999999999 SELECT 100::INTEGER::DECIMAL(38,37); -Error: 3001(EngineExecuteQuery), DataFusion error: Compute error: Overflow happened on: 100 * 10000000000000000000000000000000000000 +Error: 3001(EngineExecuteQuery), Compute error: Overflow happened on: 100 * 10000000000000000000000000000000000000 -- bigint SELECT 100::BIGINT::DECIMAL(18,3), 200::BIGINT::DECIMAL(3,0), (-100)::BIGINT::DECIMAL(3,0), 0::BIGINT::DECIMAL(3,3); @@ -238,31 +238,31 @@ SELECT 922337203685477580::BIGINT::DECIMAL(18,0), (-922337203685477580)::BIGINT: -- overflow SELECT 100::BIGINT::DECIMAL(3,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 10000000::BIGINT::DECIMAL(3,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 100000000 is too large to store in a Decimal128 of precision 3. Max is 999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 100000000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT -10000000::BIGINT::DECIMAL(3,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 100000000 is too large to store in a Decimal128 of precision 3. Max is 999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 100000000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 1::BIGINT::DECIMAL(3,3); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 100::BIGINT::DECIMAL(18,17); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 10000000000000000000 is too large to store in a Decimal128 of precision 18. Max is 999999999999999999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 10000000000000000000 is too large to store in a Decimal128 of precision 18. Max is 999999999999999999 SELECT 100::BIGINT::DECIMAL(9,7); -Error: 3001(EngineExecuteQuery), DataFusion error: Invalid argument error: 1000000000 is too large to store in a Decimal128 of precision 9. Max is 999999999 +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000000000 is too large to store in a Decimal128 of precision 9. Max is 999999999 SELECT 100::BIGINT::DECIMAL(38,37); -Error: 3001(EngineExecuteQuery), DataFusion error: Compute error: Overflow happened on: 100 * 10000000000000000000000000000000000000 +Error: 3001(EngineExecuteQuery), Compute error: Overflow happened on: 100 * 10000000000000000000000000000000000000 -- float SELECT 100::FLOAT::DECIMAL(18,3), 200::FLOAT::DECIMAL(3,0), (-300)::FLOAT::DECIMAL(3,0), 0::FLOAT::DECIMAL(3,3); @@ -292,92 +292,52 @@ SELECT 1.25::FLOAT::DECIMAL(3,2); -- overflow SELECT 100::FLOAT::DECIMAL(3,1); -+------------+ -| Int64(100) | -+------------+ -| 10.0 | -+------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 10000000::FLOAT::DECIMAL(3,1); -+-----------------+ -| Int64(10000000) | -+-----------------+ -| 10.0 | -+-----------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 100000000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT -10000000::FLOAT::DECIMAL(3,1); -+---------------------+ -| (- Int64(10000000)) | -+---------------------+ -| -10.0 | -+---------------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 100000000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 1::FLOAT::DECIMAL(3,3); -+----------+ -| Int64(1) | -+----------+ -| .100 | -+----------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 100::FLOAT::DECIMAL(18,17); -+---------------------+ -| Int64(100) | -+---------------------+ -| 1.00000000000000000 | -+---------------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 10000000000000000000 is too large to store in a Decimal128 of precision 18. Max is 999999999999999999 SELECT 100::FLOAT::DECIMAL(9,7); -+------------+ -| Int64(100) | -+------------+ -| 10.0000000 | -+------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000000000 is too large to store in a Decimal128 of precision 9. Max is 999999999 SELECT 100::FLOAT::DECIMAL(38,37); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(38, 37). Overflowing on 100.0 +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(38, 37). Overflowing on 100.0 -- Some controversial cases SELECT 17014118346046923173168730371588410572::FLOAT::DECIMAL(38,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(38, 1). Overflowing on 1.7014119e37 +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(38, 1). Overflowing on 1.7014119e37 SELECT 17014118346046923173168730371588410572::FLOAT::DECIMAL(37,0); -+-------------------------------------------------+ -| Float64(17014118346046924000000000000000000000) | -+-------------------------------------------------+ -| 1701411859957704321881461067092905164 | -+-------------------------------------------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 17014118599577043218814610670929051648 is too large to store in a Decimal128 of precision 37. Max is 9999999999999999999999999999999999999 SELECT 17014118346046923173168730371588410572::FLOAT::DECIMAL(18,0); -+-------------------------------------------------+ -| Float64(17014118346046924000000000000000000000) | -+-------------------------------------------------+ -| 170141185995770432 | -+-------------------------------------------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 17014118599577043218814610670929051648 is too large to store in a Decimal128 of precision 18. Max is 999999999999999999 SELECT 17014118346046923173168730371588410572::FLOAT::DECIMAL(9,0); -+-------------------------------------------------+ -| Float64(17014118346046924000000000000000000000) | -+-------------------------------------------------+ -| 170141185 | -+-------------------------------------------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 17014118599577043218814610670929051648 is too large to store in a Decimal128 of precision 9. Max is 999999999 SELECT 17014118346046923173168730371588410572::FLOAT::DECIMAL(4,0); -+-------------------------------------------------+ -| Float64(17014118346046924000000000000000000000) | -+-------------------------------------------------+ -| 1701 | -+-------------------------------------------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 17014118599577043218814610670929051648 is too large to store in a Decimal128 of precision 4. Max is 9999 -- double SELECT 100::DOUBLE::DECIMAL(18,3), 200::DOUBLE::DECIMAL(3,0), (-300)::DOUBLE::DECIMAL(3,0), 0::DOUBLE::DECIMAL(3,3); @@ -407,90 +367,50 @@ SELECT 1.25::DOUBLE::DECIMAL(3,2); -- overflow SELECT 100::DOUBLE::DECIMAL(3,1); -+------------+ -| Int64(100) | -+------------+ -| 10.0 | -+------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 10000000::DOUBLE::DECIMAL(3,1); -+-----------------+ -| Int64(10000000) | -+-----------------+ -| 10.0 | -+-----------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 100000000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT -10000000::DOUBLE::DECIMAL(3,1); -+---------------------+ -| (- Int64(10000000)) | -+---------------------+ -| -10.0 | -+---------------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 100000000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 1::DOUBLE::DECIMAL(3,3); -+----------+ -| Int64(1) | -+----------+ -| .100 | -+----------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000 is too large to store in a Decimal128 of precision 3. Max is 999 SELECT 100::DOUBLE::DECIMAL(18,17); -+---------------------+ -| Int64(100) | -+---------------------+ -| 1.00000000000000000 | -+---------------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 10000000000000000000 is too large to store in a Decimal128 of precision 18. Max is 999999999999999999 SELECT 100::DOUBLE::DECIMAL(9,7); -+------------+ -| Int64(100) | -+------------+ -| 10.0000000 | -+------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 1000000000 is too large to store in a Decimal128 of precision 9. Max is 999999999 SELECT 100::DOUBLE::DECIMAL(38,37); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(38, 37). Overflowing on 100.0 +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(38, 37). Overflowing on 100.0 -- Some controversial cases SELECT 17014118346046923173168730371588410572::DOUBLE::DECIMAL(38,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(38, 1). Overflowing on 1.7014118346046924e37 +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(38, 1). Overflowing on 1.7014118346046924e37 SELECT 17014118346046923173168730371588410572::DOUBLE::DECIMAL(37,0); -+-------------------------------------------------+ -| Float64(17014118346046924000000000000000000000) | -+-------------------------------------------------+ -| 1701411834604692411764202694551745331 | -+-------------------------------------------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 17014118346046924117642026945517453312 is too large to store in a Decimal128 of precision 37. Max is 9999999999999999999999999999999999999 SELECT 17014118346046923173168730371588410572::DOUBLE::DECIMAL(18,0); -+-------------------------------------------------+ -| Float64(17014118346046924000000000000000000000) | -+-------------------------------------------------+ -| 170141183460469241 | -+-------------------------------------------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 17014118346046924117642026945517453312 is too large to store in a Decimal128 of precision 18. Max is 999999999999999999 SELECT 17014118346046923173168730371588410572::DOUBLE::DECIMAL(9,0); -+-------------------------------------------------+ -| Float64(17014118346046924000000000000000000000) | -+-------------------------------------------------+ -| 170141183 | -+-------------------------------------------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 17014118346046924117642026945517453312 is too large to store in a Decimal128 of precision 9. Max is 999999999 SELECT 17014118346046923173168730371588410572::DOUBLE::DECIMAL(4,0); -+-------------------------------------------------+ -| Float64(17014118346046924000000000000000000000) | -+-------------------------------------------------+ -| 1701 | -+-------------------------------------------------+ +Error: 3001(EngineExecuteQuery), Invalid argument error: 17014118346046924117642026945517453312 is too large to store in a Decimal128 of precision 4. Max is 9999 diff --git a/tests/cases/standalone/common/types/decimal/decimal_ops.result b/tests/cases/standalone/common/types/decimal/decimal_ops.result index 3081f3811c..35de1a1d27 100644 --- a/tests/cases/standalone/common/types/decimal/decimal_ops.result +++ b/tests/cases/standalone/common/types/decimal/decimal_ops.result @@ -124,7 +124,7 @@ SELECT CEIL('0.1'::DECIMAL), CEIL('-0.1'::DECIMAL), CEIL(NULL::DECIMAL); +-------------------+--------------------+------------+ | ceil(Utf8("0.1")) | ceil(Utf8("-0.1")) | ceil(NULL) | +-------------------+--------------------+------------+ -| 1.0 | 0.0 | | +| 1.0 | -0.0 | | +-------------------+--------------------+------------+ SELECT CEIL('100.3'::DECIMAL), CEIL('-127012.3'::DECIMAL); @@ -149,7 +149,7 @@ SELECT CEIL('999.9'::DECIMAL(4,1)), CEIL('99999999.9'::DECIMAL(9,1)), CEIL('9999 +---------------------+--------------------------+-----------------------------------+-------------------------------------------------------+ | ceil(Utf8("999.9")) | ceil(Utf8("99999999.9")) | ceil(Utf8("99999999999999999.9")) | ceil(Utf8("9999999999999999999999999999999999999.9")) | +---------------------+--------------------------+-----------------------------------+-------------------------------------------------------+ -| 1000.0 | 100000000.0 | 1.0e17 | 1.0e37 | +| 1000.0 | 100000000.0 | 1e17 | 1e37 | +---------------------+--------------------------+-----------------------------------+-------------------------------------------------------+ SELECT CEIL('-999.9'::DECIMAL(4,1)), CEIL('-99999999.9'::DECIMAL(9,1)), CEIL('-99999999999999999.9'::DECIMAL(18,1)), CEIL('-9999999999999999999999999999999999999.9'::DECIMAL(38,1)); @@ -157,7 +157,7 @@ SELECT CEIL('-999.9'::DECIMAL(4,1)), CEIL('-99999999.9'::DECIMAL(9,1)), CEIL('-9 +----------------------+---------------------------+------------------------------------+--------------------------------------------------------+ | ceil(Utf8("-999.9")) | ceil(Utf8("-99999999.9")) | ceil(Utf8("-99999999999999999.9")) | ceil(Utf8("-9999999999999999999999999999999999999.9")) | +----------------------+---------------------------+------------------------------------+--------------------------------------------------------+ -| -999.0 | -99999999.0 | -1.0e17 | -1.0e37 | +| -999.0 | -99999999.0 | -1e17 | -1e37 | +----------------------+---------------------------+------------------------------------+--------------------------------------------------------+ -- test FLOOR function @@ -191,7 +191,7 @@ SELECT FLOOR('999.9'::DECIMAL(4,1)), FLOOR('99999999.9'::DECIMAL(9,1)), FLOOR('9 +----------------------+---------------------------+------------------------------------+--------------------------------------------------------+ | floor(Utf8("999.9")) | floor(Utf8("99999999.9")) | floor(Utf8("99999999999999999.9")) | floor(Utf8("9999999999999999999999999999999999999.9")) | +----------------------+---------------------------+------------------------------------+--------------------------------------------------------+ -| 999.0 | 99999999.0 | 1.0e17 | 1.0e37 | +| 999.0 | 99999999.0 | 1e17 | 1e37 | +----------------------+---------------------------+------------------------------------+--------------------------------------------------------+ SELECT FLOOR('-999.9'::DECIMAL(4,1)), FLOOR('-99999999.9'::DECIMAL(9,1)), FLOOR('-99999999999999999.9'::DECIMAL(18,1)), FLOOR('-9999999999999999999999999999999999999.9'::DECIMAL(38,1)); @@ -199,7 +199,7 @@ SELECT FLOOR('-999.9'::DECIMAL(4,1)), FLOOR('-99999999.9'::DECIMAL(9,1)), FLOOR( +-----------------------+----------------------------+-------------------------------------+---------------------------------------------------------+ | floor(Utf8("-999.9")) | floor(Utf8("-99999999.9")) | floor(Utf8("-99999999999999999.9")) | floor(Utf8("-9999999999999999999999999999999999999.9")) | +-----------------------+----------------------------+-------------------------------------+---------------------------------------------------------+ -| -1000.0 | -100000000.0 | -1.0e17 | -1.0e37 | +| -1000.0 | -100000000.0 | -1e17 | -1e37 | +-----------------------+----------------------------+-------------------------------------+---------------------------------------------------------+ -- test unary ROUND function @@ -208,7 +208,7 @@ SELECT ROUND('0.1'::DECIMAL), ROUND('-0.1'::DECIMAL), ROUND(NULL::DECIMAL); +--------------------+---------------------+-------------+ | round(Utf8("0.1")) | round(Utf8("-0.1")) | round(NULL) | +--------------------+---------------------+-------------+ -| 0.0 | 0.0 | | +| 0.0 | -0.0 | | +--------------------+---------------------+-------------+ SELECT ROUND('100.3'::DECIMAL), ROUND('-127012.3'::DECIMAL); @@ -233,7 +233,7 @@ SELECT ROUND('999.9'::DECIMAL(4,1)), ROUND('99999999.9'::DECIMAL(9,1)), ROUND('9 +----------------------+---------------------------+------------------------------------+--------------------------------------------------------+ | round(Utf8("999.9")) | round(Utf8("99999999.9")) | round(Utf8("99999999999999999.9")) | round(Utf8("9999999999999999999999999999999999999.9")) | +----------------------+---------------------------+------------------------------------+--------------------------------------------------------+ -| 1000.0 | 100000000.0 | 1.0e17 | 1.0e37 | +| 1000.0 | 100000000.0 | 1e17 | 1e37 | +----------------------+---------------------------+------------------------------------+--------------------------------------------------------+ SELECT ROUND('-999.9'::DECIMAL(4,1)), ROUND('-99999999.9'::DECIMAL(9,1)), ROUND('-99999999999999999.9'::DECIMAL(18,1)), ROUND('-9999999999999999999999999999999999999.9'::DECIMAL(38,1)); @@ -241,7 +241,7 @@ SELECT ROUND('-999.9'::DECIMAL(4,1)), ROUND('-99999999.9'::DECIMAL(9,1)), ROUND( +-----------------------+----------------------------+-------------------------------------+---------------------------------------------------------+ | round(Utf8("-999.9")) | round(Utf8("-99999999.9")) | round(Utf8("-99999999999999999.9")) | round(Utf8("-9999999999999999999999999999999999999.9")) | +-----------------------+----------------------------+-------------------------------------+---------------------------------------------------------+ -| -1000.0 | -100000000.0 | -1.0e17 | -1.0e37 | +| -1000.0 | -100000000.0 | -1e17 | -1e37 | +-----------------------+----------------------------+-------------------------------------+---------------------------------------------------------+ -- round with precision @@ -340,7 +340,7 @@ SELECT ROUND('-1049578239572094512.32415'::DECIMAL(30,10), 0)::VARCHAR, +----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+---------------------------------------------------------+ | round(Utf8("-1049578239572094512.32415"),Int64(0)) | round(Utf8("-1049578239572094512.32415"),Int64(-1)) | round(Utf8("-1049578239572094512.32415"),Int64(-2)) | round(Utf8("-1049578239572094512.32415"),Int64(-3)) | round(Utf8("-1049578239572094512.32415"),Int64(-4)) | round(Utf8("-1049578239572094512.32415"),Int64(-5)) | round(Utf8("-1049578239572094512.32415"),Int64(-6)) | round(Utf8("-1049578239572094512.32415"),Int64(-7)) | round(Utf8("-1049578239572094512.32415"),Int64(-8)) | round(Utf8("-1049578239572094512.32415"),Int64(-9)) | round(Utf8("-1049578239572094512.32415"),Int64(-10)) | round(Utf8("-1049578239572094512.32415"),Int64(-11)) | round(Utf8("-1049578239572094512.32415"),Int64(-12)) | round(Utf8("-1049578239572094512.32415"),Int64(-13)) | round(Utf8("-1049578239572094512.32415"),Int64(-14)) | round(Utf8("-1049578239572094512.32415"),Int64(-15)) | round(Utf8("-1049578239572094512.32415"),Int64(-16)) | round(Utf8("-1049578239572094512.32415"),Int64(-18)) | round(Utf8("-1049578239572094512.32415"),Int64(-19)) | round(Utf8("-1049578239572094512.32415"),Int64(-20)) | round(Utf8("-1049578239572094512.32415"),Int64(-19842)) | +----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+---------------------------------------------------------+ -| -1.0495782395720946e18 | -1.0495782395720947e18 | -1.0495782395720946e18 | -1.049578239572095e18 | -1.04957823957209e18 | -1.0495782395721e18 | -1.049578239572e18 | -1.04957823957e18 | -1.0495782396e18 | -1.0495782399999999e18 | -1.04957824e18 | -1.0495782e18 | -1.049578e18 | -1.04958e18 | -1.0496e18 | -1.0499999999999999e18 | -1.05e18 | -9.999999999999999e17 | 0.0 | 0.0 | NaN | +| -1.0495782395720946e18 | -1.0495782395720947e18 | -1.0495782395720946e18 | -1.049578239572095e18 | -1.04957823957209e18 | -1.0495782395721e18 | -1.049578239572e18 | -1.04957823957e18 | -1.0495782396e18 | -1.0495782399999999e18 | -1.04957824e18 | -1.0495782e18 | -1.049578e18 | -1.04958e18 | -1.0496e18 | -1.0499999999999999e18 | -1.05e18 | -9.999999999999999e17 | -0.0 | -0.0 | NaN | +----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+-----------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+------------------------------------------------------+---------------------------------------------------------+ SELECT ROUND(12::DECIMAL(3,0)); diff --git a/tests/cases/standalone/common/types/decimal/decimal_small_precision_behavior.result b/tests/cases/standalone/common/types/decimal/decimal_small_precision_behavior.result index 68af47cf58..0c05be4492 100644 --- a/tests/cases/standalone/common/types/decimal/decimal_small_precision_behavior.result +++ b/tests/cases/standalone/common/types/decimal/decimal_small_precision_behavior.result @@ -30,14 +30,12 @@ select '1.234499999'::DECIMAL(5,4); | 1.2345 | +---------------------+ --- arrow-rs is a little strange about the conversion behavior of negative numbers. --- issue: https://github.com/apache/arrow-datafusion/issues/8326 select '-1.023450000001'::DECIMAL(5,4); +-------------------------+ | Utf8("-1.023450000001") | +-------------------------+ -| -0.9765 | +| -1.0235 | +-------------------------+ select '-1.234499999'::DECIMAL(4,3); @@ -45,7 +43,7 @@ select '-1.234499999'::DECIMAL(4,3); +----------------------+ | Utf8("-1.234499999") | +----------------------+ -| -0.766 | +| -1.234 | +----------------------+ select '-1.23499999'::DECIMAL(4,3); @@ -53,7 +51,7 @@ select '-1.23499999'::DECIMAL(4,3); +---------------------+ | Utf8("-1.23499999") | +---------------------+ -| -0.765 | +| -1.235 | +---------------------+ select '-1.234499999'::DECIMAL(5,4); @@ -61,6 +59,6 @@ select '-1.234499999'::DECIMAL(5,4); +----------------------+ | Utf8("-1.234499999") | +----------------------+ -| -0.7655 | +| -1.2345 | +----------------------+ diff --git a/tests/cases/standalone/common/types/decimal/decimal_small_precision_behavior.sql b/tests/cases/standalone/common/types/decimal/decimal_small_precision_behavior.sql index 89fd28a385..547054c322 100644 --- a/tests/cases/standalone/common/types/decimal/decimal_small_precision_behavior.sql +++ b/tests/cases/standalone/common/types/decimal/decimal_small_precision_behavior.sql @@ -6,8 +6,6 @@ select '1.23499999'::DECIMAL(4,3); select '1.234499999'::DECIMAL(5,4); --- arrow-rs is a little strange about the conversion behavior of negative numbers. --- issue: https://github.com/apache/arrow-datafusion/issues/8326 select '-1.023450000001'::DECIMAL(5,4); select '-1.234499999'::DECIMAL(4,3); diff --git a/tests/cases/standalone/common/types/decimal/decimal_table.result b/tests/cases/standalone/common/types/decimal/decimal_table.result index aaf2a08450..03f04182a0 100644 --- a/tests/cases/standalone/common/types/decimal/decimal_table.result +++ b/tests/cases/standalone/common/types/decimal/decimal_table.result @@ -19,7 +19,7 @@ SELECT d + 1000000000000000.0 FROM decimals; +----------------------------------------+ | decimals.d + Float64(1000000000000000) | +----------------------------------------+ -| 1.0e17 | +| 1e17 | +----------------------------------------+ SELECT -1 - d FROM decimals; @@ -35,7 +35,7 @@ SELECT -1000000000000000.0 - d FROM decimals; +-----------------------------------------+ | Float64(-1000000000000000) - decimals.d | +-----------------------------------------+ -| -1.0e17 | +| -1e17 | +-----------------------------------------+ SELECT 1 * d FROM decimals; diff --git a/tests/cases/standalone/common/types/float/nan_aggregate.result b/tests/cases/standalone/common/types/float/nan_aggregate.result index 50958986ee..77636174c5 100644 --- a/tests/cases/standalone/common/types/float/nan_aggregate.result +++ b/tests/cases/standalone/common/types/float/nan_aggregate.result @@ -104,14 +104,14 @@ Affected Rows: 4 SELECT * FROM floats_doubles; -+---------+---------+-------------------------+ -| f | d | ts | -+---------+---------+-------------------------+ -| 2.0e38 | 1.0e308 | 1970-01-01T00:00:00.001 | -| 2.0e38 | 1.0e308 | 1970-01-01T00:00:00.002 | -| -1.0e38 | 0.0 | 1970-01-01T00:00:00.003 | -| -1.0e38 | 0.0 | 1970-01-01T00:00:00.004 | -+---------+---------+-------------------------+ ++-------+-------+-------------------------+ +| f | d | ts | ++-------+-------+-------------------------+ +| 2e38 | 1e308 | 1970-01-01T00:00:00.001 | +| 2e38 | 1e308 | 1970-01-01T00:00:00.002 | +| -1e38 | 0.0 | 1970-01-01T00:00:00.003 | +| -1e38 | 0.0 | 1970-01-01T00:00:00.004 | ++-------+-------+-------------------------+ -- not out of range -- SELECT SUM(f) FROM floats_doubles WHERE f > 0; diff --git a/tests/cases/standalone/common/types/float/nan_cast.result b/tests/cases/standalone/common/types/float/nan_cast.result index 585981f449..927aabf071 100644 --- a/tests/cases/standalone/common/types/float/nan_cast.result +++ b/tests/cases/standalone/common/types/float/nan_cast.result @@ -11,23 +11,23 @@ SELECT 'nan'::FLOAT; -- cannot cast nan, inf or -inf to these types SELECT 'nan'::FLOAT::INT; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Can't cast value NaN to type Int32 +Error: 3001(EngineExecuteQuery), Cast error: Can't cast value NaN to type Int32 SELECT 'nan'::FLOAT::DECIMAL(4,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(4, 1). Overflowing on NaN +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(4, 1). Overflowing on NaN SELECT 'nan'::FLOAT::DECIMAL(9,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(9, 1). Overflowing on NaN +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(9, 1). Overflowing on NaN SELECT 'nan'::FLOAT::DECIMAL(18,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(18, 1). Overflowing on NaN +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(18, 1). Overflowing on NaN SELECT 'nan'::FLOAT::DECIMAL(38,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(38, 1). Overflowing on NaN +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(38, 1). Overflowing on NaN SELECT 'inf'::FLOAT; @@ -40,23 +40,23 @@ SELECT 'inf'::FLOAT; -- cannot cast nan, inf or -inf to these types SELECT 'inf'::FLOAT::INT; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Can't cast value inf to type Int32 +Error: 3001(EngineExecuteQuery), Cast error: Can't cast value inf to type Int32 SELECT 'inf'::FLOAT::DECIMAL(4,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(4, 1). Overflowing on inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(4, 1). Overflowing on inf SELECT 'inf'::FLOAT::DECIMAL(9,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(9, 1). Overflowing on inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(9, 1). Overflowing on inf SELECT 'inf'::FLOAT::DECIMAL(18,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(18, 1). Overflowing on inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(18, 1). Overflowing on inf SELECT 'inf'::FLOAT::DECIMAL(38,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(38, 1). Overflowing on inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(38, 1). Overflowing on inf SELECT '-inf'::FLOAT; @@ -69,23 +69,23 @@ SELECT '-inf'::FLOAT; -- cannot cast nan, inf or -inf to these types SELECT '-inf'::FLOAT::INT; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Can't cast value -inf to type Int32 +Error: 3001(EngineExecuteQuery), Cast error: Can't cast value -inf to type Int32 SELECT '-inf'::FLOAT::DECIMAL(4,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(4, 1). Overflowing on -inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(4, 1). Overflowing on -inf SELECT '-inf'::FLOAT::DECIMAL(9,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(9, 1). Overflowing on -inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(9, 1). Overflowing on -inf SELECT '-inf'::FLOAT::DECIMAL(18,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(18, 1). Overflowing on -inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(18, 1). Overflowing on -inf SELECT '-inf'::FLOAT::DECIMAL(38,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(38, 1). Overflowing on -inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(38, 1). Overflowing on -inf SELECT 'nan'::DOUBLE; @@ -98,23 +98,23 @@ SELECT 'nan'::DOUBLE; -- cannot cast nan, inf or -inf to these types SELECT 'nan'::DOUBLE::INT; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Can't cast value NaN to type Int32 +Error: 3001(EngineExecuteQuery), Cast error: Can't cast value NaN to type Int32 SELECT 'nan'::DOUBLE::DECIMAL(4,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(4, 1). Overflowing on NaN +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(4, 1). Overflowing on NaN SELECT 'nan'::DOUBLE::DECIMAL(9,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(9, 1). Overflowing on NaN +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(9, 1). Overflowing on NaN SELECT 'nan'::DOUBLE::DECIMAL(18,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(18, 1). Overflowing on NaN +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(18, 1). Overflowing on NaN SELECT 'nan'::DOUBLE::DECIMAL(38,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(38, 1). Overflowing on NaN +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(38, 1). Overflowing on NaN SELECT 'inf'::DOUBLE; @@ -127,23 +127,23 @@ SELECT 'inf'::DOUBLE; -- cannot cast nan, inf or -inf to these types SELECT 'inf'::DOUBLE::INT; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Can't cast value inf to type Int32 +Error: 3001(EngineExecuteQuery), Cast error: Can't cast value inf to type Int32 SELECT 'inf'::DOUBLE::DECIMAL(4,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(4, 1). Overflowing on inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(4, 1). Overflowing on inf SELECT 'inf'::DOUBLE::DECIMAL(9,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(9, 1). Overflowing on inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(9, 1). Overflowing on inf SELECT 'inf'::DOUBLE::DECIMAL(18,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(18, 1). Overflowing on inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(18, 1). Overflowing on inf SELECT 'inf'::DOUBLE::DECIMAL(38,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(38, 1). Overflowing on inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(38, 1). Overflowing on inf SELECT '-inf'::DOUBLE; @@ -156,23 +156,23 @@ SELECT '-inf'::DOUBLE; -- cannot cast nan, inf or -inf to these types SELECT '-inf'::DOUBLE::INT; -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Can't cast value -inf to type Int32 +Error: 3001(EngineExecuteQuery), Cast error: Can't cast value -inf to type Int32 SELECT '-inf'::DOUBLE::DECIMAL(4,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(4, 1). Overflowing on -inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(4, 1). Overflowing on -inf SELECT '-inf'::DOUBLE::DECIMAL(9,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(9, 1). Overflowing on -inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(9, 1). Overflowing on -inf SELECT '-inf'::DOUBLE::DECIMAL(18,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(18, 1). Overflowing on -inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(18, 1). Overflowing on -inf SELECT '-inf'::DOUBLE::DECIMAL(38,1); -Error: 3001(EngineExecuteQuery), DataFusion error: Cast error: Cannot cast to Decimal128(38, 1). Overflowing on -inf +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast to Decimal128(38, 1). Overflowing on -inf -- we can cast nan, inf and -inf between floats and doubles, as well as to/from strings SELECT 'nan'::FLOAT::VARCHAR; diff --git a/tests/cases/standalone/common/types/timestamp/timestamp.result b/tests/cases/standalone/common/types/timestamp/timestamp.result index 2c21d1390c..91bd372303 100644 --- a/tests/cases/standalone/common/types/timestamp/timestamp.result +++ b/tests/cases/standalone/common/types/timestamp/timestamp.result @@ -32,15 +32,15 @@ SELECT timestamp ' 2017-07-23 13:10:11 '; SELECT timestamp ' 2017-07-23 13:10:11 AA'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Invalid timezone "AA": 'AA' is not a valid timezone +Error: 3001(EngineExecuteQuery), Parser error: Invalid timezone "AA": 'AA' is not a valid timezone SELECT timestamp 'AA2017-07-23 13:10:11'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from 'AA2017-07-23 13:10:11': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from 'AA2017-07-23 13:10:11': error parsing date SELECT timestamp '2017-07-23A13:10:11'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '2017-07-23A13:10:11': invalid timestamp separator +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '2017-07-23A13:10:11': invalid timestamp separator SELECT t FROM timestamp ORDER BY t; @@ -106,14 +106,14 @@ SELECT t-t FROM timestamp; +---------------------------+ | timestamp.t - timestamp.t | +---------------------------+ -| PT0S | +| P0D | | | -| PT0S | -| PT0S | -| PT0S | -| PT0S | -| PT0S | -| PT0S | +| P0D | +| P0D | +| P0D | +| P0D | +| P0D | +| P0D | +---------------------------+ SELECT EXTRACT(YEAR from TIMESTAMP '1992-01-01 01:01:01'); @@ -166,11 +166,11 @@ SELECT t::TIME FROM timestamp WHERE EXTRACT(YEAR from t)=2007 ORDER BY 1; SELECT (DATE '1992-01-01')::TIMESTAMP; -+---------------------+ -| Utf8("1992-01-01") | -+---------------------+ -| 1992-01-01T00:00:00 | -+---------------------+ ++---------------------------------------------------------------------+ +| arrow_cast(Utf8("1992-01-01"),Utf8("Timestamp(Millisecond, None)")) | ++---------------------------------------------------------------------+ +| 1992-01-01T00:00:00 | ++---------------------------------------------------------------------+ SELECT TIMESTAMP '2008-01-01 00:00:01.5'::VARCHAR; @@ -182,11 +182,11 @@ SELECT TIMESTAMP '2008-01-01 00:00:01.5'::VARCHAR; SELECT TIMESTAMP '-8-01-01 00:00:01.5'::VARCHAR; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '-8-01-01 00:00:01.5': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '-8-01-01 00:00:01.5': error parsing date SELECT TIMESTAMP '100000-01-01 00:00:01.5'::VARCHAR; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '100000-01-01 00:00:01.5': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '100000-01-01 00:00:01.5': error parsing date DROP TABLE timestamp; diff --git a/tests/cases/standalone/common/types/timestamp/timestamp_limits.result b/tests/cases/standalone/common/types/timestamp/timestamp_limits.result index 4680d298f0..a3c8c619b4 100644 --- a/tests/cases/standalone/common/types/timestamp/timestamp_limits.result +++ b/tests/cases/standalone/common/types/timestamp/timestamp_limits.result @@ -9,73 +9,73 @@ select timestamp '1970-01-01'; select '290309-12-22 (BC) 00:00:00'::timestamp; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00': error parsing date select '290309-12-21 (BC) 12:59:59.999999'::timestamp; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '290309-12-21 (BC) 12:59:59.999999': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '290309-12-21 (BC) 12:59:59.999999': error parsing date select '290309-12-22 (BC) 00:00:00'::timestamp + interval '1 day'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00': error parsing date select '290309-12-22 (BC) 00:00:00'::timestamp - interval '1 microsecond'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00': error parsing date select '290309-12-22 (BC) 00:00:00'::timestamp - interval '1 second'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00': error parsing date select '290309-12-22 (BC) 00:00:00'::timestamp - interval '1 day'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00': error parsing date select '290309-12-22 (BC) 00:00:00'::timestamp - interval '1 month'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00': error parsing date select '290309-12-22 (BC) 00:00:00'::timestamp - interval '1 year'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00': error parsing date select timestamp '294247-01-10 04:00:54.775806'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775806': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775806': error parsing date select timestamp '294247-01-10 04:00:54.775807'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775807': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775807': error parsing date select timestamp '294247-01-10 04:00:54.775806' + interval '1 microsecond'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775806': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775806': error parsing date select timestamp '294247-01-10 04:00:54.775806' + interval '1 second'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775806': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775806': error parsing date select timestamp '294247-01-10 04:00:54.775806' + interval '1 hour'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775806': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775806': error parsing date select timestamp '294247-01-10 04:00:54.775806' + interval '1 day'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775806': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775806': error parsing date select timestamp '294247-01-10 04:00:54.775806' + interval '1 month'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775806': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775806': error parsing date select timestamp '294247-01-10 04:00:54.775806' + interval '1 year'; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775806': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '294247-01-10 04:00:54.775806': error parsing date SELECT '290309-12-22 (BC) 00:00:00+07:00'::TIMESTAMP; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00+07:00': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '290309-12-22 (BC) 00:00:00+07:00': error parsing date SELECT '294247-01-10 04:00:54-07:00'::TIMESTAMP; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '294247-01-10 04:00:54-07:00': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '294247-01-10 04:00:54-07:00': error parsing date diff --git a/tests/cases/standalone/common/types/timestamp/timestamp_precision.result b/tests/cases/standalone/common/types/timestamp/timestamp_precision.result index 03126f2806..0587aa72a6 100644 --- a/tests/cases/standalone/common/types/timestamp/timestamp_precision.result +++ b/tests/cases/standalone/common/types/timestamp/timestamp_precision.result @@ -29,59 +29,59 @@ Error: 3001(EngineExecuteQuery), Execution error: Date part 'MICROSECONDS' not s -- any other precision is rounded up (e.g. 1/2 -> 3, 4/5 -> 6, 7/8 -> 9) SELECT TIMESTAMP '2020-01-01 01:23:45.123456789'::TIMESTAMP(0); -+---------------------------------------+ -| Utf8("2020-01-01 01:23:45.123456789") | -+---------------------------------------+ -| 2020-01-01T01:23:45 | -+---------------------------------------+ ++-----------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2020-01-01 01:23:45.123456789"),Utf8("Timestamp(Second, None)")) | ++-----------------------------------------------------------------------------------+ +| 2020-01-01T01:23:45 | ++-----------------------------------------------------------------------------------+ SELECT TIMESTAMP '2020-01-01 01:23:45.123456789'::TIMESTAMP(3); -+---------------------------------------+ -| Utf8("2020-01-01 01:23:45.123456789") | -+---------------------------------------+ -| 2020-01-01T01:23:45.123 | -+---------------------------------------+ ++----------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2020-01-01 01:23:45.123456789"),Utf8("Timestamp(Millisecond, None)")) | ++----------------------------------------------------------------------------------------+ +| 2020-01-01T01:23:45.123 | ++----------------------------------------------------------------------------------------+ SELECT TIMESTAMP '2020-01-01 01:23:45.123456789'::TIMESTAMP(6); -+---------------------------------------+ -| Utf8("2020-01-01 01:23:45.123456789") | -+---------------------------------------+ -| 2020-01-01T01:23:45.123456 | -+---------------------------------------+ ++----------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2020-01-01 01:23:45.123456789"),Utf8("Timestamp(Microsecond, None)")) | ++----------------------------------------------------------------------------------------+ +| 2020-01-01T01:23:45.123456 | ++----------------------------------------------------------------------------------------+ SELECT TIMESTAMP '2020-01-01 01:23:45.123456789'::TIMESTAMP(9); -+---------------------------------------+ -| Utf8("2020-01-01 01:23:45.123456789") | -+---------------------------------------+ -| 2020-01-01T01:23:45.123456789 | -+---------------------------------------+ ++---------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2020-01-01 01:23:45.123456789"),Utf8("Timestamp(Nanosecond, None)")) | ++---------------------------------------------------------------------------------------+ +| 2020-01-01T01:23:45.123456789 | ++---------------------------------------------------------------------------------------+ SELECT TIMESTAMP '2020-01-01 01:23:45.12'::TIMESTAMP(3); -+--------------------------------+ -| Utf8("2020-01-01 01:23:45.12") | -+--------------------------------+ -| 2020-01-01T01:23:45.120 | -+--------------------------------+ ++---------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2020-01-01 01:23:45.12"),Utf8("Timestamp(Millisecond, None)")) | ++---------------------------------------------------------------------------------+ +| 2020-01-01T01:23:45.120 | ++---------------------------------------------------------------------------------+ SELECT TIMESTAMP '2020-01-01 01:23:45.12345'::TIMESTAMP(6); -+-----------------------------------+ -| Utf8("2020-01-01 01:23:45.12345") | -+-----------------------------------+ -| 2020-01-01T01:23:45.123450 | -+-----------------------------------+ ++------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2020-01-01 01:23:45.12345"),Utf8("Timestamp(Microsecond, None)")) | ++------------------------------------------------------------------------------------+ +| 2020-01-01T01:23:45.123450 | ++------------------------------------------------------------------------------------+ SELECT TIMESTAMP '2020-01-01 01:23:45.12345678'::TIMESTAMP(9); -+--------------------------------------+ -| Utf8("2020-01-01 01:23:45.12345678") | -+--------------------------------------+ -| 2020-01-01T01:23:45.123456780 | -+--------------------------------------+ ++--------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2020-01-01 01:23:45.12345678"),Utf8("Timestamp(Nanosecond, None)")) | ++--------------------------------------------------------------------------------------+ +| 2020-01-01T01:23:45.123456780 | ++--------------------------------------------------------------------------------------+ DROP TABLE ts_precision; diff --git a/tests/cases/standalone/common/types/timestamp/timestamp_types.result b/tests/cases/standalone/common/types/timestamp/timestamp_types.result index e78b11dec3..e134d932c6 100644 --- a/tests/cases/standalone/common/types/timestamp/timestamp_types.result +++ b/tests/cases/standalone/common/types/timestamp/timestamp_types.result @@ -25,11 +25,11 @@ SELECT extract(YEAR from sec),extract( YEAR from milli),extract(YEAR from nano) SELECT nano::TIMESTAMP, milli::TIMESTAMP,sec::TIMESTAMP from timestamp; -+-------------------------+-------------------------+---------------------+ -| timestamp.nano | milli | timestamp.sec | -+-------------------------+-------------------------+---------------------+ -| 2008-01-01T00:00:01.889 | 2008-01-01T00:00:01.594 | 2008-01-01T00:00:01 | -+-------------------------+-------------------------+---------------------+ ++-----------------------------------------------------------------+------------------------------------------------------------------+----------------------------------------------------------------+ +| arrow_cast(timestamp.nano,Utf8("Timestamp(Millisecond, None)")) | arrow_cast(timestamp.milli,Utf8("Timestamp(Millisecond, None)")) | arrow_cast(timestamp.sec,Utf8("Timestamp(Millisecond, None)")) | ++-----------------------------------------------------------------+------------------------------------------------------------------+----------------------------------------------------------------+ +| 2008-01-01T00:00:01.889 | 2008-01-01T00:00:01.594 | 2008-01-01T00:00:01 | ++-----------------------------------------------------------------+------------------------------------------------------------------+----------------------------------------------------------------+ SELECT micro::TIMESTAMP_S as m1, micro::TIMESTAMP_MS as m2,micro::TIMESTAMP_NS as m3 from timestamp; @@ -49,7 +49,7 @@ Affected Rows: 1 select '90000-01-19 03:14:07.999999'::TIMESTAMP_US::TIMESTAMP_NS; -Error: 3001(EngineExecuteQuery), DataFusion error: Parser error: Error parsing timestamp from '90000-01-19 03:14:07.999999': error parsing date +Error: 3001(EngineExecuteQuery), Parser error: Error parsing timestamp from '90000-01-19 03:14:07.999999': error parsing date select sec::DATE from timestamp; @@ -113,63 +113,63 @@ select nano::TIME from timestamp; select sec::TIMESTAMP_MS from timestamp; -+---------------------+ -| timestamp.sec | -+---------------------+ -| 2008-01-01T00:00:01 | -| 2008-01-01T00:00:51 | -| 2008-01-01T00:00:11 | -+---------------------+ ++----------------------------------------------------------------+ +| arrow_cast(timestamp.sec,Utf8("Timestamp(Millisecond, None)")) | ++----------------------------------------------------------------+ +| 2008-01-01T00:00:01 | +| 2008-01-01T00:00:51 | +| 2008-01-01T00:00:11 | ++----------------------------------------------------------------+ select sec::TIMESTAMP_NS from timestamp; -+---------------------+ -| timestamp.sec | -+---------------------+ -| 2008-01-01T00:00:01 | -| 2008-01-01T00:00:51 | -| 2008-01-01T00:00:11 | -+---------------------+ ++---------------------------------------------------------------+ +| arrow_cast(timestamp.sec,Utf8("Timestamp(Nanosecond, None)")) | ++---------------------------------------------------------------+ +| 2008-01-01T00:00:01 | +| 2008-01-01T00:00:51 | +| 2008-01-01T00:00:11 | ++---------------------------------------------------------------+ select milli::TIMESTAMP_SEC from timestamp; -+---------------------+ -| timestamp.milli | -+---------------------+ -| 2008-01-01T00:00:01 | -| 2008-01-01T00:00:01 | -| 2008-01-01T00:00:01 | -+---------------------+ ++-------------------------------------------------------------+ +| arrow_cast(timestamp.milli,Utf8("Timestamp(Second, None)")) | ++-------------------------------------------------------------+ +| 2008-01-01T00:00:01 | +| 2008-01-01T00:00:01 | +| 2008-01-01T00:00:01 | ++-------------------------------------------------------------+ select milli::TIMESTAMP_NS from timestamp; -+-------------------------+ -| timestamp.milli | -+-------------------------+ -| 2008-01-01T00:00:01.594 | -| 2008-01-01T00:00:01.894 | -| 2008-01-01T00:00:01.794 | -+-------------------------+ ++-----------------------------------------------------------------+ +| arrow_cast(timestamp.milli,Utf8("Timestamp(Nanosecond, None)")) | ++-----------------------------------------------------------------+ +| 2008-01-01T00:00:01.594 | +| 2008-01-01T00:00:01.894 | +| 2008-01-01T00:00:01.794 | ++-----------------------------------------------------------------+ select nano::TIMESTAMP_SEC from timestamp; -+---------------------+ -| timestamp.nano | -+---------------------+ -| 2008-01-01T00:00:01 | -| 2008-01-01T00:00:01 | -| 2008-01-01T00:00:01 | -+---------------------+ ++------------------------------------------------------------+ +| arrow_cast(timestamp.nano,Utf8("Timestamp(Second, None)")) | ++------------------------------------------------------------+ +| 2008-01-01T00:00:01 | +| 2008-01-01T00:00:01 | +| 2008-01-01T00:00:01 | ++------------------------------------------------------------+ select nano::TIMESTAMP_MS from timestamp; -+-------------------------+ -| timestamp.nano | -+-------------------------+ -| 2008-01-01T00:00:01.889 | -| 2008-01-01T00:00:01.999 | -| 2008-01-01T00:00:01.899 | -+-------------------------+ ++-----------------------------------------------------------------+ +| arrow_cast(timestamp.nano,Utf8("Timestamp(Millisecond, None)")) | ++-----------------------------------------------------------------+ +| 2008-01-01T00:00:01.889 | +| 2008-01-01T00:00:01.999 | +| 2008-01-01T00:00:01.899 | ++-----------------------------------------------------------------+ select sec from timestamp order by sec; @@ -276,99 +276,99 @@ select timestamp.nano from timestamp inner join timestamp_two on (timestamp.nan select '2008-01-01 00:00:11'::TIMESTAMP_US = '2008-01-01 00:00:11'::TIMESTAMP_MS; -+-----------------------------------------------------------+ -| Utf8("2008-01-01 00:00:11") = Utf8("2008-01-01 00:00:11") | -+-----------------------------------------------------------+ -| true | -+-----------------------------------------------------------+ ++-------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Microsecond, None)")) = arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Millisecond, None)")) | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| true | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------+ select '2008-01-01 00:00:11'::TIMESTAMP_US = '2008-01-01 00:00:11'::TIMESTAMP_NS; -+-----------------------------------------------------------+ -| Utf8("2008-01-01 00:00:11") = Utf8("2008-01-01 00:00:11") | -+-----------------------------------------------------------+ -| true | -+-----------------------------------------------------------+ ++------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Microsecond, None)")) = arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Nanosecond, None)")) | ++------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| true | ++------------------------------------------------------------------------------------------------------------------------------------------------------------+ select '2008-01-01 00:00:11'::TIMESTAMP_US = '2008-01-01 00:00:11'::TIMESTAMP_S; -+-----------------------------------------------------------+ -| Utf8("2008-01-01 00:00:11") = Utf8("2008-01-01 00:00:11") | -+-----------------------------------------------------------+ -| true | -+-----------------------------------------------------------+ ++--------------------------------------------------------------------------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Microsecond, None)")) = arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Second, None)")) | ++--------------------------------------------------------------------------------------------------------------------------------------------------------+ +| true | ++--------------------------------------------------------------------------------------------------------------------------------------------------------+ select '2008-01-01 00:00:11.1'::TIMESTAMP_US = '2008-01-01 00:00:11'::TIMESTAMP_MS; -+-------------------------------------------------------------+ -| Utf8("2008-01-01 00:00:11.1") = Utf8("2008-01-01 00:00:11") | -+-------------------------------------------------------------+ -| false | -+-------------------------------------------------------------+ ++---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2008-01-01 00:00:11.1"),Utf8("Timestamp(Microsecond, None)")) = arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Millisecond, None)")) | ++---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| false | ++---------------------------------------------------------------------------------------------------------------------------------------------------------------+ select '2008-01-01 00:00:11.1'::TIMESTAMP_US = '2008-01-01 00:00:11'::TIMESTAMP_NS; -+-------------------------------------------------------------+ -| Utf8("2008-01-01 00:00:11.1") = Utf8("2008-01-01 00:00:11") | -+-------------------------------------------------------------+ -| false | -+-------------------------------------------------------------+ ++--------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2008-01-01 00:00:11.1"),Utf8("Timestamp(Microsecond, None)")) = arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Nanosecond, None)")) | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| false | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------+ select '2008-01-01 00:00:11.1'::TIMESTAMP_US = '2008-01-01 00:00:11.1'::TIMESTAMP_S; -+---------------------------------------------------------------+ -| Utf8("2008-01-01 00:00:11.1") = Utf8("2008-01-01 00:00:11.1") | -+---------------------------------------------------------------+ -| true | -+---------------------------------------------------------------+ ++------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2008-01-01 00:00:11.1"),Utf8("Timestamp(Microsecond, None)")) = arrow_cast(Utf8("2008-01-01 00:00:11.1"),Utf8("Timestamp(Second, None)")) | ++------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| true | ++------------------------------------------------------------------------------------------------------------------------------------------------------------+ select '2008-01-01 00:00:11.1'::TIMESTAMP_MS = '2008-01-01 00:00:11'::TIMESTAMP_NS; -+-------------------------------------------------------------+ -| Utf8("2008-01-01 00:00:11.1") = Utf8("2008-01-01 00:00:11") | -+-------------------------------------------------------------+ -| false | -+-------------------------------------------------------------+ ++--------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2008-01-01 00:00:11.1"),Utf8("Timestamp(Millisecond, None)")) = arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Nanosecond, None)")) | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| false | ++--------------------------------------------------------------------------------------------------------------------------------------------------------------+ select '2008-01-01 00:00:11.1'::TIMESTAMP_MS = '2008-01-01 00:00:11'::TIMESTAMP_S; -+-------------------------------------------------------------+ -| Utf8("2008-01-01 00:00:11.1") = Utf8("2008-01-01 00:00:11") | -+-------------------------------------------------------------+ -| true | -+-------------------------------------------------------------+ ++----------------------------------------------------------------------------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2008-01-01 00:00:11.1"),Utf8("Timestamp(Millisecond, None)")) = arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Second, None)")) | ++----------------------------------------------------------------------------------------------------------------------------------------------------------+ +| true | ++----------------------------------------------------------------------------------------------------------------------------------------------------------+ select '2008-01-01 00:00:11.1'::TIMESTAMP_NS = '2008-01-01 00:00:11'::TIMESTAMP_S; -+-------------------------------------------------------------+ -| Utf8("2008-01-01 00:00:11.1") = Utf8("2008-01-01 00:00:11") | -+-------------------------------------------------------------+ -| true | -+-------------------------------------------------------------+ ++---------------------------------------------------------------------------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2008-01-01 00:00:11.1"),Utf8("Timestamp(Nanosecond, None)")) = arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Second, None)")) | ++---------------------------------------------------------------------------------------------------------------------------------------------------------+ +| true | ++---------------------------------------------------------------------------------------------------------------------------------------------------------+ select '2008-01-01 00:00:11'::TIMESTAMP_MS = '2008-01-01 00:00:11'::TIMESTAMP_NS; -+-----------------------------------------------------------+ -| Utf8("2008-01-01 00:00:11") = Utf8("2008-01-01 00:00:11") | -+-----------------------------------------------------------+ -| true | -+-----------------------------------------------------------+ ++------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Millisecond, None)")) = arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Nanosecond, None)")) | ++------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| true | ++------------------------------------------------------------------------------------------------------------------------------------------------------------+ select '2008-01-01 00:00:11'::TIMESTAMP_MS = '2008-01-01 00:00:11'::TIMESTAMP_S; -+-----------------------------------------------------------+ -| Utf8("2008-01-01 00:00:11") = Utf8("2008-01-01 00:00:11") | -+-----------------------------------------------------------+ -| true | -+-----------------------------------------------------------+ ++--------------------------------------------------------------------------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Millisecond, None)")) = arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Second, None)")) | ++--------------------------------------------------------------------------------------------------------------------------------------------------------+ +| true | ++--------------------------------------------------------------------------------------------------------------------------------------------------------+ select '2008-01-01 00:00:11'::TIMESTAMP_NS = '2008-01-01 00:00:11'::TIMESTAMP_S; -+-----------------------------------------------------------+ -| Utf8("2008-01-01 00:00:11") = Utf8("2008-01-01 00:00:11") | -+-----------------------------------------------------------+ -| true | -+-----------------------------------------------------------+ ++-------------------------------------------------------------------------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Nanosecond, None)")) = arrow_cast(Utf8("2008-01-01 00:00:11"),Utf8("Timestamp(Second, None)")) | ++-------------------------------------------------------------------------------------------------------------------------------------------------------+ +| true | ++-------------------------------------------------------------------------------------------------------------------------------------------------------+ DROP TABLE timestamp; diff --git a/tests/cases/standalone/common/types/timestamp/timestamp_tz.result b/tests/cases/standalone/common/types/timestamp/timestamp_tz.result index 8084c46457..0d00d39e63 100644 --- a/tests/cases/standalone/common/types/timestamp/timestamp_tz.result +++ b/tests/cases/standalone/common/types/timestamp/timestamp_tz.result @@ -8,25 +8,25 @@ select timestamptz '2021-11-15 02:30:00'; select '2021-11-15 02:30:00'::TIMESTAMP::TIMESTAMPTZ; -+-----------------------------+ -| Utf8("2021-11-15 02:30:00") | -+-----------------------------+ -| 2021-11-15T02:30:00 | -+-----------------------------+ ++-------------------------------------------------------------------------------------------------------------------------------+ +| arrow_cast(arrow_cast(Utf8("2021-11-15 02:30:00"),Utf8("Timestamp(Millisecond, None)")),Utf8("Timestamp(Millisecond, None)")) | ++-------------------------------------------------------------------------------------------------------------------------------+ +| 2021-11-15T02:30:00 | ++-------------------------------------------------------------------------------------------------------------------------------+ SELECT '2021-04-29 10:50:09-05'::TIMESTAMPTZ::DATE; -+--------------------------------+ -| Utf8("2021-04-29 10:50:09-05") | -+--------------------------------+ -| 2021-04-29 | -+--------------------------------+ ++---------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2021-04-29 10:50:09-05"),Utf8("Timestamp(Millisecond, None)")) | ++---------------------------------------------------------------------------------+ +| 2021-04-29 | ++---------------------------------------------------------------------------------+ SELECT '2021-04-29 10:50:09-05'::TIMESTAMPTZ::TIME; -+--------------------------------+ -| Utf8("2021-04-29 10:50:09-05") | -+--------------------------------+ -| 15:50:09 | -+--------------------------------+ ++---------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2021-04-29 10:50:09-05"),Utf8("Timestamp(Millisecond, None)")) | ++---------------------------------------------------------------------------------+ +| 15:50:09 | ++---------------------------------------------------------------------------------+ diff --git a/tests/cases/standalone/limit/limit.result b/tests/cases/standalone/limit/limit.result index 0d39bee3e1..ef2bf90a20 100644 --- a/tests/cases/standalone/limit/limit.result +++ b/tests/cases/standalone/limit/limit.result @@ -9,7 +9,7 @@ EXPLAIN SELECT * FROM (SELECT SUM(number) FROM numbers LIMIT 100000000000) LIMIT | plan_type | plan | +---------------+----------------------------------+ | logical_plan | MergeScan [is_placeholder=false] | -| physical_plan | EmptyExec: produce_one_row=false | +| physical_plan | EmptyExec | | | | +---------------+----------------------------------+ @@ -19,7 +19,7 @@ EXPLAIN SELECT * FROM (SELECT SUM(number) FROM numbers LIMIT 100000000000) WHERE | plan_type | plan | +---------------+----------------------------------+ | logical_plan | MergeScan [is_placeholder=false] | -| physical_plan | EmptyExec: produce_one_row=false | +| physical_plan | EmptyExec | | | | +---------------+----------------------------------+ diff --git a/tests/cases/standalone/optimizer/filter_push_down.result b/tests/cases/standalone/optimizer/filter_push_down.result index 7e7cb8ca30..06ee859d4a 100644 --- a/tests/cases/standalone/optimizer/filter_push_down.result +++ b/tests/cases/standalone/optimizer/filter_push_down.result @@ -178,16 +178,16 @@ SELECT i FROM (SELECT * FROM integers i1 UNION SELECT * FROM integers i2) a WHER | 3 | +---+ --- TODO(LFC): Somehow the following SQL does not order by column 1 under new DataFusion occasionally. Should further investigate it. Comment it out temporarily. --- expected: --- +---+---+--------------+ --- | a | b | ROW_NUMBER() | --- +---+---+--------------+ --- | 1 | 1 | 1 | --- | 2 | 2 | 5 | --- | 3 | 3 | 9 | --- +---+---+--------------+ --- SELECT * FROM (SELECT i1.i AS a, i2.i AS b, row_number() OVER (ORDER BY i1.i, i2.i) FROM integers i1, integers i2 WHERE i1.i IS NOT NULL AND i2.i IS NOT NULL) a1 WHERE a=b ORDER BY 1; +SELECT * FROM (SELECT i1.i AS a, i2.i AS b, row_number() OVER (ORDER BY i1.i, i2.i) FROM integers i1, integers i2 WHERE i1.i IS NOT NULL AND i2.i IS NOT NULL) a1 WHERE a=b ORDER BY 1; + ++---+---+--------------------------------------------------------------------------------------------------------------------+ +| a | b | ROW_NUMBER() ORDER BY [i1.i ASC NULLS LAST, i2.i ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW | ++---+---+--------------------------------------------------------------------------------------------------------------------+ +| 1 | 1 | 1 | +| 2 | 2 | 5 | +| 3 | 3 | 9 | ++---+---+--------------------------------------------------------------------------------------------------------------------+ + -- TODO(ruihang): Invalid argument error: must either specify a row count or at least one column -- SELECT * FROM (SELECT 0=1 AS cond FROM integers i1, integers i2) a1 WHERE cond ORDER BY 1; SELECT * FROM (SELECT 0=1 AS cond FROM integers i1, integers i2 GROUP BY 1) a1 WHERE cond ORDER BY 1; diff --git a/tests/cases/standalone/optimizer/filter_push_down.sql b/tests/cases/standalone/optimizer/filter_push_down.sql index b8ff96d328..b2cd11582d 100644 --- a/tests/cases/standalone/optimizer/filter_push_down.sql +++ b/tests/cases/standalone/optimizer/filter_push_down.sql @@ -47,16 +47,7 @@ SELECT * FROM (SELECT DISTINCT i1.i AS a, i2.i AS b FROM integers i1, integers i SELECT i FROM (SELECT * FROM integers i1 UNION SELECT * FROM integers i2) a WHERE i=3; --- TODO(LFC): Somehow the following SQL does not order by column 1 under new DataFusion occasionally. Should further investigate it. Comment it out temporarily. --- expected: --- +---+---+--------------+ --- | a | b | ROW_NUMBER() | --- +---+---+--------------+ --- | 1 | 1 | 1 | --- | 2 | 2 | 5 | --- | 3 | 3 | 9 | --- +---+---+--------------+ --- SELECT * FROM (SELECT i1.i AS a, i2.i AS b, row_number() OVER (ORDER BY i1.i, i2.i) FROM integers i1, integers i2 WHERE i1.i IS NOT NULL AND i2.i IS NOT NULL) a1 WHERE a=b ORDER BY 1; +SELECT * FROM (SELECT i1.i AS a, i2.i AS b, row_number() OVER (ORDER BY i1.i, i2.i) FROM integers i1, integers i2 WHERE i1.i IS NOT NULL AND i2.i IS NOT NULL) a1 WHERE a=b ORDER BY 1; -- TODO(ruihang): Invalid argument error: must either specify a row count or at least one column -- SELECT * FROM (SELECT 0=1 AS cond FROM integers i1, integers i2) a1 WHERE cond ORDER BY 1; diff --git a/tests/runner/src/util.rs b/tests/runner/src/util.rs index 5ae63ede37..04c336e148 100644 --- a/tests/runner/src/util.rs +++ b/tests/runner/src/util.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::fmt::Display; use std::net::SocketAddr; use std::path::PathBuf; use std::process::Command; @@ -24,42 +23,6 @@ use tokio::time; /// Check port every 0.1 second. const PORT_CHECK_INTERVAL: Duration = Duration::from_millis(100); -const NULL_DATA_PLACEHOLDER: &str = "NULL"; - -/// Helper struct for iterate over column with null_mask -struct NullableColumnIter -where - N: Iterator, - B: AsRef, - D: Iterator, - T: Display, -{ - null_iter: N, - data_iter: D, -} - -impl Iterator for NullableColumnIter -where - N: Iterator, - B: AsRef, - D: Iterator, - T: Display, -{ - type Item = String; - - fn next(&mut self) -> Option { - // iter the null_mask first - if let Some(is_null) = self.null_iter.next() { - if *is_null.as_ref() { - Some(NULL_DATA_PLACEHOLDER.to_string()) - } else { - self.data_iter.next().map(|data| data.to_string()) - } - } else { - None - } - } -} /// Get the dir of test cases. This function only works when the runner is run /// under the project's dir because it depends on some envs set by cargo.