diff --git a/.github/runner-scale-sets/query-regression/README.md b/.github/runner-scale-sets/query-regression/README.md index f7a9bdc1732..fa8d436d2c2 100644 --- a/.github/runner-scale-sets/query-regression/README.md +++ b/.github/runner-scale-sets/query-regression/README.md @@ -108,7 +108,7 @@ agent, which Aliyun public Ubuntu images include. An allowlisted repository admin commenting `/query-regression` on the PR is **trust admission for that exact PR revision**. `/query-regression` runs the -six routine default cases; `/query-regression heavy` runs only the +nine routine default cases; `/query-regression heavy` runs only the high-cardinality `prom_remote_write_7913` remote-write case. `slash-command-dispatch.yml` (`issue_comment` on the default branch) parses the command and `repository_dispatch`es; `query-regression-slash.yml` admits diff --git a/.github/scripts/query-regression-run.py b/.github/scripts/query-regression-run.py index a99706c04a5..6da7c896d67 100644 --- a/.github/scripts/query-regression-run.py +++ b/.github/scripts/query-regression-run.py @@ -33,6 +33,7 @@ from typing import Any DEFAULT_CASES = [ "tests/perf/query_cases/smoke_direct_sst/case.toml", + "tests/perf/query_cases/sst_float_bss/case.toml", "tests/perf/query_cases/prom_remote_write_seeded_random/case.toml", "tests/perf/query_cases/prom_remote_write_run_heavy/case.toml", "tests/perf/query_cases/prom_remote_write_mixed_every/case.toml", diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 8cf2da32fcb..013aa320b20 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -45,6 +45,7 @@ jobs: PR_TITLE: ${{ github.event.pull_request.title }} PR_BODY: ${{ github.event.pull_request.body }} AUTHOR: ${{ github.event.pull_request.merged_by.login }} + PR_AUTHOR: ${{ github.event.pull_request.user.login }} BASE_BRANCH: ${{ github.event.pull_request.base.ref }} run: | git config user.name "github-actions[bot]" @@ -76,6 +77,7 @@ jobs: git branch -D "${head_branch}" gh issue create \ --title "Backport failure: PR #${PR_NUMBER} to ${release_branch}" \ + --assignee "${PR_AUTHOR}" \ --body "Cherry-pick of ${MERGE_COMMIT} (PR #${PR_NUMBER}) failed on ${release_branch}. Manual backport required." \ || true continue diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d7a91cadd43..d63d52a2189 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -781,10 +781,9 @@ jobs: kubectl -n kafka-cluster apply \ -f .github/actions/setup-greptimedb-cluster/kafka-wal-helper.yaml - kubectl -n kafka-cluster wait \ - --for=condition=Ready \ - pod -l app=kafka-wal-helper \ - --timeout=120s + kubectl rollout status deployment/kafka-wal-helper \ + --timeout=120s \ + -n kafka-cluster - name: Print etcd info shell: bash run: kubectl get all --show-labels -n etcd-cluster diff --git a/.github/workflows/semantic-pull-request.yml b/.github/workflows/semantic-pull-request.yml index e2958776e93..b2d3c648df1 100644 --- a/.github/workflows/semantic-pull-request.yml +++ b/.github/workflows/semantic-pull-request.yml @@ -6,9 +6,13 @@ on: - opened - reopened - edited + - synchronize + - ready_for_review + merge_group: + types: [checks_requested] concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true permissions: @@ -18,6 +22,9 @@ permissions: jobs: check: + name: Semantic PR title + # Validate before queue admission; report a skipped check for merge groups. + if: github.event_name == 'pull_request_target' runs-on: ubuntu-latest timeout-minutes: 10 steps: diff --git a/Cargo.lock b/Cargo.lock index 3876eb6d7d3..61005e91848 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -221,7 +221,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" name = "api" version = "1.3.0-alpha.1" dependencies = [ - "arrow-schema 58.3.0", + "arrow-schema 59.2.0", "common-base", "common-decimal", "common-error", @@ -323,23 +323,23 @@ dependencies = [ [[package]] name = "arrow" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" dependencies = [ - "arrow-arith 58.3.0", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-cast 58.3.0", - "arrow-csv 58.3.0", - "arrow-data 58.3.0", - "arrow-ipc 58.3.0", - "arrow-json 58.3.0", - "arrow-ord 58.3.0", - "arrow-row 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", - "arrow-string 58.3.0", + "arrow-arith 59.2.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-cast 59.2.0", + "arrow-csv 59.2.0", + "arrow-data 59.2.0", + "arrow-ipc 59.2.0", + "arrow-json 59.2.0", + "arrow-ord 59.2.0", + "arrow-row 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", + "arrow-string 59.2.0", ] [[package]] @@ -358,14 +358,14 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", "chrono", "num-traits", ] @@ -388,14 +388,14 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" dependencies = [ "ahash 0.8.12", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", "chrono", "chrono-tz", "half", @@ -418,13 +418,13 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" dependencies = [ "bytes", "half", - "num-bigint 0.4.6", + "num-bigint 0.5.1", "num-traits", ] @@ -450,18 +450,18 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-ord 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-ord 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", "atoi", - "base64 0.22.1", + "base64 0.23.1", "chrono", "comfy-table", "half", @@ -487,13 +487,13 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36" dependencies = [ - "arrow-array 58.3.0", - "arrow-cast 58.3.0", - "arrow-schema 58.3.0", + "arrow-array 59.2.0", + "arrow-cast 59.2.0", + "arrow-schema 59.2.0", "chrono", "csv", "csv-core", @@ -514,12 +514,12 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" dependencies = [ - "arrow-buffer 58.3.0", - "arrow-schema 58.3.0", + "arrow-buffer 59.2.0", + "arrow-schema 59.2.0", "half", "num-integer", "num-traits", @@ -527,16 +527,17 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28abfe8bf9f124e5fc83b334af4fa58f8d0323ad25312ccb2d1da50178415704" +checksum = "2bebfacc9d71f0728f6774164e4d4254b5e504d2b46812d0512d8290ec119a64" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-cast 58.3.0", - "arrow-ipc 58.3.0", - "arrow-schema 58.3.0", - "base64 0.22.1", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-cast 59.2.0", + "arrow-data 59.2.0", + "arrow-ipc 59.2.0", + "arrow-schema 59.2.0", + "base64 0.23.1", "bytes", "futures", "prost 0.14.1", @@ -562,17 +563,17 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", "flatbuffers", - "lz4_flex 0.13.1", + "lz4_flex 0.14.0", "zstd", ] @@ -589,7 +590,7 @@ dependencies = [ "arrow-schema 56.2.0", "chrono", "half", - "indexmap 2.13.0", + "indexmap 2.14.2", "lexical-core", "memchr", "num", @@ -600,19 +601,19 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-cast 58.3.0", - "arrow-ord 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-cast 59.2.0", + "arrow-ord 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", "chrono", "half", - "indexmap 2.13.0", + "indexmap 2.14.2", "itoa", "lexical-core", "memchr", @@ -638,25 +639,24 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", ] [[package]] name = "arrow-pg" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7acc179d3edb91fed930ea6aa6def3f73458228079857b4547096040a1cbde8" +version = "0.15.0" +source = "git+https://github.com/GreptimeTeam/datafusion-postgres.git?rev=3c77e6c32b8db80635a0d2f4b318a36b31170bc3#3c77e6c32b8db80635a0d2f4b318a36b31170bc3" dependencies = [ - "arrow 58.3.0", - "arrow-schema 58.3.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "bytes", "chrono", "datafusion", @@ -682,14 +682,14 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", "half", ] @@ -701,9 +701,9 @@ checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" [[package]] name = "arrow-schema" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" dependencies = [ "serde", "serde_core", @@ -726,15 +726,15 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" dependencies = [ "ahash 0.8.12", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", "num-traits", ] @@ -757,15 +757,15 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", "memchr", "num-traits", "regex", @@ -1290,7 +1290,7 @@ dependencies = [ "quote", "regex", "rustc-hash 2.1.1", - "shlex", + "shlex 1.3.0", "syn 2.0.117", ] @@ -1661,8 +1661,8 @@ name = "catalog" version = "1.3.0-alpha.1" dependencies = [ "api", - "arrow 58.3.0", - "arrow-schema 58.3.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "async-stream", "async-trait", "bytes", @@ -1731,13 +1731,14 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.27" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" dependencies = [ + "find-msvc-tools", "jobserver", "libc", - "shlex", + "shlex 2.0.1", ] [[package]] @@ -1781,9 +1782,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" @@ -1803,7 +1804,7 @@ version = "0.13.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fe45e18904af7af10e4312df7c97251e98af98c70f42f1f2587aecfcbee56bf" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.14.2", "lazy_static", "num-traits", "regex", @@ -1868,9 +1869,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -2316,6 +2317,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "common-batcher" +version = "1.3.0-alpha.1" +dependencies = [ + "dashmap", + "futures", + "tokio", +] + [[package]] name = "common-catalog" version = "1.3.0-alpha.1" @@ -2351,8 +2361,8 @@ dependencies = [ name = "common-datasource" version = "1.3.0-alpha.1" dependencies = [ - "arrow 58.3.0", - "arrow-schema 58.3.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "async-compression", "async-trait", "bytes", @@ -2463,9 +2473,9 @@ dependencies = [ "api", "approx 0.5.1", "arc-swap", - "arrow 58.3.0", - "arrow-cast 58.3.0", - "arrow-schema 58.3.0", + "arrow 59.2.0", + "arrow-cast 59.2.0", + "arrow-schema 59.2.0", "async-trait", "bincode", "catalog", @@ -2866,7 +2876,7 @@ dependencies = [ name = "common-sql" version = "1.3.0-alpha.1" dependencies = [ - "arrow-schema 58.3.0", + "arrow-schema 59.2.0", "common-base", "common-decimal", "common-error", @@ -2943,7 +2953,7 @@ dependencies = [ name = "common-time" version = "1.3.0-alpha.1" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "chrono", "chrono-tz", "common-error", @@ -3552,7 +3562,7 @@ checksum = "7a9bc1a22964ff6a355fbec24cf68266a0ed28f8b84c0864c386474ea3d0e479" dependencies = [ "cc", "codespan-reporting 0.13.1", - "indexmap 2.13.0", + "indexmap 2.14.2", "proc-macro2", "quote", "scratch", @@ -3567,7 +3577,7 @@ checksum = "b1f29a879d35f7906e3c9b77d7a1005a6a0787d330c09dfe4ffb5f617728cb44" dependencies = [ "clap", "codespan-reporting 0.13.1", - "indexmap 2.13.0", + "indexmap 2.14.2", "proc-macro2", "quote", "syn 2.0.117", @@ -3585,7 +3595,7 @@ version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d187e019e7b05a1f3e69a8396b70800ee867aa9fc2ab972761173ccee03742df" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.14.2", "proc-macro2", "quote", "syn 2.0.117", @@ -3747,9 +3757,9 @@ checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -3767,13 +3777,12 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "datafusion" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", - "arrow-schema 58.3.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "async-trait", - "bytes", "bzip2", "chrono", "datafusion-catalog", @@ -3803,14 +3812,13 @@ dependencies = [ "datafusion-sql", "flate2", "futures", - "itertools 0.14.0", + "indexmap 2.14.2", + "itertools 0.15.0", "liblzma", "log", "object_store", "parking_lot 0.12.4", "parquet", - "rand 0.9.4", - "regex", "sqlparser", "tempfile", "tokio", @@ -3821,10 +3829,10 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "async-trait", "dashmap", "datafusion-common", @@ -3836,7 +3844,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot 0.12.4", @@ -3845,10 +3853,10 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -3860,39 +3868,42 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", + "percent-encoding", ] [[package]] name = "datafusion-common" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "ahash 0.8.12", - "arrow 58.3.0", - "arrow-ipc 58.3.0", + "arrow 59.2.0", + "arrow-ipc 59.2.0", + "arrow-schema 59.2.0", "chrono", + "foldhash 0.2.0", "half", - "hashbrown 0.16.1", - "indexmap 2.13.0", - "itertools 0.14.0", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "itertools 0.15.0", "libc", "log", + "num-traits", "object_store", "parquet", - "paste", "recursive", "sqlparser", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ "futures", "log", @@ -3901,10 +3912,10 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "async-compression", "async-trait", "bytes", @@ -3918,14 +3929,16 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "flate2", "futures", "glob", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", + "parking_lot 0.12.4", "rand 0.9.4", "tokio", "tokio-util", @@ -3935,11 +3948,11 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", - "arrow-ipc 58.3.0", + "arrow 59.2.0", + "arrow-ipc 59.2.0", "async-trait", "bytes", "datafusion-common", @@ -3949,19 +3962,20 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "object_store", "tokio", ] [[package]] name = "datafusion-datasource-csv" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "async-trait", "bytes", "datafusion-common", @@ -3971,6 +3985,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -3980,10 +3995,10 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "async-trait", "bytes", "datafusion-common", @@ -3993,20 +4008,21 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "async-trait", "bytes", "datafusion-common", @@ -4014,15 +4030,17 @@ dependencies = [ "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-pruning", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot 0.12.4", @@ -4032,18 +4050,18 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" [[package]] name = "datafusion-execution" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", - "arrow-buffer 58.3.0", + "arrow 59.2.0", + "arrow-buffer 59.2.0", "async-trait", - "chrono", + "bytes", "dashmap", "datafusion-common", "datafusion-expr", @@ -4052,17 +4070,21 @@ dependencies = [ "log", "object_store", "parking_lot 0.12.4", + "pin-project-lite", "rand 0.9.4", "tempfile", + "tokio", + "tokio-util", "url", ] [[package]] name = "datafusion-expr" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "async-trait", "chrono", "datafusion-common", @@ -4071,9 +4093,10 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.13.0", - "itertools 0.14.0", - "paste", + "datafusion-proto-common", + "datafusion-proto-models", + "indexmap 2.14.2", + "itertools 0.15.0", "recursive", "serde_json", "sqlparser", @@ -4081,24 +4104,23 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "datafusion-common", - "indexmap 2.13.0", - "itertools 0.14.0", - "paste", + "indexmap 2.14.2", + "itertools 0.15.0", ] [[package]] name = "datafusion-functions" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", - "arrow-buffer 58.3.0", - "base64 0.22.1", + "arrow 59.2.0", + "arrow-buffer 59.2.0", + "base64 0.23.1", "blake2", "blake3", "chrono", @@ -4109,26 +4131,25 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", + "datafusion-physical-expr-common", "hex", - "itertools 0.14.0", + "itertools 0.15.0", "log", - "md-5 0.10.6", + "md-5 0.11.0", "memchr", "num-traits", "rand 0.9.4", "regex", - "sha2 0.10.9", - "unicode-segmentation", + "sha2 0.11.0", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "ahash 0.8.12", - "arrow 58.3.0", + "arrow 59.2.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -4138,18 +4159,17 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "half", + "hashbrown 0.17.1", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "ahash 0.8.12", - "arrow 58.3.0", + "arrow 59.2.0", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", @@ -4157,11 +4177,11 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", - "arrow-ord 58.3.0", + "arrow 59.2.0", + "arrow-ord 59.2.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -4172,34 +4192,34 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", - "itertools 0.14.0", + "hashbrown 0.17.1", + "itertools 0.15.0", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "parking_lot 0.12.4", - "paste", ] [[package]] name = "datafusion-functions-window" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "datafusion-common", "datafusion-doc", "datafusion-expr", @@ -4208,13 +4228,12 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -4222,27 +4241,27 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.117", + "syn 3.0.5", ] [[package]] name = "datafusion-optimizer" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "chrono", "datafusion-common", "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", - "indexmap 2.13.0", - "itertools 0.14.0", + "indexmap 2.14.2", + "itertools 0.15.0", "log", "recursive", "regex", @@ -4251,24 +4270,23 @@ dependencies = [ [[package]] name = "datafusion-orc" -version = "0.8.0" -source = "git+https://github.com/datafusion-contrib/datafusion-orc.git?rev=6c07fa282dc8d62db2aa4ded06ab55485efc811a#6c07fa282dc8d62db2aa4ded06ab55485efc811a" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "637382f27aa0c7d6f30ca18a3e37d46bdf427aa83e1b0366083dc38e362dae99" dependencies = [ "async-trait", "bytes", "datafusion", "futures", "futures-util", - "object_store", "orc-rust", "tokio", ] [[package]] name = "datafusion-pg-catalog" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b82fb8bd291b718d4226eaba85f78c73f0abdedb256a5dc91d9d9e6b0e0dab" +version = "0.18.3" +source = "git+https://github.com/GreptimeTeam/datafusion-postgres.git?rev=3c77e6c32b8db80635a0d2f4b318a36b31170bc3#3c77e6c32b8db80635a0d2f4b318a36b31170bc3" dependencies = [ "arrow-pg", "async-trait", @@ -4281,22 +4299,21 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "ahash 0.8.12", - "arrow 58.3.0", + "arrow 59.2.0", "datafusion-common", "datafusion-expr", "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", + "datafusion-proto-models", "half", - "hashbrown 0.16.1", - "indexmap 2.13.0", - "itertools 0.14.0", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "itertools 0.15.0", "parking_lot 0.12.4", - "paste", "petgraph 0.8.3", "recursive", "tokio", @@ -4304,40 +4321,41 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "datafusion-common", "datafusion-expr", "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "ahash 0.8.12", - "arrow 58.3.0", + "arrow 59.2.0", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", - "indexmap 2.13.0", - "itertools 0.14.0", + "datafusion-proto-models", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "itertools 0.15.0", "parking_lot 0.12.4", + "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -4346,20 +4364,23 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools 0.14.0", + "datafusion-session", + "itertools 0.15.0", "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "ahash 0.8.12", - "arrow 58.3.0", - "arrow-ord 58.3.0", - "arrow-schema 58.3.0", + "arrow 59.2.0", + "arrow-data 59.2.0", + "arrow-ipc 59.2.0", + "arrow-ord 59.2.0", + "arrow-schema 59.2.0", "async-trait", + "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -4369,25 +4390,28 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", "futures", "half", - "hashbrown 0.16.1", - "indexmap 2.13.0", - "itertools 0.14.0", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "itertools 0.15.0", "log", "num-traits", "parking_lot 0.12.4", "pin-project-lite", + "serde", + "serde_json", "tokio", ] [[package]] name = "datafusion-proto" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", - "chrono", + "arrow 59.2.0", "datafusion-catalog", "datafusion-catalog-listing", "datafusion-common", @@ -4403,42 +4427,52 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-proto-common", + "datafusion-proto-models", "object_store", "prost 0.14.1", - "rand 0.9.4", ] [[package]] name = "datafusion-proto-common" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "datafusion-common", "prost 0.14.1", ] [[package]] -name = "datafusion-pruning" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +name = "datafusion-proto-models" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "datafusion-common", + "datafusion-proto-common", + "prost 0.14.1", +] + +[[package]] +name = "datafusion-pruning" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" +dependencies = [ + "arrow 59.2.0", "datafusion-common", "datafusion-datasource", "datafusion-expr-common", "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools 0.14.0", "log", ] [[package]] name = "datafusion-session" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ + "arrow-schema 59.2.0", "async-trait", "datafusion-common", "datafusion-execution", @@ -4449,37 +4483,38 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "bigdecimal 0.4.8", "chrono", "datafusion-common", "datafusion-expr", "datafusion-functions-nested", - "indexmap 2.13.0", + "indexmap 2.14.2", "log", "recursive", "regex", "sqlparser", + "stacker", ] [[package]] name = "datafusion-substrait" -version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b" +version = "55.0.0" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95" dependencies = [ "async-recursion", "async-trait", "chrono", "datafusion", "half", - "itertools 0.14.0", + "itertools 0.15.0", "object_store", "pbjson-types", "prost 0.14.1", - "substrait 0.62.2", + "substrait 0.63.0", "tokio", "url", ] @@ -4563,9 +4598,9 @@ checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" name = "datatypes" version = "1.3.0-alpha.1" dependencies = [ - "arrow 58.3.0", - "arrow-array 58.3.0", - "arrow-schema 58.3.0", + "arrow 59.2.0", + "arrow-array 59.2.0", + "arrow-schema 59.2.0", "common-base", "common-decimal", "common-error", @@ -5359,6 +5394,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" + [[package]] name = "findshlibs" version = "0.10.2" @@ -5459,9 +5500,9 @@ name = "flow" version = "1.3.0-alpha.1" dependencies = [ "api", - "arrow 58.3.0", + "arrow 59.2.0", "arrow-flight", - "arrow-schema 58.3.0", + "arrow-schema 59.2.0", "async-recursion", "async-trait", "bytes", @@ -5637,6 +5678,7 @@ dependencies = [ "catalog", "client", "common-base", + "common-batcher", "common-catalog", "common-config", "common-datasource", @@ -6121,7 +6163,7 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=b74667d17827481dbb6222fa0326163627211557#b74667d17827481dbb6222fa0326163627211557" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=0c4beffaa2b3968034915867593a717c6ab0c6b4#0c4beffaa2b3968034915867593a717c6ab0c6b4" dependencies = [ "prost 0.14.1", "prost-types 0.14.1", @@ -6155,7 +6197,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.13.0", + "indexmap 2.14.2", "slab", "tokio", "tokio-util", @@ -6174,7 +6216,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.5.0", - "indexmap 2.13.0", + "indexmap 2.14.2", "slab", "tokio", "tokio-util", @@ -6987,12 +7029,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -7023,7 +7065,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash 0.8.12", - "indexmap 2.13.0", + "indexmap 2.14.2", "is-terminal", "itoa", "log", @@ -7046,7 +7088,7 @@ dependencies = [ "crossbeam-utils", "dashmap", "env_logger", - "indexmap 2.13.0", + "indexmap 2.14.2", "itoa", "log", "num-format", @@ -7130,12 +7172,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "integer-encoding" version = "4.0.2" @@ -7268,6 +7304,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -7545,7 +7590,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.14.2", ] [[package]] @@ -8157,7 +8202,7 @@ dependencies = [ "cactus", "cfgrammar", "filetime", - "indexmap 2.13.0", + "indexmap 2.14.2", "lazy_static", "lrtable", "num-traits", @@ -8240,6 +8285,12 @@ name = "lz4_flex" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" + +[[package]] +name = "lz4_flex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" dependencies = [ "twox-hash", ] @@ -8367,9 +8418,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memcomparable" @@ -8672,7 +8723,7 @@ version = "1.3.0-alpha.1" dependencies = [ "api", "aquamarine", - "arrow-schema 58.3.0", + "arrow-schema 59.2.0", "async-channel 1.9.0", "async-stream", "async-trait", @@ -9614,9 +9665,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -10053,10 +10104,11 @@ version = "1.3.0-alpha.1" dependencies = [ "ahash 0.8.12", "api", - "arrow 58.3.0", - "arrow-ipc 58.3.0", + "arrow 59.2.0", + "arrow-ipc 59.2.0", "async-stream", "async-trait", + "axum 0.8.4", "bytes", "catalog", "chrono", @@ -10117,15 +10169,16 @@ dependencies = [ "tokio-util", "tonic 0.14.2", "tracing", + "url", ] [[package]] name = "orc-rust" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b9867c4e7343218682ba11aceae1310c80735ec2beaf6124a0f8f848dad197" +checksum = "e54c163f843c4f0fcfca65b9b73ca6f93c37c8e86736d5a1e578c6e843eed0a0" dependencies = [ - "arrow 58.3.0", + "arrow 59.2.0", "async-trait", "bytemuck", "bytes", @@ -10348,18 +10401,18 @@ dependencies = [ [[package]] name = "parquet" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c" dependencies = [ "ahash 0.8.12", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-ipc 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", - "base64 0.22.1", + "arrow-array 59.2.0", + "arrow-buffer 59.2.0", + "arrow-data 59.2.0", + "arrow-ipc 59.2.0", + "arrow-schema 59.2.0", + "arrow-select 59.2.0", + "base64 0.23.1", "brotli", "bytes", "chrono", @@ -10367,19 +10420,17 @@ dependencies = [ "futures", "half", "hashbrown 0.17.1", - "lz4_flex 0.13.1", - "num-bigint 0.4.6", + "lz4_flex 0.14.0", + "num-bigint 0.5.1", "num-integer", "num-traits", "object_store", "parquet-variant", "parquet-variant-compute", "parquet-variant-json", - "paste", "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", "zstd", @@ -10387,15 +10438,15 @@ dependencies = [ [[package]] name = "parquet-variant" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74c8db065291f088a2aad8ab831853eae1871c0d311c8d0b83bbc3b7e735d0fc" +checksum = "3f7e5fff3ed0c07514a7fb8bee3f2ea5a53f36939410ecac4a466620213539a8" dependencies = [ - "arrow 58.3.0", - "arrow-schema 58.3.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "chrono", "half", - "indexmap 2.13.0", + "indexmap 2.14.2", "num-traits", "simdutf8", "uuid", @@ -10403,15 +10454,15 @@ dependencies = [ [[package]] name = "parquet-variant-compute" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a530e8d5b5e14efcb39c9a6ec55432ad11f6afb7dc4455a79be0dc615fe3cc31" +checksum = "ba4d3de89dab8d1aaaf601ae8d71bd07ea88cfca9efc1df5815b982c30f631e1" dependencies = [ - "arrow 58.3.0", - "arrow-schema 58.3.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "chrono", "half", - "indexmap 2.13.0", + "indexmap 2.14.2", "parquet-variant", "parquet-variant-json", "serde_json", @@ -10420,12 +10471,12 @@ dependencies = [ [[package]] name = "parquet-variant-json" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00ed89908289f67caa2ca078f9ff9aacd6229a313ec92b12bf4f48f613dc2b97" +checksum = "fb19dfe1bd24c17addd761ba4f7000f615e2fa12525871c7baa835dbb3d7f147" dependencies = [ - "arrow-schema 58.3.0", - "base64 0.22.1", + "arrow-schema 59.2.0", + "base64 0.23.1", "chrono", "parquet-variant", "serde_json", @@ -10639,7 +10690,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset 0.4.2", - "indexmap 2.13.0", + "indexmap 2.14.2", ] [[package]] @@ -10649,7 +10700,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset 0.5.7", - "indexmap 2.13.0", + "indexmap 2.14.2", ] [[package]] @@ -10660,7 +10711,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset 0.5.7", "hashbrown 0.15.4", - "indexmap 2.13.0", + "indexmap 2.14.2", "serde", ] @@ -10848,8 +10899,8 @@ version = "1.3.0-alpha.1" dependencies = [ "ahash 0.8.12", "api", - "arrow 58.3.0", - "arrow-schema 58.3.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "async-trait", "catalog", "chrono", @@ -10972,7 +11023,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d77244ce2d584cd84f6a15f86195b8c9b2a0dfbfd817c09e0464244091a58ed" dependencies = [ "base64 0.22.1", - "indexmap 2.13.0", + "indexmap 2.14.2", "quick-xml 0.37.5", "serde", "time", @@ -11453,7 +11504,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", - "heck 0.4.1", + "heck 0.5.0", "itertools 0.12.1", "log", "multimap", @@ -11473,7 +11524,7 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "itertools 0.14.0", "log", "multimap", @@ -11835,8 +11886,8 @@ dependencies = [ "ahash 0.8.12", "api", "arc-swap", - "arrow 58.3.0", - "arrow-schema 58.3.0", + "arrow 59.2.0", + "arrow-schema 59.2.0", "async-recursion", "async-stream", "async-trait", @@ -11863,6 +11914,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-functions", "datafusion-optimizer", + "datafusion-pg-catalog", "datafusion-physical-expr", "datafusion-proto", "datafusion-sql", @@ -12317,7 +12369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c11639076bf147be211b90e47790db89f4c22b6c8a9ca6e960833869da67166" dependencies = [ "aho-corasick", - "indexmap 2.13.0", + "indexmap 2.14.2", "itertools 0.13.0", "nohash", "regex", @@ -12777,7 +12829,7 @@ dependencies = [ "crc32c", "flate2", "futures", - "integer-encoding 4.0.2", + "integer-encoding", "lz4", "parking_lot 0.12.4", "rand 0.10.1", @@ -13485,7 +13537,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.13.0", + "indexmap 2.14.2", "schemars 0.9.0", "schemars 1.2.1", "serde_core", @@ -13512,7 +13564,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.14.2", "itoa", "ryu", "serde", @@ -13525,7 +13577,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.14.2", "itoa", "ryu", "serde", @@ -13538,11 +13590,11 @@ version = "1.3.0-alpha.1" dependencies = [ "ahash 0.8.12", "api", - "arrow 58.3.0", + "arrow 59.2.0", "arrow-flight", - "arrow-ipc 58.3.0", + "arrow-ipc 59.2.0", "arrow-pg", - "arrow-schema 58.3.0", + "arrow-schema 59.2.0", "async-trait", "auth", "axum 0.8.4", @@ -13554,6 +13606,7 @@ dependencies = [ "chrono", "client", "common-base", + "common-batcher", "common-catalog", "common-decimal", "common-error", @@ -13594,7 +13647,7 @@ dependencies = [ "humantime", "humantime-serde", "hyper 1.6.0", - "indexmap 2.13.0", + "indexmap 2.14.2", "influxdb_line_protocol", "itertools 0.14.0", "json5", @@ -13778,6 +13831,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "signal-hook-registry" version = "1.4.5" @@ -13953,7 +14012,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1961e2ef424c1424204d3a5d6975f934f56b6d50ff5732382d84ebf460e147f7" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -14042,7 +14101,7 @@ name = "sql" version = "1.3.0-alpha.1" dependencies = [ "api", - "arrow-buffer 58.3.0", + "arrow-buffer 59.2.0", "chrono", "common-base", "common-catalog", @@ -14132,13 +14191,11 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" -source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=2aefa08a8d69c96eec2d6d6703598a009bba6e4c#2aefa08a8d69c96eec2d6d6703598a009bba6e4c" +version = "0.62.0" +source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=9e9019bb1c7040ed956f654e39378dd42ab17884#9e9019bb1c7040ed956f654e39378dd42ab17884" dependencies = [ - "lazy_static", "log", "recursive", - "regex", "serde", "sqlparser_derive 0.5.0", ] @@ -14157,7 +14214,7 @@ dependencies = [ [[package]] name = "sqlparser_derive" version = "0.5.0" -source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=2aefa08a8d69c96eec2d6d6703598a009bba6e4c#2aefa08a8d69c96eec2d6d6703598a009bba6e4c" +source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=9e9019bb1c7040ed956f654e39378dd42ab17884#9e9019bb1c7040ed956f654e39378dd42ab17884" dependencies = [ "proc-macro2", "quote", @@ -14196,7 +14253,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.4", "hashlink", - "indexmap 2.13.0", + "indexmap 2.14.2", "log", "memchr", "once_cell", @@ -14369,15 +14426,15 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stacker" -version = "0.1.21" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cddb07e32ddb770749da91081d8d0ac3a16f1a569a18b20348cd371f5dead06b" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" dependencies = [ "cc", "cfg-if", "libc", "psm", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -14599,11 +14656,11 @@ dependencies = [ [[package]] name = "substrait" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0" +version = "0.63.0" +source = "git+https://github.com/GreptimeTeam/substrait-rs.git?rev=91ec978b0649417ad3da8390e7a515baec723b1b#91ec978b0649417ad3da8390e7a515baec723b1b" dependencies = [ "heck 0.5.0", + "indexmap 2.14.2", "pbjson", "pbjson-build", "pbjson-types", @@ -14700,6 +14757,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "0.1.2" @@ -15172,6 +15240,7 @@ dependencies = [ "tokio", "tokio-postgres", "tokio-stream", + "tokio-util", "tonic 0.14.2", "tower 0.5.2", "url", @@ -15240,17 +15309,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding 3.0.4", - "ordered-float 2.10.1", -] - [[package]] name = "tikv-jemalloc-ctl" version = "0.7.0" @@ -15548,7 +15606,7 @@ version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.14.2", "serde", "serde_spanned", "toml_datetime", @@ -15570,7 +15628,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.14.2", "toml_datetime", "winnow 0.5.40", ] @@ -15581,7 +15639,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.14.2", "serde", "serde_spanned", "toml_datetime", @@ -15767,7 +15825,7 @@ dependencies = [ "futures-core", "futures-util", "hdrhistogram", - "indexmap 2.13.0", + "indexmap 2.14.2", "pin-project-lite", "slab", "sync_wrapper 1.0.2", @@ -16232,13 +16290,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.21.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" dependencies = [ "getrandom 0.4.1", "js-sys", - "rand 0.9.4", + "rand 0.10.1", "serde_core", "wasm-bindgen", ] @@ -16377,7 +16435,7 @@ dependencies = [ "hostname 0.4.1", "iana-time-zone", "idna", - "indexmap 2.13.0", + "indexmap 2.14.2", "indoc", "influxdb-line-protocol", "ipcrypt-rs", @@ -16604,7 +16662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.13.0", + "indexmap 2.14.2", "wasm-encoder", "wasmparser", ] @@ -16643,7 +16701,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags 2.12.1", "hashbrown 0.15.4", - "indexmap 2.13.0", + "indexmap 2.14.2", "semver", ] @@ -17282,7 +17340,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck 0.5.0", - "indexmap 2.13.0", + "indexmap 2.14.2", "prettyplease", "syn 2.0.117", "wasm-metadata", @@ -17313,7 +17371,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags 2.12.1", - "indexmap 2.13.0", + "indexmap 2.14.2", "log", "serde", "serde_derive", @@ -17332,7 +17390,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.13.0", + "indexmap 2.14.2", "log", "semver", "serde", diff --git a/Cargo.toml b/Cargo.toml index 29fcdc74320..730901925a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "src/client", "src/cmd", "src/common/base", + "src/common/batcher", "src/common/catalog", "src/common/config", "src/common/datasource", @@ -100,13 +101,13 @@ rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] } # See for more detaiils: https://github.com/rust-lang/cargo/issues/11329 ahash = { version = "0.8", features = ["compile-time-rng"] } aquamarine = "0.6" -arrow = { version = "58.3", features = ["prettyprint"] } -arrow-array = { version = "58.3", default-features = false, features = ["chrono-tz"] } -arrow-buffer = "58.3" -arrow-cast = "58.3" -arrow-flight = "58.3" -arrow-ipc = { version = "58.3", default-features = false, features = ["lz4", "zstd"] } -arrow-schema = { version = "58.3", features = ["serde"] } +arrow = { version = "=59.2.0", features = ["prettyprint"] } +arrow-array = { version = "=59.2.0", default-features = false, features = ["chrono-tz"] } +arrow-buffer = "=59.2.0" +arrow-cast = "=59.2.0" +arrow-flight = "=59.2.0" +arrow-ipc = { version = "=59.2.0", default-features = false, features = ["lz4", "zstd"] } +arrow-schema = { version = "=59.2.0", features = ["serde"] } async-stream = "0.3" async-trait = "0.1" # Remember to update axum-extra, axum-macros when updating axum @@ -128,22 +129,22 @@ const_format = "0.2" criterion = "0.7" crossbeam-utils = "0.8" dashmap = "6.1" -datafusion = "=53.1.0" -datafusion-common = "=53.1.0" -datafusion-datasource = "=53.1.0" -datafusion-expr = "=53.1.0" -datafusion-expr-common = "=53.1.0" -datafusion-functions = "=53.1.0" -datafusion-functions-aggregate-common = "=53.1.0" -datafusion-functions-window-common = "=53.1.0" -datafusion-optimizer = "=53.1.0" -datafusion-orc = { git = "https://github.com/datafusion-contrib/datafusion-orc.git", rev = "6c07fa282dc8d62db2aa4ded06ab55485efc811a" } -datafusion-pg-catalog = "0.17.3" -datafusion-physical-expr = "=53.1.0" -datafusion-physical-plan = "=53.1.0" -datafusion-proto = "=53.1.0" -datafusion-sql = "=53.1.0" -datafusion-substrait = "=53.1.0" +datafusion = "=55.0.0" +datafusion-common = "=55.0.0" +datafusion-datasource = "=55.0.0" +datafusion-expr = "=55.0.0" +datafusion-expr-common = "=55.0.0" +datafusion-functions = "=55.0.0" +datafusion-functions-aggregate-common = "=55.0.0" +datafusion-functions-window-common = "=55.0.0" +datafusion-optimizer = "=55.0.0" +datafusion-orc = "0.10.0" +datafusion-pg-catalog = "0.18.3" +datafusion-physical-expr = "=55.0.0" +datafusion-physical-plan = "=55.0.0" +datafusion-proto = "=55.0.0" +datafusion-sql = "=55.0.0" +datafusion-substrait = "=55.0.0" datafusion_object_store = { package = "object_store", version = "0.13.2" } deadpool = "0.12" deadpool-postgres = "0.14" @@ -159,7 +160,7 @@ fs2 = "0.4" fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "b74667d17827481dbb6222fa0326163627211557" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "0c4beffaa2b3968034915867593a717c6ab0c6b4" } hex = "0.4" hostname = "0.4.0" http = "1" @@ -195,10 +196,10 @@ otel-arrow-rust = { git = "https://github.com/GreptimeTeam/otel-arrow", rev = "5 "server", ] } parking_lot = "0.12" -parquet = { version = "58.3", default-features = false, features = ["arrow", "async", "object_store"] } -parquet-variant = "58.3" -parquet-variant-compute = "58.3" -parquet-variant-json = "58.3" +parquet = { version = "=59.2.0", default-features = false, features = ["arrow", "async", "object_store"] } +parquet-variant = "=59.2.0" +parquet-variant-compute = "=59.2.0" +parquet-variant-json = "=59.2.0" paste = "1.0" pin-project = "1.0" pretty_assertions = "1.4.0" @@ -239,7 +240,7 @@ simd-json = "0.15" similar-asserts = "1.6.0" smallvec = { version = "1", features = ["serde"] } snafu = "0.8" -sqlparser = { version = "0.61.0", default-features = false, features = ["std", "visitor", "serde"] } +sqlparser = { version = "0.62.0", default-features = false, features = ["std", "visitor", "serde"] } sqlx = { version = "0.8", default-features = false, features = [ "any", "macros", @@ -287,6 +288,7 @@ cli = { path = "src/cli" } client = { path = "src/client" } cmd = { path = "src/cmd", default-features = false } common-base = { path = "src/common/base" } +common-batcher = { path = "src/common/batcher" } common-catalog = { path = "src/common/catalog" } common-config = { path = "src/common/config" } common-datasource = { path = "src/common/datasource" } @@ -353,22 +355,25 @@ git = "https://github.com/GreptimeTeam/greptime-meter.git" rev = "5618e779cf2bb4755b499c630fba4c35e91898cb" [patch.crates-io] -datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" } -sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "2aefa08a8d69c96eec2d6d6703598a009bba6e4c" } # on branch v0.61.x +substrait = { git = "https://github.com/GreptimeTeam/substrait-rs.git", rev = "91ec978b0649417ad3da8390e7a515baec723b1b" } +datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" } +datafusion-pg-catalog = { git = "https://github.com/GreptimeTeam/datafusion-postgres.git", rev = "3c77e6c32b8db80635a0d2f4b318a36b31170bc3" } +arrow-pg = { git = "https://github.com/GreptimeTeam/datafusion-postgres.git", rev = "3c77e6c32b8db80635a0d2f4b318a36b31170bc3" } +sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "9e9019bb1c7040ed956f654e39378dd42ab17884" } # Temporary: use the GreptimeTeam fork of tikv-jemalloc-sys embedding # jemalloc 5.3.1 + backport of 54f22c83 ("Initialize TSD tcache before diff --git a/config/config.md b/config/config.md index 2d85ddd7176..dde0b231066 100644 --- a/config/config.md +++ b/config/config.md @@ -165,6 +165,9 @@ | `region_engine.mito.worker_request_batch_size` | Integer | `64` | Max batch size for a worker to handle requests. | | `region_engine.mito.manifest_checkpoint_distance` | Integer | `10` | Number of meta action updated to trigger a new checkpoint for the manifest. | | `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). | +| `region_engine.mito.experimental_enable_series_index` | Bool | `false` | Under development; do not enable. Whether to enable series indexes.
Indexes are stored on the local filesystem under `{data_home}/series_index`. | +| `region_engine.mito.experimental_series_index_maintenance_interval` | String | `5m` | Interval between series-index maintenance runs. Zero uses the default of 5 min. | +| `region_engine.mito.experimental_series_index_bucket_width` | String | `5days` | Requested minimum series-index bucket width (default: 5 days), rounded up to
an exact multiple of each region's compaction time window. | | `region_engine.mito.max_background_flushes` | Integer | Auto | Max number of running background flush jobs (default: 1/2 of cpu cores). | | `region_engine.mito.max_background_compactions` | Integer | Auto | Max number of running background compaction jobs (default: 1/4 of cpu cores). | | `region_engine.mito.max_background_purges` | Integer | Auto | Max number of running background purge jobs (default: number of cpu cores). | @@ -612,6 +615,9 @@ | `region_engine.mito.experimental_manifest_keep_removed_file_count` | Integer | `256` | Number of removed files to keep in manifest's `removed_files` field before also
remove them from `removed_files`. Mostly for debugging purpose.
If set to 0, it will only use `keep_removed_file_ttl` to decide when to remove files
from `removed_files` field. | | `region_engine.mito.experimental_manifest_keep_removed_file_ttl` | String | `1h` | How long to keep removed files in the `removed_files` field of manifest
after they are removed from manifest.
files will only be removed from `removed_files` field
if both `keep_removed_file_count` and `keep_removed_file_ttl` is reached. | | `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). | +| `region_engine.mito.experimental_enable_series_index` | Bool | `false` | Under development; do not enable. Whether to enable series indexes.
Indexes are stored on the local filesystem under `{data_home}/series_index`. | +| `region_engine.mito.experimental_series_index_maintenance_interval` | String | `5m` | Interval between series-index maintenance runs. Zero uses the default of 5 min. | +| `region_engine.mito.experimental_series_index_bucket_width` | String | `5days` | Requested minimum series-index bucket width (default: 5 days), rounded up to
an exact multiple of each region's compaction time window. | | `region_engine.mito.max_background_flushes` | Integer | Auto | Max number of running background flush jobs (default: 1/2 of cpu cores). | | `region_engine.mito.max_background_compactions` | Integer | Auto | Max number of running background compaction jobs (default: 1/4 of cpu cores). | | `region_engine.mito.max_background_purges` | Integer | Auto | Max number of running background purge jobs (default: number of cpu cores). | diff --git a/config/datanode.example.toml b/config/datanode.example.toml index d378d55cc2b..67b6eea7274 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -474,6 +474,17 @@ experimental_manifest_keep_removed_file_ttl = "1h" ## Whether to compress manifest and checkpoint file by gzip (default false). compress_manifest = false +## Under development; do not enable. Whether to enable series indexes. +## Indexes are stored on the local filesystem under `{data_home}/series_index`. +#+ experimental_enable_series_index = false + +## Interval between series-index maintenance runs. Zero uses the default of 5 min. +#+ experimental_series_index_maintenance_interval = "5m" + +## Requested minimum series-index bucket width (default: 5 days), rounded up to +## an exact multiple of each region's compaction time window. +#+ experimental_series_index_bucket_width = "5days" + ## Max number of running background flush jobs (default: 1/2 of cpu cores). ## @toml2docs:none-default="Auto" #+ max_background_flushes = 4 diff --git a/config/standalone.example.toml b/config/standalone.example.toml index 0bef0793247..975e0239d08 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -635,6 +635,17 @@ manifest_checkpoint_distance = 10 ## Whether to compress manifest and checkpoint file by gzip (default false). compress_manifest = false +## Under development; do not enable. Whether to enable series indexes. +## Indexes are stored on the local filesystem under `{data_home}/series_index`. +#+ experimental_enable_series_index = false + +## Interval between series-index maintenance runs. Zero uses the default of 5 min. +#+ experimental_series_index_maintenance_interval = "5m" + +## Requested minimum series-index bucket width (default: 5 days), rounded up to +## an exact multiple of each region's compaction time window. +#+ experimental_series_index_bucket_width = "5days" + ## Max number of running background flush jobs (default: 1/2 of cpu cores). ## @toml2docs:none-default="Auto" #+ max_background_flushes = 4 diff --git a/docs/rfcs/2026-09-11-metric-export-import.md b/docs/rfcs/2026-09-11-metric-export-import.md new file mode 100644 index 00000000000..1a984211e4d --- /dev/null +++ b/docs/rfcs/2026-09-11-metric-export-import.md @@ -0,0 +1,280 @@ +--- +Feature Name: Metric Export and Import Optimization +Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/9120 +Date: 2026-09-11 +Status: Draft for discussion +--- + +# Summary + +Group the selected Metric logical tables by physical table. Query each physical +table across all its regions, order the rows by `__table_id`, and project each +logical schema into the existing logical-table Parquet files. Restore the same +V2 snapshot with batched logical-table DDL and the existing data COPY path. A +separate generic COPY fix removes repeated directory scans. + +The design preserves `manifest.version = 1` and the current V2 layout. It does +not introduce physical backup files or require importing source table IDs. +Merged data writes remain a first-release scope decision, pending representative +measurements after the COPY fix. + +# Motivation and evidence + +[Discussion #7394](https://github.com/orgs/GreptimeTeam/discussions/7394) +includes reports of exports taking days for roughly 100,000 logical tables with +only several GiB of data. Table count, physical schema width and data volume must +be measured separately; total database bytes do not describe the transfer cost. + +Local experiments support three independent optimizations: shared physical scans, +batched logical CREATE, and removal of repeated file lookup. The +[experiment results](2026-09-11-metric-export-import/experiment-results.md) +contain the measurements, controls and coverage limits. The COPY improvement +uses per-table writes; merged-write benefits remain unmeasured. + +# Terminology + +| Term | Meaning in this proposal | +| --- | --- | +| Physical table | A Metric Engine table whose data regions store rows for multiple logical tables; its schema includes their column union. | +| Logical table | A user-facing Metric table with its own schema and an association with a physical table. | +| Region | A storage partition of a physical table. A logical table's rows may span regions. | +| Export unit | The selected logical tables sharing one physical table, within one schema and V2 time chunk. This is an export grouping, not a storage object or completion checkpoint. | +| Query execution partition | A stream of work in the query plan. Frontend `target_partitions` influences the partition count; execution partitions are distinct from storage regions and export units. | +| Logical-table descriptor | Captured catalog metadata: source ID, name, physical association, column names/types/order, time index and primary key. Capturing and reusing it provides no snapshot or locking guarantee. | +| Parquet writer | The writer for one logical table's output file in an export unit. | +| Merged writes | Combining insert requests for multiple target logical tables into a region batch dispatch while retaining each table's identity, schema and routing. | + +# Physical export and logical files + +## Query and routing + +Capture the selected logical-table descriptors and reuse them for schema DDL +and data export. For each schema and V2 time chunk, group the selected logical +tables by physical table to form export units. Source schemas, selected data and +TTL effects must stay stable during export; this proposal does not provide a +transactionally consistent online backup. + +The server must authorize every selected logical table and the destination +before querying the physical table. Routing filters are not an authorization +boundary. + +For example, suppose `cpu` has `(ts, host, cpu)` and `requests` has +`(ts, host, service, requests)`, sharing `public.phy`. The physical query is +equivalent to: + +```sql +SELECT __table_id, ts, host, service, cpu, requests +FROM public.phy +WHERE ts >= '2026-09-01T00:00:00Z' + AND ts < '2026-09-02T00:00:00Z' +ORDER BY __table_id; +``` + +The implementation builds this plan through the existing query engine, using +COPY's time-filter semantics and properly resolved columns. It reads the +physical table directly; it does not first write and reread `phy.parquet`. + +```text +physical table query: selected column union + __table_id, ordered by ID + → reject rows whose IDs are absent from the captured descriptor set + → contiguous rows for cpu → project (ts, host, cpu) → cpu.parquet + → contiguous rows for requests → project (ts, host, service, requests) → requests.parquet +``` + +Only one Parquet writer is active per export unit. At an ID transition, close the +previous writer and open the next. Verify non-null UInt32 IDs and monotonic order +across input batches. Create a valid zero-row file for every selected empty table. +Never put `__table_id`, `__tsid`, or another table's columns into logical files. + +The captured catalog IDs are the routing whitelist; physical rows are not the +source of table membership. Dropped tables can leave residual rows. Predicate +pushdown for selected IDs/ranges is an optional optimization to evaluate against +residual density and index effectiveness; it does not replace routing checks. + +## Physical column union and memory + +The physical schema can contain the union of thousands of logical schemas. +Project the scan to `__table_id` plus the union needed by the selected tables. +For each ID run, project again to that logical schema **before** expanding Arrow +dictionary columns. Preserve field order, types, NULLs and empty strings. + +Logical projection reduces output conversion but does not eliminate wide scan +batches or upstream sorting. Account separately for retained scan and merge +buffers, conversion, codec reservations, row-group metadata and object-store +buffers. Bound concurrent export units as well as each unit's work. + +Input checks after a batch arrives cannot cap the scan's allocation peak, and a +writer flush threshold does not cap codec allocations. Combine schema validation +with query memory accounting, bounded conversion slices, row-group limits and +cancellation. Reject an oversized row explicitly. These controls do not provide +a hard process RSS guarantee. + +## Ordering and multiple regions + +The first release covers every selected Metric physical table, any region count, +and the Metric column types supported by existing Parquet COPY. Issue one query +per export unit covering all regions of its physical table. The query engine +provides globally ordered results through `ORDER BY __table_id`, including any +sorting and merging needed across datanodes. The exporter consumes that stream +without implementing its own region scheduler. + +In the current implementation, `MergeScanExec` declares the requested ordering +only when `output_partition_count >= regions.len()`. When the region count +exceeds frontend `target_partitions`, the frontend may need an additional sort +to satisfy SQL ordering. That sort operates on the selected column union before +per-logical-table projection. Parquet writer limits and downstream conversion +slices cannot bound its memory or spill cost. + +SQL ordering is the contract. Keep the runtime ID-monotonicity check, but do not +reject a valid plan because it contains `SortExec`. If query resources are +exhausted, cancel the query, fail the chunk and retain retry semantics. Region +count or resource exhaustion must not silently switch an enabled physical +export to per-logical-table queries. The +[ordering acceptance gate](#ordering-acceptance-gate) is required before release. + +# Snapshot and metadata contract + +Retain the current implementation's layout: + +```text +manifest.json # version 1 +schema/schemas.json # schema index +schema/ddl/public.sql # canonical database/table/view DDL +data/public/1/cpu.parquet +data/public/1/requests.parquet +``` + +Schema path segments retain existing V2 encoding. Logical data filenames retain +COPY's raw table-name semantics and import's `file_stem` matching, including dots. +There is no existing general table-filename escaping contract to reuse. Reject +path separators for the fast path; changing that contract requires a separate +compatibility design. + +Keep database, physical-table, logical/ordinary-table and view DDL in dependency +order. Physical DDL describes the table and its options; logical DDL describes +each logical schema and its physical association. Normal target CREATE procedures +reconstruct Metric data/metadata regions and logical metadata with newly allocated +IDs. Do not copy hidden metadata-region records or source IDs as restore data. +Verify effective options, column mappings and physical routes after recreation. + +# Restore path + +## Batched logical DDL + +Parse the existing DDL and batch contiguous logical CREATE statements sharing +catalog, schema and the literal `on_physical_table` value. A dot in that option +does not make it a qualified identifier. Database, physical, ordinary and view +statements are barriers and retain their order. + +Use the existing logical-table batch procedure. Process bounded batches +sequentially and validate each batch before submission. Preserve normal CREATE +validation, schema-option inheritance and target metadata resolution. Bound the +request size and the importer's DDL buffering separately. + +The production CLI needs an authenticated server operation that checks CREATE +authorization for every member and reuses the ordinary DDL path. Transport, +capability discovery and request limits must be agreed before accepting this +interface. An older server uses ordinary DDL; do not retry an ambiguously +completed batch through another path. + +## Data COPY and generic directory fix + +Read each logical Parquet file through the current COPY path and write through +normal target logical-table routing. No source-ID remapping belongs in these +files, and batch DDL does not imply batched data insertion. + +The generic fix changes explicit-file lookup from `stat → list parent → find` +to `stat → use path`, with a sandboxed file-type check for explicit local +inputs to preserve symlink filtering. Directory COPY retains listing, pattern +matching and file-type filtering. Backend authorization and error handling +remain in place. With N files in one directory, this removes roughly N² +directory-entry work. +COPY bypasses the common Lister's filename branch; the branch itself is unchanged. +This fix can ship independently of the Metric design. + +## Completion, interruption and retry + +Keep V2 completion units: export chunk completion and import `(chunk, schema)` +data tasks, with the existing DDL-completed flag. Finish and close every required +file before recording completion. A closed subset of files is not a completed +chunk. On export retry, replace only the unfinished attempt's owned outputs; +never touch completed chunks. Cancellation must close/abort writers and leave +the chunk incomplete. + +Batch DDL is not a transaction across the restore. A later failure can leave +earlier batches created. Keep DDL completion false until all statements succeed, +then use the current durable state update. Data retries retain existing COPY +semantics and may replay an incomplete task; there is no new exactly-once +guarantee or per-file checkpoint in this proposal. + +# Rationale and alternatives + +| Choice | Reason and trade-off | +| --- | --- | +| Keep logical-table Parquet files | Reuses V2 DDL, file lookup and normal target CREATE/routing with fresh IDs. Physical backup files would need a new restore contract for mixed schemas and source metadata. The retained format still has one file per logical table per time chunk. | +| Scan by physical table | Shares query and scan work among logical tables. Per-logical-table COPY is simpler but retains the table-count overhead; physical scans instead pay for the selected column union and ID routing. | +| Global ID ordering and one Parquet writer per export unit | Contiguous IDs allow a file to be closed at each table transition. Unordered routing would require many open writers or intermediate buffering/files. Ordering bounds writer state, but query sorting and merging need separate resource controls. | +| Use query-engine ordering across regions | Reuses distributed planning and SQL semantics. An exporter-owned region scheduler would add merge, cancellation and retry coordination. The chosen approach must handle both streaming merge and additional-sort plans. | +| Batch DDL while retaining per-table data COPY | Reuses logical CREATE procedures and current data-task replay semantics. Merged writes may reduce dispatch/WAL overhead, but introduce partial-success and checkpoint questions and require a separate benefit comparison. | + +# Unresolved questions + +- Which authenticated batch-DDL operation and capability-discovery mechanism + should the CLI use? Agree request byte/table limits and DDL buffering bounds. +- What are the query memory, spill-space, writer and export-unit concurrency + limits? Agree the test workloads and resource/performance thresholds before + running the ordering acceptance benchmark. +- Which released import-v2 versions and object-store backends are compatibility + targets? Test those readers against the retained V2 format. +- What name and defaults should the `experimental_` export option use? +- Should merged writes ship in the first release? Compare a bounded merged-write + PoC against corrected per-table COPY with the same batch DDL and concurrency. + Bound decoded rows/bytes and in-flight requests, preserving normal logical + validation and target routing through the existing region batch dispatch. + Cover small/larger files, wide schemas and distributed/multi-region targets; + report repeated total/data times, rows/s, bytes/s, request counts and memory. + Agree a minimum useful gain and memory ceiling before the comparison. Validate + partial success, ambiguous completion and checkpoint advancement, then record + an explicit include/defer decision. + +# Integration and release acceptance + +Use existing COPY DATABASE orchestration to form Metric export units. Retain +ordinary COPY for non-Metric tables and other formats, and direct COPY TABLE +behavior. Gate the unfinished Metric fast path behind an `experimental_` +configuration option; disabling it selects the existing path. + +The [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120) owns +the PR breakdown and implementation dependencies. Each implementation slice +must verify its own correctness and failure behavior. Release acceptance requires: + +| Area | Required evidence | +| --- | --- | +| Export | Actual CLI export with authorization, multiple physical tables and regions, dense/sparse keys, empty/residual tables, supported Metric Parquet types, and logical schema/value equivalence. | +| Restore | Authenticated remote batch DDL, per-table authorization, fresh target IDs and reconstructed metadata, dependency ordering, older-server behavior and batch failure/retry. | +| V2 lifecycle | Supported released readers; local and object-store interruption, owned-output cleanup, durable completion state and import replay. | +| Scale | Repeated release-build measurements varying table count up to approximately 100,000, rows/file, physical union width, regions and backend. Report median/spread, control execution order/cache, and include schema export and metadata discovery in end-to-end timing. | +| Open decisions | Agreed batch-DDL interface, resource limits, compatibility targets and experimental option; documented merged-write include/defer decision. | + +## Ordering acceptance gate + +The first release must pass a distributed export benchmark with region counts +**below, equal to and above frontend `target_partitions`**, using representative +row counts and narrow/wide selected column unions. Record actual execution +partition counts rather than assuming the configured target determines the plan. +Include multiple physical tables and a logical table whose rows span regions. +Exercise both streaming-merge and additional-sort plans. + +For each case, report frontend and datanode execution plans, peak memory, +query memory-accounting metrics, spill volume and elapsed time. Run with the +agreed query memory, spill-space, writer and concurrency limits. Verify global +ID order and exactly one complete file per selected logical table in each export +unit, checking schemas and typed rows outside the timed interval. + +Passing requires correct exports within the agreed resource/performance +thresholds on both plan paths. Also force resource exhaustion: cancellation +must leave the chunk incomplete, and retry with sufficient resources must +produce complete files without altering completed chunks. Writer-only memory +measurements and single-region runs cannot satisfy this gate. Until both the +normal and failure/retry cases pass, multi-region resource acceptance remains +open. diff --git a/docs/rfcs/2026-09-11-metric-export-import/experiment-results.md b/docs/rfcs/2026-09-11-metric-export-import/experiment-results.md new file mode 100644 index 00000000000..56d75530c7d --- /dev/null +++ b/docs/rfcs/2026-09-11-metric-export-import/experiment-results.md @@ -0,0 +1,109 @@ +# Metric export/import experiment results + +These experiments support three changes: physical-table export, batched logical +DDL, and removal of repeated directory listing during COPY. They measure separate +parts of export/import; their speedups should not be multiplied. + +Measurements were taken on September 10–11, 2026, on an ARM64 machine with 16 GiB +RAM, using local standalone instances, local files, warm caches and unoptimized +test builds. Each table entry represents one run. The results establish useful +optimization directions, not production throughput estimates. + +## Physical-table export + +The comparison uses 64 populated logical tables with 1,000 source rows each, +plus an empty table. A time filter selects 63,744 rows, written into 65 logical +Parquet files. Ordinary COPY DATABASE uses parallelism one. Timings cover data +export and exclude schema export and fixture preparation. Both execution orders +were measured at narrow and wide physical schemas. + +| Physical tag columns | Execution order | Ordinary COPY (ms) | Physical-table export (ms) | Speedup | +| ---: | --- | ---: | ---: | ---: | +| 4 | Ordinary COPY first | 889.4 | 262.7 | 3.39x | +| 4 | Physical export first | 893.7 | 309.5 | 2.89x | +| 67 | Ordinary COPY first | 1062.0 | 316.0 | 3.36x | +| 67 | Physical export first | 869.0 | 308.3 | 2.82x | + +Physical-table export was **2.82–3.39x faster** in these runs. Each populated +logical file still contains only its own four columns. Projecting the logical +schema before dictionary expansion kept the expanded-array peak at 0.148 MiB +and the Parquet row-group estimate at 1.112 MiB for both widths. Incoming array +estimates grew from 0.267 MiB to 2.291–2.408 MiB: logical projection does not remove +the memory cost of a wide physical schema. These estimates are not process RSS +limits, and a writer flush threshold does not cap codec allocations. + +The measured single-region query used `SeriesScan` and `SortPreservingMergeExec` +without a separate `SortExec`; `__table_id` matches a physical primary-key prefix. +Only Float64 fields, string tags and timestamps were exercised. These runs do not +establish sorting memory, spill or performance when region count exceeds +frontend `target_partitions`; see the RFC's +[ordering acceptance gate](../2026-09-11-metric-export-import.md#ordering-acceptance-gate). +The timestamp encoder reserved about 1 MiB even for a small batch, illustrating +why a writer flush threshold is not an allocation ceiling. + +## Batched logical DDL + +The restore fixture has logical tables sharing one physical table, one file per +logical table, two rows per populated table and one empty table. The 10,000-table +case contains 19,998 rows. The three paths use SQL HTTP per table, an internal call per table, and +internal batches of up to 128 tables and 1 MiB of rendered SQL, submitted +sequentially. These request bounds do not bound the whole parsed DDL list in +memory. The internal single-call control separates transport savings from +procedure batching. + +| Logical tables | SQL HTTP CREATE (s) | Internal single CREATE (s) | Batched CREATE (s) | Logical calls: single → batch | +| ---: | ---: | ---: | ---: | ---: | +| 100 | 0.318 | 0.177 | 0.056 | 100 → 1 | +| 1,000 | 3.091 | 1.601 | 0.544 | 1,000 → 8 | +| 10,000 | 29.181 | 19.524 | 5.776 | 10,000 → 79 | + +These are logical CREATE call durations. At 10,000 tables, the complete DDL phase +fell from 30.752 s with SQL HTTP to 7.019 s with batching. Total restore fell from +187.279 s to 157.492 s; unchanged data COPY still took about 150 s. Data-phase +variation is not a DDL benefit. Batching improves CREATE itself, but the original +COPY bottleneck limits its effect on total restore. + +The batch executor was in-process. A production authenticated server interface +is still needed; these times do not predict batch HTTP latency. + +## COPY file lookup + +Both variants below use batch DDL and the same COPY concurrency. Previously, +COPY listed a file's parent directory even when it already knew the filename. +With N files in one directory, repeating this for each file causes roughly N² +directory-entry work. The fix checks the known path directly; directory COPY +still lists files normally. + +| Logical tables / files | COPY before (s) | COPY after (s) | Total restore before (s) | Total restore after (s) | +| ---: | ---: | ---: | ---: | ---: | +| 100 | 0.094 | 0.071 | 0.183 | 0.164 | +| 1,000 | 2.135 | 0.612 | 2.849 | 1.289 | +| 10,000 | 146.496 | 5.481 | 153.647 | 12.300 | + +At 10,000 files, COPY improved **26.73x** and total restore improved **12.49x**. +Before the fix, backend/file lookup accounted for about 92% of cumulative active +file-COPY time. A control with profiling disabled completed COPY in 5.449 s. + +The **5.481 s result still uses per-table writes**, with only 19,998 rows. It is +not a merged-write result. Larger files, distributed targets and object stores +are needed to decide whether merged writes provide enough additional benefit. + +## Correctness and remaining coverage + +The experiments compared exported/restored schemas and typed rows with the +source. Checks covered dense/sparse keys, different logical schemas, NULL and +empty values, empty tables, dropped-table residue and fresh target IDs. The +unmodified import-v2 command used in the experiment restored the exported files +into a fresh target with different IDs. This does not establish compatibility +with all released readers. Batch-DDL checks covered two schemas, ordinary +tables/views, failure after a successful 128-table batch and retry of the original +DDL. Existing standalone COPY cases also passed. + +The internal exporter assumed a trusted caller. It could leave a partial output +directory after failure and did not implement completion publication or resume. +These limitations require production integration work. + +Production acceptance still requires release-build measurements with repeated +runs, larger workloads, distributed/multi-region execution, object stores, +released-reader compatibility and export cancellation/recovery. These are tracked +in the [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120). diff --git a/src/api/src/helper.rs b/src/api/src/helper.rs index 0a8c999461a..d8e3004aa2b 100644 --- a/src/api/src/helper.rs +++ b/src/api/src/helper.rs @@ -449,7 +449,14 @@ impl TryFrom for ColumnDataTypeWrapper { }), JsonFormat::Json2(native_type) => { if native_type.is_null() { - None + Some(ColumnDataTypeExtension { + type_ext: Some(TypeExt::JsonNativeType(Box::new( + JsonNativeTypeExtension { + datatype: ColumnDataType::Json as i32, + datatype_extension: None, + }, + ))), + }) } else { let concrete_type = ConcreteDataType::from_arrow_type(&native_type.as_arrow_type()); @@ -1291,6 +1298,32 @@ mod tests { assert!(values.bool_values.is_empty()); } + #[test] + fn test_json2_unknown_type_encoding() { + let datatype = ConcreteDataType::json2(JsonNativeType::Null); + let wrapper = ColumnDataTypeWrapper::try_from(datatype.clone()).unwrap(); + assert_eq!( + wrapper.to_parts(), + ( + ColumnDataType::Json, + Some(ColumnDataTypeExtension { + type_ext: Some(TypeExt::JsonNativeType(Box::new(JsonNativeTypeExtension { + datatype: ColumnDataType::Json as i32, + datatype_extension: None, + }))), + }), + ) + ); + assert_eq!(ConcreteDataType::from(wrapper), datatype); + + for extension in [None, Some(ColumnDataTypeExtension::default())] { + assert_eq!( + ConcreteDataType::from(ColumnDataTypeWrapper::new(ColumnDataType::Json, extension)), + datatype + ); + } + } + #[test] fn test_concrete_datatype_from_column_datatype() { assert_eq!( diff --git a/src/catalog/src/information_extension.rs b/src/catalog/src/information_extension.rs index c41c816c6d7..9d55f9f7a1e 100644 --- a/src/catalog/src/information_extension.rs +++ b/src/catalog/src/information_extension.rs @@ -27,8 +27,10 @@ use common_query::request::QueryRequest; use common_recordbatch::adapter::{AsyncRecordBatchStreamAdapter, DfRecordBatchStreamAdapter}; use common_recordbatch::util::{ChainedRecordBatchStream, LimitedRecordBatchStream}; use common_recordbatch::{DfSendableRecordBatchStream, SendableRecordBatchStream}; +use datafusion::common::Result; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::execution::TaskContext; -use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; +use datafusion::physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; use datatypes::arrow::datatypes::SchemaRef as ArrowSchemaRef; @@ -157,10 +159,6 @@ impl ExecutionPlan for DistributedInspectExec { "DistributedInspectExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> ArrowSchemaRef { self.arrow_schema.clone() } @@ -173,6 +171,13 @@ impl ExecutionPlan for DistributedInspectExec { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/src/catalog/src/system_schema/information_schema.rs b/src/catalog/src/system_schema/information_schema.rs index d0539a062c5..8c3f4bf0f5f 100644 --- a/src/catalog/src/system_schema/information_schema.rs +++ b/src/catalog/src/system_schema/information_schema.rs @@ -135,6 +135,9 @@ lazy_static! { GLOBAL_STATUS, SESSION_STATUS, PARTITIONS, + PLUGINS, + USER_PRIVILEGES, + PROCESSLIST, ]; } @@ -237,6 +240,9 @@ impl SystemSchemaProviderInner for InformationSchemaProvider { TABLE_PRIVILEGES => setup_memory_table!(TABLE_PRIVILEGES), GLOBAL_STATUS => setup_memory_table!(GLOBAL_STATUS), SESSION_STATUS => setup_memory_table!(SESSION_STATUS), + PLUGINS => setup_memory_table!(PLUGINS), + USER_PRIVILEGES => setup_memory_table!(USER_PRIVILEGES), + PROCESSLIST => setup_memory_table!(PROCESSLIST), KEY_COLUMN_USAGE => Some(Arc::new(InformationSchemaKeyColumnUsage::new( self.catalog_name.clone(), self.catalog_manager.clone(), diff --git a/src/catalog/src/system_schema/information_schema/columns.rs b/src/catalog/src/system_schema/information_schema/columns.rs index 4e68b093cd4..e92efd3bebf 100644 --- a/src/catalog/src/system_schema/information_schema/columns.rs +++ b/src/catalog/src/system_schema/information_schema/columns.rs @@ -31,7 +31,7 @@ use datatypes::scalars::ScalarVectorBuilder; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; use datatypes::value::Value; use datatypes::vectors::{ - ConstantVector, Int64Vector, Int64VectorBuilder, StringVector, StringVectorBuilder, VectorRef, + Int64Vector, Int64VectorBuilder, StringVector, StringVectorBuilder, VectorRef, }; use futures::TryStreamExt; use snafu::{OptionExt, ResultExt}; @@ -408,18 +408,9 @@ impl InformationSchemaColumnsBuilder { fn finish(&mut self) -> Result { let rows_num = self.collation_names.len(); - let privileges = Arc::new(ConstantVector::new( - Arc::new(StringVector::from(vec![DEFAULT_PRIVILEGES])), - rows_num, - )); - let empty_string = Arc::new(ConstantVector::new( - Arc::new(StringVector::from(vec![EMPTY_STR])), - rows_num, - )); - let srs_ids = Arc::new(ConstantVector::new( - Arc::new(Int64Vector::from(vec![None])), - rows_num, - )); + let privileges = Arc::new(StringVector::from(vec![DEFAULT_PRIVILEGES; rows_num])); + let empty_string = Arc::new(StringVector::from(vec![EMPTY_STR; rows_num])); + let srs_ids = Arc::new(Int64Vector::from(vec![None; rows_num])); let columns: Vec = vec![ Arc::new(self.catalog_names.finish()), diff --git a/src/catalog/src/system_schema/information_schema/information_memory_table.rs b/src/catalog/src/system_schema/information_schema/information_memory_table.rs index 56a84a0da1b..847ad06dc34 100644 --- a/src/catalog/src/system_schema/information_schema/information_memory_table.rs +++ b/src/catalog/src/system_schema/information_schema/information_memory_table.rs @@ -383,6 +383,47 @@ pub(super) fn get_schema_columns(table_name: &str) -> (SchemaRef, Vec vec![], ), + // GreptimeDB has no pluggable components. + PLUGINS => ( + string_columns(&[ + "PLUGIN_NAME", + "PLUGIN_VERSION", + "PLUGIN_STATUS", + "PLUGIN_TYPE", + "PLUGIN_TYPE_VERSION", + "PLUGIN_LIBRARY", + "PLUGIN_LIBRARY_VERSION", + "PLUGIN_AUTHOR", + "PLUGIN_DESCRIPTION", + "PLUGIN_LICENSE", + "LOAD_OPTION", + ]), + vec![], + ), + + // Privileges are not exposed through `information_schema`, same as the other + // `*_privileges` tables above. + USER_PRIVILEGES => ( + string_columns(&["GRANTEE", "TABLE_CATALOG", "PRIVILEGE_TYPE", "IS_GRANTABLE"]), + vec![], + ), + + // Sessions are reported through `information_schema.process_list` and + // `SHOW PROCESSLIST`; this table carries the MySQL column shape only. + PROCESSLIST => ( + vec![ + bigint_column("ID"), + string_column("USER"), + string_column("HOST"), + string_column("DB"), + string_column("COMMAND"), + bigint_column("TIME"), + string_column("STATE"), + string_column("INFO"), + ], + vec![], + ), + _ => unreachable!("Unknown table in information_schema: {}", table_name), }; diff --git a/src/catalog/src/system_schema/information_schema/key_column_usage.rs b/src/catalog/src/system_schema/information_schema/key_column_usage.rs index 99b0d7f6b73..4f8f116afaf 100644 --- a/src/catalog/src/system_schema/information_schema/key_column_usage.rs +++ b/src/catalog/src/system_schema/information_schema/key_column_usage.rs @@ -26,7 +26,7 @@ use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream; use datatypes::prelude::{ConcreteDataType, MutableVector, ScalarVectorBuilder, VectorRef}; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; use datatypes::value::Value; -use datatypes::vectors::{ConstantVector, StringVector, StringVectorBuilder, UInt32VectorBuilder}; +use datatypes::vectors::{StringVector, StringVectorBuilder, UInt32VectorBuilder}; use futures_util::TryStreamExt; use snafu::{OptionExt, ResultExt}; use store_api::storage::{ScanRequest, TableId}; @@ -326,10 +326,7 @@ impl InformationSchemaKeyColumnUsageBuilder { fn finish(&mut self) -> Result { let rows_num = self.table_catalog.len(); - let null_string_vector = Arc::new(ConstantVector::new( - Arc::new(StringVector::from(vec![None as Option<&str>])), - rows_num, - )); + let null_string_vector = Arc::new(StringVector::from(vec![None as Option<&str>; rows_num])); let columns: Vec = vec![ Arc::new(self.constraint_catalog.finish()), Arc::new(self.constraint_schema.finish()), diff --git a/src/catalog/src/system_schema/information_schema/partitions.rs b/src/catalog/src/system_schema/information_schema/partitions.rs index b9396fe5543..d2cce93150a 100644 --- a/src/catalog/src/system_schema/information_schema/partitions.rs +++ b/src/catalog/src/system_schema/information_schema/partitions.rs @@ -29,8 +29,8 @@ use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; use datatypes::timestamp::TimestampSecond; use datatypes::value::Value; use datatypes::vectors::{ - ConstantVector, Int64Vector, Int64VectorBuilder, MutableVector, StringVector, - StringVectorBuilder, TimestampSecondVector, TimestampSecondVectorBuilder, UInt64VectorBuilder, + Int64Vector, Int64VectorBuilder, MutableVector, StringVector, StringVectorBuilder, + TimestampSecondVector, TimestampSecondVectorBuilder, UInt64VectorBuilder, }; use futures::{StreamExt, TryStreamExt}; use partition::manager::PartitionInfo; @@ -361,22 +361,11 @@ impl InformationSchemaPartitionsBuilder { fn finish(&mut self) -> Result { let rows_num = self.catalog_names.len(); - let null_string_vector = Arc::new(ConstantVector::new( - Arc::new(StringVector::from(vec![None as Option<&str>])), - rows_num, - )); - let null_i64_vector = Arc::new(ConstantVector::new( - Arc::new(Int64Vector::from(vec![None])), - rows_num, - )); - let null_timestamp_second_vector = Arc::new(ConstantVector::new( - Arc::new(TimestampSecondVector::from(vec![None])), - rows_num, - )); - let partition_methods = Arc::new(ConstantVector::new( - Arc::new(StringVector::from(vec![Some("RANGE")])), - rows_num, - )); + let null_string_vector = Arc::new(StringVector::from(vec![None as Option<&str>; rows_num])); + let null_i64_vector = Arc::new(Int64Vector::from(vec![None; rows_num])); + let null_timestamp_second_vector = + Arc::new(TimestampSecondVector::from(vec![None; rows_num])); + let partition_methods = Arc::new(StringVector::from(vec![Some("RANGE"); rows_num])); let columns: Vec = vec![ Arc::new(self.catalog_names.finish()), diff --git a/src/catalog/src/system_schema/information_schema/ssts.rs b/src/catalog/src/system_schema/information_schema/ssts.rs index da5fcd2eed1..e364bddfd3d 100644 --- a/src/catalog/src/system_schema/information_schema/ssts.rs +++ b/src/catalog/src/system_schema/information_schema/ssts.rs @@ -336,7 +336,7 @@ mod tests { }; let plan = table.scan_to_plan(request).unwrap().unwrap(); - assert!(plan.as_any().is::()); + assert!(plan.as_ref().is::()); assert_eq!(1, plan.schema().fields().len()); } } diff --git a/src/catalog/src/system_schema/information_schema/table_constraints.rs b/src/catalog/src/system_schema/information_schema/table_constraints.rs index 8c975207f21..2208d303c2e 100644 --- a/src/catalog/src/system_schema/information_schema/table_constraints.rs +++ b/src/catalog/src/system_schema/information_schema/table_constraints.rs @@ -27,7 +27,7 @@ use datatypes::prelude::{ConcreteDataType, MutableVector}; use datatypes::scalars::ScalarVectorBuilder; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; use datatypes::value::Value; -use datatypes::vectors::{ConstantVector, StringVector, StringVectorBuilder, VectorRef}; +use datatypes::vectors::{StringVector, StringVectorBuilder, VectorRef}; use futures::TryStreamExt; use snafu::{OptionExt, ResultExt}; use store_api::storage::{ScanRequest, TableId}; @@ -242,14 +242,8 @@ impl InformationSchemaTableConstraintsBuilder { fn finish(&mut self) -> Result { let rows_num = self.constraint_names.len(); - let constraint_catalogs = Arc::new(ConstantVector::new( - Arc::new(StringVector::from(vec!["def"])), - rows_num, - )); - let enforceds = Arc::new(ConstantVector::new( - Arc::new(StringVector::from(vec!["YES"])), - rows_num, - )); + let constraint_catalogs = Arc::new(StringVector::from(vec!["def"; rows_num])); + let enforceds = Arc::new(StringVector::from(vec!["YES"; rows_num])); let columns: Vec = vec![ constraint_catalogs, diff --git a/src/catalog/src/system_schema/information_schema/table_names.rs b/src/catalog/src/system_schema/information_schema/table_names.rs index f161fecb54b..3e9b2e2d6de 100644 --- a/src/catalog/src/system_schema/information_schema/table_names.rs +++ b/src/catalog/src/system_schema/information_schema/table_names.rs @@ -55,3 +55,8 @@ pub const SSTS_INDEX_META: &str = "ssts_index_meta"; pub const TABLE_SEMANTICS: &str = "table_semantics"; pub const STATISTICS: &str = "statistics"; pub const RECYCLE_BIN: &str = "recycle_bin"; +pub const PLUGINS: &str = "plugins"; +pub const USER_PRIVILEGES: &str = "user_privileges"; +/// MySQL's session list. GreptimeDB reports its own sessions through [`PROCESS_LIST`]; +/// this table only exists so MySQL tooling finds the name and column shape it expects. +pub const PROCESSLIST: &str = "processlist"; diff --git a/src/catalog/src/table_source/dummy_catalog.rs b/src/catalog/src/table_source/dummy_catalog.rs index 20637c3a3a8..455cc8c34fa 100644 --- a/src/catalog/src/table_source/dummy_catalog.rs +++ b/src/catalog/src/table_source/dummy_catalog.rs @@ -14,7 +14,6 @@ //! Dummy catalog for region server. -use std::any::Any; use std::fmt; use std::sync::Arc; @@ -64,10 +63,6 @@ impl fmt::Debug for DummyCatalogList { } impl CatalogProviderList for DummyCatalogList { - fn as_any(&self) -> &dyn Any { - self - } - fn register_catalog( &self, _name: String, @@ -98,10 +93,6 @@ struct DummyCatalogProvider { } impl CatalogProvider for DummyCatalogProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn schema_names(&self) -> Vec { vec![] } @@ -135,10 +126,6 @@ struct DummySchemaProvider { #[async_trait] impl SchemaProvider for DummySchemaProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn table_names(&self) -> Vec { vec![] } diff --git a/src/client/src/database.rs b/src/client/src/database.rs index 881730cfaaa..e97fff6db00 100644 --- a/src/client/src/database.rs +++ b/src/client/src/database.rs @@ -825,7 +825,17 @@ impl Database { /// Ingest a stream of [RecordBatch]es that belong to a table, using Arrow Flight's "`DoPut`" /// method. The return value is also a stream, produces [DoPutResponse]s. pub async fn do_put(&self, stream: FlightDataStream) -> Result { + self.do_put_with_hints(stream, &[]).await + } + + /// Ingest a stream of [RecordBatch]es using Arrow Flight's `DoPut` with request hints. + pub async fn do_put_with_hints( + &self, + stream: FlightDataStream, + hints: &[(&str, &str)], + ) -> Result { let mut request = tonic::Request::new(stream); + Self::put_hints(request.metadata_mut(), hints)?; if let Some(AuthHeader { auth_scheme: Some(AuthScheme::Basic(Basic { username, password })), diff --git a/src/cmd/src/bin/query_perf_fixture/case.rs b/src/cmd/src/bin/query_perf_fixture/case.rs index a76c49d43d0..0f5239a6b51 100644 --- a/src/cmd/src/bin/query_perf_fixture/case.rs +++ b/src/cmd/src/bin/query_perf_fixture/case.rs @@ -105,6 +105,10 @@ pub(super) struct PromRemoteWritePlan { #[serde(default = "default_visibility_timeout_seconds")] pub(super) visibility_timeout_seconds: u64, #[serde(default)] + pub(super) base_setup_sql: Vec, + #[serde(default)] + pub(super) candidate_setup_sql: Vec, + #[serde(default)] pub(super) prom_store: PromStoreConfig, #[serde(default)] pub(super) value: ValueConfig, @@ -200,6 +204,7 @@ pub(super) enum ValuePattern { QuantizedSignal, SignalWithSporadicStalls, MixedSignalRepeated, + BoundedMixed, } impl std::fmt::Display for ValuePattern { @@ -436,6 +441,63 @@ pub(super) fn default_parallelism() -> u64 { 1 } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn remote_write_setup_sql_defaults_to_empty() { + let case: CaseFile = toml::from_str( + r#" +[scenario] +kind = "prom_remote_write_then_query" + +[scenario.remote_write] +metric = "metric" +"#, + ) + .unwrap(); + let Scenario::PromRemoteWriteThenQuery(scenario) = case.scenario else { + panic!("expected prom_remote_write_then_query scenario"); + }; + assert!(scenario.remote_write.base_setup_sql.is_empty()); + assert!(scenario.remote_write.candidate_setup_sql.is_empty()); + } + + #[test] + fn remote_write_setup_sql_roundtrips() { + let case: CaseFile = toml::from_str( + r#" +[scenario] +kind = "prom_remote_write_then_query" + +[scenario.remote_write] +metric = "metric" +base_setup_sql = ["CREATE TABLE base_table"] +candidate_setup_sql = ["CREATE TABLE candidate_table", "ALTER TABLE candidate_table SET 'x'='y'"] +"#, + ) + .unwrap(); + + let roundtrip: CaseFile = + serde_json::from_str(&serde_json::to_string(&case).unwrap()).unwrap(); + let Scenario::PromRemoteWriteThenQuery(scenario) = roundtrip.scenario else { + panic!("expected prom_remote_write_then_query scenario"); + }; + assert_eq!( + scenario.remote_write.base_setup_sql, + ["CREATE TABLE base_table"] + ); + assert_eq!( + scenario.remote_write.candidate_setup_sql, + [ + "CREATE TABLE candidate_table", + "ALTER TABLE candidate_table SET 'x'='y'" + ] + ); + } +} + impl Scenario { pub(super) fn kind(&self) -> &'static str { match self { diff --git a/src/cmd/src/bin/query_perf_fixture/direct_sst.rs b/src/cmd/src/bin/query_perf_fixture/direct_sst.rs index 746ecbd76e4..27a6b4109cd 100644 --- a/src/cmd/src/bin/query_perf_fixture/direct_sst.rs +++ b/src/cmd/src/bin/query_perf_fixture/direct_sst.rs @@ -469,6 +469,7 @@ pub(super) async fn run_direct_sst(args: DirectArgs) { write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE, row_group_size: scenario.layout.row_group_size, max_file_size: None, + float_field_encoding: Default::default(), }; let infos = match format { FormatType::Flat => writer.write_all_flat(source, Some(sequence), &opts).await, diff --git a/src/cmd/src/bin/query_perf_fixture/inspect_footer.rs b/src/cmd/src/bin/query_perf_fixture/inspect_footer.rs index d2a2e8006cb..19e972db3b3 100644 --- a/src/cmd/src/bin/query_perf_fixture/inspect_footer.rs +++ b/src/cmd/src/bin/query_perf_fixture/inspect_footer.rs @@ -14,18 +14,21 @@ use std::collections::BTreeSet; use std::fs; +use std::ops::Range; use std::path::{Path, PathBuf}; use std::sync::Arc; use clap::Args as ClapArgs; use datafusion_object_store::path::Path as StorePath; -use datafusion_object_store::{ObjectMeta, ObjectStore}; -use futures::{StreamExt, TryStreamExt}; +use datafusion_object_store::{ObjectMeta, ObjectStore, ObjectStoreExt}; +use futures::future::BoxFuture; +use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt}; use object_store::config::ObjectStoreConfig; use object_store::factory::new_raw_object_store; use object_store::services::Fs; -use parquet::arrow::async_reader::ParquetObjectReader; -use parquet::file::metadata::ParquetMetaDataReader; +use parquet::arrow::async_reader::AsyncFileReader; +use parquet::errors::{ParquetError, Result as ParquetResult}; +use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader}; use serde::{Deserialize, Serialize}; /// Same shape as `query_regression_runner::model::DestinationConfig`. The two @@ -100,6 +103,70 @@ struct ListedFile { relative_path: String, } +/// An asynchronous Parquet reader backed directly by an object store. +/// +/// The file size is retained from the listing so footer reads use bounded +/// ranges without an additional stat/head request. +#[derive(Clone, Debug)] +struct ObjectStoreReader { + store: Arc, + path: StorePath, + file_size: u64, +} + +impl ObjectStoreReader { + fn new(store: Arc, path: StorePath, file_size: u64) -> Self { + Self { + store, + path, + file_size, + } + } +} + +fn to_parquet_error(error: datafusion_object_store::Error) -> ParquetError { + ParquetError::External(Box::new(error)) +} + +impl AsyncFileReader for ObjectStoreReader { + fn get_bytes( + &mut self, + range: Range, + ) -> BoxFuture<'_, ParquetResult> { + self.store + .get_range(&self.path, range) + .map_err(to_parquet_error) + .boxed() + } + + fn get_byte_ranges( + &mut self, + ranges: Vec>, + ) -> BoxFuture<'_, ParquetResult>> { + async move { + self.store + .get_ranges(&self.path, &ranges) + .await + .map_err(to_parquet_error) + } + .boxed() + } + + fn get_metadata<'a>( + &'a mut self, + _options: Option<&'a parquet::arrow::arrow_reader::ArrowReaderOptions>, + ) -> BoxFuture<'a, ParquetResult>> { + let file_size = self.file_size; + async move { + let metadata = ParquetMetaDataReader::new() + .load_and_finish(self, file_size) + .await?; + Ok(Arc::new(metadata)) + } + .boxed() + } +} + pub(super) async fn run_inspect_footer( args: InspectFooterArgs, ) -> Result<(), Box> { @@ -217,11 +284,8 @@ async fn inspect_file( file: &ListedFile, column: &str, ) -> Result> { - let mut reader = - ParquetObjectReader::new(store, file.location.clone()).with_file_size(file.size); - let metadata = ParquetMetaDataReader::new() - .load_and_finish(&mut reader, file.size) - .await?; + let mut reader = ObjectStoreReader::new(store, file.location.clone(), file.size); + let metadata = reader.get_metadata(None).await?; let file_metadata = metadata.file_metadata(); let row_groups = metadata.row_groups(); let mut columns = Vec::new(); diff --git a/src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs b/src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs index 1f36b0c2d23..a59178f4a79 100644 --- a/src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs +++ b/src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs @@ -197,8 +197,35 @@ fn deterministic_prom_value(args: &PromRemoteWriteArgs, series_idx: u64, sample_ args.value_base + (series_idx % 97) as f64 + local as f64 * args.value_step } } + ValuePattern::BoundedMixed => bounded_mixed_value(args, series_idx, local), } } + +fn bounded_mixed_value(args: &PromRemoteWriteArgs, series_idx: u64, local: u64) -> f64 { + const SPANS: [i64; 4] = [10, 1_000, 100_000, 10_000_000]; + let series = splitmix64(series_idx ^ args.value_seed); + let span = SPANS[(series >> 62) as usize] * (1 + ((series >> 32) % 10) as i64); + let baseline = (series % args.value_cardinality.max(1)) as f64 * span as f64 / 10.0; + let run_length = args.value_run_length.max(1); + let group = local / run_length; + let phase = local % run_length; + let anchor = bounded_mixed_anchor(series, group, span); + let next_anchor = bounded_mixed_anchor(series, group + 1, span); + let signal = baseline + anchor + (next_anchor - anchor) * phase as f64 / run_length as f64; + let value = (args.value_base + args.value_step * signal).round(); + if series_idx % args.value_mixed_every.max(1) == args.value_mixed_every.max(1) - 1 { + let numerator = splitmix64(series ^ local) % 999 + 1; + value + numerator as f64 / 1_000.0 + } else { + value + } +} + +fn bounded_mixed_anchor(series: u64, group: u64, span: i64) -> f64 { + let limit = span - 1; + (splitmix64(series ^ group) % (2 * limit as u64 + 1)) as i64 as f64 - limit as f64 +} + fn splitmix64(mut value: u64) -> u64 { value = value.wrapping_add(0x9e3779b97f4a7c15); let mut mixed = value; @@ -307,7 +334,7 @@ mod tests { for chunk_idx in 0..10 { let mut chunk = args(ValuePattern::SeededRandom); chunk.samples_per_series = 1_440; - chunk.value_sample_offset = chunk_idx * 1_440; + chunk.value_sample_offset = chunk_idx * 1_439; chunk.value_total_samples_per_series = Some(14_400); for sample_idx in 0..chunk.samples_per_series { assert_eq!( @@ -319,6 +346,118 @@ mod tests { } } + #[test] + fn bounded_mixed_has_bounded_95_5_per_series_values() { + let mut args = args(ValuePattern::BoundedMixed); + args.series_count = 1_000; + args.samples_per_series = 4_320; + args.value_cardinality = 1_000; + args.value_seed = 42; + args.value_run_length = 60; + args.value_mixed_every = 20; + + let mut integral_series = 0; + let mut fractional_series = 0; + let mut spans = HashSet::new(); + let mut ranges = HashSet::new(); + for series_idx in 0..args.series_count { + let series = splitmix64(series_idx ^ args.value_seed); + let span = [10_i64, 1_000, 100_000, 10_000_000][(series >> 62) as usize] + * (1 + ((series >> 32) % 10) as i64); + let baseline = (series % args.value_cardinality.max(1)) as f64 * span as f64 / 10.0; + let values = (0..args.samples_per_series) + .map(|sample_idx| deterministic_prom_value(&args, series_idx, sample_idx)) + .collect::>(); + + assert!(values.iter().all(|value| value.is_finite())); + assert!( + values + .iter() + .all(|value| *value >= baseline - span as f64 - 1.0 + && *value <= baseline + span as f64 + 1.0) + ); + assert!( + values.windows(2).any(|pair| pair[1] > pair[0]), + "series {series_idx} never rises" + ); + assert!( + values.windows(2).any(|pair| pair[1] < pair[0]), + "series {series_idx} never falls" + ); + ranges.insert(( + values + .iter() + .copied() + .fold(f64::INFINITY, f64::min) + .to_bits(), + values + .iter() + .copied() + .fold(f64::NEG_INFINITY, f64::max) + .to_bits(), + )); + spans.insert(span); + if series_idx % args.value_mixed_every == args.value_mixed_every - 1 { + fractional_series += 1; + assert!(values.iter().all(|value| value.fract() != 0.0)); + } else { + integral_series += 1; + assert!(values.iter().all(|value| value.fract() == 0.0)); + } + } + + assert_eq!((integral_series, fractional_series), (950, 50)); + assert!(spans.len() > 4); + assert!(ranges.len() > 1); + } + + #[test] + fn bounded_mixed_is_seeded_and_chunk_bit_identical() { + let mut monolithic = args(ValuePattern::BoundedMixed); + monolithic.samples_per_series = 14_400; + monolithic.value_total_samples_per_series = Some(14_400); + monolithic.value_cardinality = 1_000; + monolithic.value_seed = 42; + monolithic.value_run_length = 60; + monolithic.value_mixed_every = 20; + + let original = (0..600) + .map(|sample_idx| deterministic_prom_value(&monolithic, 19, sample_idx)) + .collect::>(); + monolithic.value_seed = 43; + assert_ne!( + original, + (0..600) + .map(|sample_idx| deterministic_prom_value(&monolithic, 19, sample_idx)) + .collect::>() + ); + + monolithic.value_seed = 42; + for series_idx in [0, 19] { + for chunk_idx in 0..10 { + let mut chunk = args(ValuePattern::BoundedMixed); + chunk.samples_per_series = 1_440; + chunk.value_sample_offset = chunk_idx * 1_439; + chunk.value_total_samples_per_series = Some(14_400); + chunk.value_cardinality = 1_000; + chunk.value_seed = 42; + chunk.value_run_length = 60; + chunk.value_mixed_every = 20; + for sample_idx in 0..chunk.samples_per_series { + assert_eq!( + deterministic_prom_value(&chunk, series_idx, sample_idx).to_bits(), + deterministic_prom_value( + &monolithic, + series_idx, + chunk.value_sample_offset + sample_idx + ) + .to_bits() + ); + } + } + } + } + #[test] fn values_continue_across_series_boundaries() { let mut args = args(ValuePattern::Unique); diff --git a/src/cmd/src/bin/query_regression_runner/model.rs b/src/cmd/src/bin/query_regression_runner/model.rs index 6831a5256a4..5fafdb5635b 100644 --- a/src/cmd/src/bin/query_regression_runner/model.rs +++ b/src/cmd/src/bin/query_regression_runner/model.rs @@ -116,6 +116,10 @@ pub(super) struct RemoteWrite { pub(super) sample_chunk_size: Option, pub(super) flush_every_sample_chunks: u64, pub(super) visibility_timeout_seconds: u64, + #[serde(default)] + pub(super) base_setup_sql: Vec, + #[serde(default)] + pub(super) candidate_setup_sql: Vec, pub(super) prom_store: PromStore, pub(super) value: RemoteValue, pub(super) storage: Option, diff --git a/src/cmd/src/bin/query_regression_runner/remote/ingest.rs b/src/cmd/src/bin/query_regression_runner/remote/ingest.rs index f563402afcd..271e27fac42 100644 --- a/src/cmd/src/bin/query_regression_runner/remote/ingest.rs +++ b/src/cmd/src/bin/query_regression_runner/remote/ingest.rs @@ -57,6 +57,7 @@ pub(super) async fn run_prepare_remote(args: PrepareRemoteArgs) -> Result<()> { args.base_http_port, &args.fixture_generator, &remote, + &remote.base_setup_sql, &client, ) .await?; @@ -65,6 +66,7 @@ pub(super) async fn run_prepare_remote(args: PrepareRemoteArgs) -> Result<()> { args.candidate_http_port, &args.fixture_generator, &remote, + &remote.candidate_setup_sql, &client, ) .await?; @@ -88,6 +90,7 @@ async fn prepare_remote_target( port: u16, generator: &Path, remote: &RemoteWrite, + setup_sql: &[String], client: &Client, ) -> Result { let create_database = http_post_sql( @@ -107,6 +110,7 @@ async fn prepare_remote_target( ) .into()); } + let setup = run_setup_sql(client, port, &remote.database, name, setup_sql).await?; let (remote_write, flushes) = ingest_remote_write(generator, port, remote, client).await?; let expected_rows = remote .series_count @@ -124,6 +128,7 @@ async fn prepare_remote_target( Ok(json!({ "name": name, "create_database": create_database, + "setup_sql": setup, "remote_write": remote_write, "flushes": flushes, "visibility": visibility, @@ -131,6 +136,28 @@ async fn prepare_remote_target( })) } +async fn run_setup_sql( + client: &Client, + port: u16, + database: &str, + target: &str, + setup_sql: &[String], +) -> Result> { + let mut results = Vec::with_capacity(setup_sql.len()); + for (index, statement) in setup_sql.iter().enumerate() { + let result = http_post_sql(client, port, statement, database).await; + if !result["ok"].as_bool().unwrap_or(false) { + return Err(format!( + "setup SQL statement {} failed for {target}: {result}", + index + 1 + ) + .into()); + } + results.push(result); + } + Ok(results) +} + #[derive(Clone, Debug, PartialEq, Eq)] struct SampleChunk { index: u64, @@ -434,9 +461,110 @@ fn json_number(value: f64) -> String { #[cfg(test)] mod tests { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::{TcpListener, TcpStream}; + use super::*; use crate::query_regression_runner::model::{PromStore, RemoteValue}; + async fn read_http_request(stream: &mut TcpStream) -> Vec { + let mut request = Vec::new(); + let mut chunk = [0; 1024]; + loop { + let read = stream.read(&mut chunk).await.unwrap(); + assert_ne!( + read, 0, + "HTTP client closed request before sending its body" + ); + request.extend_from_slice(&chunk[..read]); + let Some(header_end) = request.windows(4).position(|window| window == b"\r\n\r\n") + else { + continue; + }; + let headers = std::str::from_utf8(&request[..header_end]).unwrap(); + let content_length = headers + .lines() + .find_map(|line| line.strip_prefix("content-length: ")) + .or_else(|| { + headers + .lines() + .find_map(|line| line.strip_prefix("Content-Length: ")) + }) + .unwrap() + .parse::() + .unwrap(); + if request.len() >= header_end + 4 + content_length { + return request; + } + } + } + + async fn spawn_sql_server( + responses: Vec, + ) -> (u16, tokio::task::JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = tokio::spawn(async move { + let mut requests = Vec::with_capacity(responses.len()); + for body in responses { + let (mut stream, _) = listener.accept().await.unwrap(); + let request = read_http_request(&mut stream).await; + requests.push(String::from_utf8(request).unwrap()); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\ + Connection: close\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await.unwrap(); + } + requests + }); + (port, server) + } + + #[tokio::test] + async fn setup_sql_runs_in_order_with_configured_database() { + let (port, server) = spawn_sql_server(vec!["{\"code\":0}".to_string(); 2]).await; + let client = Client::new(); + let setup = vec![ + "CREATE TABLE physical (ts TIMESTAMP TIME INDEX) ENGINE=metric".to_string(), + "ALTER TABLE physical SET 'compaction.twcs.trigger_file_num'='100'".to_string(), + ]; + + let results = run_setup_sql(&client, port, "perf", "candidate", &setup) + .await + .unwrap(); + + assert_eq!(results.len(), 2); + let requests = server.await.unwrap(); + assert_eq!(requests.len(), 2); + assert!(requests[0].contains("db=perf")); + assert!(requests[0].contains("sql=CREATE")); + assert!(requests[1].contains("db=perf")); + assert!(requests[1].contains("sql=ALTER")); + } + + #[tokio::test] + async fn setup_sql_stops_at_the_failing_statement() { + let (port, server) = + spawn_sql_server(vec!["{\"code\":0}".to_string(), "{\"code\":1}".to_string()]).await; + let client = Client::new(); + let setup = vec![ + "CREATE TABLE physical (ts TIMESTAMP TIME INDEX) ENGINE=metric".to_string(), + "invalid SQL".to_string(), + "ALTER TABLE physical SET 'compaction.twcs.trigger_file_num'='100'".to_string(), + ]; + + let error = run_setup_sql(&client, port, "perf", "base", &setup) + .await + .unwrap_err(); + + assert!(error.to_string().contains("statement 2 failed for base")); + let requests = server.await.unwrap(); + assert_eq!(requests.len(), 2); + assert!(requests[1].contains("sql=invalid+SQL")); + } + #[test] fn schedules_remote_sample_chunks_and_flushes() { let remote = RemoteWrite { @@ -452,6 +580,8 @@ mod tests { sample_chunk_size: Some(2), flush_every_sample_chunks: 2, visibility_timeout_seconds: 30, + base_setup_sql: Vec::new(), + candidate_setup_sql: Vec::new(), prom_store: PromStore { pending_rows_flush_interval: "1s".to_string(), max_batch_rows: 1, diff --git a/src/cmd/src/bin/query_regression_runner/remote/read_bench.rs b/src/cmd/src/bin/query_regression_runner/remote/read_bench.rs index 6b20a31c181..b9a05f6a69a 100644 --- a/src/cmd/src/bin/query_regression_runner/remote/read_bench.rs +++ b/src/cmd/src/bin/query_regression_runner/remote/read_bench.rs @@ -306,13 +306,25 @@ fn run_bench_command(command: Vec, mut run: Value) -> Result { } fn parse_average_duration(stdout: &str) -> Option { - Regex::new(r"(?i)Average duration[^0-9]*([0-9.]+)\s*ms") - .ok()? - .captures(stdout)? - .get(1)? - .as_str() - .parse() - .ok() + let captures = Regex::new( + r"(?i)Average:\s+(?:\x1b\[[0-9;]*m)*\d+(?:\x1b\[[0-9;]*m)*\s+rows(?:,\s+(?:\x1b\[[0-9;]*m)*\d+(?:\x1b\[[0-9;]*m)*\s+record batches)?\s+in\s+([0-9]+(?:\.[0-9]+)?)\s*(ns|µs|us|ms|s)\s+over\s+\d+\s+iterations|Average duration[^0-9]*([0-9]+(?:\.[0-9]+)?)\s*ms", + ) + .ok()? + .captures(stdout)?; + let (value, unit) = match (captures.get(1), captures.get(2), captures.get(3)) { + (Some(value), Some(unit), _) => (value.as_str(), unit.as_str()), + (_, _, Some(value)) => (value.as_str(), "ms"), + _ => return None, + }; + let value = value.parse::().ok()?; + let milliseconds = match unit.to_ascii_lowercase().as_str() { + "ns" => value / 1_000_000.0, + "µs" | "us" => value / 1_000.0, + "ms" => value, + "s" => value * 1_000.0, + _ => return None, + }; + milliseconds.is_finite().then_some(milliseconds) } fn bench_median(runs: &[Value]) -> Option { @@ -328,12 +340,51 @@ mod tests { use super::*; #[test] - fn parses_average_duration_and_groups_scan_paths() { - assert_eq!( - parse_average_duration("work\nAverage duration: 12.5 ms\n"), - Some(12.5) - ); - assert_eq!(parse_average_duration("completed"), None); + fn parses_average_durations() { + let cases = [ + ( + "parquetbench nanoseconds", + "ℹ Average: 42 rows, 2 record batches in 500ns over 4 iterations", + Some(0.0005), + ), + ( + "scanbench colored microseconds", + "\x1b[1;32mSummary\x1b[0m Average: \x1b[36m12\x1b[0m rows in 250µs over 2 iterations", + Some(0.25), + ), + ( + "scanbench seconds", + "Summary Average: 12 rows in 2.5s over 2 iterations", + Some(2500.0), + ), + ( + "legacy milliseconds", + "Average duration: 12.5 ms", + Some(12.5), + ), + ("absent", "Benchmark completed!", None), + ( + "malformed", + "Summary Average: 12 rows in unknown over 2 iterations", + None, + ), + ]; + + for (name, stdout, expected) in cases { + let actual = parse_average_duration(stdout); + match (actual, expected) { + (Some(actual), Some(expected)) => assert!( + (actual - expected).abs() < f64::EPSILON, + "{name}: expected {expected}, got {actual}" + ), + (None, None) => {} + (actual, expected) => panic!("{name}: expected {expected:?}, got {actual:?}"), + } + } + } + + #[test] + fn groups_scan_paths() { let targets = vec![ BenchTarget { relative_path: "data/a.parquet".to_string(), diff --git a/src/common/batcher/Cargo.toml b/src/common/batcher/Cargo.toml new file mode 100644 index 00000000000..5d6ace4b2d3 --- /dev/null +++ b/src/common/batcher/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "common-batcher" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +dashmap.workspace = true +futures.workspace = true +tokio.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } diff --git a/src/common/batcher/src/flush_limiter.rs b/src/common/batcher/src/flush_limiter.rs new file mode 100644 index 00000000000..98b6b2a2f01 --- /dev/null +++ b/src/common/batcher/src/flush_limiter.rs @@ -0,0 +1,73 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use tokio::sync::{AcquireError, OwnedSemaphorePermit, Semaphore}; + +/// Shared execution budget. One permit represents one complete flush, not one RPC. +#[derive(Clone)] +pub struct FlushLimiter { + semaphore: Arc, +} + +impl FlushLimiter { + /// Returns `None` if the limit is zero or exceeds Tokio's supported semaphore capacity. + pub fn try_new(max_concurrent_flushes: usize) -> Option { + let permits = max_concurrent_flushes; + ((1..=Semaphore::MAX_PERMITS).contains(&permits)).then(|| Self { + semaphore: Arc::new(Semaphore::new(permits)), + }) + } + + /// The caller must retain the returned permit until its flush finishes. + pub async fn acquire(&self) -> Result { + self.semaphore.clone().acquire_owned().await + } +} + +#[cfg(test)] +mod tests { + use std::future::Future; + + use super::*; + + #[test] + fn test_invalid_capacity() { + for capacity in [0, Semaphore::MAX_PERMITS + 1, usize::MAX] { + assert!(FlushLimiter::try_new(capacity).is_none()); + } + for capacity in [1, Semaphore::MAX_PERMITS] { + assert!(FlushLimiter::try_new(capacity).is_some()); + } + } + + #[tokio::test] + async fn test_clones_share_flush_budget() { + let limiter = FlushLimiter::try_new(1).unwrap(); + let other = limiter.clone(); + let permit = limiter.acquire().await.unwrap(); + let waiting = other.acquire(); + tokio::pin!(waiting); + // Poll admission without a sleep or an assumption about scheduler timing. + assert!( + std::future::poll_fn(|cx| { + std::task::Poll::Ready(waiting.as_mut().poll(cx).is_pending()) + }) + .await + ); + drop(permit); + let _next = waiting.await.unwrap(); + } +} diff --git a/src/common/batcher/src/flush_policy.rs b/src/common/batcher/src/flush_policy.rs new file mode 100644 index 00000000000..14804f815e2 --- /dev/null +++ b/src/common/batcher/src/flush_policy.rs @@ -0,0 +1,43 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod timing; + +use tokio::time::Instant; + +use crate::pending_batch::PendingBatch; + +/// The event that caused the worker to reconsider flushing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FlushTrigger { + /// A complete submission was appended to the batch. + Submission, + /// The flush timer reached its deadline. + Deadline, +} + +/// Decides when a pending batch is ready without owning its payloads or execution. +/// +/// A deadline wakeup must either allow flushing or produce a future deadline; +/// otherwise the caller would repeatedly poll an expired timer. +pub trait FlushPolicy { + /// Returns the next timer deadline, or `None` when no timer is needed. + fn deadline(&self, batch: &PendingBatch) -> Option; + + /// Returns whether the batch should be flushed for this event at `now`. + /// + /// Workers never flush empty batches, regardless of this decision. + fn should_flush(&self, batch: &PendingBatch, now: Instant, trigger: FlushTrigger) + -> bool; +} diff --git a/src/common/batcher/src/flush_policy/timing.rs b/src/common/batcher/src/flush_policy/timing.rs new file mode 100644 index 00000000000..56a9172eaa0 --- /dev/null +++ b/src/common/batcher/src/flush_policy/timing.rs @@ -0,0 +1,157 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Duration; + +use tokio::time::Instant; + +use crate::flush_policy::{FlushPolicy, FlushTrigger}; +use crate::pending_batch::PendingBatch; + +/// Decides whether the first-submission deadline or row threshold requires a flush. +/// +/// The row threshold is checked after appending a complete submission. It is not a +/// hard batch-size limit and never splits a submission. +#[derive(Debug, Clone, Copy)] +pub struct TimingFlushPolicy { + flush_interval: Duration, + max_batch_rows: usize, +} + +impl TimingFlushPolicy { + /// Creates a policy, rejecting zero limits, zero intervals, and unrepresentable deadlines. + pub fn try_new(flush_interval: Duration, max_batch_rows: usize) -> Option { + if max_batch_rows == 0 || !Self::validate(flush_interval) { + return None; + } + Some(Self { + flush_interval, + max_batch_rows, + }) + } + + /// Checks that the interval is nonzero and its deadline is representable. + pub fn validate(flush_interval: Duration) -> bool { + !flush_interval.is_zero() && Instant::now().checked_add(flush_interval).is_some() + } + + /// Checks only the row trigger when a caller drives deadline events separately. + pub fn reached_row_threshold(&self, batch: &PendingBatch) -> bool { + !batch.is_empty() && batch.total_rows() >= self.max_batch_rows + } +} + +impl FlushPolicy for TimingFlushPolicy { + /// Returns the deadline anchored to the first submission, or `None` for an empty batch. + /// + /// An unrepresentable deadline falls back to the first submission time so that + /// extreme timestamps cannot leave a nonempty batch waiting indefinitely. + fn deadline(&self, batch: &PendingBatch) -> Option { + batch + .first_submitted_at() + .map(|first| first.checked_add(self.flush_interval).unwrap_or(first)) + } + + /// Returns whether a nonempty batch has reached either trigger. + fn should_flush( + &self, + batch: &PendingBatch, + now: Instant, + trigger: FlushTrigger, + ) -> bool { + !batch.is_empty() + && (self.reached_row_threshold(batch) + || (trigger == FlushTrigger::Deadline + && self.deadline(batch).is_some_and(|deadline| now >= deadline))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_validate_matches_construction() { + for interval in [Duration::ZERO, Duration::from_secs(1), Duration::MAX] { + assert_eq!( + TimingFlushPolicy::validate(interval), + TimingFlushPolicy::try_new(interval, 1).is_some() + ); + } + } + + #[test] + fn test_first_submission_deadline() { + let policy = TimingFlushPolicy::try_new(Duration::from_millis(10), 100).unwrap(); + let mut batch = PendingBatch::new(); + let first = Instant::now(); + assert_eq!(policy.deadline(&batch), None); + assert!(!policy.should_flush(&batch, first, FlushTrigger::Deadline)); + batch.push(1, 1, first); + batch.push(2, 1, first + Duration::from_millis(9)); + assert_eq!( + policy.deadline(&batch), + Some(first + Duration::from_millis(10)) + ); + assert!(!policy.should_flush( + &batch, + first + Duration::from_millis(9), + FlushTrigger::Deadline + )); + assert!(policy.should_flush( + &batch, + first + Duration::from_millis(10), + FlushTrigger::Deadline + )); + assert!(!policy.reached_row_threshold(&batch)); + assert!(!policy.should_flush( + &batch, + first + Duration::from_millis(10), + FlushTrigger::Submission, + )); + assert_eq!(batch.take(), vec![1, 2]); + let next = first + Duration::from_millis(20); + batch.push(3, 0, next); + assert!(!policy.should_flush(&batch, next, FlushTrigger::Deadline)); + assert!(policy.should_flush( + &batch, + next + Duration::from_millis(10), + FlushTrigger::Deadline + )); + } + + #[test] + fn test_row_threshold_preserves_complete_submissions() { + let policy = TimingFlushPolicy::try_new(Duration::from_secs(1), 3).unwrap(); + let now = Instant::now(); + for rows in [2, 3, 4] { + let mut batch = PendingBatch::new(); + batch.push(vec![0; rows], rows, now); + assert_eq!( + policy.should_flush(&batch, now, FlushTrigger::Submission), + rows >= 3 + ); + assert_eq!(policy.reached_row_threshold(&batch), rows >= 3); + assert_eq!(batch.take(), vec![vec![0; rows]]); + } + } + + #[test] + fn test_invalid_interval() { + assert!(TimingFlushPolicy::try_new(Duration::from_secs(1), 0).is_none()); + let max_batch_rows = 1; + assert!(TimingFlushPolicy::try_new(Duration::ZERO, max_batch_rows).is_none()); + assert!(TimingFlushPolicy::try_new(Duration::MAX, max_batch_rows).is_none()); + } +} diff --git a/src/common/batcher/src/flush_timer.rs b/src/common/batcher/src/flush_timer.rs new file mode 100644 index 00000000000..7ae8173201a --- /dev/null +++ b/src/common/batcher/src/flush_timer.rs @@ -0,0 +1,141 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::pin::Pin; + +use tokio::time::{Instant, Sleep, sleep_until}; + +/// Maintains one reusable timer for the current flush deadline. +/// +/// Creating a timer does not require a Tokio runtime. Arming it requires a Tokio +/// runtime with time enabled. Disabling retains the allocation for the next arm. +#[derive(Default)] +pub struct FlushTimer { + deadline: Option, + sleep: Option>>, +} + +impl FlushTimer { + /// Creates a disabled timer without allocating a Tokio sleep. + pub fn new() -> Self { + Self::default() + } + + /// Arms or disables the timer, resetting its sleep only when the deadline changes. + /// + /// # Panics + /// + /// Arming for the first time panics outside a Tokio runtime with time enabled. + pub fn set_deadline(&mut self, deadline: Option) { + if self.deadline == deadline { + return; + } + self.deadline = deadline; + if let Some(deadline) = deadline { + match &mut self.sleep { + Some(sleep) => sleep.as_mut().reset(deadline), + None => self.sleep = Some(Box::pin(sleep_until(deadline))), + } + } + } + + /// Waits until the armed deadline, or indefinitely while disabled. + /// + /// Cancelling this wait does not change the deadline. An elapsed timer stays + /// ready until the caller changes or disables its deadline. + pub async fn wait(&mut self) { + if self.deadline.is_some() + && let Some(sleep) = &mut self.sleep + { + sleep.as_mut().await; + } else { + std::future::pending::<()>().await; + } + } +} + +#[cfg(test)] +mod tests { + use std::future::{Future, poll_fn}; + use std::task::Poll; + use std::time::Duration; + + use super::*; + + async fn is_ready(timer: &mut FlushTimer) -> bool { + let wait = timer.wait(); + tokio::pin!(wait); + poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx).is_ready())).await + } + + #[test] + fn test_create_without_runtime() { + let mut timer = FlushTimer::new(); + timer.set_deadline(None); + assert!(timer.sleep.is_none()); + } + + #[tokio::test(start_paused = true)] + async fn test_deadline_and_cancelled_wait() { + let mut timer = FlushTimer::new(); + assert!(!is_ready(&mut timer).await); + let deadline = Instant::now() + Duration::from_millis(10); + timer.set_deadline(Some(deadline)); + assert!(!is_ready(&mut timer).await); + tokio::time::advance(Duration::from_millis(9)).await; + // Each poll drops its wait future. Repeating the same deadline must not + // move the timer forward or replace its sleep allocation. + let sleep = timer.sleep.as_ref().unwrap().as_ref().get_ref() as *const Sleep; + timer.set_deadline(Some(deadline)); + assert_eq!( + sleep, + timer.sleep.as_ref().unwrap().as_ref().get_ref() as *const Sleep + ); + assert!(!is_ready(&mut timer).await); + tokio::time::advance(Duration::from_millis(1)).await; + assert!(is_ready(&mut timer).await); + } + + #[tokio::test(start_paused = true)] + async fn test_disable_and_rearm_reuses_sleep() { + let mut timer = FlushTimer::new(); + timer.set_deadline(Some(Instant::now())); + assert!(is_ready(&mut timer).await); + let sleep = timer.sleep.as_ref().unwrap().as_ref().get_ref() as *const Sleep; + timer.set_deadline(None); + assert!(!is_ready(&mut timer).await); + assert!(!is_ready(&mut timer).await); + timer.set_deadline(Some(Instant::now() + Duration::from_millis(5))); + assert_eq!( + sleep, + timer.sleep.as_ref().unwrap().as_ref().get_ref() as *const Sleep + ); + assert!(!is_ready(&mut timer).await); + tokio::time::advance(Duration::from_millis(5)).await; + assert!(is_ready(&mut timer).await); + } + + #[tokio::test(start_paused = true)] + async fn test_change_armed_deadline() { + let mut timer = FlushTimer::new(); + let now = Instant::now(); + timer.set_deadline(Some(now + Duration::from_millis(5))); + assert!(!is_ready(&mut timer).await); + timer.set_deadline(Some(now + Duration::from_millis(10))); + tokio::time::advance(Duration::from_millis(5)).await; + assert!(!is_ready(&mut timer).await); + timer.set_deadline(Some(Instant::now())); + assert!(is_ready(&mut timer).await); + } +} diff --git a/src/mito2/src/sst/parquet/json_align/mod.rs b/src/common/batcher/src/lib.rs similarity index 61% rename from src/mito2/src/sst/parquet/json_align/mod.rs rename to src/common/batcher/src/lib.rs index 54c97bdbb89..9ad29097aff 100644 --- a/src/mito2/src/sst/parquet/json_align/mod.rs +++ b/src/common/batcher/src/lib.rs @@ -12,13 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use datatypes::arrow::record_batch::RecordBatch; -use futures::stream::BoxStream; +//! Protocol-independent building blocks for timing-based batching. +//! +//! Payload preparation, grouping keys, write execution and request completion +//! remain the caller's responsibility. -mod stream; - -pub(crate) use stream::NestedSchemaAligner; - -use crate::error::Result; - -pub(crate) type ProjectedRecordBatchStream = BoxStream<'static, Result>; +pub mod flush_limiter; +pub mod flush_policy; +pub mod flush_timer; +pub mod notifier; +pub mod pending_batch; +pub mod pending_worker; +pub mod request_limiter; +pub mod worker_registry; diff --git a/src/common/batcher/src/notifier.rs b/src/common/batcher/src/notifier.rs new file mode 100644 index 00000000000..efc795b938a --- /dev/null +++ b/src/common/batcher/src/notifier.rs @@ -0,0 +1,145 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::future::Future; +use std::num::NonZeroUsize; + +use futures::{StreamExt, stream}; +use tokio::sync::Semaphore; +use tokio::sync::mpsc::error::TrySendError; +use tokio::sync::mpsc::{self, Receiver, Sender}; + +/// Best-effort notification admission, independent of the write result. +/// +/// The caller owns the notification payload, delivery handler and failure metrics. +pub struct Notifier { + sender: Sender, +} + +impl Clone for Notifier { + fn clone(&self) -> Self { + Self { + sender: self.sender.clone(), + } + } +} + +impl Notifier { + /// Returns `None` if the queue capacity is zero or exceeds Tokio's supported capacity. + pub fn try_new(capacity: usize) -> Option<(Self, Receiver)> { + if capacity == 0 || capacity > Semaphore::MAX_PERMITS { + return None; + } + let (sender, receiver) = mpsc::channel(capacity); + Some((Self { sender }, receiver)) + } + + /// Configured queue capacity, independent of its current occupancy. + pub fn max_capacity(&self) -> usize { + self.sender.max_capacity() + } + + /// Never waits for delivery. Returns the payload on a full or closed queue + /// so the caller can record domain-specific diagnostics. + pub fn try_notify(&self, notification: T) -> Result<(), TrySendError> { + self.sender.try_send(notification) + } +} + +/// Delivers queued notifications with bounded concurrency. +/// +/// The caller schedules this future on its runtime. Closing all senders drains +/// queued and in-flight notifications. Handler failures are handled by the caller; +/// this worker does not retry or propagate them into write completion. +pub async fn run_notifier( + receiver: Receiver, + max_concurrent_notifications: NonZeroUsize, + handler: F, +) where + F: FnMut(T) -> Fut, + Fut: Future, +{ + stream::unfold(receiver, |mut receiver| async move { + receiver.recv().await.map(|item| (item, receiver)) + }) + .for_each_concurrent(max_concurrent_notifications.get(), handler) + .await; +} + +#[cfg(test)] +mod tests { + use std::future::{Future, poll_fn}; + use std::num::NonZeroUsize; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::task::Poll; + + use tokio::sync::Semaphore; + use tokio::sync::mpsc::error::TrySendError; + + use crate::notifier::{Notifier, run_notifier}; + + #[test] + fn test_admission() { + assert!(Notifier::::try_new(0).is_none()); + assert!(Notifier::::try_new(usize::MAX).is_none()); + let (notifier, receiver) = Notifier::try_new(1).unwrap(); + let other = notifier.clone(); + assert_eq!(other.max_capacity(), 1); + assert_eq!(notifier.try_notify(1), Ok(())); + assert_eq!(other.try_notify(2), Err(TrySendError::Full(2))); + drop(receiver); + assert_eq!(other.try_notify(3), Err(TrySendError::Closed(3))); + } + + #[tokio::test] + async fn test_concurrency_and_drain() { + // One-factor comparison: only delivery concurrency changes. + for _ in 0..2 { + for concurrency in [1, 2] { + let (notifier, receiver) = Notifier::try_new(3).unwrap(); + for item in 0..3 { + notifier.try_notify(item).unwrap(); + } + drop(notifier); + let started = Arc::new(AtomicUsize::new(0)); + let completed = Arc::new(AtomicUsize::new(0)); + let gate = Arc::new(Semaphore::new(0)); + let worker = + run_notifier(receiver, NonZeroUsize::new(concurrency).unwrap(), |_| { + let started = started.clone(); + let completed = completed.clone(); + let gate = gate.clone(); + async move { + started.fetch_add(1, Ordering::SeqCst); + gate.acquire().await.unwrap().forget(); + completed.fetch_add(1, Ordering::SeqCst); + } + }); + tokio::pin!(worker); + poll_fn(|cx| { + assert!(worker.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + assert_eq!(started.load(Ordering::SeqCst), concurrency); + assert_eq!(completed.load(Ordering::SeqCst), 0); + gate.add_permits(3); + worker.await; + assert_eq!(started.load(Ordering::SeqCst), 3); + assert_eq!(completed.load(Ordering::SeqCst), 3); + } + } + } +} diff --git a/src/common/batcher/src/pending_batch.rs b/src/common/batcher/src/pending_batch.rs new file mode 100644 index 00000000000..44dc6565fa1 --- /dev/null +++ b/src/common/batcher/src/pending_batch.rs @@ -0,0 +1,115 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use tokio::time::Instant; + +/// Accumulates complete submissions in arrival order, without interpreting their data. +#[derive(Debug)] +pub struct PendingBatch { + items: Vec, + total_rows: usize, + first_submitted_at: Option, +} + +impl Default for PendingBatch { + fn default() -> Self { + Self { + items: Vec::new(), + total_rows: 0, + first_submitted_at: None, + } + } +} + +impl PendingBatch { + /// Creates an empty batch. + pub fn new() -> Self { + Self::default() + } + + /// Appends a complete submission without moving the first submission's timestamp. + pub fn push(&mut self, item: T, total_rows: usize, now: Instant) { + self.first_submitted_at.get_or_insert(now); + self.items.push(item); + // Saturation preserves threshold decisions even for caller-supplied oversized weights. + self.total_rows = self.total_rows.saturating_add(total_rows); + } + + /// Returns the accumulated row count, saturated at `usize::MAX`. + pub fn total_rows(&self) -> usize { + self.total_rows + } + + /// Returns the number of complete submissions. + pub fn len(&self) -> usize { + self.items.len() + } + + /// Returns whether there are no submissions, including zero-row submissions. + pub fn is_empty(&self) -> bool { + self.items.is_empty() + } + + /// Returns the time the first submission was appended to the current batch. + pub fn first_submitted_at(&self) -> Option { + self.first_submitted_at + } + + /// Takes all submissions in arrival order and resets the batch's accounting. + pub fn take(&mut self) -> Vec { + self.total_rows = 0; + self.first_submitted_at = None; + std::mem::take(&mut self.items) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[test] + fn test_complete_submissions_and_reset() { + let mut batch = PendingBatch::new(); + let first = Instant::now(); + assert!(batch.is_empty()); + assert_eq!(batch.first_submitted_at(), None); + batch.push(vec![1, 2], 2, first); + batch.push(vec![3, 4, 5], 3, first + Duration::from_secs(1)); + assert_eq!(batch.len(), 2); + assert_eq!(batch.total_rows(), 5); + assert_eq!(batch.first_submitted_at(), Some(first)); + assert_eq!(batch.take(), vec![vec![1, 2], vec![3, 4, 5]]); + assert!(batch.is_empty()); + assert_eq!(batch.total_rows(), 0); + assert_eq!(batch.first_submitted_at(), None); + let next = first + Duration::from_secs(2); + batch.push(vec![6], 1, next); + assert_eq!(batch.first_submitted_at(), Some(next)); + } + + #[test] + fn test_zero_rows_and_saturating_count() { + let mut batch = PendingBatch::new(); + let now = Instant::now(); + batch.push(1, 0, now); + assert!(!batch.is_empty()); + assert_eq!(batch.total_rows(), 0); + batch.push(2, usize::MAX, now); + batch.push(3, 1, now); + assert_eq!(batch.total_rows(), usize::MAX); + assert_eq!(batch.take(), vec![1, 2, 3]); + } +} diff --git a/src/common/batcher/src/pending_worker.rs b/src/common/batcher/src/pending_worker.rs new file mode 100644 index 00000000000..2adda17a5ee --- /dev/null +++ b/src/common/batcher/src/pending_worker.rs @@ -0,0 +1,260 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use tokio::time::Instant; + +use crate::flush_policy::{FlushPolicy, FlushTrigger}; +use crate::flush_timer::FlushTimer; +use crate::pending_batch::PendingBatch; + +/// Batch state for one grouping key, independent of the caller's event loop. +/// +/// The caller owns channels, idle/shutdown decisions, execution limits, task +/// spawning and business completion. Taking a batch transfers its items to the +/// caller; no background task is started by this worker. +pub struct PendingWorker { + batch: PendingBatch, + flush_policy: P, + flush_timer: FlushTimer, +} + +impl PendingWorker { + /// Creates an empty worker without starting a timer or runtime task. + pub fn new(flush_policy: P) -> Self { + Self { + batch: PendingBatch::new(), + flush_policy, + flush_timer: FlushTimer::new(), + } + } + + /// Returns whether the worker has no submissions, including zero-row items. + pub fn is_empty(&self) -> bool { + self.batch.is_empty() + } + + /// Returns the pending row count before ownership is transferred to the caller. + pub fn total_rows(&self) -> usize { + self.batch.total_rows() + } + + /// Waits for a timer wakeup without taking or executing the batch. + /// + /// Cancellation is safe: selecting another event does not lose items or + /// reset the deadline. After a wakeup, call [`Self::take_ready`] with + /// [`FlushTrigger::Deadline`]. An empty or unarmed worker stays pending. + pub async fn wait_flush(&mut self) { + self.flush_timer.wait().await; + } + + /// Takes all pending submissions regardless of policy, for example on shutdown. + /// + /// Returns `None` for an empty batch. The caller decides whether to execute + /// inline, acquire a flush permit or discard the returned items. + pub fn take_pending(&mut self) -> Option> { + self.flush_timer.set_deadline(None); + if self.batch.is_empty() { + None + } else { + Some(self.batch.take()) + } + } +} + +impl PendingWorker { + /// Appends one complete submission and arms the policy's deadline. + /// + /// Call [`Self::take_ready`] with [`FlushTrigger::Submission`] afterwards to + /// check whether this submission triggered a flush. Timer-backed policies + /// require a Tokio runtime with time enabled. + /// + /// # Panics + /// + /// Panics if arming a new timer outside a Tokio runtime with time enabled. + pub fn submit(&mut self, item: T, total_rows: usize) { + self.batch.push(item, total_rows, Instant::now()); + self.refresh_deadline(); + } + + /// Takes the batch only when the policy permits flushing for this event. + /// + /// Like [`Self::submit`], this may arm a timer and requires a time-enabled + /// Tokio runtime when the policy starts using deadlines. + pub fn take_ready(&mut self, trigger: FlushTrigger) -> Option> { + if !self.batch.is_empty() + && self + .flush_policy + .should_flush(&self.batch, Instant::now(), trigger) + { + self.take_pending() + } else { + self.refresh_deadline(); + None + } + } + + fn refresh_deadline(&mut self) { + let deadline = if self.batch.is_empty() { + None + } else { + self.flush_policy.deadline(&self.batch) + }; + self.flush_timer.set_deadline(deadline); + } +} + +#[cfg(test)] +mod tests { + use std::future::Future; + use std::task::Poll; + use std::time::Duration; + + use super::*; + use crate::flush_policy::timing::TimingFlushPolicy; + + fn worker(rows: usize) -> PendingWorker { + PendingWorker::new(TimingFlushPolicy::try_new(Duration::from_millis(10), rows).unwrap()) + } + + async fn assert_wait_pending(worker: &mut PendingWorker) { + let wait = worker.wait_flush(); + tokio::pin!(wait); + assert!(std::future::poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx).is_pending())).await); + } + + #[tokio::test(start_paused = true)] + async fn test_first_deadline_and_cancelled_wait() { + let mut worker = worker(100); + let start = Instant::now(); + worker.submit(1, 1); + assert_wait_pending(&mut worker).await; + tokio::time::advance(Duration::from_millis(5)).await; + worker.submit(2, 1); + assert!(worker.take_ready(FlushTrigger::Submission).is_none()); + assert_wait_pending(&mut worker).await; + worker.wait_flush().await; + assert_eq!(start + Duration::from_millis(10), Instant::now()); + assert_eq!(Some(vec![1, 2]), worker.take_ready(FlushTrigger::Deadline)); + assert!(worker.is_empty()); + assert_eq!(0, worker.total_rows()); + assert_wait_pending(&mut worker).await; + } + + #[tokio::test(start_paused = true)] + async fn test_submission_does_not_consume_expired_deadline() { + let mut worker = worker(100); + worker.submit(1, 1); + tokio::time::advance(Duration::from_millis(10)).await; + worker.submit(2, 1); + assert!(worker.take_ready(FlushTrigger::Submission).is_none()); + worker.wait_flush().await; + assert_eq!(Some(vec![1, 2]), worker.take_ready(FlushTrigger::Deadline)); + } + + #[tokio::test(start_paused = true)] + async fn test_size_flush_and_rearm() { + let mut worker = worker(2); + worker.submit(1, 3); + assert_eq!(3, worker.total_rows()); + assert_eq!(Some(vec![1]), worker.take_ready(FlushTrigger::Submission)); + tokio::time::advance(Duration::from_secs(1)).await; + assert_wait_pending(&mut worker).await; + let start = Instant::now(); + worker.submit(2, 1); + worker.wait_flush().await; + assert_eq!(start + Duration::from_millis(10), Instant::now()); + assert_eq!(Some(vec![2]), worker.take_ready(FlushTrigger::Deadline)); + } + + #[tokio::test(start_paused = true)] + async fn test_take_pending_without_waiting() { + let mut worker = worker(100); + assert_eq!(None, worker.take_pending()); + worker.submit(1, 0); + assert!(!worker.is_empty()); + let now = Instant::now(); + assert_eq!(Some(vec![1]), worker.take_pending()); + assert_eq!(now, Instant::now()); + tokio::time::advance(Duration::from_secs(1)).await; + assert_wait_pending(&mut worker).await; + } + + #[tokio::test(start_paused = true)] + async fn test_row_threshold_ablation() { + for (rows, early) in [(2, true), (100, false)] { + let mut worker = worker(rows); + let start = Instant::now(); + worker.submit(1, 1); + tokio::time::advance(Duration::from_millis(5)).await; + worker.submit(2, 1); + let batch = worker.take_ready(FlushTrigger::Submission); + if early { + assert_eq!(Some(vec![1, 2]), batch); + assert_eq!(start + Duration::from_millis(5), Instant::now()); + } else { + assert!(batch.is_none()); + worker.wait_flush().await; + assert_eq!(start + Duration::from_millis(10), Instant::now()); + assert_eq!(Some(vec![1, 2]), worker.take_ready(FlushTrigger::Deadline)); + } + } + } + + #[tokio::test] + async fn test_non_timing_policy() { + struct TwoSubmissionsPolicy; + impl FlushPolicy for TwoSubmissionsPolicy { + fn deadline(&self, _: &PendingBatch) -> Option { + None + } + fn should_flush( + &self, + batch: &PendingBatch, + _: Instant, + _: FlushTrigger, + ) -> bool { + batch.len() >= 2 + } + } + let mut worker = PendingWorker::new(TwoSubmissionsPolicy); + worker.submit(1, 0); + assert_wait_pending(&mut worker).await; + assert!(worker.take_ready(FlushTrigger::Submission).is_none()); + worker.submit(2, 0); + assert_eq!( + Some(vec![1, 2]), + worker.take_ready(FlushTrigger::Submission) + ); + } + + #[tokio::test(start_paused = true)] + async fn test_caller_owns_event_loop() { + let mut worker = worker(100); + let (tx, mut rx) = tokio::sync::mpsc::channel(1); + tx.send(1).await.unwrap(); + tokio::select! { + _ = worker.wait_flush() => panic!("empty worker must not wake"), + item = rx.recv() => worker.submit(item.unwrap(), 1), + } + assert!(worker.take_ready(FlushTrigger::Submission).is_none()); + let start = Instant::now(); + tokio::select! { + _ = worker.wait_flush() => { + assert_eq!(Some(vec![1]), worker.take_ready(FlushTrigger::Deadline)); + } + _ = rx.recv() => panic!("sender is still open without another submission"), + } + assert_eq!(start + Duration::from_millis(10), Instant::now()); + } +} diff --git a/src/common/batcher/src/request_limiter.rs b/src/common/batcher/src/request_limiter.rs new file mode 100644 index 00000000000..c488bec02d9 --- /dev/null +++ b/src/common/batcher/src/request_limiter.rs @@ -0,0 +1,115 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use tokio::sync::{AcquireError, OwnedSemaphorePermit, Semaphore}; + +/// Limits unfinished original requests independently of their number of submissions. +/// +/// Acquire once for each original request, then clone its permit into all of that +/// request's submissions. Capacity is released when the last owner drops it. +#[derive(Clone)] +pub struct RequestLimiter { + semaphore: Arc, +} + +impl RequestLimiter { + /// Returns `None` if the limit is zero or exceeds Tokio's supported semaphore capacity. + pub fn try_new(max_inflight_requests: usize) -> Option { + let permits = max_inflight_requests; + ((1..=Semaphore::MAX_PERMITS).contains(&permits)).then(|| Self { + semaphore: Arc::new(Semaphore::new(permits)), + }) + } + + /// Waits for one request slot and returns a permit shared by its submissions. + /// + /// Cancelling a pending acquisition does not consume a slot. Once acquired, + /// callers must retain the permit until every submission has finished, even + /// if the original caller stops waiting for its response. + pub async fn acquire(&self) -> Result, AcquireError> { + self.semaphore.clone().acquire_owned().await.map(Arc::new) + } +} + +#[cfg(test)] +mod tests { + use std::future::{Future, poll_fn}; + use std::pin::Pin; + use std::task::Poll; + + use tokio::sync::Semaphore; + + use crate::request_limiter::RequestLimiter; + + async fn is_pending(mut future: Pin<&mut F>) -> bool { + poll_fn(|cx| Poll::Ready(future.as_mut().poll(cx).is_pending())).await + } + + #[test] + fn test_capacity_boundaries() { + for capacity in [1, Semaphore::MAX_PERMITS] { + assert!(RequestLimiter::try_new(capacity).is_some()); + } + for capacity in [0, Semaphore::MAX_PERMITS + 1, usize::MAX] { + assert!(RequestLimiter::try_new(capacity).is_none()); + } + } + + #[tokio::test] + async fn test_last_submission_releases_request_slot() { + let limiter = RequestLimiter::try_new(1).unwrap(); + let other = limiter.clone(); + let request = limiter.acquire().await.unwrap(); + let first_submission = request.clone(); + let second_submission = request.clone(); + let mut waiting = Box::pin(other.acquire()); + assert!(is_pending(waiting.as_mut()).await); + // The caller and one submission finish, but another submission remains. + drop(request); + drop(first_submission); + assert!(is_pending(waiting.as_mut()).await); + drop(second_submission); + let next = waiting.await.unwrap(); + assert_eq!(next.num_permits(), 1); + } + + #[tokio::test] + async fn test_cancelled_acquisition_does_not_leak_capacity() { + let limiter = RequestLimiter::try_new(1).unwrap(); + let permit = limiter.acquire().await.unwrap(); + let mut cancelled = Box::pin(limiter.acquire()); + assert!(is_pending(cancelled.as_mut()).await); + drop(cancelled); + let mut next = Box::pin(limiter.acquire()); + assert!(is_pending(next.as_mut()).await); + drop(permit); + let next = next.await.unwrap(); + drop(next); + let _available = limiter.acquire().await.unwrap(); + } + + #[tokio::test] + async fn test_independent_requests_consume_separate_slots() { + let limiter = RequestLimiter::try_new(2).unwrap(); + let first = limiter.acquire().await.unwrap(); + let second = limiter.acquire().await.unwrap(); + let mut third = Box::pin(limiter.acquire()); + assert!(is_pending(third.as_mut()).await); + drop(first); + let _third = third.await.unwrap(); + drop(second); + } +} diff --git a/src/common/batcher/src/worker_registry.rs b/src/common/batcher/src/worker_registry.rs new file mode 100644 index 00000000000..f5dbf48e480 --- /dev/null +++ b/src/common/batcher/src/worker_registry.rs @@ -0,0 +1,265 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::hash::Hash; + +use dashmap::DashMap; +use dashmap::mapref::entry::Entry; +use tokio::sync::mpsc::{self, Receiver, Sender}; + +/// Registers worker senders by key, without owning workers or their execution. +/// +/// Closed senders are replaced on lookup/creation. Cleanup checks channel +/// identity under the same lock as replacement, so an old worker cannot remove +/// a replacement registered under its key. +pub struct WorkerRegistry { + workers: DashMap>, +} + +impl Default for WorkerRegistry { + fn default() -> Self { + Self { + workers: DashMap::new(), + } + } +} + +impl WorkerRegistry { + pub fn new() -> Self { + Self::default() + } + + /// Returns a live sender, if one is currently registered. + /// The receiver can close after this method returns; callers must handle a + /// failed send and retry worker lookup without discarding the unsent item. + pub async fn get(&self, key: &K) -> Option> { + self.workers + .get(key) + .filter(|tx| !tx.is_closed()) + .map(|tx| tx.value().clone()) + } + + /// Returns the registered sender and, only when created, its receiver. + /// + /// Start the new worker before the next await so cancellation cannot leave + /// a registered channel without a consumer. Existing channels retain their + /// original capacity. New channel capacity must satisfy `mpsc::channel`. + pub async fn get_or_create(&self, key: K, capacity: usize) -> (Sender, Option>) { + let mut receiver = None; + let sender = self + .get_or_insert_with(key, || { + let (sender, rx) = mpsc::channel(capacity); + receiver = Some(rx); + sender + }) + .await; + (sender, receiver) + } + + /// Reuses a live sender or atomically creates its replacement. + /// + /// `create` runs synchronously under the registry shard lock. It should only + /// prepare the sender and capture any initialization state (such as the + /// receiver) for the caller; start the worker after this method returns. + /// Do not block or reenter the registry from `create`. + pub async fn get_or_insert_with(&self, key: K, create: F) -> Sender + where + F: FnOnce() -> Sender, + { + if let Some(tx) = self.get(&key).await { + return tx; + } + + match self.workers.entry(key) { + Entry::Occupied(mut entry) => { + if entry.get().is_closed() { + entry.insert(create()); + } + entry.get().clone() + } + Entry::Vacant(entry) => entry.insert(create()).value().clone(), + } + } + + /// Removes the key only if it still points to this worker's channel. + pub async fn remove_if_same(&self, key: &K, tx: &Sender) -> bool { + self.workers + .remove_if(key, |_, current| current.same_channel(tx)) + .is_some() + } + + /// Number of registered entries, including senders whose receivers closed. + pub async fn len(&self) -> usize { + self.workers.len() + } + + pub async fn is_empty(&self) -> bool { + self.workers.is_empty() + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use tokio::sync::{Barrier, mpsc}; + + use crate::worker_registry::WorkerRegistry; + + #[tokio::test] + async fn test_live_lookup_uses_read_lock() { + let registry = Arc::new(WorkerRegistry::<_, ()>::new()); + let (sender, _receiver) = registry.get_or_create(1, 1).await; + let guard = registry.workers.get(&1).unwrap(); + let (result_tx, result_rx) = std::sync::mpsc::channel(); + let worker_registry = registry.clone(); + let runtime = tokio::runtime::Handle::current(); + let thread = std::thread::spawn(move || { + let result = runtime.block_on(worker_registry.get_or_create(1, 1)); + let _ = result_tx.send(result); + }); + // Release the read guard before joining, even if lookup needs a write lock. + let result = result_rx.recv_timeout(std::time::Duration::from_secs(5)); + drop(guard); + thread.join().unwrap(); + let (reused, receiver) = result.expect("live lookup must not wait for an exclusive lock"); + assert!(sender.same_channel(&reused)); + assert!(receiver.is_none()); + } + + #[tokio::test] + async fn test_reuses_live_sender_and_returns_receiver_to_caller() { + let registry = WorkerRegistry::new(); + assert!(registry.is_empty().await); + assert!(registry.get(&1).await.is_none()); + let mut receiver = None; + let first = registry + .get_or_insert_with(1, || { + let (tx, rx) = mpsc::channel(1); + receiver = Some(rx); + tx + }) + .await; + let second = registry + .get_or_insert_with(1, || panic!("live worker must be reused")) + .await; + assert!(first.same_channel(&second)); + assert!(registry.get(&1).await.unwrap().same_channel(&first)); + first.send(7).await.unwrap(); + assert_eq!(Some(7), receiver.as_mut().unwrap().recv().await); + assert_eq!(1, registry.len().await); + assert!(registry.remove_if_same(&1, &first).await); + assert!(!registry.remove_if_same(&1, &first).await); + assert!(registry.is_empty().await); + } + + #[tokio::test] + async fn test_old_cleanup_does_not_remove_replacement() { + let registry = WorkerRegistry::<_, ()>::new(); + let (first, mut receiver) = mpsc::channel(1); + registry.get_or_insert_with("table", || first.clone()).await; + receiver.close(); + assert!(registry.get(&"table").await.is_none()); + // Closed entries remain accounted until replacement or explicit removal. + assert_eq!(1, registry.len().await); + let (second, _receiver) = mpsc::channel(1); + registry + .get_or_insert_with("table", || second.clone()) + .await; + assert!(!registry.remove_if_same(&"table", &first).await); + assert!(registry.get(&"table").await.unwrap().same_channel(&second)); + assert_eq!(1, registry.len().await); + assert!(registry.remove_if_same(&"table", &second).await); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_concurrent_lookup_initializes_once() { + let registry = Arc::new(WorkerRegistry::<_, ()>::new()); + let barrier = Arc::new(Barrier::new(3)); + let count = Arc::new(AtomicUsize::new(0)); + let mut tasks = Vec::new(); + for _ in 0..2 { + let registry = registry.clone(); + let barrier = barrier.clone(); + let count = count.clone(); + tasks.push(tokio::spawn(async move { + barrier.wait().await; + let mut receiver = None; + let tx = registry + .get_or_insert_with(1, || { + count.fetch_add(1, Ordering::Relaxed); + let (tx, rx) = mpsc::channel(1); + receiver = Some(rx); + tx + }) + .await; + // Keep the newly created receiver alive while both lookups run. + barrier.wait().await; + (tx, receiver) + })); + } + barrier.wait().await; + barrier.wait().await; + let first = tasks.remove(0).await.unwrap(); + let second = tasks.remove(0).await.unwrap(); + assert!(first.0.same_channel(&second.0)); + assert_eq!(1, count.load(Ordering::Relaxed)); + assert_eq!(1, registry.len().await); + } + #[tokio::test] + async fn test_channel_creation_replacement_and_cleanup() { + let registry = WorkerRegistry::<_, usize>::new(); + let (first, receiver) = registry.get_or_create(1, 2).await; + let mut receiver = receiver.unwrap(); + let (reused, absent) = registry.get_or_create(1, 3).await; + assert!(absent.is_none()); + assert!(first.same_channel(&reused)); + assert_eq!(2, reused.max_capacity()); + first.send(7).await.unwrap(); + assert_eq!(Some(7), receiver.recv().await); + receiver.close(); + let (replacement, receiver) = registry.get_or_create(1, 3).await; + assert!(receiver.is_some()); + assert_eq!(3, replacement.max_capacity()); + assert!(!first.same_channel(&replacement)); + assert!(!registry.remove_if_same(&1, &first).await); + assert!(registry.get(&1).await.unwrap().same_channel(&replacement)); + assert!(registry.remove_if_same(&1, &replacement).await); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_concurrent_channel_creation_returns_one_receiver() { + let registry = Arc::new(WorkerRegistry::<_, ()>::new()); + let barrier = Arc::new(Barrier::new(3)); + let mut tasks = Vec::new(); + for _ in 0..2 { + let registry = registry.clone(); + let barrier = barrier.clone(); + tasks.push(tokio::spawn(async move { + barrier.wait().await; + let channel = registry.get_or_create(1, 2).await; + barrier.wait().await; + channel + })); + } + barrier.wait().await; + barrier.wait().await; + let first = tasks.remove(0).await.unwrap(); + let second = tasks.remove(0).await.unwrap(); + assert!(first.0.same_channel(&second.0)); + assert_ne!(first.1.is_some(), second.1.is_some()); + assert_eq!(1, registry.len().await); + } +} diff --git a/src/common/catalog/src/consts.rs b/src/common/catalog/src/consts.rs index bc1ae5f38f9..dd1b5fc3f03 100644 --- a/src/common/catalog/src/consts.rs +++ b/src/common/catalog/src/consts.rs @@ -122,6 +122,12 @@ pub const INFORMATION_SCHEMA_STATISTICS_TABLE_ID: u32 = 43; pub const INFORMATION_SCHEMA_RECYCLE_BIN_TABLE_ID: u32 = 44; /// id for information_schema.flow_statistics pub const INFORMATION_SCHEMA_FLOW_STATISTICS_TABLE_ID: u32 = 45; +/// id for information_schema.PLUGINS +pub const INFORMATION_SCHEMA_PLUGINS_TABLE_ID: u32 = 47; +/// id for information_schema.USER_PRIVILEGES +pub const INFORMATION_SCHEMA_USER_PRIVILEGES_TABLE_ID: u32 = 48; +/// id for information_schema.PROCESSLIST (for mysql) +pub const INFORMATION_SCHEMA_PROCESSLIST_TABLE_ID: u32 = 49; // ----- End of information_schema tables ----- diff --git a/src/common/datasource/Cargo.toml b/src/common/datasource/Cargo.toml index 8b4053db2fc..d1ca609c606 100644 --- a/src/common/datasource/Cargo.toml +++ b/src/common/datasource/Cargo.toml @@ -34,7 +34,7 @@ futures.workspace = true lazy_static.workspace = true object-store.workspace = true object_store_opendal.workspace = true -orc-rust = { version = "0.8", default-features = false, features = ["async"] } +orc-rust = { version = "0.9", default-features = false, features = ["async"] } parquet.workspace = true paste.workspace = true regex.workspace = true @@ -47,3 +47,4 @@ url.workspace = true [dev-dependencies] common-test-util.workspace = true +object-store = { workspace = true, features = ["testing"] } diff --git a/src/common/datasource/src/error.rs b/src/common/datasource/src/error.rs index dbb0dec417c..1cc04d5a3de 100644 --- a/src/common/datasource/src/error.rs +++ b/src/common/datasource/src/error.rs @@ -27,6 +27,15 @@ use url::ParseError; #[snafu(visibility(pub))] #[stack_trace_debug] pub enum Error { + #[snafu(display("Parquet write cancelled"))] + ParquetWriteCancelled {}, + + #[snafu(display("Parquet writer limits must be positive"))] + InvalidParquetWriterLimits {}, + + #[snafu(display("Parquet writer resource limit exceeded: {reason}"))] + ParquetWriterResource { reason: String }, + #[snafu(display("Unsupported compression type: {}", compression_type))] UnsupportedCompressionType { compression_type: String, @@ -275,6 +284,9 @@ impl ErrorExt for Error { fn status_code(&self) -> StatusCode { use Error::*; match self { + ParquetWriteCancelled {} => StatusCode::Cancelled, + InvalidParquetWriterLimits {} => StatusCode::InvalidArguments, + ParquetWriterResource { .. } => StatusCode::Suspended, BuildBackend { .. } | ListObjects { .. } | ReadObject { .. } diff --git a/src/common/datasource/src/file_format.rs b/src/common/datasource/src/file_format.rs index 61f838899ef..7ae60c615e7 100644 --- a/src/common/datasource/src/file_format.rs +++ b/src/common/datasource/src/file_format.rs @@ -35,7 +35,7 @@ use datafusion::datasource::file_format::file_compression_type::FileCompressionT use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::object_store::ObjectStoreUrl; use datafusion::datasource::physical_plan::{ - FileGroup, FileOpenFuture, FileScanConfigBuilder, FileSource, FileStream, + FileGroup, FileOpenFuture, FileScanConfigBuilder, FileSource, FileStreamBuilder, }; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::physical_plan::SendableRecordBatchStream; @@ -321,7 +321,11 @@ pub async fn file_to_stream( let store = Arc::new(object_store_opendal::OpendalStore::new(store.clone())); let file_opener = config.file_source().create_file_opener(store, &config, 0)?; - let stream = FileStream::new(&config, 0, file_opener, &ExecutionPlanMetricsSet::new())?; + let stream = FileStreamBuilder::new(&config) + .with_partition(0) + .with_file_opener(file_opener) + .with_metrics(&ExecutionPlanMetricsSet::new()) + .build()?; Ok(Box::pin(stream)) } diff --git a/src/common/datasource/src/file_format/parquet.rs b/src/common/datasource/src/file_format/parquet.rs index 43b281077e0..0032de6b31e 100644 --- a/src/common/datasource/src/file_format/parquet.rs +++ b/src/common/datasource/src/file_format/parquet.rs @@ -29,22 +29,17 @@ use datafusion::parquet::file::metadata::{ use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion_datasource::PartitionedFile; -use datatypes::schema::SchemaRef; use futures::StreamExt; use futures::future::BoxFuture; use object_store::{FuturesAsyncReader, ObjectStore}; -use parquet::arrow::AsyncArrowWriter; use parquet::arrow::arrow_reader::ArrowReaderOptions; -use parquet::basic::{Compression, Encoding, ZstdLevel}; -use parquet::file::properties::{WriterProperties, WriterPropertiesBuilder}; -use parquet::schema::types::ColumnPath; use snafu::ResultExt; -use tokio_util::compat::{Compat, FuturesAsyncReadCompatExt, FuturesAsyncWriteCompatExt}; +use tokio_util::compat::{Compat, FuturesAsyncReadCompatExt}; -use crate::DEFAULT_WRITE_BUFFER_SIZE; use crate::buffered_writer::{ArrowWriterCloser, DfRecordBatchEncoder}; -use crate::error::{self, Result, WriteObjectSnafu, WriteParquetSnafu}; +use crate::error::{self, Result}; use crate::file_format::FileFormat; +use crate::parquet_writer::ParquetFileWriter; use crate::share_buffer::SharedBuffer; #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -212,61 +207,23 @@ impl ArrowWriterCloser for ArrowWriter { /// Returns number of rows written. pub async fn stream_to_parquet( mut stream: SendableRecordBatchStream, - schema: datatypes::schema::SchemaRef, store: ObjectStore, path: &str, concurrency: usize, ) -> Result { - let write_props = column_wise_config( - WriterProperties::builder() - .set_compression(Compression::ZSTD(ZstdLevel::default())) - .set_statistics_truncate_length(None) - .set_column_index_truncate_length(None), - schema, - ) - .build(); - let inner_writer = store - .writer_with(path) - .concurrent(concurrency) - .chunk(DEFAULT_WRITE_BUFFER_SIZE.as_bytes() as usize) - .await - .map(|w| w.into_futures_async_write().compat_write()) - .context(WriteObjectSnafu { path })?; - - let mut writer = AsyncArrowWriter::try_new(inner_writer, stream.schema(), Some(write_props)) - .context(WriteParquetSnafu { path })?; + let mut writer = + ParquetFileWriter::open(stream.schema(), store, path, concurrency, None).await?; let mut rows_written = 0; - while let Some(batch) = stream.next().await { let batch = batch.context(error::ReadRecordBatchSnafu)?; - writer - .write(&batch) - .await - .context(WriteParquetSnafu { path })?; - rows_written += batch.num_rows(); + let rows = batch.num_rows(); + writer.write(batch, None).await?; + rows_written += rows; } - writer.close().await.context(WriteParquetSnafu { path })?; + writer.finish(None).await?; Ok(rows_written) } -/// Customizes per-column properties. -fn column_wise_config( - mut props: WriterPropertiesBuilder, - schema: SchemaRef, -) -> WriterPropertiesBuilder { - // Disable dictionary for timestamp column, since for increasing timestamp column, - // the dictionary pages will be larger than data pages. - for col in schema.column_schemas() { - if col.data_type.is_timestamp() { - let path = ColumnPath::new(vec![col.name.clone()]); - props = props - .set_column_dictionary_enabled(path.clone(), false) - .set_column_encoding(path, Encoding::DELTA_BINARY_PACKED) - } - } - props -} - #[cfg(test)] mod tests { use common_test_util::find_workspace_path; diff --git a/src/common/datasource/src/file_format/tests.rs b/src/common/datasource/src/file_format/tests.rs index a925f73d48c..27b86e1b114 100644 --- a/src/common/datasource/src/file_format/tests.rs +++ b/src/common/datasource/src/file_format/tests.rs @@ -19,7 +19,7 @@ use std::{assert_matches, vec}; use common_test_util::find_workspace_path; use datafusion::assert_batches_eq; use datafusion::datasource::physical_plan::{ - CsvSource, FileScanConfig, FileSource, FileStream, JsonSource, ParquetSource, + CsvSource, FileScanConfig, FileSource, FileStreamBuilder, JsonSource, ParquetSource, }; use datafusion::datasource::source::DataSourceExec; use datafusion::execution::context::TaskContext; @@ -50,16 +50,15 @@ impl Test<'_> { .create_file_opener(store, &self.config, 0) .unwrap(); - let result = FileStream::new( - &self.config, - 0, - file_opener, - &ExecutionPlanMetricsSet::new(), - ) - .unwrap() - .map(|b| b.unwrap()) - .collect::>() - .await; + let result = FileStreamBuilder::new(&self.config) + .with_partition(0) + .with_file_opener(file_opener) + .with_metrics(&ExecutionPlanMetricsSet::new()) + .build() + .unwrap() + .map(|b| b.unwrap()) + .collect::>() + .await; assert_batches_eq!(self.expected, &result); } diff --git a/src/common/datasource/src/object_store.rs b/src/common/datasource/src/object_store.rs index 97bf779413e..37dd11aafdb 100644 --- a/src/common/datasource/src/object_store.rs +++ b/src/common/datasource/src/object_store.rs @@ -50,6 +50,22 @@ pub const AZBLOB_SCHEMA: &str = "AZBLOB"; pub struct BuiltBackend { pub object_store: ObjectStore, pub object_path: Option, + local_root: Option, +} + +impl BuiltBackend { + /// Checks an explicit input using the same file-type filter as directory listing. + pub async fn is_file(&self, path: &str) -> object_store::Result { + // Stat first preserves errors for dangling or inaccessible symlink targets. + if self.object_store.stat(path).await?.mode() != object_store::EntryMode::FILE { + return Ok(false); + } + if let Some(root) = &self.local_root { + root.is_file(path).await + } else { + Ok(true) + } + } } /// Controls whether SQL paths may access the local filesystem. @@ -379,6 +395,7 @@ async fn build_backend_inner( return Ok(BuiltBackend { object_store: build_fs_backend(&root)?, object_path, + local_root: Some(root), }); } @@ -418,6 +435,7 @@ async fn build_backend_inner( Ok(BuiltBackend { object_store, object_path, + local_root: None, }) } diff --git a/src/common/datasource/src/parquet_writer.rs b/src/common/datasource/src/parquet_writer.rs index 088d17fb831..ae1fcec04a9 100644 --- a/src/common/datasource/src/parquet_writer.rs +++ b/src/common/datasource/src/parquet_writer.rs @@ -12,11 +12,215 @@ // See the License for the specific language governing permissions and // limitations under the License. +use arrow::datatypes::{DataType, SchemaRef}; +use arrow::record_batch::RecordBatch; use bytes::Bytes; use futures::future::BoxFuture; -use object_store::Writer; +use object_store::{ObjectStore, Writer}; +use parquet::arrow::ArrowWriter; use parquet::arrow::async_writer::AsyncFileWriter; +use parquet::basic::{Compression, Encoding, ZstdLevel}; use parquet::errors::ParquetError; +use parquet::file::properties::WriterProperties; +use parquet::schema::types::ColumnPath; +use snafu::{IntoError, ResultExt, ensure}; +use tokio_util::sync::CancellationToken; + +use crate::DEFAULT_WRITE_BUFFER_SIZE; +use crate::error::{self, Result}; + +/// Limits for one Parquet file. Flush thresholds are not hard memory caps. +#[derive(Clone, Copy, Debug)] +pub struct ParquetWriterLimits { + /// Maximum rows in a row group. + pub row_group_rows: usize, + /// Flush when encoder memory or encoded size reaches this threshold. + pub flush_threshold_bytes: usize, + /// Maximum row groups retained in the file footer. + pub max_row_groups: usize, +} + +/// Encodes batches on a blocking runtime and writes one object-store file. +/// Callers must await each operation before aborting; dropping an in-flight +/// operation can leave a filesystem worker running after cleanup. +pub struct ParquetFileWriter { + encoder: Option>>, + sink: Writer, + store: ObjectStore, + path: String, + limits: Option, +} + +impl ParquetFileWriter { + /// Open a file using COPY's encoding settings. Destination ownership and + /// overwrite policy belong to the caller. None preserves Parquet's defaults. + pub async fn open( + schema: SchemaRef, + store: ObjectStore, + path: &str, + concurrency: usize, + limits: Option, + ) -> Result { + let mut props = WriterProperties::builder() + .set_compression(Compression::ZSTD(ZstdLevel::default())) + .set_statistics_truncate_length(None) + .set_column_index_truncate_length(None); + if let Some(limits) = limits { + ensure!( + limits.row_group_rows > 0 + && limits.flush_threshold_bytes > 0 + && limits.max_row_groups > 0, + error::InvalidParquetWriterLimitsSnafu + ); + props = props + .set_max_row_group_row_count(Some(limits.row_group_rows)) + .set_max_row_group_bytes(None); + } + for field in schema.fields() { + if matches!(field.data_type(), DataType::Timestamp(_, _)) { + let column = ColumnPath::new(vec![field.name().clone()]); + props = props + .set_column_dictionary_enabled(column.clone(), false) + .set_column_encoding(column, Encoding::DELTA_BINARY_PACKED); + } + } + let encoder = ArrowWriter::try_new(Vec::new(), schema, Some(props.build())) + .context(error::WriteParquetSnafu { path })?; + let sink = store + .writer_with(path) + .concurrent(concurrency) + .chunk(DEFAULT_WRITE_BUFFER_SIZE.as_bytes() as usize) + .await + .context(error::WriteObjectSnafu { path })?; + Ok(Self { + encoder: Some(encoder), + sink, + store, + path: path.to_owned(), + limits, + }) + } + + /// Write a batch, enforcing file limits across batch and row-group boundaries. + pub async fn write( + &mut self, + batch: RecordBatch, + cancellation: Option<&CancellationToken>, + ) -> Result<()> { + let mut offset = 0; + while offset < batch.num_rows() { + check_cancelled(cancellation)?; + let mut encoder = self.encoder.take().ok_or_else(|| { + error::WriteParquetSnafu { path: &self.path } + .into_error(ParquetError::General("Parquet writer is closed".into())) + })?; + let len = self.limits.map_or(batch.num_rows() - offset, |limits| { + (limits.row_group_rows - encoder.in_progress_rows()).min(batch.num_rows() - offset) + }); + let slice = batch.slice(offset, len); + let limits = self.limits; + let path = self.path.clone(); + let (encoder, bytes) = common_runtime::spawn_blocking_global(move || { + if let Some(limits) = limits { + ensure!( + encoder.flushed_row_groups().len() < limits.max_row_groups, + error::ParquetWriterResourceSnafu { + reason: "Parquet row-group metadata budget exceeded" + } + ); + } + encoder + .write(&slice) + .context(error::WriteParquetSnafu { path: &path })?; + if limits.is_some_and(|limits| { + encoder.memory_size() >= limits.flush_threshold_bytes + || encoder.in_progress_size() >= limits.flush_threshold_bytes + }) { + encoder + .flush() + .context(error::WriteParquetSnafu { path: &path })?; + } + // Draining preserves ArrowWriter's cumulative file offsets. + let bytes = std::mem::take(encoder.inner_mut()); + Ok::<_, error::Error>((encoder, bytes)) + }) + .await + .context(error::JoinHandleSnafu)??; + self.encoder = Some(encoder); + check_cancelled(cancellation)?; + self.write_bytes(bytes).await?; + check_cancelled(cancellation)?; + offset += len; + } + Ok(()) + } + + async fn write_bytes(&mut self, bytes: Vec) -> Result<()> { + if !bytes.is_empty() { + self.sink + .write(bytes) + .await + .context(error::WriteObjectSnafu { path: &self.path })?; + } + Ok(()) + } + + /// Write the footer and close the file. Retains the handle for cleanup on error. + pub async fn finish(&mut self, cancellation: Option<&CancellationToken>) -> Result<()> { + check_cancelled(cancellation)?; + let mut encoder = self.encoder.take().ok_or_else(|| { + error::WriteParquetSnafu { path: &self.path } + .into_error(ParquetError::General("Parquet writer is closed".into())) + })?; + let path = self.path.clone(); + let bytes = common_runtime::spawn_blocking_global(move || { + encoder + .finish() + .context(error::WriteParquetSnafu { path: &path })?; + Ok::<_, error::Error>(std::mem::take(encoder.inner_mut())) + }) + .await + .context(error::JoinHandleSnafu)??; + self.write_bytes(bytes).await?; + check_cancelled(cancellation)?; + self.sink + .close() + .await + .context(error::WriteObjectSnafu { path: &self.path })?; + check_cancelled(cancellation)?; + Ok(()) + } + + /// Abort an exclusively owned file after all in-flight operations have completed. + /// If the backend cannot abort, deletion assumes this attempt owns the path. + pub async fn abort(mut self) -> Result<()> { + let result = self.sink.abort().await; + if result + .as_ref() + .is_err_and(|e| e.kind() == object_store::ErrorKind::Unsupported) + { + let store = self.store.clone(); + let path = self.path.clone(); + // Secure filesystem writers require dropping the handle before deletion. + drop(self); + store + .delete(&path) + .await + .context(error::WriteObjectSnafu { path })?; + } else { + result.context(error::WriteObjectSnafu { path: &self.path })?; + } + Ok(()) + } +} + +fn check_cancelled(cancellation: Option<&CancellationToken>) -> Result<()> { + ensure!( + cancellation.is_none_or(|token| !token.is_cancelled()), + error::ParquetWriteCancelledSnafu + ); + Ok(()) +} /// Bridges opendal [Writer] with parquet [AsyncFileWriter]. pub struct AsyncWriter { @@ -50,3 +254,244 @@ impl AsyncFileWriter for AsyncWriter { }) } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow::array::{Int64Array, TimestampMillisecondArray}; + use common_error::ext::ErrorExt; + use common_error::status_code::StatusCode; + use datafusion::physical_plan::stream::RecordBatchStreamAdapter; + use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + + use super::*; + use crate::file_format::parquet::stream_to_parquet; + + fn batch() -> RecordBatch { + RecordBatch::try_from_iter([ + ( + "value", + Arc::new(Int64Array::from(vec![Some(1), None, Some(3), Some(4)])) + as arrow::array::ArrayRef, + ), + ( + "ts", + Arc::new(TimestampMillisecondArray::from(vec![1, 2, 3, 4])), + ), + ]) + .unwrap() + } + + async fn read(store: &ObjectStore, path: &str) -> ParquetRecordBatchReaderBuilder { + ParquetRecordBatchReaderBuilder::try_new(store.read(path).await.unwrap().to_bytes()) + .unwrap() + } + + #[tokio::test] + async fn row_and_byte_limits_split_batches() { + let store = ObjectStore::new(object_store::services::Memory::default()).unwrap(); + let batch = batch(); + for (row_group_rows, flush_threshold_bytes, split) in [(2, usize::MAX, 3), (100, 1, 2)] { + let mut writer = ParquetFileWriter::open( + batch.schema(), + store.clone(), + "groups.parquet", + 1, + Some(ParquetWriterLimits { + row_group_rows, + flush_threshold_bytes, + max_row_groups: 2, + }), + ) + .await + .unwrap(); + writer.write(batch.slice(0, split), None).await.unwrap(); + writer + .write(batch.slice(split, batch.num_rows() - split), None) + .await + .unwrap(); + writer.finish(None).await.unwrap(); + let reader = read(&store, "groups.parquet").await; + assert_eq!(reader.metadata().num_row_groups(), 2); + for group in reader.metadata().row_groups() { + assert_eq!(group.num_rows(), 2); + let encodings = group.column(1).encodings().collect::>(); + assert!(encodings.contains(&Encoding::DELTA_BINARY_PACKED)); + assert!(!encodings.contains(&Encoding::RLE_DICTIONARY)); + assert_eq!( + group.column(0).compression(), + Compression::ZSTD(ZstdLevel::default()) + ); + } + let actual = reader + .build() + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!( + arrow::compute::concat_batches(&batch.schema(), &actual).unwrap(), + batch + ); + } + } + + #[tokio::test] + async fn oversized_batch_stops_at_footer_limit_and_can_abort() { + let store = ObjectStore::new(object_store::services::Memory::default()).unwrap(); + let batch = batch(); + let mut writer = ParquetFileWriter::open( + batch.schema(), + store.clone(), + "limited.parquet", + 1, + Some(ParquetWriterLimits { + row_group_rows: 2, + flush_threshold_bytes: usize::MAX, + max_row_groups: 1, + }), + ) + .await + .unwrap(); + let err = writer.write(batch, None).await.unwrap_err(); + assert!(matches!(err, error::Error::ParquetWriterResource { .. })); + assert_eq!(err.status_code(), StatusCode::Suspended); + writer.abort().await.unwrap(); + assert!(!store.exists("limited.parquet").await.unwrap()); + } + + #[tokio::test] + async fn copy_stream_preserves_values_and_empty_schema() { + let store = ObjectStore::new(object_store::services::Memory::default()).unwrap(); + let batch = batch(); + for (path, batches) in [ + ("copy.parquet", vec![batch.clone()]), + ("empty.parquet", vec![]), + ] { + let stream = RecordBatchStreamAdapter::new( + batch.schema(), + futures::stream::iter(batches.clone().into_iter().map(Ok)), + ); + assert_eq!( + stream_to_parquet(Box::pin(stream), store.clone(), path, 2) + .await + .unwrap(), + batches.iter().map(RecordBatch::num_rows).sum::() + ); + let reader = read(&store, path).await; + assert_eq!(reader.schema().fields(), batch.schema().fields()); + let actual = reader + .build() + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(actual, batches); + } + } + + #[tokio::test] + async fn failed_copy_preserves_untouched_securefs_destination() { + let directory = common_test_util::temp_dir::create_temp_dir("copy_existing"); + let path = directory.path().join("existing.parquet"); + std::fs::write(&path, b"original bytes").unwrap(); + let access = crate::object_store::LocalFileAccess::sandboxed(directory.path()).unwrap(); + let store = crate::object_store::build_backend_for_write( + &format!("{}/", directory.path().display()), + &Default::default(), + &access, + ) + .await + .unwrap(); + let stream = RecordBatchStreamAdapter::new( + batch().schema(), + futures::stream::iter(vec![Err(datafusion::error::DataFusionError::Execution( + "injected input failure".into(), + ))]), + ); + let err = stream_to_parquet(Box::pin(stream), store, "existing.parquet", 1) + .await + .unwrap_err(); + assert!(matches!(err, error::Error::ReadRecordBatch { .. })); + assert_eq!(std::fs::read(path).unwrap(), b"original bytes"); + } + + struct PausedFooterWriter { + inner: object_store::layers::mock::oio::Writer, + paused: bool, + started: Arc, + release: Arc, + closed: Arc, + } + + impl object_store::layers::mock::oio::Write for PausedFooterWriter { + async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> { + if !self.paused { + self.paused = true; + self.started.notify_one(); + self.release.notified().await; + } + self.inner.write(bytes).await + } + async fn close(&mut self) -> object_store::Result { + self.closed.store(true, std::sync::atomic::Ordering::SeqCst); + self.inner.close().await + } + async fn abort(&mut self) -> object_store::Result<()> { + self.inner.abort().await + } + } + + #[tokio::test] + async fn cancellation_during_footer_write_waits_then_aborts_before_close() { + use object_store::layers::mock::{MockLayerBuilder, MockWriterFactory}; + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let closed = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let factory: MockWriterFactory = Arc::new({ + let (started, release, closed) = (started.clone(), release.clone(), closed.clone()); + move |_, _, inner| { + Box::new(PausedFooterWriter { + inner, + paused: false, + started: started.clone(), + release: release.clone(), + closed: closed.clone(), + }) + } + }); + let store = ObjectStore::new(object_store::services::Memory::default()) + .unwrap() + .layer( + MockLayerBuilder::default() + .writer_factory(factory) + .build() + .unwrap(), + ); + let mut writer = + ParquetFileWriter::open(batch().schema(), store.clone(), "cancel.parquet", 1, None) + .await + .unwrap(); + // Force footer bytes through the sink before its close operation. + writer.sink = store.writer_with("cancel.parquet").chunk(1).await.unwrap(); + let cancellation = CancellationToken::new(); + let result = { + let finish = writer.finish(Some(&cancellation)); + tokio::pin!(finish); + tokio::select! { + result = &mut finish => panic!("finished before footer write: {result:?}"), + _ = started.notified() => {}, + } + cancellation.cancel(); + assert!(futures::poll!(&mut finish).is_pending()); + release.notify_one(); + finish.await + }; + assert!(matches!( + result, + Err(error::Error::ParquetWriteCancelled {}) + )); + assert!(!closed.load(std::sync::atomic::Ordering::SeqCst)); + writer.abort().await.unwrap(); + assert!(!store.exists("cancel.parquet").await.unwrap()); + } +} diff --git a/src/common/datasource/src/test_util.rs b/src/common/datasource/src/test_util.rs index 9dd93a4ee45..4cefda88890 100644 --- a/src/common/datasource/src/test_util.rs +++ b/src/common/datasource/src/test_util.rs @@ -20,7 +20,7 @@ use datafusion::datasource::file_format::file_compression_type::FileCompressionT use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::object_store::ObjectStoreUrl; use datafusion::datasource::physical_plan::{ - CsvSource, FileGroup, FileScanConfig, FileScanConfigBuilder, FileSource, FileStream, + CsvSource, FileGroup, FileScanConfig, FileScanConfigBuilder, FileSource, FileStreamBuilder, JsonOpener, JsonSource, }; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; @@ -110,13 +110,12 @@ pub async fn setup_stream_to_json_test(origin_path: &str, threshold: impl Fn(usi let size = store.read(origin_path).await.unwrap().len(); let config = scan_config(None, origin_path, Arc::new(JsonSource::new(schema))); - let stream = FileStream::new( - &config, - 0, - Arc::new(json_opener), - &ExecutionPlanMetricsSet::new(), - ) - .unwrap(); + let stream = FileStreamBuilder::new(&config) + .with_partition(0) + .with_file_opener(Arc::new(json_opener)) + .with_metrics(&ExecutionPlanMetricsSet::new()) + .build() + .unwrap(); let (tmp_store, dir) = test_tmp_store("test_stream_to_json"); @@ -162,7 +161,12 @@ pub async fn setup_stream_to_csv_test( 0, ) .unwrap(); - let stream = FileStream::new(&config, 0, csv_opener, &ExecutionPlanMetricsSet::new()).unwrap(); + let stream = FileStreamBuilder::new(&config) + .with_partition(0) + .with_file_opener(csv_opener) + .with_metrics(&ExecutionPlanMetricsSet::new()) + .build() + .unwrap(); let (tmp_store, dir) = test_tmp_store("test_stream_to_csv"); diff --git a/src/common/function/src/aggrs/aggr_wrapper.rs b/src/common/function/src/aggrs/aggr_wrapper.rs index c2e8b567caa..70651eee41d 100644 --- a/src/common/function/src/aggrs/aggr_wrapper.rs +++ b/src/common/function/src/aggrs/aggr_wrapper.rs @@ -33,15 +33,15 @@ use datafusion::functions_aggregate::count::Count; use datafusion::functions_aggregate::min_max::{Max, Min}; use datafusion::optimizer::AnalyzerRule; use datafusion::optimizer::analyzer::type_coercion::TypeCoercion; -use datafusion::physical_planner::create_aggregate_expr_and_maybe_filter; use datafusion_common::{Column, ScalarValue}; use datafusion_expr::expr::{AggregateFunction, AggregateFunctionParams}; use datafusion_expr::function::StateFieldsArgs; +use datafusion_expr::physical_planning_context::PhysicalPlanningContext; use datafusion_expr::{ Accumulator, Aggregate, AggregateUDF, AggregateUDFImpl, EmitTo, Expr, ExprSchemable, GroupsAccumulator, LogicalPlan, Signature, }; -use datafusion_physical_expr::aggregate::AggregateFunctionExpr; +use datafusion_physical_expr::aggregate::{AggregateFunctionExpr, LoweredAggregateBuilder}; use datatypes::arrow::datatypes::{DataType, Field}; use crate::aggrs::aggr_wrapper::fix_order::FixStateUdafOrderingAnalyzer; @@ -210,12 +210,25 @@ impl StateMergeHelper { lower_aggr_exprs.push(expr); // then create the merge function using the physical expression of the original aggregate function - let (original_phy_expr, _filter, _ordering) = create_aggregate_expr_and_maybe_filter( + let (name, human_display) = match aggr_expr { + Expr::Alias(alias) => (alias.name.clone(), aggr_expr.human_display().to_string()), + Expr::AggregateFunction(_) => ( + aggr_expr.schema_name().to_string(), + aggr_expr.human_display().to_string(), + ), + _ => unreachable!("aggregate expression was validated above"), + }; + let original_phy_expr = LoweredAggregateBuilder::new( aggr_expr, aggr.input.schema(), aggr.input.schema().as_arrow(), &Default::default(), - )?; + &PhysicalPlanningContext::default(), + ) + .with_name(name) + .with_human_display(human_display) + .build()? + .aggregate; let merge_func = MergeWrapper::new( (*aggr_func.func).clone(), @@ -371,9 +384,6 @@ impl AggregateUDFImpl for StateWrapper { Ok(Box::new(StateGroupsAccum::new(inner, state_type)?)) } - fn as_any(&self) -> &dyn std::any::Any { - self - } fn name(&self) -> &str { self.name.as_str() } @@ -443,7 +453,7 @@ impl AggregateUDFImpl for StateWrapper { &self, statistics_args: &datafusion_expr::StatisticsArgs, ) -> Option { - let inner = self.inner().inner().as_any(); + let inner = self.inner().inner(); // only count/min/max need special handling here, for getting result from statistics // the result of count/min/max is also the result of count_state so can return directly let can_use_stat = inner.is::() || inner.is::() || inner.is::(); @@ -572,11 +582,10 @@ impl GroupsAccumulator for StateGroupsAccum { &mut self, values: &[ArrayRef], group_indices: &[usize], - opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> datafusion_common::Result<()> { self.inner - .merge_batch(values, group_indices, opt_filter, total_num_groups) + .merge_batch(values, group_indices, total_num_groups) } fn evaluate(&mut self, emit_to: EmitTo) -> datafusion_common::Result { @@ -596,10 +605,6 @@ impl GroupsAccumulator for StateGroupsAccum { self.inner.convert_to_state(values, opt_filter) } - fn supports_convert_to_state(&self) -> bool { - self.inner.supports_convert_to_state() - } - fn size(&self) -> usize { self.inner.size() } @@ -800,10 +805,6 @@ impl AggregateUDFImpl for DeltaMergeWrapper { })) } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn name(&self) -> &str { &self.name } @@ -973,9 +974,6 @@ impl AggregateUDFImpl for MergeWrapper { Ok(Box::new(MergeAccum::new(inner_accum, &fields))) } - fn as_any(&self) -> &dyn std::any::Any { - self - } fn name(&self) -> &str { self.name.as_str() } diff --git a/src/common/function/src/aggrs/aggr_wrapper/fix_order.rs b/src/common/function/src/aggrs/aggr_wrapper/fix_order.rs index 35aa35b5837..480b0b5a490 100644 --- a/src/common/function/src/aggrs/aggr_wrapper/fix_order.rs +++ b/src/common/function/src/aggrs/aggr_wrapper/fix_order.rs @@ -144,7 +144,6 @@ fn rewrite_expr( let Some(old_state_wrapper) = aggregate_function .func .inner() - .as_any() .downcast_ref::() else { return Ok(Transformed::no(Expr::AggregateFunction(aggregate_function))); diff --git a/src/common/function/src/aggrs/aggr_wrapper/tests.rs b/src/common/function/src/aggrs/aggr_wrapper/tests.rs index 3f59139f05b..4274a03cd89 100644 --- a/src/common/function/src/aggrs/aggr_wrapper/tests.rs +++ b/src/common/function/src/aggrs/aggr_wrapper/tests.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::pin::Pin; use std::sync::{Arc, Mutex}; use std::task::{Context, Poll}; @@ -25,6 +24,7 @@ use arrow::record_batch::RecordBatch; use arrow_schema::SchemaRef; use common_telemetry::init_default_ut_logging; use datafusion::catalog::{Session, TableProvider}; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::datasource::DefaultTableSource; use datafusion::execution::{RecordBatchStream, SendableRecordBatchStream, TaskContext}; use datafusion::functions_aggregate::average::avg_udaf; @@ -39,16 +39,16 @@ use datafusion::physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner}; use datafusion::prelude::SessionContext; use datafusion_common::arrow::array::AsArray; use datafusion_common::arrow::datatypes::{Float64Type, UInt64Type}; -use datafusion_common::{Column, TableReference}; +use datafusion_common::{Column, Result, TableReference}; use datafusion_expr::expr::{AggregateFunction, NullTreatment}; use datafusion_expr::function::AccumulatorArgs; use datafusion_expr::{ Aggregate, AggregateUDFImpl, ColumnarValue, Expr, LogicalPlan, ScalarFunctionArgs, SortExpr, - TableScan, TypeSignature, lit, + TableScanBuilder, TypeSignature, lit, }; use datafusion_physical_expr::aggregate::AggregateExprBuilder; use datafusion_physical_expr::expressions::{Column as PhysicalColumn, col, lit as physical_lit}; -use datafusion_physical_expr::{EquivalenceProperties, Partitioning}; +use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr}; use futures::{Stream, StreamExt as _}; use hyperloglogplus::HyperLogLog; use pretty_assertions::assert_eq; @@ -97,10 +97,6 @@ impl ExecutionPlan for MockInputExec { "MockInputExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.properties } @@ -109,6 +105,13 @@ impl ExecutionPlan for MockInputExec { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, _children: Vec>, @@ -203,10 +206,6 @@ impl Default for DummyTableProvider { #[async_trait::async_trait] impl TableProvider for DummyTableProvider { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> Arc { self.schema.clone() } @@ -237,14 +236,12 @@ fn dummy_table_scan() -> LogicalPlan { let table_provider = Arc::new(DummyTableProvider::default()); let table_source = DefaultTableSource::new(table_provider); LogicalPlan::TableScan( - TableScan::try_new( - TableReference::bare("Number"), - Arc::new(table_source), - None, - vec![], - None, - ) - .unwrap(), + TableScanBuilder::new(TableReference::bare("Number"), Arc::new(table_source)) + .with_projection(None) + .with_filters(vec![]) + .with_fetch(None) + .build() + .unwrap(), ) } @@ -252,14 +249,12 @@ fn dummy_table_scan_with_ts() -> LogicalPlan { let table_provider = Arc::new(DummyTableProvider::with_ts(None)); let table_source = DefaultTableSource::new(table_provider); LogicalPlan::TableScan( - TableScan::try_new( - TableReference::bare("Number"), - Arc::new(table_source), - None, - vec![], - None, - ) - .unwrap(), + TableScanBuilder::new(TableReference::bare("Number"), Arc::new(table_source)) + .with_projection(None) + .with_filters(vec![]) + .with_fetch(None) + .build() + .unwrap(), ) } @@ -381,10 +376,7 @@ async fn test_sum_udaf() { .create_physical_plan(&res.lower_state, &ctx.state()) .await .unwrap(); - let aggr_exec = phy_aggr_state_plan - .as_any() - .downcast_ref::() - .unwrap(); + let aggr_exec = phy_aggr_state_plan.downcast_ref::().unwrap(); let aggr_func_expr = &aggr_exec.aggr_expr()[0]; let mut state_accum = aggr_func_expr.create_accumulator().unwrap(); @@ -414,10 +406,7 @@ async fn test_sum_udaf() { .create_physical_plan(&res.upper_merge, &ctx.state()) .await .unwrap(); - let aggr_exec = phy_aggr_merge_plan - .as_any() - .downcast_ref::() - .unwrap(); + let aggr_exec = phy_aggr_merge_plan.downcast_ref::().unwrap(); let aggr_func_expr = &aggr_exec.aggr_expr()[0]; let mut merge_accum = aggr_func_expr.create_accumulator().unwrap(); @@ -543,10 +532,7 @@ async fn test_avg_udaf() { .create_physical_plan(&coerced_aggr_state_plan, &ctx.state()) .await .unwrap(); - let aggr_exec = phy_aggr_state_plan - .as_any() - .downcast_ref::() - .unwrap(); + let aggr_exec = phy_aggr_state_plan.downcast_ref::().unwrap(); let aggr_func_expr = &aggr_exec.aggr_expr()[0]; let mut state_accum = aggr_func_expr.create_accumulator().unwrap(); @@ -582,10 +568,7 @@ async fn test_avg_udaf() { .create_physical_plan(&res.upper_merge, &ctx.state()) .await .unwrap(); - let aggr_exec = phy_aggr_merge_plan - .as_any() - .downcast_ref::() - .unwrap(); + let aggr_exec = phy_aggr_merge_plan.downcast_ref::().unwrap(); let aggr_func_expr = &aggr_exec.aggr_expr()[0]; let mut merge_accum = aggr_func_expr.create_accumulator().unwrap(); @@ -703,10 +686,7 @@ async fn test_last_value_order_by_udaf() { .create_physical_plan(&fixed_aggr_state_plan, &ctx.state()) .await .unwrap(); - let aggr_exec = phy_aggr_state_plan - .as_any() - .downcast_ref::() - .unwrap(); + let aggr_exec = phy_aggr_state_plan.downcast_ref::().unwrap(); let aggr_func_expr = &aggr_exec.aggr_expr()[0]; let merge_input_fields = vec![Arc::new(Field::new( @@ -796,10 +776,7 @@ async fn test_last_value_order_by_udaf() { .create_physical_plan(&res.upper_merge, &ctx.state()) .await .unwrap(); - let aggr_exec = phy_aggr_merge_plan - .as_any() - .downcast_ref::() - .unwrap(); + let aggr_exec = phy_aggr_merge_plan.downcast_ref::().unwrap(); let aggr_func_expr = &aggr_exec.aggr_expr()[0]; let mut merge_accum = aggr_func_expr.create_accumulator().unwrap(); @@ -900,7 +877,7 @@ fn test_avg_state_groups_accumulator_state_merge_evaluate() { .update_batch(&merged_values, &merged_group_indices, None, 3) .unwrap(); merged_accum - .merge_batch(&source_state, &[1, 2, 0], None, 3) + .merge_batch(&source_state, &[1, 2, 0], 3) .unwrap(); let result = merged_accum.evaluate(EmitTo::All).unwrap(); @@ -1324,14 +1301,12 @@ async fn test_udaf_correct_eval_result() { ); let table_source = DefaultTableSource::new(Arc::new(table_provider)); let logical_plan = LogicalPlan::TableScan( - TableScan::try_new( - test_table_ref.clone(), - Arc::new(table_source), - None, - vec![], - None, - ) - .unwrap(), + TableScanBuilder::new(test_table_ref.clone(), Arc::new(table_source)) + .with_projection(None) + .with_filters(vec![]) + .with_fetch(None) + .build() + .unwrap(), ); let args = case.args; diff --git a/src/common/function/src/aggrs/approximate/uddsketch.rs b/src/common/function/src/aggrs/approximate/uddsketch.rs index f7d1558d13c..6af8211615b 100644 --- a/src/common/function/src/aggrs/approximate/uddsketch.rs +++ b/src/common/function/src/aggrs/approximate/uddsketch.rs @@ -123,7 +123,6 @@ impl UddSketchState { fn downcast_accumulator_args(args: AccumulatorArgs) -> DfResult<(u32, f64)> { let bucket_size = match args.exprs[0] - .as_any() .downcast_ref::() .map(|lit| lit.value()) { @@ -140,7 +139,6 @@ fn downcast_accumulator_args(args: AccumulatorArgs) -> DfResult<(u32, f64)> { }; let error_rate = match args.exprs[1] - .as_any() .downcast_ref::() .map(|lit| lit.value()) { diff --git a/src/common/function/src/aggrs/count_hash.rs b/src/common/function/src/aggrs/count_hash.rs index bd00aeb0123..28ed854638e 100644 --- a/src/common/function/src/aggrs/count_hash.rs +++ b/src/common/function/src/aggrs/count_hash.rs @@ -26,7 +26,7 @@ use std::sync::Arc; use ahash::RandomState; use datafusion_common::cast::as_list_array; use datafusion_common::error::Result; -use datafusion_common::hash_utils::create_hashes; +use datafusion_common::hash_utils::create_hashes_with_hasher; use datafusion_common::utils::SingleRowListArrayBuilder; use datafusion_common::{ScalarValue, internal_err, not_impl_err}; use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs}; @@ -74,10 +74,6 @@ pub struct CountHash { } impl AggregateUDFImpl for CountHash { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn name(&self) -> &str { "count_hash" } @@ -208,7 +204,7 @@ impl GroupsAccumulator for CountHashGroupAccumulator { let array = &values[0]; self.batch_hashes.clear(); self.batch_hashes.resize(array.len(), 0); - let hashes = create_hashes( + let hashes = create_hashes_with_hasher( &[ArrayRef::clone(array)], &self.random_state, &mut self.batch_hashes, @@ -279,7 +275,6 @@ impl GroupsAccumulator for CountHashGroupAccumulator { &mut self, values: &[ArrayRef], group_indices: &[usize], - _opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> Result<()> { assert_eq!( @@ -364,10 +359,6 @@ impl GroupsAccumulator for CountHashGroupAccumulator { Ok(vec![Arc::new(list_array)]) } - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { // Base size of the struct let mut size = size_of::(); @@ -423,7 +414,7 @@ impl Accumulator for CountHashAccumulator { self.batch_hashes.clear(); self.batch_hashes.resize(arr.len(), 0); - let hashes = create_hashes( + let hashes = create_hashes_with_hasher( &[ArrayRef::clone(arr)], &self.random_state, &mut self.batch_hashes, @@ -522,6 +513,28 @@ mod tests { Ok(()) } + #[test] + fn test_count_hash_accumulator_typed_null_state_merge() -> Result<()> { + let typed_nulls = Arc::new(Int32Array::from(vec![None, None])) as ArrayRef; + + let mut fresh = create_test_accumulator(); + fresh.update_batch(&[typed_nulls])?; + let fresh_state = fresh.state()?; + assert_eq!(fresh.evaluate()?, ScalarValue::Int64(Some(1))); + + let persisted_state = Arc::new( + SingleRowListArrayBuilder::new(Arc::new(UInt64Array::from(vec![0])) as ArrayRef) + .build_list_array(), + ) as ArrayRef; + let mut restored = create_test_accumulator(); + restored.merge_batch(&[persisted_state])?; + + assert_eq!(restored.evaluate()?, fresh.evaluate()?); + assert_eq!(restored.state()?, fresh_state); + + Ok(()) + } + #[test] fn test_count_hash_accumulator_merge() -> Result<()> { // Accumulator 1 @@ -622,7 +635,7 @@ mod tests { // We will merge acc1's group 0 into acc2's group 0 // and acc1's group 1 into acc2's group 2 let merge_group_indices = vec![0, 2]; - acc2.merge_batch(&state1, &merge_group_indices, None, 3)?; + acc2.merge_batch(&state1, &merge_group_indices, 3)?; let result_array = acc2.evaluate(EmitTo::All)?; let result = result_array.as_any().downcast_ref::().unwrap(); diff --git a/src/common/function/src/aggrs/vector/avg.rs b/src/common/function/src/aggrs/vector/avg.rs index ddf1823d283..822a7c16efe 100644 --- a/src/common/function/src/aggrs/vector/avg.rs +++ b/src/common/function/src/aggrs/vector/avg.rs @@ -183,8 +183,6 @@ mod tests { use std::sync::Arc; use arrow::array::StringArray; - use datatypes::scalars::ScalarVector; - use datatypes::vectors::{ConstantVector, StringVector, Vector}; use super::*; @@ -252,14 +250,12 @@ mod tests { vec_avg.evaluate().unwrap() ); - // test update with constant vector + // test update with repeated values let mut vec_avg = VectorAvg::default(); - let v: Vec = vec![ - Arc::new(ConstantVector::new( - Arc::new(StringVector::from_vec(vec!["[1.0,2.0,3.0]".to_string()])), - 4, - )) - .to_arrow_array(), + let v = vec![ + ScalarValue::Utf8(Some("[1.0,2.0,3.0]".to_string())) + .to_array_of_size(4) + .unwrap(), ]; vec_avg.update_batch(&v).unwrap(); assert_eq!( diff --git a/src/common/function/src/aggrs/vector/product.rs b/src/common/function/src/aggrs/vector/product.rs index d8f201966ab..11cfa4cb1a8 100644 --- a/src/common/function/src/aggrs/vector/product.rs +++ b/src/common/function/src/aggrs/vector/product.rs @@ -159,9 +159,6 @@ impl Accumulator for VectorProduct { mod tests { use std::sync::Arc; - use datatypes::scalars::ScalarVector; - use datatypes::vectors::{ConstantVector, StringVector, Vector}; - use super::*; #[test] @@ -213,14 +210,12 @@ mod tests { vec_product.update_batch(&v).unwrap(); assert_eq!(ScalarValue::Binary(None), vec_product.evaluate().unwrap()); - // test update with constant vector + // test update with repeated values let mut vec_product = VectorProduct::default(); - let v: Vec = vec![ - Arc::new(ConstantVector::new( - Arc::new(StringVector::from_vec(vec!["[1.0,2.0,3.0]".to_string()])), - 4, - )) - .to_arrow_array(), + let v = vec![ + ScalarValue::Utf8(Some("[1.0,2.0,3.0]".to_string())) + .to_array_of_size(4) + .unwrap(), ]; vec_product.update_batch(&v).unwrap(); diff --git a/src/common/function/src/aggrs/vector/sum.rs b/src/common/function/src/aggrs/vector/sum.rs index 3f875a8412c..0ee8f0df7bf 100644 --- a/src/common/function/src/aggrs/vector/sum.rs +++ b/src/common/function/src/aggrs/vector/sum.rs @@ -171,8 +171,6 @@ mod tests { use std::sync::Arc; use arrow::array::StringArray; - use datatypes::scalars::ScalarVector; - use datatypes::vectors::{ConstantVector, StringVector, Vector}; use super::*; @@ -225,14 +223,12 @@ mod tests { vec_sum.update_batch(&v).unwrap(); assert_eq!(ScalarValue::Binary(None), vec_sum.evaluate().unwrap()); - // test update with constant vector + // test update with repeated values let mut vec_sum = VectorSum::default(); - let v: Vec = vec![ - Arc::new(ConstantVector::new( - Arc::new(StringVector::from_vec(vec!["[1.0,2.0,3.0]".to_string()])), - 4, - )) - .to_arrow_array(), + let v = vec![ + ScalarValue::Utf8(Some("[1.0,2.0,3.0]".to_string())) + .to_array_of_size(4) + .unwrap(), ]; vec_sum.update_batch(&v).unwrap(); assert_eq!( diff --git a/src/common/function/src/helper.rs b/src/common/function/src/helper.rs index 6e643443ece..bbd589d407a 100644 --- a/src/common/function/src/helper.rs +++ b/src/common/function/src/helper.rs @@ -23,6 +23,33 @@ use datatypes::types::cast::cast; use datatypes::value::ValueRef; use snafu::{OptionExt, ResultExt}; +/// Integer types accepted by geospatial function signatures. +pub(crate) const INTEGER_TYPES: &[DataType] = &[ + DataType::Int8, + DataType::Int16, + DataType::Int32, + DataType::Int64, + DataType::UInt8, + DataType::UInt16, + DataType::UInt32, + DataType::UInt64, +]; + +/// Legacy primitive numeric signature types; Decimal values are coerced to `Float64`. +pub(crate) const NUMERICS: &[DataType] = &[ + DataType::Int8, + DataType::Int16, + DataType::Int32, + DataType::Int64, + DataType::UInt8, + DataType::UInt16, + DataType::UInt32, + DataType::UInt64, + DataType::Float16, + DataType::Float32, + DataType::Float64, +]; + /// Create a function signature with oneof signatures of interleaving two arguments. pub(crate) fn one_of_sigs2(args1: Vec, args2: Vec) -> Signature { let mut sigs = Vec::with_capacity(args1.len() * args2.len()); diff --git a/src/common/function/src/scalars/anomaly/iqr.rs b/src/common/function/src/scalars/anomaly/iqr.rs index cf25166de2b..423adc8373a 100644 --- a/src/common/function/src/scalars/anomaly/iqr.rs +++ b/src/common/function/src/scalars/anomaly/iqr.rs @@ -24,7 +24,6 @@ //! When IQR = 0 (constant quartiles), returns 0.0 if value is on the fence, //! or +inf if value is outside. -use std::any::Any; use std::fmt::Debug; use std::ops::Range; use std::sync::Arc; @@ -32,11 +31,11 @@ use std::sync::Arc; use arrow::array::{Array, ArrayRef, Float64Array}; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::{DataFusionError, Result, ScalarValue}; -use datafusion_expr::type_coercion::aggregates::NUMERICS; use datafusion_expr::{PartitionEvaluator, Signature, Volatility, WindowUDFImpl}; use datafusion_functions_window_common::field::WindowUDFFieldArgs; use datafusion_functions_window_common::partition::PartitionEvaluatorArgs; +use crate::helper::NUMERICS; use crate::scalars::anomaly::utils::{cast_to_f64, collect_window_values, percentile_sorted}; /// Minimum valid samples for IQR (linear-interpolated Q1 != Q3 is possible at n >= 3). @@ -56,10 +55,6 @@ impl AnomalyScoreIqr { } impl WindowUDFImpl for AnomalyScoreIqr { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "anomaly_score_iqr" } diff --git a/src/common/function/src/scalars/anomaly/mad.rs b/src/common/function/src/scalars/anomaly/mad.rs index 8bf97df0dc7..9f686879471 100644 --- a/src/common/function/src/scalars/anomaly/mad.rs +++ b/src/common/function/src/scalars/anomaly/mad.rs @@ -20,7 +20,6 @@ //! When MAD = 0 (majority-constant window), returns 0.0 if value equals //! median, or +inf otherwise. -use std::any::Any; use std::fmt::Debug; use std::ops::Range; use std::sync::Arc; @@ -28,11 +27,11 @@ use std::sync::Arc; use arrow::array::{Array, ArrayRef, Float64Array}; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::{DataFusionError, Result, ScalarValue}; -use datafusion_expr::type_coercion::aggregates::NUMERICS; use datafusion_expr::{PartitionEvaluator, Signature, Volatility, WindowUDFImpl}; use datafusion_functions_window_common::field::WindowUDFFieldArgs; use datafusion_functions_window_common::partition::PartitionEvaluatorArgs; +use crate::helper::NUMERICS; use crate::scalars::anomaly::utils::{ anomaly_ratio, cast_to_f64, collect_window_values, median_f64, }; @@ -57,10 +56,6 @@ impl AnomalyScoreMad { } impl WindowUDFImpl for AnomalyScoreMad { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "anomaly_score_mad" } diff --git a/src/common/function/src/scalars/anomaly/zscore.rs b/src/common/function/src/scalars/anomaly/zscore.rs index f5852ce9c30..610bab8cab5 100644 --- a/src/common/function/src/scalars/anomaly/zscore.rs +++ b/src/common/function/src/scalars/anomaly/zscore.rs @@ -19,7 +19,6 @@ //! When stddev = 0 (constant window), returns 0.0 if value equals mean, //! or +inf otherwise. -use std::any::Any; use std::fmt::Debug; use std::ops::Range; use std::sync::Arc; @@ -27,11 +26,11 @@ use std::sync::Arc; use arrow::array::{Array, ArrayRef, Float64Array}; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::{DataFusionError, Result, ScalarValue}; -use datafusion_expr::type_coercion::aggregates::NUMERICS; use datafusion_expr::{PartitionEvaluator, Signature, Volatility, WindowUDFImpl}; use datafusion_functions_window_common::field::WindowUDFFieldArgs; use datafusion_functions_window_common::partition::PartitionEvaluatorArgs; +use crate::helper::NUMERICS; use crate::scalars::anomaly::utils::{anomaly_ratio, cast_to_f64, collect_window_values}; /// Minimum valid samples for zscore (stddev requires n >= 2). @@ -51,10 +50,6 @@ impl AnomalyScoreZscore { } impl WindowUDFImpl for AnomalyScoreZscore { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "anomaly_score_zscore" } diff --git a/src/common/function/src/scalars/expression.rs b/src/common/function/src/scalars/expression.rs index 63ed40fc8f7..777c91e7c0c 100644 --- a/src/common/function/src/scalars/expression.rs +++ b/src/common/function/src/scalars/expression.rs @@ -12,13 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod binary; mod ctx; mod if_func; mod is_null; mod unary; -pub use binary::scalar_binary_op; pub use ctx::EvalContext; pub use unary::scalar_unary_op; diff --git a/src/common/function/src/scalars/expression/binary.rs b/src/common/function/src/scalars/expression/binary.rs deleted file mode 100644 index 4d9f6268a89..00000000000 --- a/src/common/function/src/scalars/expression/binary.rs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::iter; - -use common_query::error::Result; -use datatypes::prelude::*; -use datatypes::vectors::{ConstantVector, Helper}; - -use crate::scalars::expression::ctx::EvalContext; - -pub fn scalar_binary_op( - l: &VectorRef, - r: &VectorRef, - f: F, - ctx: &mut EvalContext, -) -> Result<::VectorType> -where - F: Fn(Option>, Option>, &mut EvalContext) -> Option, -{ - debug_assert!( - l.len() == r.len(), - "Size of vectors must match to apply binary expression" - ); - - let result = match (l.is_const(), r.is_const()) { - (false, true) => { - let left: &::VectorType = unsafe { Helper::static_cast(l) }; - let right: &ConstantVector = unsafe { Helper::static_cast(r) }; - let right: &::VectorType = unsafe { Helper::static_cast(right.inner()) }; - let b = right.get_data(0); - - let it = left.iter_data().map(|a| f(a, b.clone(), ctx)); - ::VectorType::from_owned_iterator(it) - } - - (false, false) => { - let left: &::VectorType = unsafe { Helper::static_cast(l) }; - let right: &::VectorType = unsafe { Helper::static_cast(r) }; - - let it = left - .iter_data() - .zip(right.iter_data()) - .map(|(a, b)| f(a, b, ctx)); - ::VectorType::from_owned_iterator(it) - } - - (true, false) => { - let left: &ConstantVector = unsafe { Helper::static_cast(l) }; - let left: &::VectorType = unsafe { Helper::static_cast(left.inner()) }; - let a = left.get_data(0); - - let right: &::VectorType = unsafe { Helper::static_cast(r) }; - let it = right.iter_data().map(|b| f(a.clone(), b, ctx)); - ::VectorType::from_owned_iterator(it) - } - - (true, true) => { - let left: &ConstantVector = unsafe { Helper::static_cast(l) }; - let left: &::VectorType = unsafe { Helper::static_cast(left.inner()) }; - let a = left.get_data(0); - - let right: &ConstantVector = unsafe { Helper::static_cast(r) }; - let right: &::VectorType = unsafe { Helper::static_cast(right.inner()) }; - let b = right.get_data(0); - - let it = iter::repeat(a) - .zip(iter::repeat(b)) - .map(|(a, b)| f(a, b, ctx)) - .take(left.len()); - ::VectorType::from_owned_iterator(it) - } - }; - - if let Some(error) = ctx.error.take() { - return Err(error); - } - Ok(result) -} diff --git a/src/common/function/src/scalars/geo/geohash.rs b/src/common/function/src/scalars/geo/geohash.rs index 90bb958246c..7d91de3565a 100644 --- a/src/common/function/src/scalars/geo/geohash.rs +++ b/src/common/function/src/scalars/geo/geohash.rs @@ -22,12 +22,12 @@ use datafusion::arrow::array::{Array, AsArray, ListBuilder, StringViewBuilder}; use datafusion::arrow::datatypes::{DataType, Field, Float64Type, UInt8Type}; use datafusion::logical_expr::ColumnarValue; use datafusion_common::DataFusionError; -use datafusion_expr::type_coercion::aggregates::INTEGERS; use datafusion_expr::{ScalarFunctionArgs, Signature, TypeSignature, Volatility}; use geohash::Coord; use snafu::ResultExt; use crate::function::{Function, extract_args}; +use crate::helper::INTEGER_TYPES; use crate::scalars::geo::helpers; fn ensure_resolution_usize(v: u8) -> datafusion_common::Result { @@ -49,7 +49,7 @@ impl Default for GeohashFunction { fn default() -> Self { let mut signatures = Vec::new(); for coord_type in &[DataType::Float32, DataType::Float64] { - for resolution_type in INTEGERS { + for resolution_type in INTEGER_TYPES { signatures.push(TypeSignature::Exact(vec![ // latitude coord_type.clone(), @@ -146,7 +146,7 @@ impl Default for GeohashNeighboursFunction { fn default() -> Self { let mut signatures = Vec::new(); for coord_type in &[DataType::Float32, DataType::Float64] { - for resolution_type in INTEGERS { + for resolution_type in INTEGER_TYPES { signatures.push(TypeSignature::Exact(vec![ // latitude coord_type.clone(), diff --git a/src/common/function/src/scalars/geo/h3.rs b/src/common/function/src/scalars/geo/h3.rs index c6630525df2..7792f3a9f1b 100644 --- a/src/common/function/src/scalars/geo/h3.rs +++ b/src/common/function/src/scalars/geo/h3.rs @@ -26,7 +26,6 @@ use datafusion::arrow::compute; use datafusion::arrow::datatypes::{Float64Type, Int64Type, UInt8Type, UInt64Type}; use datafusion::logical_expr::ColumnarValue; use datafusion_common::{DataFusionError, ScalarValue}; -use datafusion_expr::type_coercion::aggregates::INTEGERS; use datafusion_expr::{ScalarFunctionArgs, Signature, TypeSignature, Volatility}; use datatypes::arrow::datatypes::{DataType, Field}; use derive_more::Display; @@ -34,6 +33,7 @@ use h3o::{CellIndex, LatLng, Resolution}; use snafu::prelude::*; use crate::function::{Function, extract_args}; +use crate::helper::INTEGER_TYPES; use crate::scalars::geo::helpers; static CELL_TYPES: LazyLock> = @@ -42,11 +42,11 @@ static CELL_TYPES: LazyLock> = static COORDINATE_TYPES: LazyLock> = LazyLock::new(|| vec![DataType::Float32, DataType::Float64]); -static RESOLUTION_TYPES: &[DataType] = INTEGERS; +static RESOLUTION_TYPES: &[DataType] = INTEGER_TYPES; -static DISTANCE_TYPES: &[DataType] = INTEGERS; +static DISTANCE_TYPES: &[DataType] = INTEGER_TYPES; -static POSITION_TYPES: &[DataType] = INTEGERS; +static POSITION_TYPES: &[DataType] = INTEGER_TYPES; /// Function that returns [h3] encoding cellid for a given geospatial coordinate. /// diff --git a/src/common/function/src/scalars/geo/s2.rs b/src/common/function/src/scalars/geo/s2.rs index e4c2848dce2..88fef9ac324 100644 --- a/src/common/function/src/scalars/geo/s2.rs +++ b/src/common/function/src/scalars/geo/s2.rs @@ -25,6 +25,7 @@ use s2::latlng::LatLng; use snafu::ensure; use crate::function::{Function, extract_args}; +use crate::helper::INTEGER_TYPES; use crate::scalars::geo::helpers; use crate::scalars::geo::helpers::ensure_and_coerce; @@ -34,7 +35,7 @@ static CELL_TYPES: LazyLock> = static COORDINATE_TYPES: LazyLock> = LazyLock::new(|| vec![DataType::Float32, DataType::Float64]); -static LEVEL_TYPES: &[DataType] = datafusion_expr::type_coercion::aggregates::INTEGERS; +static LEVEL_TYPES: &[DataType] = INTEGER_TYPES; /// Function that returns [s2] encoding cellid for a given geospatial coordinate. /// diff --git a/src/common/function/src/scalars/json/json_get.rs b/src/common/function/src/scalars/json/json_get.rs index 2357bc88f91..fcce7c9ee1c 100644 --- a/src/common/function/src/scalars/json/json_get.rs +++ b/src/common/function/src/scalars/json/json_get.rs @@ -532,7 +532,7 @@ mod tests { use datafusion_common::arrow::datatypes::{Float64Type, Int64Type}; use datatypes::extension::json::Json2ExtensionType; use datatypes::types::parse_string_to_jsonb; - use serde_json::json; + use serde_json::{Value, json}; use super::*; @@ -595,6 +595,34 @@ mod tests { }) } + fn assert_json_or_string_eq(actual: Option<&str>, expected: Option<&str>) { + let is_json_container = |value: &str| { + matches!( + serde_json::from_str::(value), + Ok(Value::Object(_) | Value::Array(_)) + ) + }; + + match (actual, expected) { + (Some(actual), Some(expected)) + if is_json_container(actual) || is_json_container(expected) => + { + let actual_value = serde_json::from_str::(actual).unwrap_or_else(|error| { + panic!("failed to parse actual JSON result {actual:?}: {error}") + }); + let expected_value = + serde_json::from_str::(expected).unwrap_or_else(|error| { + panic!("failed to parse expected JSON result {expected:?}: {error}") + }); + assert_eq!( + actual_value, expected_value, + "JSON result mismatch: actual {actual:?}, expected {expected:?}" + ); + } + _ => assert_eq!(actual, expected), + } + } + #[test] fn test_json_get_int() { let json_get_int = JsonGetInt::default(); @@ -895,7 +923,7 @@ mod tests { let result = result.as_string_view(); assert_eq!(1, result.len()); let actual = result.is_valid(0).then(|| result.value(0)); - assert_eq!(actual, expect); + assert_json_or_string_eq(actual, expect); } } @@ -1033,7 +1061,7 @@ mod tests { let result = result.as_string_view(); assert_eq!(1, result.len()); let actual = result.is_valid(0).then(|| result.value(0)); - assert_eq!(actual, expect); + assert_json_or_string_eq(actual, expect); } let json_strings = [ diff --git a/src/common/function/src/scalars/json/json_get_rewriter.rs b/src/common/function/src/scalars/json/json_get_rewriter.rs index 0143ee05d57..b686e4f68af 100644 --- a/src/common/function/src/scalars/json/json_get_rewriter.rs +++ b/src/common/function/src/scalars/json/json_get_rewriter.rs @@ -59,10 +59,8 @@ impl FunctionRewrite for JsonGetRewriter { // json_get(column, path, ) // ) fn inject_type_from_cast_expr(cast: Cast) -> Result> { - let Cast { - expr, - mut data_type, - } = cast; + let Cast { expr, field } = cast; + let mut data_type = field.data_type().clone(); let mut json_get = match *expr { Expr::ScalarFunction(f) @@ -73,7 +71,7 @@ fn inject_type_from_cast_expr(cast: Cast) -> Result> { expr => { return Ok(Transformed::no(Expr::Cast(Cast { expr: Box::new(expr), - data_type, + field, }))); } }; @@ -204,10 +202,7 @@ mod tests { }); // Create a cast expression: json_get(...)::int8 - let cast_expr = Expr::Cast(Cast { - expr: Box::new(json_expr), - data_type: DataType::Int8, - }); + let cast_expr = Expr::Cast(Cast::new(Box::new(json_expr), DataType::Int8)); // Apply the rewriter let result = rewriter.rewrite(cast_expr, &schema, &config).unwrap(); @@ -279,10 +274,7 @@ mod tests { // Create an arrow cast function: cast(json_get(...), 'Int64') // Note: ArrowCastFunc doesn't exist in this codebase, so this test uses a simple cast instead - let arrow_cast_expr = Expr::Cast(Cast { - expr: Box::new(json_get_expr), - data_type: DataType::Int64, - }); + let arrow_cast_expr = Expr::Cast(Cast::new(Box::new(json_get_expr), DataType::Int64)); // Apply the rewriter let result = rewriter.rewrite(arrow_cast_expr, &schema, &config).unwrap(); diff --git a/src/common/function/src/scalars/matches.rs b/src/common/function/src/scalars/matches.rs index b5de60dc85f..14889957ee9 100644 --- a/src/common/function/src/scalars/matches.rs +++ b/src/common/function/src/scalars/matches.rs @@ -24,6 +24,7 @@ use datafusion::execution::SessionStateBuilder; use datafusion::logical_expr::{self, ColumnarValue, Expr, Volatility}; use datafusion::physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner}; use datafusion_common::DataFusionError; +use datafusion_expr::physical_planning_context::PhysicalPlanningContext; use datafusion_expr::{ScalarFunctionArgs, Signature}; use datatypes::arrow::array::RecordBatch; use datatypes::arrow::datatypes::{DataType, Field}; @@ -110,8 +111,12 @@ impl MatchesFunction { let input_schema = Self::input_schema(); let session_state = SessionStateBuilder::new().with_default_features().build(); let planner = DefaultPhysicalPlanner::default(); - let physical_expr = - planner.create_physical_expr(&like_expr, &input_schema, &session_state)?; + let physical_expr = planner.create_physical_expr( + &like_expr, + &input_schema, + &session_state, + &PhysicalPlanningContext::default(), + )?; let arrow_schema = Arc::new(input_schema.as_arrow().clone()); let input_record_batch = RecordBatch::try_new(arrow_schema, vec![data_array]).unwrap(); diff --git a/src/common/function/src/scalars/math/clamp.rs b/src/common/function/src/scalars/math/clamp.rs index 14774c0b61a..a76f17a99dc 100644 --- a/src/common/function/src/scalars/math/clamp.rs +++ b/src/common/function/src/scalars/math/clamp.rs @@ -19,10 +19,10 @@ use datafusion::arrow::array::{Array, ArrayRef, AsArray, PrimitiveArray}; use datafusion::arrow::datatypes::DataType as ArrowDataType; use datafusion::logical_expr::{ColumnarValue, Volatility}; use datafusion_common::{DataFusionError, ScalarValue, utils}; -use datafusion_expr::type_coercion::aggregates::NUMERICS; use datafusion_expr::{ScalarFunctionArgs, Signature}; use crate::function::Function; +use crate::helper::NUMERICS; #[derive(Clone, Debug)] pub struct ClampFunction { @@ -338,7 +338,9 @@ mod test { use arrow_schema::Field; use datafusion_common::config::ConfigOptions; - use datatypes::arrow::array::{ArrayRef, Float64Array, Int64Array, UInt64Array}; + use datatypes::arrow::array::{ + ArrayRef, Decimal128Array, Float64Array, Int64Array, UInt64Array, + }; use datatypes::arrow_array::StringArray; use super::*; @@ -370,6 +372,79 @@ mod test { impl_test_eval!(ClampMinFunction); impl_test_eval!(ClampMaxFunction); + fn decimal_array(values: Vec) -> ColumnarValue { + ColumnarValue::Array(Arc::new( + Decimal128Array::from(values) + .with_precision_and_scale(10, 2) + .unwrap(), + )) + } + + fn decimal_scalar(value: i128) -> ColumnarValue { + ColumnarValue::Scalar(ScalarValue::Decimal128(Some(value), 10, 2)) + } + + #[allow(deprecated)] + fn evaluate_decimal( + function: &dyn Function, + args: Vec, + ) -> datafusion_common::Result { + let input_types = args + .iter() + .map(ColumnarValue::data_type) + .collect::>(); + let planned_types = datafusion_expr::type_coercion::functions::data_types( + function.name(), + &input_types, + function.signature(), + )?; + let args = args + .into_iter() + .zip(planned_types) + .map(|(arg, planned_type)| arg.cast_to(&planned_type, None)) + .collect::>>()?; + function + .invoke_with_args(ScalarFunctionArgs { + args, + arg_fields: vec![], + number_rows: 3, + return_field: Arc::new(Field::new("x", ArrowDataType::Float64, false)), + config_options: Arc::new(ConfigOptions::new()), + }) + .and_then(|value| value.to_array(3)) + } + + #[test] + fn clamp_decimal_coercion_executes_as_float64() { + let test_cases = [ + ( + Box::new(ClampFunction::default()) as Box, + vec![ + decimal_array(vec![100, 300, 500]), + decimal_scalar(200), + decimal_scalar(400), + ], + vec![2.0, 3.0, 4.0], + ), + ( + Box::new(ClampMinFunction::default()), + vec![decimal_array(vec![100, 300, 500]), decimal_scalar(200)], + vec![2.0, 3.0, 5.0], + ), + ( + Box::new(ClampMaxFunction::default()), + vec![decimal_array(vec![100, 300, 500]), decimal_scalar(200)], + vec![1.0, 2.0, 2.0], + ), + ]; + + for (function, args, expected) in test_cases { + let result = evaluate_decimal(function.as_ref(), args).unwrap(); + let expected: ArrayRef = Arc::new(Float64Array::from(expected)); + assert_eq!(expected.as_ref(), result.as_ref()); + } + } + #[test] fn clamp_i64() { let inputs = [ diff --git a/src/common/function/src/scalars/math/modulo.rs b/src/common/function/src/scalars/math/modulo.rs index 869b8efacad..23ad52cb891 100644 --- a/src/common/function/src/scalars/math/modulo.rs +++ b/src/common/function/src/scalars/math/modulo.rs @@ -18,10 +18,10 @@ use std::fmt::Display; use datafusion_common::arrow::compute; use datafusion_common::arrow::compute::kernels::numeric; use datafusion_common::arrow::datatypes::DataType; -use datafusion_expr::type_coercion::aggregates::NUMERICS; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, Volatility}; use crate::function::{Function, extract_args}; +use crate::helper::NUMERICS; const NAME: &str = "mod"; @@ -90,12 +90,74 @@ mod tests { use std::sync::Arc; use arrow_schema::Field; + use datafusion_common::ScalarValue; use datafusion_common::arrow::array::{ - AsArray, Float64Array, Int32Array, StringViewArray, UInt32Array, + AsArray, Decimal128Array, Float64Array, Int32Array, StringViewArray, UInt32Array, }; use datafusion_common::arrow::datatypes::{Float64Type, Int64Type, UInt64Type}; use super::*; + fn decimal_array(values: Vec) -> ColumnarValue { + ColumnarValue::Array(Arc::new( + Decimal128Array::from(values) + .with_precision_and_scale(10, 2) + .unwrap(), + )) + } + + fn decimal_scalar(value: i128) -> ColumnarValue { + ColumnarValue::Scalar(ScalarValue::Decimal128(Some(value), 10, 2)) + } + + #[test] + #[allow(deprecated)] + fn modulo_decimal_coercion_executes_as_float64() { + let function = ModuloFunction::default(); + let test_cases = [ + ( + vec![decimal_array(vec![500, 600]), decimal_scalar(200)], + vec![1.0, 0.0], + ), + ( + vec![decimal_scalar(500), decimal_array(vec![200, 300])], + vec![1.0, 2.0], + ), + ]; + + for (args, expected) in test_cases { + let input_types = args + .iter() + .map(ColumnarValue::data_type) + .collect::>(); + let planned_types = datafusion_expr::type_coercion::functions::data_types( + function.name(), + &input_types, + function.signature(), + ) + .unwrap(); + assert_eq!(vec![DataType::Float64; 2], planned_types); + let args = args + .into_iter() + .zip(planned_types) + .map(|(arg, planned_type)| arg.cast_to(&planned_type, None)) + .collect::>>() + .unwrap(); + let result = function + .invoke_with_args(ScalarFunctionArgs { + args, + arg_fields: vec![], + number_rows: 2, + return_field: Arc::new(Field::new("x", DataType::Float64, false)), + config_options: Arc::new(Default::default()), + }) + .unwrap() + .to_array(2) + .unwrap(); + let result = result.as_primitive::(); + assert_eq!(&Float64Array::from(expected), result); + } + } + #[test] fn test_mod_function_signed() { let function = ModuloFunction::default(); diff --git a/src/common/function/src/scalars/math/rate.rs b/src/common/function/src/scalars/math/rate.rs index b2a45b20369..b33667a06cd 100644 --- a/src/common/function/src/scalars/math/rate.rs +++ b/src/common/function/src/scalars/math/rate.rs @@ -18,11 +18,11 @@ use common_query::error; use datafusion::arrow::compute::kernels::numeric; use datafusion_common::arrow::compute::kernels::cast; use datafusion_common::arrow::datatypes::DataType; -use datafusion_expr::type_coercion::aggregates::NUMERICS; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, Volatility}; use snafu::ResultExt; use crate::function::{Function, extract_args}; +use crate::helper::NUMERICS; /// generates rates from a sequence of adjacent data points. #[derive(Clone, Debug)] @@ -96,12 +96,13 @@ mod tests { let rate = RateFunction::default(); assert_eq!("rate", rate.name()); assert_eq!(DataType::Float64, rate.return_type(&[]).unwrap()); - assert!(matches!(rate.signature(), - Signature { - type_signature: TypeSignature::Uniform(2, valid_types), - volatility: Volatility::Immutable, - .. - } if valid_types == NUMERICS + assert!(matches!( + rate.signature(), + Signature { + type_signature: TypeSignature::Uniform(2, valid_types), + volatility: Volatility::Immutable, + .. + } if valid_types == NUMERICS )); let values = vec![1.0, 3.0, 6.0]; let ts = vec![0, 1, 2]; diff --git a/src/common/function/src/scalars/test.rs b/src/common/function/src/scalars/test.rs index 8768d546a32..e7f59a65925 100644 --- a/src/common/function/src/scalars/test.rs +++ b/src/common/function/src/scalars/test.rs @@ -14,13 +14,43 @@ use std::fmt; +use common_query::error::Result; use datafusion::logical_expr::ColumnarValue; use datafusion_expr::{ScalarFunctionArgs, Signature, Volatility}; use datatypes::arrow::datatypes::DataType; +use datatypes::prelude::{Scalar, ScalarVector, VectorRef}; use datatypes::vectors::{Helper, Vector}; use crate::function::{Function, extract_args}; -use crate::scalars::expression::{EvalContext, scalar_binary_op}; +use crate::scalars::expression::EvalContext; + +fn scalar_binary_op( + l: &VectorRef, + r: &VectorRef, + f: F, + ctx: &mut EvalContext, +) -> Result<::VectorType> +where + F: Fn(Option>, Option>, &mut EvalContext) -> Option, +{ + debug_assert!( + l.len() == r.len(), + "Size of vectors must match to apply binary expression" + ); + + let left: &::VectorType = unsafe { Helper::static_cast(l) }; + let right: &::VectorType = unsafe { Helper::static_cast(r) }; + let result = ::VectorType::from_owned_iterator( + left.iter_data() + .zip(right.iter_data()) + .map(|(a, b)| f(a, b, ctx)), + ); + + if let Some(error) = ctx.error.take() { + return Err(error); + } + Ok(result) +} #[derive(Clone)] pub(crate) struct TestAndFunction { diff --git a/src/common/function/src/scalars/udf.rs b/src/common/function/src/scalars/udf.rs index 638f7c38af9..48c59ac230a 100644 --- a/src/common/function/src/scalars/udf.rs +++ b/src/common/function/src/scalars/udf.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::fmt::{Debug, Formatter}; use std::hash::{Hash, Hasher}; @@ -49,10 +48,6 @@ impl Hash for ScalarUdf { } impl ScalarUDFImpl for ScalarUdf { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.function.name() } diff --git a/src/common/function/src/system.rs b/src/common/function/src/system.rs index 0c1e5bee367..5d3a76ac4dc 100644 --- a/src/common/function/src/system.rs +++ b/src/common/function/src/system.rs @@ -21,7 +21,8 @@ mod version; use build::BuildFunction; use database::{ - ConnectionIdFunction, DatabaseFunction, PgBackendPidFunction, ReadPreferenceFunction, + ConnectionIdFunction, CurrentUserFunction, DatabaseFunction, PgBackendPidFunction, + ReadPreferenceFunction, SchemaFunction, SystemUserFunction, UserFunction, }; use pg_catalog::PGCatalogFunction; use procedure_state::ProcedureStateFunction; @@ -37,6 +38,10 @@ impl SystemFunction { registry.register_scalar(BuildFunction::default()); registry.register_scalar(VersionFunction::default()); registry.register_scalar(DatabaseFunction::default()); + registry.register_scalar(SchemaFunction::default()); + registry.register_scalar(UserFunction::default()); + registry.register_scalar(CurrentUserFunction::default()); + registry.register_scalar(SystemUserFunction::default()); registry.register_scalar(ReadPreferenceFunction::default()); registry.register_scalar(PgBackendPidFunction::default()); registry.register_scalar(ConnectionIdFunction::default()); diff --git a/src/common/function/src/system/database.rs b/src/common/function/src/system/database.rs index b20ab1f883e..1884c5525f0 100644 --- a/src/common/function/src/system/database.rs +++ b/src/common/function/src/system/database.rs @@ -20,39 +20,91 @@ use crate::function::{Function, find_function_context}; use crate::system::define_nullary_udf; define_nullary_udf!(DatabaseFunction); +define_nullary_udf!(SchemaFunction); +define_nullary_udf!(UserFunction); +define_nullary_udf!(CurrentUserFunction); +define_nullary_udf!(SystemUserFunction); define_nullary_udf!(ReadPreferenceFunction); define_nullary_udf!(PgBackendPidFunction); define_nullary_udf!(ConnectionIdFunction); const DATABASE_FUNCTION_NAME: &str = "database"; +const SCHEMA_FUNCTION_NAME: &str = "schema"; +const USER_FUNCTION_NAME: &str = "user"; +const CURRENT_USER_FUNCTION_NAME: &str = "current_user"; +const SYSTEM_USER_FUNCTION_NAME: &str = "system_user"; const READ_PREFERENCE_FUNCTION_NAME: &str = "read_preference"; const PG_BACKEND_PID: &str = "pg_backend_pid"; const CONNECTION_ID: &str = "connection_id"; -impl Function for DatabaseFunction { - fn name(&self) -> &str { - DATABASE_FUNCTION_NAME - } +macro_rules! impl_current_schema_function { + ($name: ident, $fn_name: expr) => { + impl Function for $name { + fn name(&self) -> &str { + $fn_name + } - fn return_type(&self, _: &[DataType]) -> datafusion_common::Result { - Ok(DataType::Utf8View) - } + fn return_type(&self, _: &[DataType]) -> datafusion_common::Result { + Ok(DataType::Utf8View) + } - fn signature(&self) -> &Signature { - &self.signature - } + fn signature(&self) -> &Signature { + &self.signature + } - fn invoke_with_args( - &self, - args: ScalarFunctionArgs, - ) -> datafusion_common::Result { - let func_ctx = find_function_context(&args)?; - let db = func_ctx.query_ctx.current_schema(); + fn invoke_with_args( + &self, + args: ScalarFunctionArgs, + ) -> datafusion_common::Result { + let func_ctx = find_function_context(&args)?; + let db = func_ctx.query_ctx.current_schema(); - Ok(ColumnarValue::Scalar(ScalarValue::Utf8View(Some(db)))) - } + Ok(ColumnarValue::Scalar(ScalarValue::Utf8View(Some(db)))) + } + } + }; } +impl_current_schema_function!(DatabaseFunction, DATABASE_FUNCTION_NAME); +// MySQL's `SCHEMA()` is a synonym for `DATABASE()`. +impl_current_schema_function!(SchemaFunction, SCHEMA_FUNCTION_NAME); + +macro_rules! impl_current_user_function { + ($name: ident, $fn_name: expr) => { + impl Function for $name { + fn name(&self) -> &str { + $fn_name + } + + fn return_type(&self, _: &[DataType]) -> datafusion_common::Result { + Ok(DataType::Utf8View) + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn invoke_with_args( + &self, + args: ScalarFunctionArgs, + ) -> datafusion_common::Result { + let func_ctx = find_function_context(&args)?; + let user = func_ctx.query_ctx.current_user(); + + Ok(ColumnarValue::Scalar(ScalarValue::Utf8View(Some( + user.username().to_string(), + )))) + } + } + }; +} + +// GreptimeDB has no notion of a user switching identity mid-session, so `USER()`, +// `CURRENT_USER()` and `SYSTEM_USER()` all report the authenticated user. +impl_current_user_function!(UserFunction, USER_FUNCTION_NAME); +impl_current_user_function!(CurrentUserFunction, CURRENT_USER_FUNCTION_NAME); +impl_current_user_function!(SystemUserFunction, SYSTEM_USER_FUNCTION_NAME); + impl Function for ReadPreferenceFunction { fn name(&self) -> &str { READ_PREFERENCE_FUNCTION_NAME diff --git a/src/common/function/src/system/pg_catalog.rs b/src/common/function/src/system/pg_catalog.rs index 96bcc3fe9d6..3029361c202 100644 --- a/src/common/function/src/system/pg_catalog.rs +++ b/src/common/function/src/system/pg_catalog.rs @@ -389,6 +389,9 @@ impl PGCatalogFunction { registry.register(pg_catalog::create_pg_get_partition_ancestors_udf()); registry.register(pg_catalog::quote_ident_udf::create_quote_ident_udf()); registry.register(pg_catalog::quote_ident_udf::create_parse_ident_udf()); + // Register array bound UDFs used by pg_catalog views. + registry.register(pg_catalog::array_bounds_udf::create_array_upper_udf()); + registry.register(pg_catalog::array_bounds_udf::create_array_lower_udf()); registry.register_scalar(ObjDescriptionFunction::new()); registry.register_scalar(ColDescriptionFunction::new()); registry.register_scalar(ShobjDescriptionFunction::new()); diff --git a/src/common/grpc/src/flight.rs b/src/common/grpc/src/flight.rs index 1245a86efb1..4f5a47cb535 100644 --- a/src/common/grpc/src/flight.rs +++ b/src/common/grpc/src/flight.rs @@ -37,6 +37,35 @@ use vec1::{Vec1, vec1}; use crate::error; use crate::error::{DecodeFlightDataSnafu, InvalidFlightDataSnafu, Result}; +/// Encodes a batch into the schema, record-batch header and payload used by bulk inserts. +/// +/// Uses the default Flight compression and rejects dictionary batches, whose +/// additional Flight messages cannot be represented by this three-part format. +pub fn record_batch_to_ipc( + record_batch: DfRecordBatch, +) -> Result<(ProstBytes, ProstBytes, ProstBytes)> { + let mut encoder = FlightEncoder::default(); + let schema = encoder.encode_schema(record_batch.schema().as_ref()); + let mut iter = encoder + .encode(FlightMessage::RecordBatch(record_batch)) + .into_iter(); + let flight_data = iter.next().context(InvalidFlightDataSnafu { + reason: "Failed to encode empty flight data", + })?; + if iter.next().is_some() { + return error::NotSupportedSnafu { + feat: "bulk insert RecordBatch with dictionary arrays", + } + .fail(); + } + + Ok(( + schema.data_header, + flight_data.data_header, + flight_data.data_body, + )) +} + /// Flight metadata key used to carry flow query extensions as JSON pairs. pub const FLOW_EXTENSIONS_METADATA_KEY: &str = "x-greptime-flow-extensions"; /// Flight metadata key used to carry query snapshot read upper bounds as JSON. @@ -374,8 +403,52 @@ mod test { use datatypes::arrow::buffer::OffsetBuffer; use datatypes::arrow::datatypes::{DataType, Field, Schema}; - use super::*; use crate::Error; + use crate::flight::*; + + #[test] + fn test_record_batch_to_ipc_preserves_wire_bytes() { + let schema = Arc::new(Schema::new(vec![Field::new("n", DataType::Int32, true)])); + for values in [vec![], vec![Some(1), None, Some(3)]] { + let batch = + DfRecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(values))]) + .unwrap(); + // Reproduce the previous Prom encoding sequence and compare all bytes. + let mut encoder = FlightEncoder::default(); + let expected_schema = encoder.encode_schema(batch.schema().as_ref()); + let messages = encoder.encode(FlightMessage::RecordBatch(batch.clone())); + assert_eq!(messages.len(), 1); + let expected_batch = messages.first(); + assert_eq!( + record_batch_to_ipc(batch).unwrap(), + ( + expected_schema.data_header, + expected_batch.data_header.clone(), + expected_batch.data_body.clone(), + ) + ); + } + } + + #[test] + fn test_record_batch_to_ipc_rejects_dictionary() { + let schema = Arc::new(Schema::new(vec![Field::new_dictionary( + "tag", + DataType::UInt32, + DataType::Utf8, + true, + )])); + let dictionary = DictionaryArray::new( + UInt32Array::from_value(0, 3), + Arc::new(StringArray::from_iter_values(["x"])), + ); + let batch = DfRecordBatch::try_new(schema, vec![Arc::new(dictionary)]).unwrap(); + assert!(matches!( + record_batch_to_ipc(batch), + Err(Error::NotSupported { feat }) + if feat == "bulk insert RecordBatch with dictionary arrays" + )); + } #[test] fn test_try_decode() -> Result<()> { diff --git a/src/common/macro/src/admin_fn.rs b/src/common/macro/src/admin_fn.rs index c580389f3c6..3575ebfe6c2 100644 --- a/src/common/macro/src/admin_fn.rs +++ b/src/common/macro/src/admin_fn.rs @@ -239,10 +239,6 @@ fn build_struct( // Implement DataFusion's ScalarUDFImpl trait impl datafusion::logical_expr::ScalarUDFImpl for #name { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn name(&self) -> &str { #display_name } @@ -304,7 +300,7 @@ fn build_struct( .and_then(|cv| match cv { common_query::prelude::ColumnarValue::Vector(v) => Ok(v), common_query::prelude::ColumnarValue::Scalar(s) => { - datatypes::vectors::Helper::try_from_scalar_value(s, args.number_rows) + datatypes::vectors::Helper::try_from_scalar_value(s, args.number_rows, None) .context(common_query::error::FromScalarValueSnafu) } }) diff --git a/src/common/meta/src/ddl/alter_table.rs b/src/common/meta/src/ddl/alter_table.rs index 8e24f3ae2ab..e61250b47f6 100644 --- a/src/common/meta/src/ddl/alter_table.rs +++ b/src/common/meta/src/ddl/alter_table.rs @@ -31,6 +31,7 @@ use common_procedure::{ EventTrigger, LockKey, PoisonKey, PoisonKeys, Procedure, ProcedureId, Status, StringKey, }; use common_telemetry::{error, info, warn}; +use common_wal::options::WalOptions; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, ensure}; use store_api::metadata::ColumnMetadata; @@ -47,12 +48,12 @@ use crate::ddl::event::table::{ TableDdlEvent, TableDdlEventType, TableDdlLocator, alter_table_kind_name, }; use crate::ddl::utils::{ - MultipleResults, extract_column_metadatas, handle_multiple_results, map_to_procedure_error, - sync_follower_regions, + MultipleResults, extract_column_metadatas, extract_region_wal_options, handle_multiple_results, + map_to_procedure_error, sync_follower_regions, }; use crate::error::{ AbortProcedureSnafu, ConvertAlterTableRequestSnafu, NoLeaderSnafu, PutPoisonSnafu, Result, - RetryLaterSnafu, UnsupportedSnafu, + RetryLaterSnafu, UnexpectedSnafu, UnsupportedSnafu, }; use crate::key::table_info::TableInfoValue; use crate::key::{DeserializedValueWithBytes, RegionDistribution}; @@ -148,11 +149,11 @@ impl AlterTableProcedure { // Safety: Checked in `AlterTableProcedure::new`. let alter_kind = self.data.task.alter_table.kind.as_ref().unwrap(); - if enables_skip_wal(alter_kind) { + if sets_skip_wal(alter_kind) { ensure!( - only_enables_skip_wal(alter_kind), + only_sets_skip_wal(alter_kind), UnsupportedSnafu { - operation: "combining skip_wal = 'true' with other table options".to_string() + operation: "combining skip_wal with other table options".to_string() } ); let engine = table_info_value.table_info.meta.engine.as_str(); @@ -162,7 +163,8 @@ impl AlterTableProcedure { operation: format!("setting skip_wal on {engine} engine tables") } ); - // Persist the irreversible intent before any region stops writing WAL. + // Keep skip-WAL changes serialized with region migration and use the same + // route snapshot for validation and RPC delivery. let (physical_table_id, physical_table_route) = self .context .table_metadata_manager @@ -199,6 +201,37 @@ impl AlterTableProcedure { table_id: physical_table_id } ); + // validate_alter_table_expr() has already parsed this single skip-WAL option. + let Some(skip_wal) = skip_wal_value(alter_kind) else { + return UnexpectedSnafu { + err_msg: "missing or invalid skip_wal value after validation".to_string(), + } + .fail(); + }; + if !skip_wal { + let datanode_table_values = self + .context + .table_metadata_manager + .datanode_table_manager() + .regions(physical_table_id, &physical_table_route) + .await?; + let region_wal_options = extract_region_wal_options(&datanode_table_values)?; + ensure!( + current_region_ids.iter().all(|region_id| { + matches!( + region_wal_options + .get(®ion_id.region_number()) + .cloned() + .unwrap_or_default(), + WalOptions::RaftEngine | WalOptions::Kafka(_) + ) + }), + UnsupportedSnafu { + operation: "setting skip_wal = 'false' without an existing WAL provider" + .to_string() + } + ); + } self.data.region_distribution = Some(region_distribution(&physical_table_route.region_routes)); } @@ -265,8 +298,8 @@ impl AlterTableProcedure { MultipleResults::PartialRetryable(error) => Err(error), MultipleResults::PartialNonRetryable(error) | MultipleResults::AllNonRetryable(error) => { - // The metadata already enables skip-WAL. Retry the idempotent request - // so later attempts can update the remaining replicas. + // The metadata already contains the requested skip-WAL value. Retry the + // idempotent request so later attempts can update the remaining replicas. Err(BoxedError::new(error)).context(RetryLaterSnafu { clean_poisons: true, }) @@ -450,24 +483,35 @@ impl AlterTableProcedure { } } -fn enables_skip_wal(alter_kind: &Kind) -> bool { +fn sets_skip_wal(alter_kind: &Kind) -> bool { let Kind::SetTableOptions(SetTableOptions { table_options }) = alter_kind else { return false; }; table_options .iter() - .any(|option| option.key == SKIP_WAL_KEY && option.value == "true") + .any(|option| option.key == SKIP_WAL_KEY) } -pub(crate) fn only_enables_skip_wal(alter_kind: &Kind) -> bool { +pub(crate) fn only_sets_skip_wal(alter_kind: &Kind) -> bool { let Kind::SetTableOptions(SetTableOptions { table_options }) = alter_kind else { return false; }; - table_options.len() == 1 - && table_options[0].key == SKIP_WAL_KEY - && table_options[0].value == "true" + table_options.len() == 1 && table_options[0].key == SKIP_WAL_KEY +} + +fn skip_wal_value(alter_kind: &Kind) -> Option { + let Kind::SetTableOptions(SetTableOptions { table_options }) = alter_kind else { + return None; + }; + let [option] = table_options.as_slice() else { + return None; + }; + if option.key != SKIP_WAL_KEY { + return None; + } + option.value.parse().ok() } fn is_metadata_only_alter(alter_kind: &Kind) -> Result { @@ -487,7 +531,7 @@ enum AlterTableFlow { impl AlterTableFlow { fn from_kind(kind: &Kind) -> Result { - Ok(if only_enables_skip_wal(kind) { + Ok(if only_sets_skip_wal(kind) { Self::MetadataFirst } else if is_metadata_only_alter(kind)? { Self::MetadataOnly @@ -611,7 +655,7 @@ pub struct AlterTableData { table_info_value: Option>, /// Region distribution for table in case we need to update region options. region_distribution: Option, - /// Region locks held by irreversible region-option alters. + /// Region locks held by skip-WAL alters. #[serde(default)] region_locks: Vec, } diff --git a/src/common/meta/src/ddl/alter_table/executor.rs b/src/common/meta/src/ddl/alter_table/executor.rs index 65fa45d0a8f..53bc4b3d05b 100644 --- a/src/common/meta/src/ddl/alter_table/executor.rs +++ b/src/common/meta/src/ddl/alter_table/executor.rs @@ -174,7 +174,7 @@ impl AlterTableExecutor { .await } - /// Alters all replicas for the irreversible skip-WAL flow. + /// Alters all replicas for the skip-WAL flow. pub(crate) async fn on_alter_skip_wal_regions( &self, node_manager: &NodeManagerRef, diff --git a/src/common/meta/src/ddl/create_flow.rs b/src/common/meta/src/ddl/create_flow.rs index 2679e61e742..e36c89fbb34 100644 --- a/src/common/meta/src/ddl/create_flow.rs +++ b/src/common/meta/src/ddl/create_flow.rs @@ -568,6 +568,21 @@ pub fn validate_flow_options(flow_task: &CreateFlowTask) -> Result<()> { } } + if let Some(value) = flow_task + .flow_options + .get(FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY) + && value != FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE + { + value.parse::().map_err(|_| { + UnexpectedSnafu { + err_msg: format!( + "Invalid flow option {FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY}: {value}" + ), + } + .build() + })?; + } + defer_on_missing_source(flow_task)?; get_flow_type_from_options(flow_task)?; Ok(()) @@ -770,6 +785,9 @@ pub enum FlowType { pub const FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY: &str = "experimental_enable_incremental_read"; +/// Reserved internal value for flows requiring exact sequence ranges. +pub const FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE: &str = + "__greptime_internal_exact_sequence_range"; impl FlowType { pub const BATCHING: &str = "batching"; diff --git a/src/common/meta/src/ddl/tests/alter_table.rs b/src/common/meta/src/ddl/tests/alter_table.rs index f0ceaf592e1..305601af5be 100644 --- a/src/common/meta/src/ddl/tests/alter_table.rs +++ b/src/common/meta/src/ddl/tests/alter_table.rs @@ -19,7 +19,7 @@ use std::sync::Arc; use api::region::RegionResponse; use api::v1::alter_table_expr::Kind; use api::v1::region::sync_request::ManifestInfo; -use api::v1::region::{RegionRequest, region_request}; +use api::v1::region::{RegionRequest, alter_request, region_request}; use api::v1::{ AddColumn, AddColumns, AlterTableExpr, ColumnDataType, ColumnDef as PbColumnDef, DropColumn, DropColumns, SemanticType, SetTableOptions, @@ -30,6 +30,7 @@ use common_error::status_code::StatusCode; use common_procedure::store::poison_store::PoisonStore; use common_procedure::{Procedure, ProcedureId, Status}; use common_procedure_test::{MockContextProvider, execute_procedure_until_done}; +use common_wal::options::{KafkaWalOptions, WalOptions}; use datatypes::prelude::ConcreteDataType; use datatypes::schema::ColumnSchema; use store_api::metadata::ColumnMetadata; @@ -164,6 +165,29 @@ fn assert_alter_request( assert_eq!(req.region_id, expected_region_id); } +fn assert_skip_wal_alter_request( + peer: Peer, + request: RegionRequest, + expected_peer_id: u64, + expected_region_id: RegionId, + expected_skip_wal: bool, +) { + assert_eq!(peer.id, expected_peer_id); + let Some(region_request::Body::Alter(req)) = request.body else { + unreachable!(); + }; + assert_eq!(req.region_id, expected_region_id); + let Some(alter_request::Kind::SetTableOptions(options)) = req.kind else { + unreachable!(); + }; + assert_eq!(1, options.table_options.len()); + assert_eq!(SKIP_WAL_KEY, options.table_options[0].key); + assert_eq!( + expected_skip_wal.to_string(), + options.table_options[0].value + ); +} + fn assert_sync_request( peer: Peer, request: RegionRequest, @@ -729,39 +753,42 @@ async fn test_skip_wal_rejects_file_engine_table() { fn test_skip_wal_holds_region_locks() { let table_id = 1024; let region_ids = vec![RegionId::new(table_id, 2), RegionId::new(table_id, 1)]; - let task = AlterTableTask { - alter_table: AlterTableExpr { - catalog_name: DEFAULT_CATALOG_NAME.to_string(), - schema_name: DEFAULT_SCHEMA_NAME.to_string(), - table_name: "foo".to_string(), - kind: Some(Kind::SetTableOptions(SetTableOptions { - table_options: vec![api::v1::Option { - key: SKIP_WAL_KEY.to_string(), - value: "true".to_string(), - }], - })), - }, - }; let context = new_ddl_context(Arc::new(MockDatanodeManager::new(()))); - let procedure = AlterTableProcedure::new_with_region_locks( - table_id, - task, - region_ids.clone(), - context.clone(), - ) - .unwrap(); + for value in ["true", "false"] { + let task = AlterTableTask { + alter_table: AlterTableExpr { + catalog_name: DEFAULT_CATALOG_NAME.to_string(), + schema_name: DEFAULT_SCHEMA_NAME.to_string(), + table_name: "foo".to_string(), + kind: Some(Kind::SetTableOptions(SetTableOptions { + table_options: vec![api::v1::Option { + key: SKIP_WAL_KEY.to_string(), + value: value.to_string(), + }], + })), + }, + }; + let procedure = AlterTableProcedure::new_with_region_locks( + table_id, + task, + region_ids.clone(), + context.clone(), + ) + .unwrap(); - let lock_key = procedure.lock_key(); - for region_id in ®ion_ids { - assert!( - lock_key - .keys_to_lock() - .any(|key| key == &RegionLock::Write(*region_id).into()) - ); + let lock_key = procedure.lock_key(); + for region_id in ®ion_ids { + assert!( + lock_key + .keys_to_lock() + .any(|key| key == &RegionLock::Write(*region_id).into()) + ); + } + + let recovered = + AlterTableProcedure::from_json(&procedure.dump().unwrap(), context.clone()).unwrap(); + assert_eq!(lock_key, recovered.lock_key()); } - - let recovered = AlterTableProcedure::from_json(&procedure.dump().unwrap(), context).unwrap(); - assert_eq!(lock_key, recovered.lock_key()); } #[tokio::test] @@ -875,6 +902,252 @@ async fn test_skip_wal_rejects_no_leader_before_updating_metadata() { assert!(!table_info.meta.options.skip_wal); } +#[tokio::test] +async fn test_enable_wal_rejects_noop_provider_before_updating_metadata() { + let ddl_context = new_ddl_context(Arc::new(MockDatanodeManager::new(()))); + let table_name = "foo"; + let table_id = 1024; + let mut task = test_create_table_task(table_name, table_id); + task.table_info.meta.options.skip_wal = true; + task.table_info + .meta + .options + .extra_options + .insert(SKIP_WAL_KEY.to_string(), "true".to_string()); + let table_route = prepare_table_route(table_id); + let region_locks = table_route + .region_routes() + .unwrap() + .iter() + .map(|route| route.region.id) + .collect(); + let region_wal_options = HashMap::from([ + (1, WalOptions::Noop), + (2, WalOptions::Noop), + (3, WalOptions::Noop), + ]); + ddl_context + .table_metadata_manager + .create_table_metadata(task.table_info, table_route, region_wal_options) + .await + .unwrap(); + + let alter_task = AlterTableTask { + alter_table: AlterTableExpr { + catalog_name: DEFAULT_CATALOG_NAME.to_string(), + schema_name: DEFAULT_SCHEMA_NAME.to_string(), + table_name: table_name.to_string(), + kind: Some(Kind::SetTableOptions(SetTableOptions { + table_options: vec![api::v1::Option { + key: SKIP_WAL_KEY.to_string(), + value: "false".to_string(), + }], + })), + }, + }; + let mut procedure = AlterTableProcedure::new_with_region_locks( + table_id, + alter_task, + region_locks, + ddl_context.clone(), + ) + .unwrap(); + + let error = procedure.on_prepare().await.unwrap_err(); + assert_matches!(error, Error::Unsupported { .. }); + assert!( + error + .to_string() + .contains("without an existing WAL provider") + ); + let table_info = ddl_context + .table_metadata_manager + .table_info_manager() + .get(table_id) + .await + .unwrap() + .unwrap() + .into_inner() + .table_info; + assert!(table_info.meta.options.skip_wal); +} + +#[tokio::test] +async fn test_enable_wal_accepts_legacy_raft_regions_without_wal_options() { + let ddl_context = new_ddl_context(Arc::new(MockDatanodeManager::new(()))); + let table_name = "legacy_raft"; + let table_id = 1024; + let mut task = test_create_table_task(table_name, table_id); + task.table_info.meta.options.skip_wal = true; + task.table_info + .meta + .options + .extra_options + .insert(SKIP_WAL_KEY.to_string(), "true".to_string()); + let table_route = prepare_table_route(table_id); + let region_locks = table_route + .region_routes() + .unwrap() + .iter() + .map(|route| route.region.id) + .collect(); + ddl_context + .table_metadata_manager + .create_table_metadata(task.table_info, table_route, HashMap::new()) + .await + .unwrap(); + + let alter_task = AlterTableTask { + alter_table: AlterTableExpr { + catalog_name: DEFAULT_CATALOG_NAME.to_string(), + schema_name: DEFAULT_SCHEMA_NAME.to_string(), + table_name: table_name.to_string(), + kind: Some(Kind::SetTableOptions(SetTableOptions { + table_options: vec![api::v1::Option { + key: SKIP_WAL_KEY.to_string(), + value: "false".to_string(), + }], + })), + }, + }; + let mut procedure = AlterTableProcedure::new_with_region_locks( + table_id, + alter_task, + region_locks, + ddl_context.clone(), + ) + .unwrap(); + + procedure.on_prepare().await.unwrap(); + procedure.on_update_metadata().await.unwrap(); + let table_info = ddl_context + .table_metadata_manager + .table_info_manager() + .get(table_id) + .await + .unwrap() + .unwrap() + .into_inner() + .table_info; + assert!(!table_info.meta.options.skip_wal); +} + +#[tokio::test] +async fn test_enable_wal_updates_metadata_and_region_request() { + let (tx, mut rx) = mpsc::channel(8); + let node_manager = Arc::new(MockDatanodeManager::new(DatanodeWatcher::new(tx))); + let ddl_context = new_ddl_context(node_manager); + let table_name = "foo"; + let table_id = 1024; + let mut task = test_create_table_task(table_name, table_id); + task.table_info.meta.options.skip_wal = true; + task.table_info + .meta + .options + .extra_options + .insert(SKIP_WAL_KEY.to_string(), "true".to_string()); + let table_route = prepare_table_route(table_id); + let region_locks = table_route + .region_routes() + .unwrap() + .iter() + .map(|route| route.region.id) + .collect(); + let region_wal_options = HashMap::from([ + (1, WalOptions::RaftEngine), + ( + 2, + WalOptions::Kafka(KafkaWalOptions::new("topic-2".to_string())), + ), + ( + 3, + WalOptions::Kafka(KafkaWalOptions::new("topic-3".to_string())), + ), + ]); + ddl_context + .table_metadata_manager + .create_table_metadata(task.table_info, table_route, region_wal_options) + .await + .unwrap(); + + let alter_task = AlterTableTask { + alter_table: AlterTableExpr { + catalog_name: DEFAULT_CATALOG_NAME.to_string(), + schema_name: DEFAULT_SCHEMA_NAME.to_string(), + table_name: table_name.to_string(), + kind: Some(Kind::SetTableOptions(SetTableOptions { + table_options: vec![api::v1::Option { + key: SKIP_WAL_KEY.to_string(), + value: "false".to_string(), + }], + })), + }, + }; + let mut procedure = AlterTableProcedure::new_with_region_locks( + table_id, + alter_task, + region_locks, + ddl_context.clone(), + ) + .unwrap(); + + procedure.on_prepare().await.unwrap(); + let persisted: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap(); + assert_eq!("UpdateMetadata", persisted["state"]); + + let Some(alter_request::Kind::SetTableOptions(options)) = + procedure.make_region_alter_kind().unwrap() + else { + unreachable!(); + }; + assert_eq!(1, options.table_options.len()); + assert_eq!(SKIP_WAL_KEY, options.table_options[0].key); + assert_eq!("false", options.table_options[0].value); + + procedure.on_update_metadata().await.unwrap(); + let table_info = ddl_context + .table_metadata_manager + .table_info_manager() + .get(table_id) + .await + .unwrap() + .unwrap() + .into_inner() + .table_info; + assert!(!table_info.meta.options.skip_wal); + assert_eq!( + Some("false"), + table_info + .meta + .options + .extra_options + .get(SKIP_WAL_KEY) + .map(String::as_str) + ); + + let provider = Arc::new(MockContextProvider::default()); + procedure + .submit_alter_region_requests(ProcedureId::random(), provider.as_ref()) + .await + .unwrap(); + let mut requests = Vec::new(); + while let Ok(request) = rx.try_recv() { + requests.push(request); + } + requests.sort_unstable_by_key(|(peer, _)| peer.id); + let expected = [ + (1, RegionId::new(table_id, 1)), + (2, RegionId::new(table_id, 2)), + (3, RegionId::new(table_id, 3)), + (4, RegionId::new(table_id, 2)), + (5, RegionId::new(table_id, 1)), + ]; + assert_eq!(expected.len(), requests.len()); + for ((peer, request), (peer_id, region_id)) in requests.into_iter().zip(expected) { + assert_skip_wal_alter_request(peer, request, peer_id, region_id, false); + } +} + #[tokio::test] async fn test_skip_wal_updates_metadata_before_all_replicas() { let (tx, mut rx) = mpsc::channel(8); diff --git a/src/common/meta/src/ddl_manager.rs b/src/common/meta/src/ddl_manager.rs index 30911b7b272..51b0a90f304 100644 --- a/src/common/meta/src/ddl_manager.rs +++ b/src/common/meta/src/ddl_manager.rs @@ -35,7 +35,7 @@ use table::table_name::TableName; use crate::ddl::alter_database::AlterDatabaseProcedure; use crate::ddl::alter_logical_tables::AlterLogicalTablesProcedure; -use crate::ddl::alter_table::{AlterTableProcedure, RegionRouteChanged, only_enables_skip_wal}; +use crate::ddl::alter_table::{AlterTableProcedure, RegionRouteChanged, only_sets_skip_wal}; use crate::ddl::comment_on::CommentOnProcedure; use crate::ddl::create_database::{CreateDatabaseMetadataCommitterRef, CreateDatabaseProcedure}; use crate::ddl::create_flow::CreateFlowProcedure; @@ -433,7 +433,7 @@ impl DdlManager { .alter_table .kind .as_ref() - .is_some_and(only_enables_skip_wal); + .is_some_and(only_sets_skip_wal); let mut route_change_retries = 0; loop { diff --git a/src/common/meta/src/error.rs b/src/common/meta/src/error.rs index 3bd1475cb40..3f4868ffba0 100644 --- a/src/common/meta/src/error.rs +++ b/src/common/meta/src/error.rs @@ -269,6 +269,20 @@ pub enum Error { error: prost::DecodeError, }, + #[snafu(display("Failed to decode packed file references"))] + DecodePackedFileRefs { + #[snafu(implicit)] + location: Location, + #[snafu(source)] + error: base64::DecodeError, + }, + + #[snafu(display("Invalid packed file references framing"))] + InvalidPackedFileRefs { + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Failed to encode object into json"))] EncodeJson { #[snafu(implicit)] @@ -1212,6 +1226,8 @@ impl ErrorExt for Error { | PayloadNotExist { .. } | ConvertRawKey { .. } | DecodeProto { .. } + | DecodePackedFileRefs { .. } + | InvalidPackedFileRefs { .. } | BuildTableMeta { .. } | TableRouteNotFound { .. } | TableRepartNotFound { .. } diff --git a/src/common/meta/src/heartbeat/handler.rs b/src/common/meta/src/heartbeat/handler.rs index ecc735083b5..fb5f16653a9 100644 --- a/src/common/meta/src/heartbeat/handler.rs +++ b/src/common/meta/src/heartbeat/handler.rs @@ -94,7 +94,21 @@ impl HeartbeatResponseHandlerExecutor for HandlerGroupExecutor { Ok(HandleControl::Done) => break, Ok(HandleControl::Continue) => {} Err(e) => { - error!(e;"Error while handling: {:?}", ctx.response); + let mailbox_message_id = ctx + .response + .mailbox_message + .as_ref() + .map(|message| message.id); + let json_payload_len = + ctx.response.mailbox_message.as_ref().and_then(|message| { + message.payload.as_ref().map(|payload| match payload { + api::v1::meta::mailbox_message::Payload::Json(json) => json.len(), + }) + }); + error!( + %e; + "Error while handling heartbeat response: mailbox_message_id={mailbox_message_id:?}, json_payload_len={json_payload_len:?}" + ); break; } } @@ -102,3 +116,86 @@ impl HeartbeatResponseHandlerExecutor for HandlerGroupExecutor { Ok(()) } } + +#[cfg(test)] +mod error_log_tests { + use std::fmt::Debug; + use std::sync::{Arc, Mutex}; + + use api::v1::meta::mailbox_message::Payload; + use api::v1::meta::{HeartbeatResponse, MailboxMessage}; + use common_telemetry::tracing::field::{Field, Visit}; + use common_telemetry::tracing::{Event, Subscriber}; + use common_telemetry::tracing_subscriber::layer::{Context, SubscriberExt}; + use common_telemetry::{tracing, tracing_subscriber}; + + use super::parse_mailbox_message::ParseMailboxMessageHandler; + use super::*; + use crate::heartbeat::mailbox::HeartbeatMailbox; + + #[derive(Clone, Default)] + struct LogCapture(Arc>>); + + impl tracing_subscriber::Layer for LogCapture + where + S: Subscriber, + { + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + let mut visitor = FieldVisitor::default(); + event.record(&mut visitor); + self.0.lock().unwrap().push(visitor.fields.join(", ")); + } + } + + #[derive(Default)] + struct FieldVisitor { + fields: Vec, + } + + impl Visit for FieldVisitor { + fn record_debug(&mut self, field: &Field, value: &dyn Debug) { + self.fields.push(format!("{}={value:?}", field.name())); + } + + fn record_str(&mut self, field: &Field, value: &str) { + self.fields.push(format!("{}={value:?}", field.name())); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn test_handler_error_log_excludes_mailbox_payload() { + let payload_sentinel = "MALFORMED_PACKED_PAYLOAD_MUST_NOT_BE_LOGGED".repeat(256); + let payload = format!( + r#"{{"PackedGcRegions":{{"regions":[],"packed_file_refs_manifest":"{}"}}"#, + payload_sentinel + ); + let payload_len = payload.len(); + let capture = LogCapture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _guard = tracing::subscriber::set_default(subscriber); + let (mailbox_tx, _) = tokio::sync::mpsc::channel(1); + let ctx = HeartbeatResponseHandlerContext::new( + Arc::new(HeartbeatMailbox::new(mailbox_tx)), + HeartbeatResponse { + mailbox_message: Some(MailboxMessage { + id: 42, + subject: "unsafe subject".to_string(), + to: "unsafe recipient".to_string(), + from: "unsafe sender".to_string(), + payload: Some(Payload::Json(payload)), + ..Default::default() + }), + ..Default::default() + }, + ); + let executor = HandlerGroupExecutor::new(vec![Arc::new(ParseMailboxMessageHandler)]); + + executor.handle(ctx).await.unwrap(); + + let logs = capture.0.lock().unwrap().join("\n"); + assert!(logs.contains("Error while handling heartbeat response")); + assert!(logs.contains("mailbox_message_id=Some(42)")); + assert!(logs.contains(&format!("json_payload_len=Some({payload_len})"))); + assert!(!logs.contains(&payload_sentinel)); + } +} diff --git a/src/common/meta/src/instruction.rs b/src/common/meta/src/instruction.rs index ba4f2e32ab7..60ebe14e37c 100644 --- a/src/common/meta/src/instruction.rs +++ b/src/common/meta/src/instruction.rs @@ -16,16 +16,19 @@ use std::collections::{HashMap, HashSet}; use std::fmt::{Display, Formatter}; use std::time::Duration; +use base64::Engine; use common_error::ext::{ErrorExt, RetryHint}; use common_error::status_code::StatusCode; use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use snafu::ResultExt as _; use store_api::region_engine::SyncRegionFromRequest; use store_api::region_request::{RegionFlushReason, RegionRequirements}; -use store_api::storage::{FileRefsManifest, GcReport, RegionId, RegionNumber}; +use store_api::storage::{FileId, FileRef, FileRefsManifest, GcReport, RegionId, RegionNumber}; use strum::Display; use table::metadata::TableId; use table::table_name::TableName; +use crate::error::{DecodePackedFileRefsSnafu, InvalidPackedFileRefsSnafu}; use crate::flow_name::FlowName; use crate::key::schema_name::SchemaName; use crate::key::{FlowId, FlowPartitionId}; @@ -579,6 +582,24 @@ impl Display for GetFileRefs { } } +/// Instruction to get file references using the packed manifest format. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct GetPackedFileRefs { + /// List of region IDs to get file references from active FileHandles (in-memory). + pub query_regions: Vec, + /// Mapping from the src region IDs (whose file references to look for) to + /// the dst region IDs (where to read the manifests). + /// Key: The source region IDs (where files originally came from). + /// Value: The set of destination region IDs (whose manifests need to be read). + pub related_regions: HashMap>, +} + +impl Display for GetPackedFileRefs { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "GetPackedFileRefs(region_ids={:?})", self.query_regions) + } +} + /// Instruction to trigger garbage collection for a region. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct GcRegions { @@ -590,6 +611,147 @@ pub struct GcRegions { pub full_file_listing: bool, } +/// Instruction to trigger garbage collection with a packed file-reference manifest. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct PackedGcRegions { + /// The region ID to perform GC on. + pub regions: Vec, + /// Packed manifest used by the packed-only GC protocol. + pub packed_file_refs_manifest: PackedFileRefsManifest, + /// Whether to perform a full file listing to find orphan files. + pub full_file_listing: bool, +} + +impl Display for PackedGcRegions { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let file_refs_count = self.packed_file_refs_manifest.file_refs.len(); + write!( + f, + "PackedGcRegions(regions={:?}, file_refs_count={}, full_file_listing={})", + self.regions, file_refs_count, self.full_file_listing + ) + } +} + +/// A packed, JSON-compatible encoding of a file reference manifest. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +pub struct PackedFileRefsManifest { + /// Required packed file references for each region. + pub file_refs: HashMap, + #[serde(default)] + pub manifest_version: HashMap, + #[serde(default)] + pub cross_region_refs: HashMap>, +} + +/// Packed file references for one region, encoded with STANDARD Base64. +/// +/// `files` is required and contains 16-byte UUID records for references whose +/// `index_version` is `None`. `indexed` is required and contains 24-byte records +/// for references whose `index_version` is `Some`: a 16-byte UUID followed by +/// an 8-byte big-endian `u64` version. Decoding fails when either field is not +/// valid STANDARD Base64 or its decoded bytes do not align to the corresponding +/// record width. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +pub struct PackedRegionFileRefs { + /// Required STANDARD Base64-encoded 16-byte UUID records. + pub files: String, + /// Required STANDARD Base64-encoded 16-byte UUID and 8-byte version records. + pub indexed: String, +} + +impl PackedFileRefsManifest { + /// Packs a file reference manifest using required [`PackedRegionFileRefs`] + /// STANDARD Base64 fields. + /// + /// References whose `index_version` is `None` are encoded as 16-byte UUID + /// `files` records; references whose `index_version` is `Some` are encoded + /// as 24-byte `indexed` records: a 16-byte UUID followed by an 8-byte + /// big-endian `u64` version. + pub fn from_manifest(manifest: &FileRefsManifest) -> Self { + Self { + file_refs: manifest + .file_refs + .iter() + .map(|(region, refs)| (*region, PackedRegionFileRefs::from_refs(refs))) + .collect(), + manifest_version: manifest.manifest_version.clone(), + cross_region_refs: manifest.cross_region_refs.clone(), + } + } + + /// Decodes this packed representation into a file reference manifest. + /// + /// Returns a typed error for invalid STANDARD Base64 or malformed record + /// framing without retaining encoded payloads. + pub fn into_manifest(self) -> crate::error::Result { + let mut file_refs = HashMap::new(); + for (region, encoded) in self.file_refs { + file_refs.insert(region, encoded.into_refs(region)?); + } + Ok(FileRefsManifest { + file_refs, + manifest_version: self.manifest_version, + cross_region_refs: self.cross_region_refs, + }) + } +} + +impl PackedRegionFileRefs { + /// Packs file references as the documented STANDARD Base64 record streams. + pub fn from_refs(refs: &HashSet) -> Self { + let engine = base64::engine::general_purpose::STANDARD; + let mut files = Vec::new(); + let mut indexed = Vec::new(); + for file_ref in refs { + match file_ref.index_version { + None => files.extend_from_slice(file_ref.file_id.as_bytes()), + Some(version) => { + indexed.extend_from_slice(file_ref.file_id.as_bytes()); + indexed.extend_from_slice(&version.to_be_bytes()); + } + } + } + Self { + files: engine.encode(files), + indexed: engine.encode(indexed), + } + } + + /// Decodes the documented STANDARD Base64 record streams for `region`. + fn into_refs(self, region: RegionId) -> crate::error::Result> { + let engine = base64::engine::general_purpose::STANDARD; + let files = engine + .decode(self.files) + .context(DecodePackedFileRefsSnafu)?; + let indexed = engine + .decode(self.indexed) + .context(DecodePackedFileRefsSnafu)?; + if files.len() % 16 != 0 || indexed.len() % 24 != 0 { + return InvalidPackedFileRefsSnafu.fail(); + } + + let mut refs = HashSet::new(); + for bytes in files.chunks_exact(16) { + let mut id = [0; 16]; + id.copy_from_slice(bytes); + refs.insert(FileRef::new(region, FileId::from_bytes(id), None)); + } + for bytes in indexed.chunks_exact(24) { + let mut id = [0; 16]; + id.copy_from_slice(&bytes[..16]); + let mut version = [0; 8]; + version.copy_from_slice(&bytes[16..]); + refs.insert(FileRef::new( + region, + FileId::from_bytes(id), + Some(u64::from_be_bytes(version)), + )); + } + Ok(refs) + } +} + impl Display for GcRegions { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( @@ -625,6 +787,26 @@ impl Display for GetFileRefsReply { } } +/// Reply for GetPackedFileRefs instruction. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct GetPackedFileRefsReply { + pub packed_file_refs_manifest: PackedFileRefsManifest, + pub success: bool, + pub error: Option, +} + +impl Display for GetPackedFileRefsReply { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "GetPackedFileRefsReply(success={}, file_refs_count={}, error={:?})", + self.success, + self.packed_file_refs_manifest.file_refs.len(), + self.error + ) + } +} + /// Reply for GC instruction. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct GcRegionsReply { @@ -827,8 +1009,12 @@ pub enum Instruction { FlushRegions(FlushRegions), /// Gets file references for regions. GetFileRefs(GetFileRefs), + /// Gets file references using the packed manifest format. + GetPackedFileRefs(GetPackedFileRefs), /// Triggers garbage collection for a region. GcRegions(GcRegions), + /// Triggers garbage collection with a packed file-reference manifest. + PackedGcRegions(PackedGcRegions), /// Temporary suspend serving reads or writes Suspend, /// Makes regions enter staging state. @@ -1111,6 +1297,7 @@ pub enum InstructionReply { DowngradeRegions(DowngradeRegionsReply), FlushRegions(FlushRegionReply), GetFileRefs(GetFileRefsReply), + GetPackedFileRefs(GetPackedFileRefsReply), GcRegions(GcRegionsReply), EnterStagingRegions(EnterStagingRegionsReply), SyncRegions(SyncRegionsReply), @@ -1131,6 +1318,9 @@ impl Display for InstructionReply { } Self::FlushRegions(reply) => write!(f, "InstructionReply::FlushRegions({})", reply), Self::GetFileRefs(reply) => write!(f, "InstructionReply::GetFileRefs({})", reply), + Self::GetPackedFileRefs(reply) => { + write!(f, "InstructionReply::GetPackedFileRefs({})", reply) + } Self::GcRegions(reply) => write!(f, "InstructionReply::GcRegion({})", reply), Self::EnterStagingRegions(reply) => { write!( @@ -1221,6 +1411,7 @@ impl InstructionReply { #[cfg(test)] mod tests { use std::collections::HashSet; + use std::error::Error as _; use common_error::mock::MockError; use common_wal::options::WalOptions; @@ -1771,6 +1962,161 @@ mod tests { } } + #[test] + fn test_legacy_gc_file_refs_compatibility() { + #[derive(Debug, Deserialize)] + struct LegacyGetFileRefs { + query_regions: Vec, + related_regions: HashMap>, + } + + #[derive(Debug, Deserialize)] + struct LegacyGetFileRefsReply { + file_refs_manifest: FileRefsManifest, + success: bool, + error: Option, + } + + #[derive(Debug, Deserialize)] + enum LegacyInstruction { + GetFileRefs(LegacyGetFileRefs), + } + + let get_file_refs = Instruction::GetFileRefs(GetFileRefs { + query_regions: vec![RegionId::new(7, 3)], + related_regions: HashMap::new(), + }); + let get_file_refs_json = serde_json::to_string(&get_file_refs).unwrap(); + let legacy_get_file_refs: LegacyInstruction = + serde_json::from_str(&get_file_refs_json).unwrap(); + let LegacyInstruction::GetFileRefs(legacy_get_file_refs) = legacy_get_file_refs; + assert_eq!(legacy_get_file_refs.query_regions.len(), 1); + assert!(legacy_get_file_refs.related_regions.is_empty()); + + let reply = GetFileRefsReply { + file_refs_manifest: FileRefsManifest::default(), + success: true, + error: None, + }; + let reply_json = serde_json::to_string(&reply).unwrap(); + let legacy_reply: LegacyGetFileRefsReply = serde_json::from_str(&reply_json).unwrap(); + let current_reply: GetFileRefsReply = serde_json::from_str(&reply_json).unwrap(); + assert!(legacy_reply.success); + assert!(legacy_reply.error.is_none()); + assert!(legacy_reply.file_refs_manifest.file_refs.is_empty()); + assert_eq!(current_reply, reply); + + let packed = Instruction::PackedGcRegions(PackedGcRegions { + regions: vec![], + packed_file_refs_manifest: PackedFileRefsManifest::default(), + full_file_listing: false, + }); + let packed_json = serde_json::to_string(&packed).unwrap(); + assert!(serde_json::from_str::(&packed_json).is_err()); + } + + #[test] + fn test_packed_gc_regions_round_trip_is_distinct() { + let instruction = Instruction::PackedGcRegions(PackedGcRegions { + regions: vec![RegionId::new(7, 3)], + packed_file_refs_manifest: PackedFileRefsManifest::default(), + full_file_listing: true, + }); + let serialized = serde_json::to_string(&instruction).unwrap(); + assert!(serialized.contains("PackedGcRegions")); + assert_eq!( + serde_json::from_str::(&serialized).unwrap(), + instruction + ); + } + + #[test] + fn test_packed_file_refs_manifest_round_trip_and_validation() { + let region = RegionId::new(7, 3); + let file_id = FileId::parse_str("00000000-0000-0000-0000-000000000001").unwrap(); + let mut manifest = FileRefsManifest::default(); + manifest.file_refs.insert( + region, + HashSet::from([ + FileRef::new(region, file_id, None), + FileRef::new(region, file_id, Some(0)), + FileRef::new(region, file_id, Some(u64::MAX)), + ]), + ); + manifest.manifest_version.insert(region, 42); + manifest + .cross_region_refs + .insert(region, HashSet::from([RegionId::new(7, 4)])); + let packed = PackedFileRefsManifest::from_manifest(&manifest); + assert_eq!(packed.clone().into_manifest().unwrap(), manifest); + let mut missing_files = serde_json::to_value(&packed).unwrap(); + missing_files + .get_mut("file_refs") + .and_then(serde_json::Value::as_object_mut) + .and_then(|file_refs| file_refs.values_mut().next()) + .and_then(serde_json::Value::as_object_mut) + .unwrap() + .remove("files"); + assert!(serde_json::from_value::(missing_files).is_err()); + + let mut missing_indexed = serde_json::to_value(&packed).unwrap(); + missing_indexed + .get_mut("file_refs") + .and_then(serde_json::Value::as_object_mut) + .and_then(|file_refs| file_refs.values_mut().next()) + .and_then(serde_json::Value::as_object_mut) + .unwrap() + .remove("indexed"); + assert!(serde_json::from_value::(missing_indexed).is_err()); + + let mut malformed = packed.clone(); + malformed.file_refs.get_mut(®ion).unwrap().indexed = "!".to_string(); + let err = malformed.into_manifest().unwrap_err(); + assert!(matches!( + &err, + crate::error::Error::DecodePackedFileRefs { .. } + )); + assert!( + err.source() + .is_some_and(|source| source.is::()) + ); + assert_eq!(err.status_code(), StatusCode::Unexpected); + assert_eq!(err.retry_hint(), RetryHint::NonRetryable); + + let mut malformed = packed; + malformed.file_refs.get_mut(®ion).unwrap().indexed = + base64::engine::general_purpose::STANDARD.encode([0; 1]); + let err = malformed.into_manifest().unwrap_err(); + assert!(matches!( + &err, + crate::error::Error::InvalidPackedFileRefs { .. } + )); + assert!(err.source().is_none()); + assert_eq!(err.status_code(), StatusCode::Unexpected); + assert_eq!(err.retry_hint(), RetryHint::NonRetryable); + } + + #[test] + fn test_packed_file_refs_is_smaller_than_legacy() { + let region = RegionId::new(7, 3); + let mut manifest = FileRefsManifest::default(); + manifest.file_refs.insert( + region, + (0..500) + .map(|i| { + let id = + FileId::parse_str(&format!("00000000-0000-0000-0000-{i:012}")).unwrap(); + FileRef::new(region, id, None) + }) + .collect(), + ); + let legacy = serde_json::to_vec(&manifest).unwrap().len(); + let packed = serde_json::to_vec(&PackedFileRefsManifest::from_manifest(&manifest)) + .unwrap() + .len(); + assert!(packed * 2 < legacy, "packed={packed}, legacy={legacy}"); + } + #[test] fn test_serialize_get_file_refs_instruction_reply() { let mut manifest = FileRefsManifest::default(); diff --git a/src/common/query/src/error.rs b/src/common/query/src/error.rs index dd2b29adf3a..4c53be86b6e 100644 --- a/src/common/query/src/error.rs +++ b/src/common/query/src/error.rs @@ -270,18 +270,158 @@ pub fn datafusion_status_code( e: &DataFusionError, default_status: Option, ) -> StatusCode { - match e { + let mut error = e; + loop { + error = match error { + DataFusionError::Shared(inner) => inner, + DataFusionError::Context(_, inner) | DataFusionError::Diagnostic(_, inner) => inner, + _ => break, + }; + } + + match error { DataFusionError::Internal(_) => StatusCode::Internal, DataFusionError::NotImplemented(_) => StatusCode::Unsupported, DataFusionError::Plan(_) => StatusCode::PlanQuery, - DataFusionError::External(e) => { - if let Some(ext) = (*e).downcast_ref::() { + DataFusionError::External(error) => { + if let Some(ext) = (*error).downcast_ref::() { + ext.status_code() + } else if let Some(ext) = (*error).downcast_ref::() { ext.status_code() } else { default_status.unwrap_or(StatusCode::EngineExecuteQuery) } } - DataFusionError::Diagnostic(_, e) => datafusion_status_code::(e, default_status), _ => default_status.unwrap_or(StatusCode::EngineExecuteQuery), } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use common_error::ext::PlainError; + + use super::*; + + #[test] + fn test_datafusion_status_code_preserves_external_errors_through_wrappers() { + let boxed_error = |status| { + DataFusionError::External(Box::new(BoxedError::new(PlainError::new( + "neutral error".to_string(), + status, + )))) + }; + + for status in [ + StatusCode::RequestOutdated, + StatusCode::Unknown, + StatusCode::Unsupported, + ] { + let errors = [ + boxed_error(status), + DataFusionError::Shared(Arc::new(boxed_error(status))), + DataFusionError::Context("context".to_string(), Box::new(boxed_error(status))), + DataFusionError::Diagnostic( + Box::new(datafusion_common::Diagnostic::new_error("diagnostic", None)), + Box::new(boxed_error(status)), + ), + DataFusionError::Shared(Arc::new(DataFusionError::Context( + "context".to_string(), + Box::new(DataFusionError::Diagnostic( + Box::new(datafusion_common::Diagnostic::new_error("diagnostic", None)), + Box::new(boxed_error(status)), + )), + ))), + ]; + + for error in errors { + assert_eq!(datafusion_status_code::(&error, None), status); + assert_eq!( + datafusion_status_code::(&error, Some(StatusCode::PlanQuery)), + status + ); + } + } + + let direct_error = DataFusionError::External(Box::new(Error::DynFilterPayloadTooLarge { + payload_size_bytes: 2, + max_payload_bytes: 1, + location: Location::default(), + })); + assert_eq!( + datafusion_status_code::(&direct_error, Some(StatusCode::Internal)), + StatusCode::PlanQuery + ); + + let boundary_error = + DataFusionError::Shared(Arc::new(DataFusionError::External(Box::new( + BoxedError::new(common_recordbatch::error::Error::PhysicalExpr { + error: DataFusionError::NotImplemented("inner error".to_string()), + location: Location::default(), + }), + )))); + assert_eq!( + datafusion_status_code::(&boundary_error, Some(StatusCode::PlanQuery)), + StatusCode::Internal + ); + } + + #[test] + fn test_datafusion_status_code_uses_default_for_untyped_errors() { + let wrap = |error| { + DataFusionError::Shared(Arc::new(DataFusionError::Context( + "context".to_string(), + Box::new(DataFusionError::Diagnostic( + Box::new(datafusion_common::Diagnostic::new_error("diagnostic", None)), + Box::new(error), + )), + ))) + }; + let errors = || { + [ + ( + DataFusionError::External(Box::new(std::io::Error::other("neutral io error"))), + StatusCode::EngineExecuteQuery, + StatusCode::Unknown, + ), + ( + DataFusionError::Internal("neutral internal error".to_string()), + StatusCode::Internal, + StatusCode::Internal, + ), + ( + DataFusionError::NotImplemented("neutral not implemented error".to_string()), + StatusCode::Unsupported, + StatusCode::Unsupported, + ), + ( + DataFusionError::Plan("neutral plan error".to_string()), + StatusCode::PlanQuery, + StatusCode::PlanQuery, + ), + ( + DataFusionError::External(Box::new(DataFusionError::Internal( + "inner error".to_string(), + ))), + StatusCode::EngineExecuteQuery, + StatusCode::Unknown, + ), + ] + }; + + for (error, none_expected, default_expected) in + errors() + .into_iter() + .chain(errors().map(|(error, none_expected, default_expected)| { + (wrap(error), none_expected, default_expected) + })) + { + assert_eq!(datafusion_status_code::(&error, None), none_expected); + assert_eq!( + datafusion_status_code::(&error, Some(StatusCode::Unknown)), + default_expected + ); + } + } +} diff --git a/src/common/query/src/request.rs b/src/common/query/src/request.rs index 6dc62b0a89f..56683818618 100644 --- a/src/common/query/src/request.rs +++ b/src/common/query/src/request.rs @@ -137,9 +137,7 @@ fn portable_remote_dyn_filter_expr( bounds_only: bool, ) -> DataFusionResult> { expr.transform_up(|node| { - if node.as_any().is::() - || (bounds_only && node.as_any().is::()) - { + if node.is::() || (bounds_only && node.is::()) { Ok(Transformed::yes(lit(true))) } else { Ok(Transformed::no(node)) @@ -183,7 +181,7 @@ fn validate_payload_size( fn validate_supported_payload_expr(expr: &Arc) -> DataFusionResult<()> { expr.apply(|node| { - if node.as_any().is::() { + if node.is::() { return Err(DataFusionError::Plan( "HashTableLookupExpr cannot be encoded into DynFilterPayload::Datafusion" .to_string(), @@ -207,7 +205,7 @@ fn validate_decoded_payload_expr( input_schema: &datafusion::arrow::datatypes::Schema, ) -> DataFusionResult<()> { expr.apply(|node| { - if let Some(column) = node.as_any().downcast_ref::() { + if let Some(column) = node.downcast_ref::() { let Some(field) = input_schema.fields().get(column.index()) else { return Err(DataFusionError::Plan(format!( "Decoded Column '{}' references out-of-bounds index {} for input schema of size {}", @@ -391,8 +389,8 @@ mod tests { .decode_datafusion_expr(&TaskContext::default(), &schema, 1024) .unwrap(); - let original = expr.as_any().downcast_ref::().unwrap(); - let decoded = decoded.as_any().downcast_ref::().unwrap(); + let original = expr.downcast_ref::().unwrap(); + let decoded = decoded.downcast_ref::().unwrap(); assert_eq!(decoded.name(), original.name()); assert_eq!(decoded.index(), original.index()); @@ -457,7 +455,7 @@ mod tests { )) as Arc; let lookup = Arc::new(HashTableLookupExpr::new( vec![Arc::clone(&device_id)], - SeededRandomState::with_seeds(0, 0, 0, 0), + SeededRandomState::with_seed(0), Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(0)))), "hash_lookup".to_string(), )) as Arc; @@ -539,10 +537,10 @@ mod tests { )); } - fn contains_expr(expr: &Arc) -> bool { + fn contains_expr(expr: &Arc) -> bool { let mut found = false; expr.apply(|node| { - if node.as_any().is::() { + if node.is::() { found = true; Ok(TreeNodeRecursion::Stop) } else { diff --git a/src/common/query/src/request/initial_remote_dyn_filter_reg.rs b/src/common/query/src/request/initial_remote_dyn_filter_reg.rs index a4a00d5face..3706ae25df5 100644 --- a/src/common/query/src/request/initial_remote_dyn_filter_reg.rs +++ b/src/common/query/src/request/initial_remote_dyn_filter_reg.rs @@ -371,7 +371,7 @@ mod tests { let decoded = reg .decode_children(&TaskContext::default(), &schema, 1024) .unwrap(); - let decoded = decoded[0].as_any().downcast_ref::().unwrap(); + let decoded = decoded[0].downcast_ref::().unwrap(); assert_eq!(reg.filter_id, "filter-1"); assert_eq!(decoded.name(), "host"); diff --git a/src/common/query/src/stream.rs b/src/common/query/src/stream.rs index 1af777ab15f..0a46164d9b4 100644 --- a/src/common/query/src/stream.rs +++ b/src/common/query/src/stream.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::fmt::{Debug, Formatter}; use std::sync::{Arc, Mutex}; @@ -22,8 +21,11 @@ use datafusion::execution::SendableRecordBatchStream as DfSendableRecordBatchStr use datafusion::execution::context::TaskContext; use datafusion::physical_expr::{EquivalenceProperties, Partitioning, PhysicalSortExpr}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, PhysicalExpr, PlanProperties, +}; use datafusion_common::DataFusionError; +use datafusion_common::tree_node::TreeNodeRecursion; use datatypes::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datatypes::schema::SchemaRef; @@ -99,10 +101,6 @@ impl DisplayAs for StreamScanAdapter { } impl ExecutionPlan for StreamScanAdapter { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> ArrowSchemaRef { self.arrow_schema.clone() } @@ -115,6 +113,13 @@ impl ExecutionPlan for StreamScanAdapter { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> datafusion_common::Result, + ) -> datafusion_common::Result { + Ok(TreeNodeRecursion::Continue) + } + // DataFusion will swap children unconditionally. // But since this node is leaf node, it's safe to just return self. fn with_new_children( diff --git a/src/common/recordbatch/src/adapter.rs b/src/common/recordbatch/src/adapter.rs index ca9619c013a..e10bf440833 100644 --- a/src/common/recordbatch/src/adapter.rs +++ b/src/common/recordbatch/src/adapter.rs @@ -29,6 +29,7 @@ use datafusion::arrow::datatypes::SchemaRef as DfSchemaRef; use datafusion::error::Result as DfResult; use datafusion::execution::context::ExecutionProps; use datafusion::logical_expr::Expr; +use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext; use datafusion::logical_expr::utils::conjunction; use datafusion::physical_expr::create_physical_expr; use datafusion::physical_plan::metrics::{BaselineMetrics, MetricValue}; @@ -98,8 +99,13 @@ where .to_dfschema_ref() .context(error::PhysicalExprSnafu)?; - let filters = create_physical_expr(&expr, &df_schema, &ExecutionProps::new()) - .context(error::PhysicalExprSnafu)?; + let filters = create_physical_expr( + &expr, + &df_schema, + &ExecutionProps::new(), + &PhysicalPlanningContext::default(), + ) + .context(error::PhysicalExprSnafu)?; Some(filters) } else { None @@ -931,7 +937,6 @@ fn convert_map_to_json_binary( #[cfg(test)] mod test { - use std::any::Any; use std::time::Duration; use common_error::ext::BoxedError; @@ -942,6 +947,7 @@ mod test { use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricBuilder, MetricsSet}; use datafusion::physical_plan::{DisplayAs, PlanProperties}; + use datafusion_common::tree_node::TreeNodeRecursion; use datatypes::arrow::array::{ArrayRef, MapArray, StringArray, StructArray}; use datatypes::arrow::buffer::OffsetBuffer; use datatypes::arrow::datatypes::Field; @@ -1021,10 +1027,6 @@ mod test { REGION_SCAN_EXEC_NAME } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.properties } @@ -1033,6 +1035,15 @@ mod test { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> datafusion_common::Result, + ) -> datafusion_common::Result { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, _children: Vec>, diff --git a/src/common/recordbatch/src/error.rs b/src/common/recordbatch/src/error.rs index 469320c19cf..bddb8ae8032 100644 --- a/src/common/recordbatch/src/error.rs +++ b/src/common/recordbatch/src/error.rs @@ -205,7 +205,26 @@ impl ErrorExt for Error { | Error::PhysicalExpr { .. } | Error::RecordBatchSliceIndexOverflow { .. } => StatusCode::Internal, - Error::PollStream { .. } => StatusCode::EngineExecuteQuery, + Error::PollStream { error, .. } => { + let mut error = error; + loop { + error = match error { + datafusion::error::DataFusionError::Shared(inner) => inner, + datafusion::error::DataFusionError::Context(_, inner) + | datafusion::error::DataFusionError::Diagnostic(_, inner) => inner, + _ => break, + }; + } + + match error { + datafusion::error::DataFusionError::External(source) => source + .downcast_ref::() + .map_or(StatusCode::EngineExecuteQuery, |source| { + source.status_code() + }), + _ => StatusCode::EngineExecuteQuery, + } + } Error::ArrowCompute { .. } => StatusCode::IllegalState, @@ -242,3 +261,106 @@ impl ErrorExt for Error { } } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use common_error::ext::PlainError; + use datafusion::error::DataFusionError; + + use super::*; + + #[test] + fn poll_stream_status_code_preserves_boxed_error_through_wrappers() { + let boxed_error = |status| { + DataFusionError::External(Box::new(BoxedError::new(PlainError::new( + "neutral error".to_string(), + status, + )))) + }; + + for status in [ + StatusCode::RequestOutdated, + StatusCode::Unknown, + StatusCode::Unsupported, + ] { + let errors = [ + boxed_error(status), + DataFusionError::Shared(Arc::new(boxed_error(status))), + DataFusionError::Context("context".to_string(), Box::new(boxed_error(status))), + DataFusionError::Diagnostic( + Box::new(datafusion::common::Diagnostic::new_error( + "diagnostic", + None, + )), + Box::new(boxed_error(status)), + ), + DataFusionError::Shared(Arc::new(DataFusionError::Context( + "context".to_string(), + Box::new(DataFusionError::Diagnostic( + Box::new(datafusion::common::Diagnostic::new_error( + "diagnostic", + None, + )), + Box::new(boxed_error(status)), + )), + ))), + ]; + + for error in errors { + let error = Error::PollStream { + error, + location: Location::default(), + }; + assert_eq!(error.status_code(), status); + } + } + + let error = Error::PollStream { + error: DataFusionError::Shared(Arc::new(DataFusionError::External(Box::new( + BoxedError::new(Error::PhysicalExpr { + error: DataFusionError::NotImplemented("inner error".to_string()), + location: Location::default(), + }), + )))), + location: Location::default(), + }; + assert_eq!(error.status_code(), StatusCode::Internal); + } + + #[test] + fn poll_stream_status_code_defaults_for_other_datafusion_errors() { + let wrap = |error| { + DataFusionError::Shared(Arc::new(DataFusionError::Context( + "context".to_string(), + Box::new(DataFusionError::Diagnostic( + Box::new(datafusion::common::Diagnostic::new_error( + "diagnostic", + None, + )), + Box::new(error), + )), + ))) + }; + let errors = || { + [ + DataFusionError::External(Box::new(std::io::Error::other("neutral io error"))), + DataFusionError::Internal("neutral internal error".to_string()), + DataFusionError::NotImplemented("neutral not implemented error".to_string()), + DataFusionError::Plan("neutral plan error".to_string()), + DataFusionError::External(Box::new(DataFusionError::Internal( + "inner error".to_string(), + ))), + ] + }; + + for error in errors().into_iter().chain(errors().map(wrap)) { + let error = Error::PollStream { + error, + location: Location::default(), + }; + assert_eq!(error.status_code(), StatusCode::EngineExecuteQuery); + } + } +} diff --git a/src/common/recordbatch/src/filter.rs b/src/common/recordbatch/src/filter.rs index 1b398d6cc45..26c28aa3fa0 100644 --- a/src/common/recordbatch/src/filter.rs +++ b/src/common/recordbatch/src/filter.rs @@ -616,6 +616,7 @@ mod test { use std::sync::Arc; use datafusion::execution::context::ExecutionProps; + use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext; use datafusion::logical_expr::{BinaryExpr, col, lit}; use datafusion::physical_expr::create_physical_expr; use datafusion_common::{Column, DFSchema}; @@ -737,7 +738,13 @@ mod test { ]); let df_schema = DFSchema::try_from(schema.clone()).unwrap(); let props = ExecutionProps::new(); - let physical_expr = create_physical_expr(&expr, &df_schema, &props).unwrap(); + let physical_expr = create_physical_expr( + &expr, + &df_schema, + &props, + &PhysicalPlanningContext::default(), + ) + .unwrap(); let batch = RecordBatch::try_new( Arc::new(schema), vec![ @@ -783,7 +790,13 @@ mod test { let schema = Schema::new(vec![Field::new("col", DataType::Utf8, false)]); let df_schema = DFSchema::try_from(schema.clone()).unwrap(); let props = ExecutionProps::new(); - let physical_expr = create_physical_expr(&col_or_expr, &df_schema, &props).unwrap(); + let physical_expr = create_physical_expr( + &col_or_expr, + &df_schema, + &props, + &PhysicalPlanningContext::default(), + ) + .unwrap(); // Create test data let col_data = Arc::new(datatypes::arrow::array::StringArray::from(vec![ diff --git a/src/common/recordbatch/src/lib.rs b/src/common/recordbatch/src/lib.rs index 72918e9e5e8..4fb37cd5b11 100644 --- a/src/common/recordbatch/src/lib.rs +++ b/src/common/recordbatch/src/lib.rs @@ -44,7 +44,9 @@ pub use datatypes::arrow::record_batch::RecordBatch as DfRecordBatch; use datatypes::arrow::util::display::{ ArrayFormatter, ArrayFormatterFactory, DisplayIndex, FormatOptions, FormatResult, }; -use datatypes::arrow::util::pretty::pretty_format_batches_with_options; +use datatypes::arrow::util::pretty::{ + pretty_format_batches_with_options, pretty_format_batches_with_schema, +}; use datatypes::extension::json::is_any_json_extension_type; use datatypes::prelude::{ConcreteDataType, DataType, VectorRef}; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; @@ -396,12 +398,19 @@ impl RecordBatches { .iter() .map(|x| x.df_record_batch().clone()) .collect::>(); - let options = - FormatOptions::default().with_formatter_factory(Some(&BinaryFormatterFactory)); - let result = - pretty_format_batches_with_options(df_batches, &options).context(error::FormatSnafu)?; + let result: String = if df_batches.is_empty() { + pretty_format_batches_with_schema(self.schema.arrow_schema().clone(), df_batches) + .context(error::FormatSnafu)? + .to_string() + } else { + let options = + FormatOptions::default().with_formatter_factory(Some(&BinaryFormatterFactory)); + pretty_format_batches_with_options(df_batches, &options) + .context(error::FormatSnafu)? + .to_string() + }; - Ok(result.to_string()) + Ok(result) } pub fn try_new(schema: SchemaRef, batches: Vec) -> Result { @@ -1082,6 +1091,35 @@ mod tests { assert_eq!(r.take(), expected); } + #[tokio::test] + async fn test_recordbatches_pretty_print_empty_batches_preserves_schema() { + let schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("unit", ConcreteDataType::string_datatype(), false), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ), + ColumnSchema::new( + "lhs.degrees(val) + rhs.radians(val)", + ConcreteDataType::float64_datatype(), + false, + ), + ])); + let batches = + RecordBatches::try_collect(Box::pin(EmptyRecordBatchStream::new(schema.clone()))) + .await + .unwrap(); + + assert_eq!(schema, batches.schema()); + let expected = "\ ++------+----+-------------------------------------+ +| unit | ts | lhs.degrees(val) + rhs.radians(val) | ++------+----+-------------------------------------+ ++------+----+-------------------------------------+"; + assert_eq!(expected, batches.pretty_print().unwrap()); + } + #[test] fn test_recordbatches_try_new() { let column_a = ColumnSchema::new("a", ConcreteDataType::int32_datatype(), false); diff --git a/src/datanode/src/heartbeat.rs b/src/datanode/src/heartbeat.rs index 0e0c51624e3..f8d92212728 100644 --- a/src/datanode/src/heartbeat.rs +++ b/src/datanode/src/heartbeat.rs @@ -18,6 +18,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use api::v1::meta::heartbeat_request::NodeWorkloads; +use api::v1::meta::mailbox_message::Payload; use api::v1::meta::{DatanodeWorkloads, HeartbeatRequest, NodeInfo, Peer, RegionRole, RegionStat}; use common_base::Plugins; use common_meta::cache_invalidator::CacheInvalidatorRef; @@ -146,7 +147,15 @@ impl HeartbeatTask { None }) { if let Some(msg) = res.mailbox_message.as_ref() { - info!("Received mailbox message: {msg:?}, meta_client id: {client_id:?}"); + let payload_len = msg + .payload + .as_ref() + .map(|Payload::Json(payload)| payload.len()) + .unwrap_or_default(); + info!( + message_id = msg.id, + payload_len, client_id, "Received mailbox message" + ); } if let Some(lease) = res.region_lease.as_ref() { metrics::LAST_RECEIVED_HEARTBEAT_ELAPSED @@ -193,7 +202,8 @@ impl HeartbeatTask { ctx: HeartbeatResponseHandlerContext, handler_executor: HeartbeatResponseHandlerExecutorRef, ) -> Result<()> { - trace!("Heartbeat response: {:?}", ctx.response); + let mailbox_message_id = ctx.response.mailbox_message.as_ref().map(|msg| msg.id); + trace!(?mailbox_message_id, "Handling heartbeat response"); handler_executor .handle(ctx) .await @@ -379,7 +389,8 @@ impl HeartbeatTask { .set(last_sent.elapsed().as_millis() as i64); // Resets the timer. last_sent = Instant::now(); - debug!("Sending heartbeat request: {:?}", req); + let mailbox_message_id = req.mailbox_message.as_ref().map(|msg| msg.id); + debug!(?mailbox_message_id, "Sending heartbeat request"); if let Err(e) = tx.send(req).await { error!(e; "Failed to send heartbeat to metasrv"); match Self::create_streams( diff --git a/src/datanode/src/heartbeat/handler.rs b/src/datanode/src/heartbeat/handler.rs index 71797592c3b..2d18d56004b 100644 --- a/src/datanode/src/heartbeat/handler.rs +++ b/src/datanode/src/heartbeat/handler.rs @@ -41,9 +41,9 @@ use crate::heartbeat::handler::apply_staging_manifest::ApplyStagingManifestsHand use crate::heartbeat::handler::close_region::CloseRegionsHandler; use crate::heartbeat::handler::downgrade_region::DowngradeRegionsHandler; use crate::heartbeat::handler::enter_staging::EnterStagingRegionsHandler; -use crate::heartbeat::handler::file_ref::GetFileRefsHandler; +use crate::heartbeat::handler::file_ref::{GetFileRefsHandler, GetPackedFileRefsHandler}; use crate::heartbeat::handler::flush_region::FlushRegionsHandler; -use crate::heartbeat::handler::gc_worker::GcRegionsHandler; +use crate::heartbeat::handler::gc_worker::{GcRegionsHandler, PackedGcRegionsHandler}; use crate::heartbeat::handler::open_region::OpenRegionsHandler; use crate::heartbeat::handler::remap_manifest::RemapManifestHandler; use crate::heartbeat::handler::sync_region::SyncRegionHandler; @@ -135,7 +135,11 @@ impl RegionHeartbeatResponseHandler { .into(), ))), Instruction::GetFileRefs(_) => Ok(Some(Box::new(GetFileRefsHandler.into()))), + Instruction::GetPackedFileRefs(_) => { + Ok(Some(Box::new(GetPackedFileRefsHandler.into()))) + } Instruction::GcRegions(_) => Ok(Some(Box::new(GcRegionsHandler.into()))), + Instruction::PackedGcRegions(_) => Ok(Some(Box::new(PackedGcRegionsHandler.into()))), Instruction::InvalidateCaches(_) => InvalidHeartbeatResponseSnafu.fail(), Instruction::Suspend => Ok(None), Instruction::EnterStagingRegions(_) => { @@ -159,7 +163,9 @@ pub enum InstructionHandlers { DowngradeRegions(DowngradeRegionsHandler), UpgradeRegions(UpgradeRegionsHandler), GetFileRefs(GetFileRefsHandler), + GetPackedFileRefs(GetPackedFileRefsHandler), GcRegions(GcRegionsHandler), + PackedGcRegions(PackedGcRegionsHandler), EnterStagingRegions(EnterStagingRegionsHandler), SyncRegions(SyncRegionHandler), RemapManifest(RemapManifestHandler), @@ -192,7 +198,9 @@ impl_from_handler!( DowngradeRegionsHandler => DowngradeRegions, UpgradeRegionsHandler => UpgradeRegions, GetFileRefsHandler => GetFileRefs, + GetPackedFileRefsHandler => GetPackedFileRefs, GcRegionsHandler => GcRegions, + PackedGcRegionsHandler => PackedGcRegions, EnterStagingRegionsHandler => EnterStagingRegions, SyncRegionHandler => SyncRegions, RemapManifestHandler => RemapManifest, @@ -240,7 +248,9 @@ dispatch_instr!( DowngradeRegions => DowngradeRegions, UpgradeRegions => UpgradeRegions, GetFileRefs => GetFileRefs, + GetPackedFileRefs => GetPackedFileRefs, GcRegions => GcRegions, + PackedGcRegions => PackedGcRegions, EnterStagingRegions => EnterStagingRegions, SyncRegions => SyncRegions, RemapManifest => RemapManifest, diff --git a/src/datanode/src/heartbeat/handler/file_ref.rs b/src/datanode/src/heartbeat/handler/file_ref.rs index ef9cbd4158e..554f4119b05 100644 --- a/src/datanode/src/heartbeat/handler/file_ref.rs +++ b/src/datanode/src/heartbeat/handler/file_ref.rs @@ -13,7 +13,10 @@ // limitations under the License. use common_error::ext::ErrorExt; -use common_meta::instruction::{GetFileRefs, GetFileRefsReply, InstructionError, InstructionReply}; +use common_meta::instruction::{ + GetFileRefs, GetFileRefsReply, GetPackedFileRefs, GetPackedFileRefsReply, InstructionError, + InstructionReply, PackedFileRefsManifest, +}; use store_api::storage::FileRefsManifest; use crate::heartbeat::handler::{HandlerContext, InstructionHandler}; @@ -65,3 +68,54 @@ impl InstructionHandler for GetFileRefsHandler { } } } + +pub struct GetPackedFileRefsHandler; + +#[async_trait::async_trait] +impl InstructionHandler for GetPackedFileRefsHandler { + type Instruction = GetPackedFileRefs; + + async fn handle( + &self, + ctx: &HandlerContext, + get_file_refs: Self::Instruction, + ) -> Option { + let region_server = &ctx.region_server; + let Some(mito_engine) = region_server.mito_engine() else { + return Some(InstructionReply::GetPackedFileRefs( + GetPackedFileRefsReply { + packed_file_refs_manifest: PackedFileRefsManifest::default(), + success: false, + error: Some(InstructionError::legacy_internal_retryable( + "MitoEngine not found", + )), + }, + )); + }; + match mito_engine + .get_snapshot_of_file_refs(get_file_refs.query_regions, get_file_refs.related_regions) + .await + { + Ok(all_file_refs) => Some(InstructionReply::GetPackedFileRefs( + GetPackedFileRefsReply { + packed_file_refs_manifest: PackedFileRefsManifest::from_manifest( + &all_file_refs, + ), + success: true, + error: None, + }, + )), + Err(e) => Some(InstructionReply::GetPackedFileRefs( + GetPackedFileRefsReply { + packed_file_refs_manifest: PackedFileRefsManifest::default(), + success: false, + error: Some(InstructionError::new( + e.status_code(), + format!("Failed to get file refs: {}", e.output_msg()), + e.retry_hint(), + )), + }, + )), + } + } +} diff --git a/src/datanode/src/heartbeat/handler/gc_worker.rs b/src/datanode/src/heartbeat/handler/gc_worker.rs index b32a1c29ad4..e63d21b6e5b 100644 --- a/src/datanode/src/heartbeat/handler/gc_worker.rs +++ b/src/datanode/src/heartbeat/handler/gc_worker.rs @@ -15,7 +15,7 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; -use common_meta::instruction::{GcRegions, GcRegionsReply, InstructionReply}; +use common_meta::instruction::{GcRegions, GcRegionsReply, InstructionReply, PackedGcRegions}; use common_meta::key::table_info::TableInfoManager; use common_meta::key::table_route::TableRouteManager; use common_telemetry::{debug, warn}; @@ -34,6 +34,7 @@ use crate::error::{GcMitoEngineSnafu, GetMetadataSnafu, Result, UnexpectedSnafu} use crate::heartbeat::handler::{HandlerContext, InstructionHandler}; pub struct GcRegionsHandler; +pub struct PackedGcRegionsHandler; #[async_trait::async_trait] impl InstructionHandler for GcRegionsHandler { @@ -67,7 +68,7 @@ impl InstructionHandler for GcRegionsHandler { .push(rid); } - let file_refs_manifest = gc_regions.file_refs_manifest.clone(); + let file_refs_manifest = gc_regions.file_refs_manifest; let full_file_listing = gc_regions.full_file_listing; let ctx_clone = ctx.clone(); @@ -138,6 +139,36 @@ impl InstructionHandler for GcRegionsHandler { } } +#[async_trait::async_trait] +impl InstructionHandler for PackedGcRegionsHandler { + type Instruction = PackedGcRegions; + + async fn handle( + &self, + ctx: &HandlerContext, + instruction: Self::Instruction, + ) -> Option { + let manifest = match instruction.packed_file_refs_manifest.into_manifest() { + Ok(manifest) => manifest, + Err(err) => { + return Some(InstructionReply::GcRegions(GcRegionsReply { + result: Err(common_meta::instruction::InstructionError::from_error(&err)), + })); + } + }; + GcRegionsHandler + .handle( + ctx, + GcRegions { + regions: instruction.regions, + file_refs_manifest: manifest, + full_file_listing: instruction.full_file_listing, + }, + ) + .await + } +} + impl GcRegionsHandler { /// Create a GC worker for the given table and region IDs. async fn create_gc_worker( diff --git a/src/datanode/src/region_server/catalog.rs b/src/datanode/src/region_server/catalog.rs index a4df422b759..264408342e9 100644 --- a/src/datanode/src/region_server/catalog.rs +++ b/src/datanode/src/region_server/catalog.rs @@ -179,9 +179,6 @@ impl NameAwareCatalogList { } impl CatalogProviderList for NameAwareCatalogList { - fn as_any(&self) -> &dyn std::any::Any { - self - } fn register_catalog( &self, _name: String, @@ -203,9 +200,6 @@ struct NameAwareCatalogProvider { } impl CatalogProvider for NameAwareCatalogProvider { - fn as_any(&self) -> &dyn std::any::Any { - self - } fn schema_names(&self) -> Vec { vec![] } @@ -229,9 +223,6 @@ impl std::fmt::Debug for NameAwareSchemaProvider { #[async_trait::async_trait] impl SchemaProvider for NameAwareSchemaProvider { - fn as_any(&self) -> &dyn std::any::Any { - self - } fn table_names(&self) -> Vec { vec![] } diff --git a/src/datatypes/src/schema/constraint.rs b/src/datatypes/src/schema/constraint.rs index 60025be68e4..d946a92bec5 100644 --- a/src/datatypes/src/schema/constraint.rs +++ b/src/datatypes/src/schema/constraint.rs @@ -23,7 +23,7 @@ use crate::error::{self, Result}; use crate::types::cast; use crate::value::Value; use crate::vectors::operations::VectorOp; -use crate::vectors::{TimestampMillisecondVector, VectorRef}; +use crate::vectors::{Helper, TimestampMillisecondVector, VectorRef}; pub const CURRENT_TIMESTAMP: &str = "current_timestamp"; pub const CURRENT_TIMESTAMP_FN: &str = "current_timestamp()"; @@ -151,15 +151,20 @@ impl ColumnDefaultConstraint { ColumnDefaultConstraint::Value(v) => { ensure!(is_nullable || !v.is_null(), error::NullDefaultSnafu); - // TODO(yingwen): - // 1. For null value, we could use NullVector once it supports custom logical type. - // 2. For non null value, we could use ConstantVector, but it would cause all codes - // attempt to downcast the vector fail if they don't check whether the vector is const - // first. - let mut mutable_vector = data_type.create_mutable_vector(1); - mutable_vector.try_push_value_ref(&v.as_value_ref())?; - let base_vector = mutable_vector.to_vector(); - Ok(base_vector.replicate(&[num_rows])) + if let Ok(vector) = v.try_to_scalar_value(data_type).and_then(|scalar| { + Helper::try_from_scalar_value(scalar, num_rows, Some(data_type)) + }) { + return Ok(vector); + } + + // Some extension values, such as JSON nested in a struct, cannot safely + // round-trip through ScalarValue. Preserve their logical type with the + // type-specific vector builder instead. + let mut mutable_vector = data_type.create_mutable_vector(num_rows); + for _ in 0..num_rows { + mutable_vector.try_push_value_ref(&v.as_value_ref())?; + } + Ok(mutable_vector.to_vector()) } } } @@ -391,6 +396,82 @@ mod tests { assert_eq!(Value::Int32(10), v); } + #[test] + fn test_struct_default_null_fields_and_json() { + use crate::types::{StructField, StructType}; + use crate::value::StructValue; + + let inner_type = StructType::from([StructField::new( + "x", + ConcreteDataType::int32_datatype(), + true, + )]); + let inner = Value::Struct(StructValue::new(vec![Value::Null], inner_type)); + let json = crate::json::JsonSettings::default() + .encode(serde_json::json!({"answer": 42})) + .unwrap(); + let nested_type = StructType::from([StructField::new("nested", inner.data_type(), true)]); + let json_type = StructType::from([StructField::new("json", json.data_type(), true)]); + let values = [ + inner.clone(), + Value::Struct(StructValue::new(vec![inner], nested_type)), + Value::Struct(StructValue::new(vec![], StructType::default())), + Value::Struct(StructValue::new(vec![json], json_type)), + ]; + for value in values { + let data_type = value.data_type(); + // JSON children are read back as their underlying struct values. + let expected = serde_json::Value::try_from(value.clone()).unwrap(); + for num_rows in [1, 3] { + let vector = ColumnDefaultConstraint::Value(value.clone()) + .create_default_vector(&data_type, false, num_rows) + .unwrap(); + assert_eq!(data_type, vector.data_type()); + assert_eq!( + data_type.as_arrow_type(), + *vector.to_arrow_array().data_type() + ); + assert_eq!(num_rows, vector.len()); + assert_eq!(0, vector.null_count()); + for row in 0..num_rows { + assert_eq!( + expected, + serde_json::Value::try_from(vector.get(row)).unwrap() + ); + } + } + } + } + + #[test] + fn test_string_default_preserves_batch_schema() { + use arrow::datatypes::{Field, Schema}; + use arrow::record_batch::RecordBatch; + + for data_type in [ + ConcreteDataType::large_string_datatype(), + ConcreteDataType::utf8_view_datatype(), + ] { + let schema = Arc::new(Schema::new(vec![Field::new( + "tag", + data_type.as_arrow_type(), + true, + )])); + for value in [Value::from("greptime"), Value::Null] { + let vector = ColumnDefaultConstraint::Value(value.clone()) + .create_default_vector(&data_type, true, 3) + .unwrap(); + let batch = + RecordBatch::try_new(schema.clone(), vec![vector.to_arrow_array()]).unwrap(); + assert_eq!(3, batch.num_rows()); + assert_eq!(data_type, vector.data_type()); + for row in 0..batch.num_rows() { + assert_eq!(value, vector.get(row)); + } + } + } + } + #[test] fn test_create_default_vector_by_func() { let constraint = ColumnDefaultConstraint::Function(CURRENT_TIMESTAMP.to_string()); diff --git a/src/datatypes/src/timestamp.rs b/src/datatypes/src/timestamp.rs index bf6fdaa1623..ac6ca1444f1 100644 --- a/src/datatypes/src/timestamp.rs +++ b/src/datatypes/src/timestamp.rs @@ -13,7 +13,7 @@ // limitations under the License. use arrow_array::{ - ArrayRef, PrimitiveArray, TimestampMicrosecondArray, TimestampMillisecondArray, + Array, ArrayRef, PrimitiveArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, }; use arrow_schema::DataType; @@ -179,8 +179,24 @@ pub fn timestamp_array_to_primitive( Some((ts_primitive, *unit)) } +/// Appends non-null timestamps in the source array's native time unit. +/// +/// Returns `None` for a non-timestamp array without changing `timestamps`. +pub fn append_timestamps(ts_array: &ArrayRef, timestamps: &mut Vec) -> Option<()> { + let (values, _) = timestamp_array_to_primitive(ts_array)?; + if values.null_count() == 0 { + timestamps.extend_from_slice(values.values()); + } else { + timestamps.extend(values.iter().flatten()); + } + Some(()) +} + #[cfg(test)] mod tests { + use std::sync::Arc; + + use arrow_array::Int64Array; use common_time::timezone::set_default_timezone; use super::*; @@ -213,4 +229,46 @@ mod tests { assert_eq!(ts, ts.as_scalar_ref()); assert_eq!(ts, ts.to_owned_scalar()); } + + #[test] + fn test_append_timestamps() { + let cases = [ + vec![Some(i64::MIN), Some(-1), Some(0), Some(i64::MAX)], + vec![Some(-1), None, Some(2), None], + vec![None, None], + vec![], + ]; + for values in cases { + let arrays: [ArrayRef; 4] = [ + Arc::new(TimestampSecondArray::from(values.clone())), + Arc::new(TimestampMillisecondArray::from(values.clone())), + Arc::new(TimestampMicrosecondArray::from(values.clone())), + Arc::new(TimestampNanosecondArray::from(values.clone())), + ]; + let mut expected = vec![42]; + expected.extend(values.iter().flatten().copied()); + for array in arrays { + for _ in 0..2 { + let (primitive, _) = timestamp_array_to_primitive(&array).unwrap(); + let mut reference = vec![42]; + reference.extend(primitive.iter().flatten()); + + let mut timestamps = vec![42]; + assert_eq!(append_timestamps(&array, &mut timestamps), Some(())); + assert_eq!(timestamps, expected); + assert_eq!(timestamps, reference); + } + } + } + } + + #[test] + fn test_append_timestamps_invalid_array_preserves_prefix() { + for values in [vec![], vec![Some(1), None]] { + let array: ArrayRef = Arc::new(Int64Array::from(values)); + let mut timestamps = vec![42, -1]; + assert_eq!(append_timestamps(&array, &mut timestamps), None); + assert_eq!(timestamps, vec![42, -1]); + } + } } diff --git a/src/datatypes/src/types/cast.rs b/src/datatypes/src/types/cast.rs index d7a27a45e10..2f92713c26f 100644 --- a/src/datatypes/src/types/cast.rs +++ b/src/datatypes/src/types/cast.rs @@ -26,7 +26,7 @@ pub fn cast(src_value: Value, dest_type: &ConcreteDataType) -> Result { } let src_type = src_value.data_type(); let scalar_value = src_value.try_to_scalar_value(&src_type)?; - let new_value = Helper::try_from_scalar_value(scalar_value, 1)? + let new_value = Helper::try_from_scalar_value(scalar_value, 1, None)? .cast(dest_type)? .get(0); Ok(new_value) diff --git a/src/datatypes/src/value.rs b/src/datatypes/src/value.rs index 9ad9e1d46ef..56a551484f2 100644 --- a/src/datatypes/src/value.rs +++ b/src/datatypes/src/value.rs @@ -1085,11 +1085,21 @@ impl StructValue { } fn try_to_scalar_value(&self, output_type: &StructType) -> Result { + let output_fields = output_type.fields(); + ensure!( + self.items.len() == output_fields.len(), + InconsistentStructFieldsAndItemsSnafu { + field_len: output_fields.len(), + item_len: self.items.len() + } + ); let arrays = self .items .iter() - .map(|value| { - let scalar_value = value.try_to_scalar_value(&value.data_type())?; + .zip(output_fields.iter()) + .map(|(value, field)| { + // Null values need the declared field type to produce a typed null array. + let scalar_value = value.try_to_scalar_value(field.data_type())?; scalar_value .to_array() .context(ConvertScalarToArrowArraySnafu) @@ -1097,7 +1107,13 @@ impl StructValue { .collect::>>>()?; let fields = output_type.as_arrow_fields(); - let struct_array = StructArray::new(fields, arrays, None); + let struct_array = + StructArray::try_new_with_length(fields, arrays, None, 1).map_err(|error| { + error::ToScalarValueSnafu { + reason: error.to_string(), + } + .build() + })?; Ok(ScalarValue::Struct(Arc::new(struct_array))) } } @@ -1224,6 +1240,8 @@ impl TryFrom for Value { | ScalarValue::Decimal256(_, _, _) | ScalarValue::FixedSizeList(_) | ScalarValue::LargeList(_) + | ScalarValue::ListView(_) + | ScalarValue::LargeListView(_) | ScalarValue::Union(_, _, _) | ScalarValue::Float16(_) | ScalarValue::Utf8View(_) @@ -3082,6 +3100,66 @@ pub(crate) mod tests { } } + #[test] + fn test_struct_scalar_null_fields() { + let struct_type = StructType::from([ + StructField::new("x", ConcreteDataType::int32_datatype(), true), + StructField::new("name", ConcreteDataType::string_datatype(), true), + ]); + let value = StructValue::new(vec![Value::Null, Value::from("hello")], struct_type.clone()); + let ScalarValue::Struct(array) = value.try_to_scalar_value(&struct_type).unwrap() else { + panic!("Expected struct scalar"); + }; + assert_eq!(1, array.len()); + assert_eq!(0, array.null_count()); + assert_eq!(&struct_type.as_arrow_fields(), array.fields()); + assert_eq!( + ScalarValue::Int32(None), + ScalarValue::try_from_array(array.column(0), 0).unwrap() + ); + + let nested_type = StructType::from([StructField::new( + "nested", + ConcreteDataType::struct_datatype(struct_type), + true, + )]); + for child in [Value::Struct(value), Value::Null] { + let nested = StructValue::new(vec![child.clone()], nested_type.clone()); + let ScalarValue::Struct(array) = nested.try_to_scalar_value(&nested_type).unwrap() + else { + panic!("Expected struct scalar"); + }; + let vector = crate::vectors::Helper::try_into_vector(array.column(0).clone()).unwrap(); + assert_eq!(child, vector.get(0)); + } + + let empty_type = StructType::default(); + let empty = StructValue::new(vec![], empty_type.clone()); + let ScalarValue::Struct(array) = empty.try_to_scalar_value(&empty_type).unwrap() else { + panic!("Expected struct scalar"); + }; + assert_eq!(1, array.len()); + assert_eq!(0, array.num_columns()); + assert_eq!(0, array.null_count()); + } + + #[test] + fn test_struct_scalar_invalid_fields() { + let struct_type = StructType::from([StructField::new( + "x", + ConcreteDataType::int32_datatype(), + false, + )]); + for child in [Value::Null, Value::from("wrong type")] { + let value = StructValue::new(vec![child], struct_type.clone()); + assert!(value.try_to_scalar_value(&struct_type).is_err()); + } + let value = StructValue::new(vec![Value::Int32(1)], struct_type.clone()); + assert!(value.try_to_scalar_value(&StructType::default()).is_err()); + let empty = StructValue::new(vec![], StructType::default()); + assert!(empty.try_to_scalar_value(&struct_type).is_err()); + } + #[test] fn test_timestamp_to_scalar_value() { assert_eq!( diff --git a/src/datatypes/src/vectors.rs b/src/datatypes/src/vectors.rs index 593bec833da..38db5940df9 100644 --- a/src/datatypes/src/vectors.rs +++ b/src/datatypes/src/vectors.rs @@ -27,7 +27,6 @@ use crate::vectors::operations::VectorOp; mod binary; mod boolean; -mod constant; mod date; mod decimal; mod dictionary; @@ -48,7 +47,6 @@ mod validity; pub use binary::{BinaryVector, BinaryVectorBuilder}; pub use boolean::{BooleanVector, BooleanVectorBuilder}; -pub use constant::ConstantVector; pub use date::{DateVector, DateVectorBuilder}; pub use decimal::{Decimal128Vector, Decimal128VectorBuilder}; pub(crate) use dictionary::StringDictionaryVectorBuilder; @@ -126,11 +124,6 @@ pub trait Vector: Send + Sync + Serializable + Debug + VectorOp { /// This is `O(1)`. fn null_count(&self) -> usize; - /// Returns true when it's a ConstantColumn - fn is_const(&self) -> bool { - false - } - /// Returns whether row is null. fn is_null(&self, row: usize) -> bool; diff --git a/src/datatypes/src/vectors/binary.rs b/src/datatypes/src/vectors/binary.rs index 6d9954fb80f..77344b46512 100644 --- a/src/datatypes/src/vectors/binary.rs +++ b/src/datatypes/src/vectors/binary.rs @@ -479,7 +479,6 @@ mod tests { assert_eq!(2, v.len()); assert_eq!("BinaryVector", v.vector_type_name()); - assert!(!v.is_const()); assert!(v.validity().is_all_valid()); assert!(!v.only_null()); assert_eq!(128, v.memory_size()); diff --git a/src/datatypes/src/vectors/boolean.rs b/src/datatypes/src/vectors/boolean.rs index 4ea7a4591b9..48625da516b 100644 --- a/src/datatypes/src/vectors/boolean.rs +++ b/src/datatypes/src/vectors/boolean.rs @@ -40,10 +40,6 @@ impl BooleanVector { pub fn as_boolean_array(&self) -> &BooleanArray { &self.array } - - pub(crate) fn false_count(&self) -> usize { - self.array.false_count() - } } impl From> for BooleanVector { @@ -251,7 +247,6 @@ mod tests { let v = BooleanVector::from(bools.clone()); assert_eq!(9, v.len()); assert_eq!("BooleanVector", v.vector_type_name()); - assert!(!v.is_const()); assert!(v.validity().is_all_valid()); assert!(!v.only_null()); assert_eq!(2, v.memory_size()); diff --git a/src/datatypes/src/vectors/constant.rs b/src/datatypes/src/vectors/constant.rs deleted file mode 100644 index 3a0adc7b62a..00000000000 --- a/src/datatypes/src/vectors/constant.rs +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::any::Any; -use std::fmt; -use std::sync::Arc; - -use arrow::array::{Array, ArrayRef, UInt32Array}; -use snafu::{ResultExt, ensure}; - -use crate::data_type::ConcreteDataType; -use crate::error::{self, Result, SerializeSnafu}; -use crate::serialize::Serializable; -use crate::value::{Value, ValueRef}; -use crate::vectors::{BooleanVector, Helper, UInt32Vector, Validity, Vector, VectorRef}; - -#[derive(Clone)] -pub struct ConstantVector { - length: usize, - vector: VectorRef, -} - -impl ConstantVector { - /// Create a new [ConstantVector]. - /// - /// # Panics - /// Panics if `vector.len() != 1`. - pub fn new(vector: VectorRef, length: usize) -> Self { - assert_eq!(1, vector.len()); - - // Avoid const recursion. - if vector.is_const() { - let vec: &ConstantVector = unsafe { Helper::static_cast(&vector) }; - return Self::new(vec.inner().clone(), length); - } - Self { vector, length } - } - - pub fn inner(&self) -> &VectorRef { - &self.vector - } - - /// Returns the constant value. - pub fn get_constant_ref(&self) -> ValueRef<'_> { - self.vector.get_ref(0) - } - - pub(crate) fn replicate_vector(&self, offsets: &[usize]) -> VectorRef { - assert_eq!(offsets.len(), self.len()); - - if offsets.is_empty() { - return self.slice(0, 0); - } - - Arc::new(ConstantVector::new( - self.vector.clone(), - *offsets.last().unwrap(), - )) - } - - pub(crate) fn filter_vector(&self, filter: &BooleanVector) -> Result { - let length = self.len() - filter.false_count(); - if length == self.len() { - return Ok(Arc::new(self.clone())); - } - Ok(Arc::new(ConstantVector::new(self.inner().clone(), length))) - } - - pub(crate) fn cast_vector(&self, to_type: &ConcreteDataType) -> Result { - Ok(Arc::new(ConstantVector::new( - self.inner().cast(to_type)?, - self.length, - ))) - } - - pub(crate) fn take_vector(&self, indices: &UInt32Vector) -> Result { - if indices.is_empty() { - return Ok(self.slice(0, 0)); - } - ensure!( - indices.null_count() == 0, - error::UnsupportedOperationSnafu { - op: "taking a null index", - vector_type: self.vector_type_name(), - } - ); - - let len = self.len(); - let arr = indices.to_arrow_array(); - let indices_arr = arr.as_any().downcast_ref::().unwrap(); - if !arrow::compute::min_boolean( - &arrow::compute::kernels::cmp::lt(indices_arr, &UInt32Array::new_scalar(len as u32)) - .unwrap(), - ) - .unwrap() - { - panic!( - "Array index out of bounds, cannot take index out of the length of the array: {len}" - ); - } - - Ok(Arc::new(ConstantVector::new( - self.inner().clone(), - indices.len(), - ))) - } -} - -impl Vector for ConstantVector { - fn data_type(&self) -> ConcreteDataType { - self.vector.data_type() - } - - fn vector_type_name(&self) -> String { - "ConstantVector".to_string() - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn len(&self) -> usize { - self.length - } - - fn to_arrow_array(&self) -> ArrayRef { - let v = self.vector.replicate(&[self.length]); - v.to_arrow_array() - } - - fn to_boxed_arrow_array(&self) -> Box { - let v = self.vector.replicate(&[self.length]); - v.to_boxed_arrow_array() - } - - fn is_const(&self) -> bool { - true - } - - fn validity(&self) -> Validity { - if self.vector.is_null(0) { - Validity::all_null(self.length) - } else { - Validity::all_valid(self.length) - } - } - - fn memory_size(&self) -> usize { - self.vector.memory_size() - } - - fn is_null(&self, _row: usize) -> bool { - self.vector.is_null(0) - } - - fn only_null(&self) -> bool { - self.vector.is_null(0) - } - - fn slice(&self, _offset: usize, length: usize) -> VectorRef { - Arc::new(Self { - vector: self.vector.clone(), - length, - }) - } - - fn get(&self, _index: usize) -> Value { - self.vector.get(0) - } - - fn get_ref(&self, _index: usize) -> ValueRef<'_> { - self.vector.get_ref(0) - } - - fn null_count(&self) -> usize { - if self.only_null() { self.len() } else { 0 } - } -} - -impl fmt::Debug for ConstantVector { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "ConstantVector([{:?}; {}])", self.get(0), self.len()) - } -} - -impl Serializable for ConstantVector { - fn serialize_to_json(&self) -> Result> { - std::iter::repeat_n(self.get(0), self.len()) - .map(serde_json::Value::try_from) - .collect::>() - .context(SerializeSnafu) - } -} - -#[cfg(test)] -mod tests { - use arrow::datatypes::DataType as ArrowDataType; - - use super::*; - use crate::vectors::Int32Vector; - - #[test] - fn test_constant_vector_misc() { - let a = Int32Vector::from_slice(vec![1]); - let c = ConstantVector::new(Arc::new(a), 10); - - assert_eq!("ConstantVector", c.vector_type_name()); - assert!(c.is_const()); - assert_eq!(10, c.len()); - assert!(c.validity().is_all_valid()); - assert!(!c.only_null()); - assert_eq!(4, c.memory_size()); - - for i in 0..10 { - assert!(!c.is_null(i)); - assert_eq!(Value::Int32(1), c.get(i)); - } - - let arrow_arr = c.to_arrow_array(); - assert_eq!(10, arrow_arr.len()); - assert_eq!(&ArrowDataType::Int32, arrow_arr.data_type()); - } - - #[test] - fn test_debug_null_array() { - let a = Int32Vector::from_slice(vec![1]); - let c = ConstantVector::new(Arc::new(a), 10); - - let s = format!("{c:?}"); - assert_eq!(s, "ConstantVector([Int32(1); 10])"); - } - - #[test] - fn test_serialize_json() { - let a = Int32Vector::from_slice(vec![1]); - let c = ConstantVector::new(Arc::new(a), 10); - - let s = serde_json::to_string(&c.serialize_to_json().unwrap()).unwrap(); - assert_eq!(s, "[1,1,1,1,1,1,1,1,1,1]"); - } -} diff --git a/src/datatypes/src/vectors/decimal.rs b/src/datatypes/src/vectors/decimal.rs index 46fa73297a5..34d066f0725 100644 --- a/src/datatypes/src/vectors/decimal.rs +++ b/src/datatypes/src/vectors/decimal.rs @@ -385,43 +385,6 @@ impl Decimal128VectorBuilder { vectors::impl_try_from_arrow_array_for_vector!(Decimal128Array, Decimal128Vector); -pub(crate) fn replicate_decimal128( - vector: &Decimal128Vector, - offsets: &[usize], -) -> Decimal128Vector { - assert_eq!(offsets.len(), vector.len()); - - if offsets.is_empty() { - return vector.get_slice(0, 0); - } - - // Safety: safe to unwrap because we the vector ensures precision and scale are valid. - let mut builder = Decimal128VectorBuilder::with_capacity(*offsets.last().unwrap()) - .with_precision_and_scale(vector.precision(), vector.scale()) - .unwrap(); - - let mut previous_offset = 0; - - for (offset, value) in offsets.iter().zip(vector.array.iter()) { - let repeat_times = *offset - previous_offset; - match value { - Some(data) => { - unsafe { - // Safety: std::iter::Repeat and std::iter::Take implement TrustedLen. - builder - .mutable_array - .append_trusted_len_iter(std::iter::repeat_n(data, repeat_times)); - } - } - None => { - builder.mutable_array.append_nulls(repeat_times); - } - } - previous_offset = *offset; - } - builder.finish() -} - #[cfg(test)] pub mod tests { use arrow_array::Decimal128Array; diff --git a/src/datatypes/src/vectors/dictionary.rs b/src/datatypes/src/vectors/dictionary.rs index d41daf4638f..5f117b0ae88 100644 --- a/src/datatypes/src/vectors/dictionary.rs +++ b/src/datatypes/src/vectors/dictionary.rs @@ -17,8 +17,7 @@ use std::fmt; use std::sync::Arc; use arrow::array::{ - Array, ArrayBuilder, ArrayRef, DictionaryArray, PrimitiveArray, PrimitiveBuilder, - StringDictionaryBuilder, + Array, ArrayBuilder, ArrayRef, DictionaryArray, PrimitiveArray, StringDictionaryBuilder, }; use arrow::datatypes::{ArrowDictionaryKeyType, ArrowNativeType, UInt32Type}; use serde_json::Value as JsonValue; @@ -320,36 +319,6 @@ impl<'a, K: ArrowDictionaryKeyType> Iterator for DictionaryIter<'a, K> { } impl VectorOp for DictionaryVector { - fn replicate(&self, offsets: &[usize]) -> VectorRef { - let keys = self.array.keys(); - let mut replicated_keys = PrimitiveBuilder::new(); - - let mut previous_offset = 0; - let mut key_iter = keys.iter().chain(std::iter::repeat(None)); - for &offset in offsets { - let key = key_iter.next().unwrap(); - - // repeat this key (offset - previous_offset) times - let repeat_count = offset - previous_offset; - for _ in 0..repeat_count { - replicated_keys.append_option(key); - } - - previous_offset = offset; - } - - let new_keys = replicated_keys.finish(); - let new_array = DictionaryArray::try_new(new_keys, self.values().clone()) - .expect("Failed to create replicated dictionary array"); - - Arc::new(Self { - array: new_array, - key_type: self.key_type.clone(), - item_type: self.item_type.clone(), - item_vector: self.item_vector.clone(), - }) - } - fn filter(&self, filter: &vectors::BooleanVector) -> Result { let key_array: ArrayRef = Arc::new(self.array.keys().clone()); let key_vector = Helper::try_into_vector(&key_array)?; @@ -485,21 +454,6 @@ mod tests { assert_eq!(sliced.get(2), Value::Null); } - #[test] - fn test_replicate() { - let dict_vec = create_test_dictionary(); - - // Replicate with offsets [0, 2, 5] - should get values at these indices - let offsets = vec![0, 2, 5]; - let replicated = dict_vec.replicate(&offsets); - assert_eq!(replicated.len(), 5); - assert_eq!(replicated.get(0), Value::String("b".to_string().into())); - assert_eq!(replicated.get(1), Value::String("b".to_string().into())); - assert_eq!(replicated.get(2), Value::String("c".to_string().into())); - assert_eq!(replicated.get(3), Value::String("c".to_string().into())); - assert_eq!(replicated.get(4), Value::String("c".to_string().into())); - } - #[test] fn test_filter() { let dict_vec = create_test_dictionary(); diff --git a/src/datatypes/src/vectors/eq.rs b/src/datatypes/src/vectors/eq.rs index e8e942d3ff4..937e4587c35 100644 --- a/src/datatypes/src/vectors/eq.rs +++ b/src/datatypes/src/vectors/eq.rs @@ -18,7 +18,6 @@ use common_time::interval::IntervalUnit; use crate::data_type::DataType; use crate::types::{DurationType, TimeType, TimestampType}; -use crate::vectors::constant::ConstantVector; use crate::vectors::struct_vector::StructVector; use crate::vectors::{ BinaryVector, BooleanVector, DateVector, Decimal128Vector, DurationMicrosecondVector, @@ -58,23 +57,6 @@ fn equal(lhs: &dyn Vector, rhs: &dyn Vector) -> bool { return false; } - if lhs.is_const() || rhs.is_const() { - // Length has been checked before, so we only need to compare inner - // vector here. - return equal( - &**lhs - .as_any() - .downcast_ref::() - .unwrap() - .inner(), - &**rhs - .as_any() - .downcast_ref::() - .unwrap() - .inner(), - ); - } - use crate::data_type::ConcreteDataType::*; let lhs_type = lhs.data_type(); @@ -191,10 +173,6 @@ mod tests { Some(b"world".to_vec()), ]))); assert_vector_ref_eq(Arc::new(BooleanVector::from(vec![true, false]))); - assert_vector_ref_eq(Arc::new(ConstantVector::new( - Arc::new(BooleanVector::from(vec![true])), - 5, - ))); assert_vector_ref_eq(Arc::new(BooleanVector::from(vec![true, false]))); assert_vector_ref_eq(Arc::new(DateVector::from(vec![Some(100), Some(120)]))); assert_vector_ref_eq(Arc::new(TimestampSecondVector::from_values([100, 120]))); @@ -259,22 +237,6 @@ mod tests { ]))); } - // Regression: second arm must downcast `rhs` (was `lhs`), or same-length ConstantVectors - // with different inners compare equal. - #[test] - fn test_constant_vector_eq_compares_both_inners() { - assert_vector_ref_ne( - Arc::new(ConstantVector::new( - Arc::new(BooleanVector::from(vec![true])), - 5, - )), - Arc::new(ConstantVector::new( - Arc::new(BooleanVector::from(vec![false])), - 5, - )), - ); - } - #[test] fn test_vector_ne() { assert_vector_ref_ne( @@ -289,36 +251,6 @@ mod tests { Arc::new(Int32Vector::from_slice([1, 2, 3, 4])), Arc::new(BooleanVector::from(vec![true, true])), ); - assert_vector_ref_ne( - Arc::new(ConstantVector::new( - Arc::new(BooleanVector::from(vec![true])), - 5, - )), - Arc::new(ConstantVector::new( - Arc::new(BooleanVector::from(vec![true])), - 4, - )), - ); - assert_vector_ref_ne( - Arc::new(ConstantVector::new( - Arc::new(BooleanVector::from(vec![true])), - 5, - )), - Arc::new(ConstantVector::new( - Arc::new(BooleanVector::from(vec![false])), - 4, - )), - ); - assert_vector_ref_ne( - Arc::new(ConstantVector::new( - Arc::new(BooleanVector::from(vec![true])), - 5, - )), - Arc::new(ConstantVector::new( - Arc::new(Int32Vector::from_slice(vec![1])), - 4, - )), - ); assert_vector_ref_ne(Arc::new(NullVector::new(5)), Arc::new(NullVector::new(8))); assert_vector_ref_ne( diff --git a/src/datatypes/src/vectors/helper.rs b/src/datatypes/src/vectors/helper.rs index 685b07419f9..68fd794b5a2 100644 --- a/src/datatypes/src/vectors/helper.rs +++ b/src/datatypes/src/vectors/helper.rs @@ -30,21 +30,21 @@ use datafusion_common::ScalarValue; use snafu::{OptionExt, ResultExt}; use crate::data_type::ConcreteDataType; -use crate::error::{self, ConvertArrowArrayToScalarsSnafu, Result}; +use crate::error::{self, ConvertScalarToArrowArraySnafu, Result}; use crate::prelude::DataType; -use crate::scalars::{Scalar, ScalarVectorBuilder}; +use crate::scalars::Scalar; use crate::types::StructType; -use crate::value::{ListValue, ListValueRef, Value}; +use crate::value::Value; use crate::vectors::struct_vector::StructVector; use crate::vectors::{ - BinaryVector, BooleanVector, ConstantVector, DateVector, Decimal128Vector, DictionaryVector, + BinaryVector, BooleanVector, DateVector, Decimal128Vector, DictionaryVector, DurationMicrosecondVector, DurationMillisecondVector, DurationNanosecondVector, DurationSecondVector, Float32Vector, Float64Vector, Int8Vector, Int16Vector, Int32Vector, Int64Vector, IntervalDayTimeVector, IntervalMonthDayNanoVector, IntervalYearMonthVector, - ListVector, ListVectorBuilder, MutableVector, NullVector, StringVector, TimeMicrosecondVector, - TimeMillisecondVector, TimeNanosecondVector, TimeSecondVector, TimestampMicrosecondVector, - TimestampMillisecondVector, TimestampNanosecondVector, TimestampSecondVector, UInt8Vector, - UInt16Vector, UInt32Vector, UInt64Vector, Vector, VectorRef, + ListVector, NullVector, StringVector, TimeMicrosecondVector, TimeMillisecondVector, + TimeNanosecondVector, TimeSecondVector, TimestampMicrosecondVector, TimestampMillisecondVector, + TimestampNanosecondVector, TimestampSecondVector, UInt8Vector, UInt16Vector, UInt32Vector, + UInt64Vector, Vector, VectorRef, }; /// Helper functions for `Vector`. @@ -102,154 +102,75 @@ impl Helper { }) } - /// Try to cast an arrow scalar value into vector - pub fn try_from_scalar_value(value: ScalarValue, length: usize) -> Result { - let vector = match value { - ScalarValue::Null => ConstantVector::new(Arc::new(NullVector::new(1)), length), - ScalarValue::Boolean(v) => { - ConstantVector::new(Arc::new(BooleanVector::from(vec![v])), length) + /// Materializes an Arrow scalar as a vector of the given length. + /// + /// With a type hint, casts the array to the requested representation and returns + /// an error if the vector cannot preserve that concrete type. Without a hint, + /// normalizes the scalar to a supported GreptimeDB representation. + pub fn try_from_scalar_value( + value: ScalarValue, + length: usize, + type_hint: Option<&ConcreteDataType>, + ) -> Result { + if let Some(data_type) = type_hint { + let mut array = value + .to_array_of_size(length) + .context(ConvertScalarToArrowArraySnafu)?; + let arrow_type = data_type.as_arrow_type(); + if array.data_type() != &arrow_type { + array = compute::cast(&array, &arrow_type).context(error::ArrowComputeSnafu)?; } - ScalarValue::Float16(v) => ConstantVector::new( - Arc::new(Float32Vector::from(vec![v.map(f32::from)])), - length, - ), - ScalarValue::Float32(v) => { - ConstantVector::new(Arc::new(Float32Vector::from(vec![v])), length) + let vector = Self::try_into_vector(array)?; + if &vector.data_type() != data_type { + return error::CastTypeSnafu { + msg: format!( + "Scalar materialization produced {:?}, expected {data_type:?}", + vector.data_type() + ), + } + .fail(); } - ScalarValue::Float64(v) => { - ConstantVector::new(Arc::new(Float64Vector::from(vec![v])), length) + return Ok(vector); + } + + let value = match value { + // GreptimeDB doesn't support Float16 vectors. + ScalarValue::Float16(v) => ScalarValue::Float32(v.map(f32::from)), + ScalarValue::LargeUtf8(v) => ScalarValue::Utf8(v), + ScalarValue::LargeBinary(v) | ScalarValue::FixedSizeBinary(_, v) => { + ScalarValue::Binary(v) } - ScalarValue::Int8(v) => { - ConstantVector::new(Arc::new(Int8Vector::from(vec![v])), length) - } - ScalarValue::Int16(v) => { - ConstantVector::new(Arc::new(Int16Vector::from(vec![v])), length) - } - ScalarValue::Int32(v) => { - ConstantVector::new(Arc::new(Int32Vector::from(vec![v])), length) - } - ScalarValue::Int64(v) => { - ConstantVector::new(Arc::new(Int64Vector::from(vec![v])), length) - } - ScalarValue::UInt8(v) => { - ConstantVector::new(Arc::new(UInt8Vector::from(vec![v])), length) - } - ScalarValue::UInt16(v) => { - ConstantVector::new(Arc::new(UInt16Vector::from(vec![v])), length) - } - ScalarValue::UInt32(v) => { - ConstantVector::new(Arc::new(UInt32Vector::from(vec![v])), length) - } - ScalarValue::UInt64(v) => { - ConstantVector::new(Arc::new(UInt64Vector::from(vec![v])), length) - } - ScalarValue::Utf8(v) | ScalarValue::LargeUtf8(v) => { - ConstantVector::new(Arc::new(StringVector::from(vec![v])), length) - } - ScalarValue::Binary(v) - | ScalarValue::LargeBinary(v) - | ScalarValue::FixedSizeBinary(_, v) => { - ConstantVector::new(Arc::new(BinaryVector::from(vec![v])), length) - } - ScalarValue::List(array) => { - let item_type = Arc::new(ConcreteDataType::try_from(&array.value_type())?); - let mut builder = ListVectorBuilder::with_type_capacity(item_type.clone(), 1); - let scalar_values = ScalarValue::convert_array_to_scalar_vec(array.as_ref()) - .context(ConvertArrowArrayToScalarsSnafu)?; - let values = scalar_values - .into_iter() - .flat_map(|v| v.unwrap_or_else(|| vec![ScalarValue::Null])) - .map(ScalarValue::try_into) - .collect::>>()?; - builder.push(Some(ListValueRef::Ref { - val: &ListValue::new(values, item_type), - })); - let list_vector = builder.to_vector(); - ConstantVector::new(list_vector, length) - } - ScalarValue::Date32(v) => { - ConstantVector::new(Arc::new(DateVector::from(vec![v])), length) - } - ScalarValue::TimestampSecond(v, _) => { - // Timezone is unimplemented now. - ConstantVector::new(Arc::new(TimestampSecondVector::from(vec![v])), length) - } - ScalarValue::TimestampMillisecond(v, _) => { - // Timezone is unimplemented now. - ConstantVector::new(Arc::new(TimestampMillisecondVector::from(vec![v])), length) - } - ScalarValue::TimestampMicrosecond(v, _) => { - // Timezone is unimplemented now. - ConstantVector::new(Arc::new(TimestampMicrosecondVector::from(vec![v])), length) - } - ScalarValue::TimestampNanosecond(v, _) => { - // Timezone is unimplemented now. - ConstantVector::new(Arc::new(TimestampNanosecondVector::from(vec![v])), length) - } - ScalarValue::Time32Second(v) => { - ConstantVector::new(Arc::new(TimeSecondVector::from(vec![v])), length) - } - ScalarValue::Time32Millisecond(v) => { - ConstantVector::new(Arc::new(TimeMillisecondVector::from(vec![v])), length) - } - ScalarValue::Time64Microsecond(v) => { - ConstantVector::new(Arc::new(TimeMicrosecondVector::from(vec![v])), length) - } - ScalarValue::Time64Nanosecond(v) => { - ConstantVector::new(Arc::new(TimeNanosecondVector::from(vec![v])), length) - } - ScalarValue::IntervalYearMonth(v) => { - ConstantVector::new(Arc::new(IntervalYearMonthVector::from(vec![v])), length) - } - ScalarValue::IntervalDayTime(v) => { - ConstantVector::new(Arc::new(IntervalDayTimeVector::from(vec![v])), length) - } - ScalarValue::IntervalMonthDayNano(v) => { - ConstantVector::new(Arc::new(IntervalMonthDayNanoVector::from(vec![v])), length) - } - ScalarValue::DurationSecond(v) => { - ConstantVector::new(Arc::new(DurationSecondVector::from(vec![v])), length) - } - ScalarValue::DurationMillisecond(v) => { - ConstantVector::new(Arc::new(DurationMillisecondVector::from(vec![v])), length) - } - ScalarValue::DurationMicrosecond(v) => { - ConstantVector::new(Arc::new(DurationMicrosecondVector::from(vec![v])), length) - } - ScalarValue::DurationNanosecond(v) => { - ConstantVector::new(Arc::new(DurationNanosecondVector::from(vec![v])), length) - } - ScalarValue::Decimal128(v, p, s) => { - let vector = Decimal128Vector::from(vec![v]).with_precision_and_scale(p, s)?; - ConstantVector::new(Arc::new(vector), length) - } - ScalarValue::Struct(v) => { - let struct_type = StructType::from(v.fields()); - ConstantVector::new( - Arc::new(StructVector::try_new(struct_type, (*v).clone())?), - length, - ) - } - ScalarValue::Decimal32(_, _, _) + // Timezones are not supported by GreptimeDB vectors. + ScalarValue::TimestampSecond(v, _) => ScalarValue::TimestampSecond(v, None), + ScalarValue::TimestampMillisecond(v, _) => ScalarValue::TimestampMillisecond(v, None), + ScalarValue::TimestampMicrosecond(v, _) => ScalarValue::TimestampMicrosecond(v, None), + ScalarValue::TimestampNanosecond(v, _) => ScalarValue::TimestampNanosecond(v, None), + value @ (ScalarValue::Decimal32(_, _, _) | ScalarValue::Decimal64(_, _, _) | ScalarValue::Decimal256(_, _, _) | ScalarValue::FixedSizeList(_) | ScalarValue::LargeList(_) + | ScalarValue::ListView(_) + | ScalarValue::LargeListView(_) | ScalarValue::Dictionary(_, _) | ScalarValue::Union(_, _, _) | ScalarValue::Utf8View(_) | ScalarValue::BinaryView(_) | ScalarValue::Map(_) | ScalarValue::Date64(_) - | ScalarValue::RunEndEncoded(_, _, _) => { + | ScalarValue::RunEndEncoded(_, _, _)) => { return error::ConversionSnafu { from: format!("Unsupported scalar value: {value}"), } .fail(); } + value => value, }; - Ok(Arc::new(vector)) + let array = value + .to_array_of_size(length) + .context(ConvertScalarToArrowArraySnafu)?; + Self::try_into_vector(array) } /// Try to cast an arrow array into vector @@ -457,12 +378,15 @@ mod tests { }; use arrow::buffer::Buffer; use arrow::datatypes::{Int32Type, IntervalMonthDayNano}; - use arrow_array::{BinaryArray, DictionaryArray, FixedSizeBinaryArray, LargeStringArray}; - use arrow_schema::DataType; + use arrow_array::{ + BinaryArray, DictionaryArray, FixedSizeBinaryArray, LargeStringArray, StructArray, + }; + use arrow_schema::{DataType, Field, Fields}; use common_decimal::Decimal128; use common_time::time::Time; use common_time::timestamp::TimeUnit; use common_time::{Date, Duration}; + use datafusion_common::scalar::ScalarStructBuilder; use super::*; use crate::value::Value; @@ -496,7 +420,7 @@ mod tests { #[test] fn test_try_from_scalar_date_value() { - let vector = Helper::try_from_scalar_value(ScalarValue::Date32(Some(42)), 3).unwrap(); + let vector = Helper::try_from_scalar_value(ScalarValue::Date32(Some(42)), 3, None).unwrap(); assert_eq!(ConcreteDataType::date_datatype(), vector.data_type()); assert_eq!(3, vector.len()); for i in 0..vector.len() { @@ -507,7 +431,7 @@ mod tests { #[test] fn test_try_from_scalar_duration_value() { let vector = - Helper::try_from_scalar_value(ScalarValue::DurationSecond(Some(42)), 3).unwrap(); + Helper::try_from_scalar_value(ScalarValue::DurationSecond(Some(42)), 3, None).unwrap(); assert_eq!( ConcreteDataType::duration_second_datatype(), vector.data_type() @@ -524,7 +448,8 @@ mod tests { #[test] fn test_try_from_scalar_decimal128_value() { let vector = - Helper::try_from_scalar_value(ScalarValue::Decimal128(Some(42), 3, 1), 3).unwrap(); + Helper::try_from_scalar_value(ScalarValue::Decimal128(Some(42), 3, 1), 3, None) + .unwrap(); assert_eq!( ConcreteDataType::decimal128_datatype(3, 1), vector.data_type() @@ -542,7 +467,7 @@ mod tests { &ArrowDataType::Int32, true, )); - let vector = Helper::try_from_scalar_value(value, 3).unwrap(); + let vector = Helper::try_from_scalar_value(value, 3, None).unwrap(); assert_eq!( ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::int32_datatype())), vector.data_type() @@ -555,6 +480,101 @@ mod tests { } } + #[test] + fn test_try_from_scalar_value_materializes_values() { + let vector = Helper::try_from_scalar_value(ScalarValue::Int32(Some(42)), 4, None).unwrap(); + assert_eq!(ConcreteDataType::int32_datatype(), vector.data_type()); + assert_eq!(4, vector.len()); + assert_eq!(0, vector.null_count()); + for i in 0..vector.len() { + assert_eq!(Value::Int32(42), vector.get(i)); + } + + let empty = Helper::try_from_scalar_value(ScalarValue::Int32(Some(42)), 0, None).unwrap(); + assert_eq!(ConcreteDataType::int32_datatype(), empty.data_type()); + assert!(empty.is_empty()); + + let nulls = Helper::try_from_scalar_value(ScalarValue::Int32(None), 3, None).unwrap(); + assert_eq!(3, nulls.len()); + assert_eq!(3, nulls.null_count()); + for i in 0..nulls.len() { + assert_eq!(Value::Null, nulls.get(i)); + } + } + + #[test] + fn test_try_from_scalar_struct_value() { + let fields = Fields::from(vec![ + Field::new("id", ArrowDataType::Int32, false), + Field::new("name", ArrowDataType::Utf8, true), + ]); + let value = ScalarValue::Struct(Arc::new(StructArray::new( + fields.clone(), + vec![ + ScalarValue::Int32(Some(7)).to_array().unwrap(), + ScalarValue::Utf8(Some("greptime".to_string())) + .to_array() + .unwrap(), + ], + None, + ))); + + let vector = Helper::try_from_scalar_value(value, 3, None).unwrap(); + assert_eq!( + ConcreteDataType::struct_datatype(StructType::from(&fields)), + vector.data_type() + ); + assert_eq!(3, vector.len()); + for i in 0..vector.len() { + let Value::Struct(value) = vector.get(i) else { + panic!("expected struct value"); + }; + assert_eq!( + &[Value::Int32(7), Value::String("greptime".into())], + value.items() + ); + } + + let null = ScalarStructBuilder::new_null(fields); + let vector = Helper::try_from_scalar_value(null, 2, None).unwrap(); + assert_eq!(2, vector.len()); + assert_eq!(2, vector.null_count()); + assert_eq!(Value::Null, vector.get(0)); + assert_eq!(Value::Null, vector.get(1)); + } + + #[test] + fn test_try_from_scalar_value_normalizes_arrow_types() { + let string = Helper::try_from_scalar_value( + ScalarValue::LargeUtf8(Some("greptime".to_string())), + 2, + None, + ) + .unwrap(); + assert_eq!(ConcreteDataType::string_datatype(), string.data_type()); + assert_eq!(&ArrowDataType::Utf8, string.to_arrow_array().data_type()); + + let binary = Helper::try_from_scalar_value( + ScalarValue::FixedSizeBinary(2, Some(vec![1, 2])), + 2, + None, + ) + .unwrap(); + assert_eq!(ConcreteDataType::binary_datatype(), binary.data_type()); + assert_eq!(&ArrowDataType::Binary, binary.to_arrow_array().data_type()); + + let timestamp = Helper::try_from_scalar_value( + ScalarValue::TimestampMillisecond(Some(42), Some("UTC".into())), + 2, + None, + ) + .unwrap(); + assert_eq!( + &ArrowDataType::Timestamp(arrow::datatypes::TimeUnit::Millisecond, None), + timestamp.to_arrow_array().data_type() + ); + } + #[test] fn test_like_utf8() { fn assert_vector(expected: Vec<&str>, actual: &VectorRef) { @@ -745,7 +765,8 @@ mod tests { #[test] fn test_try_from_scalar_time_value() { - let vector = Helper::try_from_scalar_value(ScalarValue::Time32Second(Some(42)), 3).unwrap(); + let vector = + Helper::try_from_scalar_value(ScalarValue::Time32Second(Some(42)), 3, None).unwrap(); assert_eq!(ConcreteDataType::time_second_datatype(), vector.data_type()); assert_eq!(3, vector.len()); for i in 0..vector.len() { @@ -758,6 +779,7 @@ mod tests { let vector = Helper::try_from_scalar_value( ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNano::new(1, 1, 2000))), 3, + None, ) .unwrap(); diff --git a/src/datatypes/src/vectors/json/array.rs b/src/datatypes/src/vectors/json/array.rs index 153d6b490ef..e8b75dee259 100644 --- a/src/datatypes/src/vectors/json/array.rs +++ b/src/datatypes/src/vectors/json/array.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::cmp::Ordering; use std::sync::Arc; use arrow::compute::{can_cast_types, cast}; @@ -21,13 +20,12 @@ use arrow_array::types::{ Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type, UInt8Type, UInt16Type, UInt32Type, UInt64Type, }; -use arrow_array::{Array, ArrayRef, GenericListArray, ListArray, StructArray, new_null_array}; -use arrow_schema::{DataType, Field, FieldRef}; -use common_telemetry::trace; +use arrow_array::{Array, ArrayRef, GenericListArray, StructArray, new_null_array}; +use arrow_schema::{DataType, Field}; use serde_json::Value; use snafu::{OptionExt, ResultExt}; -use crate::arrow_array::{MutableBinaryArray, binary_array_value, string_array_value}; +use crate::arrow_array::{binary_array_value, string_array_value}; use crate::data_type::ConcreteDataType; use crate::error::{ AlignJsonArraySnafu, ArrowComputeSnafu, InvalidJsonSnafu, InvalidJsonbSnafu, Result, @@ -200,171 +198,6 @@ impl JsonArray<'_> { Ok(values) } - /// Normalizes a JSON2 array to the wider `expect` data type without losing - /// information. - /// - /// This is mainly used for write/flush-time JSON2 schema alignment: - /// - fields missing from the source are filled with typed null arrays; - /// - fields present in the source must also exist in `expect`; - /// - fields present in both are widened recursively when their types differ. - /// - /// Narrowing conversions and any other conversions that may lose information - /// are rejected. - pub fn widen_to(&self, expect: &DataType) -> Result { - let data_type = self.inner.data_type(); - - if data_type == expect { - return Ok(self.inner.clone()); - } - - trace!( - "Try aligning JSON array {} to data type {}", - data_type, expect - ); - - let struct_array = self.inner.as_struct_opt().context(AlignJsonArraySnafu { - reason: "expect struct array", - })?; - let array_fields = struct_array.fields(); - let array_columns = struct_array.columns(); - let DataType::Struct(expect_fields) = expect else { - return AlignJsonArraySnafu { - reason: "expect struct datatype", - } - .fail(); - }; - let mut aligned = Vec::with_capacity(expect_fields.len()); - - // Compare the fields in the JSON array and the to-be-aligned schema, amending with null - // arrays on the way. It's very important to note that fields in the JSON array and those - // in the JSON type are both **SORTED**, which can be guaranteed because the fields in the - // JSON type implementation are sorted. - debug_assert!(expect_fields.iter().map(|f| f.name()).is_sorted()); - debug_assert!(array_fields.iter().map(|f| f.name()).is_sorted()); - - let mut i = 0; // point to the expect fields - let mut j = 0; // point to the array fields - while i < expect_fields.len() && j < array_fields.len() { - let expect_field = &expect_fields[i]; - let array_field = &array_fields[j]; - match expect_field.name().cmp(array_field.name()) { - Ordering::Equal => { - if expect_field.data_type() == array_field.data_type() { - aligned.push(array_columns[j].clone()); - } else { - let expect_type = expect_field.data_type(); - let array_type = array_field.data_type(); - let array = match (expect_type, array_type) { - (DataType::Struct(_), DataType::Struct(_)) => { - JsonArray::from(&array_columns[j]).widen_to(expect_type)? - } - (DataType::List(expect_item), DataType::List(array_item)) => { - let list_array = array_columns[j].as_list::(); - widen_list(list_array, array_item, expect_item)? - } - _ => JsonArray::from(&array_columns[j]).widen_scalar_to(expect_type)?, - }; - aligned.push(array); - } - i += 1; - j += 1; - } - Ordering::Less => { - aligned.push(new_null_array(expect_field.data_type(), struct_array.len())); - i += 1; - } - Ordering::Greater => { - return AlignJsonArraySnafu { - reason: format!( - "source field {} does not exist in target schema", - array_field.name() - ), - } - .fail(); - } - } - } - if j < array_fields.len() { - return AlignJsonArraySnafu { - reason: format!( - "source field {} does not exist in target schema", - array_fields[j].name() - ), - } - .fail(); - } - if i < expect_fields.len() { - for field in &expect_fields[i..] { - aligned.push(new_null_array(field.data_type(), struct_array.len())); - } - } - - let json_array = StructArray::try_new_with_length( - expect_fields.clone(), - aligned, - struct_array.nulls().cloned(), - struct_array.len(), - ) - .map_err(|e| { - AlignJsonArraySnafu { - reason: e.to_string(), - } - .build() - })?; - Ok(Arc::new(json_array)) - } - - /// Widens an array to the merged JSON2 physical type without losing information. - /// - /// Supported conversions: - /// - identical types are returned unchanged; - /// - null arrays become typed null arrays; - /// - concrete JSON values are encoded as JSONB when the target type is binary. - /// - /// All other conversions are rejected. - fn widen_scalar_to(&self, to_type: &DataType) -> Result { - let from_type = self.inner.data_type(); - if from_type == to_type { - return Ok(self.inner.clone()); - } - - if from_type == &DataType::Null { - return Ok(new_null_array(to_type, self.inner.len())); - } - - if !from_type.is_binary() && to_type.is_binary() { - return self.encode_variant(); - } - - AlignJsonArraySnafu { - reason: format!("unable to widen {from_type} to {to_type}"), - } - .fail() - } - - fn encode_variant(&self) -> Result { - let len = self.inner.len(); - let mut encoded = Vec::with_capacity(len); - let mut total_bytes = 0; - - for i in 0..len { - let value = self.try_get_value(i)?; - if value.is_null() { - encoded.push(None); - } else { - let bytes = encode_serde_json_as_jsonb(value); - total_bytes += bytes.len(); - encoded.push(Some(bytes)); - } - } - - let mut builder = MutableBinaryArray::with_capacity(len, total_bytes); - for value in encoded { - builder.append_option(value); - } - Ok(Arc::new(builder.finish())) - } - /// Projects this JSON array to `target` for query evaluation. /// /// Unlike [`Self::widen_to`], projection tolerates lossy conversions: @@ -567,28 +400,6 @@ fn project_json_value_to_type(value: Value, to_type: &ConcreteDataType) -> Resul Ok(to_type.try_cast(value).unwrap_or(GreptimeValue::Null)) } -fn widen_list(list_array: &ListArray, actual: &FieldRef, expected: &FieldRef) -> Result { - let item_aligned = match (actual.data_type(), expected.data_type()) { - (DataType::Struct(_), DataType::Struct(_)) => { - JsonArray::from(list_array.values()).widen_to(expected.data_type())? - } - (DataType::List(actual), DataType::List(expected)) => { - let list_array = list_array.values().as_list::(); - widen_list(list_array, actual, expected)? - } - _ => JsonArray::from(list_array.values()).widen_scalar_to(expected.data_type())?, - }; - Ok(Arc::new( - GenericListArray::::try_new( - expected.clone(), - list_array.offsets().clone(), - item_aligned, - list_array.nulls().cloned(), - ) - .context(ArrowComputeSnafu)?, - )) -} - impl<'a> From<&'a ArrayRef> for JsonArray<'a> { fn from(inner: &'a ArrayRef) -> Self { Self { inner } @@ -740,262 +551,6 @@ mod test { Ok(()) } - #[test] - fn test_widen_null_to_any_type() -> Result<()> { - let nulls = new_null_array(&DataType::Null, 2); - let target_types = [ - DataType::Boolean, - DataType::UInt64, - DataType::Utf8View, - DataType::Binary, - DataType::List(Arc::new(Field::new_list_field(DataType::Int64, true))), - DataType::Struct(Fields::from(vec![Field::new( - "value", - DataType::Int64, - true, - )])), - ]; - - for target_type in target_types { - let widened = JsonArray::from(&nulls).widen_scalar_to(&target_type)?; - assert_eq!(&target_type, widened.data_type()); - assert_eq!(2, widened.len()); - assert_eq!(2, widened.null_count()); - } - - Ok(()) - } - - #[test] - fn test_widen_non_null_to_utf8_view_fails() { - let bools: ArrayRef = Arc::new(BooleanArray::from(vec![true])); - let err = JsonArray::from(&bools) - .widen_scalar_to(&DataType::Utf8View) - .unwrap_err(); - - assert_eq!( - "Failed to align JSON array, reason: unable to widen Boolean to Utf8View", - err.to_string() - ); - } - - #[test] - fn test_widen_variant_to_non_binary_fails() { - let value = jsonb::parse_value(b"true").unwrap().to_vec(); - let variants: ArrayRef = Arc::new(BinaryArray::from(vec![value.as_slice()])); - let err = JsonArray::from(&variants) - .widen_scalar_to(&DataType::Boolean) - .unwrap_err(); - - assert_eq!( - "Failed to align JSON array, reason: unable to widen Binary to Boolean", - err.to_string() - ); - } - - #[test] - fn test_widen_between_number_types_fails() { - let values: ArrayRef = Arc::new(UInt64Array::from(vec![1])); - let err = JsonArray::from(&values) - .widen_scalar_to(&DataType::Int64) - .unwrap_err(); - - assert_eq!( - "Failed to align JSON array, reason: unable to widen UInt64 to Int64", - err.to_string() - ); - } - - #[test] - fn test_widen_numbers_to_variant_preserves_values() -> Result<()> { - let cases: [(ArrayRef, Value); 3] = [ - (Arc::new(UInt64Array::from(vec![u64::MAX])), json!(u64::MAX)), - (Arc::new(Int64Array::from(vec![i64::MIN])), json!(i64::MIN)), - (Arc::new(Float64Array::from(vec![1.25])), json!(1.25)), - ]; - - for (values, expected) in cases { - let widened = JsonArray::from(&values).widen_scalar_to(&DataType::Binary)?; - assert_eq!(&DataType::Binary, widened.data_type()); - assert_eq!(expected, JsonArray::from(&widened).try_get_value(0)?); - } - - Ok(()) - } - - #[test] - fn test_align_json_array() -> Result<()> { - struct TestCase { - json_array: ArrayRef, - schema_type: DataType, - expected: std::result::Result, - } - - impl TestCase { - fn new( - json_array: StructArray, - schema_type: Fields, - expected: std::result::Result, String>, - ) -> Self { - Self { - json_array: Arc::new(json_array), - schema_type: DataType::Struct(schema_type.clone()), - expected: expected - .map(|x| Arc::new(StructArray::new(schema_type, x, None)) as ArrayRef), - } - } - - fn test(self) -> Result<()> { - let result = JsonArray::from(&self.json_array).widen_to(&self.schema_type); - match (result, self.expected) { - (Ok(json_array), Ok(expected)) => assert_eq!(&json_array, &expected), - (Ok(json_array), Err(e)) => { - panic!("expecting error {e} but actually get: {json_array:?}") - } - (Err(e), Err(expected)) => assert_eq!(e.to_string(), expected), - (Err(e), Ok(_)) => return Err(e), - } - Ok(()) - } - } - - // Test empty json array can be aligned with a complex json type. - TestCase::new( - StructArray::new_empty_fields(2, None), - Fields::from(vec![ - Field::new("int", DataType::Int64, true), - Field::new_struct( - "nested", - vec![Field::new("bool", DataType::Boolean, true)], - true, - ), - Field::new("string", DataType::Utf8, true), - ]), - Ok(vec![ - Arc::new(Int64Array::new_null(2)) as ArrayRef, - Arc::new(StructArray::new_null( - Fields::from(vec![Arc::new(Field::new("bool", DataType::Boolean, true))]), - 2, - )), - Arc::new(StringArray::new_null(2)), - ]), - ) - .test()?; - - // Test simple json array alignment. - TestCase::new( - StructArray::from(vec![( - Arc::new(Field::new("float", DataType::Float64, true)), - Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0])) as ArrayRef, - )]), - Fields::from(vec![ - Field::new("float", DataType::Float64, true), - Field::new("string", DataType::Utf8, true), - ]), - Ok(vec![ - Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0])) as ArrayRef, - Arc::new(StringArray::new_null(3)), - ]), - ) - .test()?; - - // Test complex json array alignment. - TestCase::new( - StructArray::from(vec![ - ( - Arc::new(Field::new_list( - "list", - Field::new_list_field(DataType::Int64, true), - true, - )), - Arc::new(ListArray::from_iter_primitive::(vec![ - Some(vec![Some(1)]), - None, - Some(vec![Some(2), Some(3)]), - ])) as ArrayRef, - ), - ( - Arc::new(Field::new_struct( - "nested", - vec![Field::new("int", DataType::Int64, true)], - true, - )), - Arc::new(StructArray::from(vec![( - Arc::new(Field::new("int", DataType::Int64, true)), - Arc::new(Int64Array::from(vec![-1, -2, -3])) as ArrayRef, - )])), - ), - ( - Arc::new(Field::new("string", DataType::Utf8, true)), - Arc::new(StringArray::from(vec!["a", "b", "c"])), - ), - ]), - Fields::from(vec![ - Field::new("bool", DataType::Boolean, true), - Field::new_list("list", Field::new_list_field(DataType::Int64, true), true), - Field::new_struct( - "nested", - vec![ - Field::new("float", DataType::Float64, true), - Field::new("int", DataType::Int64, true), - ], - true, - ), - Field::new("string", DataType::Utf8, true), - ]), - Ok(vec![ - Arc::new(BooleanArray::new_null(3)) as ArrayRef, - Arc::new(ListArray::from_iter_primitive::(vec![ - Some(vec![Some(1)]), - None, - Some(vec![Some(2), Some(3)]), - ])), - Arc::new(StructArray::from(vec![ - ( - Arc::new(Field::new("float", DataType::Float64, true)), - Arc::new(Float64Array::new_null(3)) as ArrayRef, - ), - ( - Arc::new(Field::new("int", DataType::Int64, true)), - Arc::new(Int64Array::from(vec![-1, -2, -3])), - ), - ])), - Arc::new(StringArray::from(vec!["a", "b", "c"])), - ]), - ) - .test()?; - - // Source fields that do not exist in the target schema must not be discarded. - TestCase::new( - StructArray::from(vec![( - Arc::new(Field::new("a", DataType::Boolean, true)), - Arc::new(BooleanArray::from(vec![true])) as ArrayRef, - )]), - Fields::from(vec![Field::new("b", DataType::Boolean, true)]), - Err( - "Failed to align JSON array, reason: source field a does not exist in target schema" - .to_string(), - ), - ) - .test()?; - - // Trailing source fields must also be rejected after all target fields are processed. - TestCase::new( - StructArray::from(vec![( - Arc::new(Field::new("b", DataType::Boolean, true)), - Arc::new(BooleanArray::from(vec![true])) as ArrayRef, - )]), - Fields::from(vec![Field::new("a", DataType::Boolean, true)]), - Err( - "Failed to align JSON array, reason: source field b does not exist in target schema" - .to_string(), - ), - ) - .test()?; - - Ok(()) - } - #[test] fn test_align_variant_to_struct() -> Result<()> { let encode = |json: &[u8]| jsonb::parse_value(json).unwrap().to_vec(); diff --git a/src/datatypes/src/vectors/null.rs b/src/datatypes/src/vectors/null.rs index e049eb205c6..2510824ce0d 100644 --- a/src/datatypes/src/vectors/null.rs +++ b/src/datatypes/src/vectors/null.rs @@ -260,16 +260,6 @@ impl ScalarVectorBuilder for NullVectorBuilder { } } -pub(crate) fn replicate_null(vector: &NullVector, offsets: &[usize]) -> VectorRef { - assert_eq!(offsets.len(), vector.len()); - - if offsets.is_empty() { - return vector.slice(0, 0); - } - - Arc::new(NullVector::new(*offsets.last().unwrap())) -} - #[cfg(test)] mod tests { use serde_json; @@ -290,7 +280,6 @@ mod tests { assert_eq!(vector2.null_count(), 16); assert_eq!("NullVector", v.vector_type_name()); - assert!(!v.is_const()); assert!(v.validity().is_all_null()); assert!(v.only_null()); diff --git a/src/datatypes/src/vectors/operations.rs b/src/datatypes/src/vectors/operations.rs index 1dc9e21a0d2..f75ff060f3e 100644 --- a/src/datatypes/src/vectors/operations.rs +++ b/src/datatypes/src/vectors/operations.rs @@ -14,14 +14,12 @@ mod cast; mod filter; -mod replicate; mod take; use std::sync::Arc; use crate::error::{self, Result}; use crate::types::LogicalPrimitiveType; -use crate::vectors::constant::ConstantVector; use crate::vectors::{ BinaryVector, BooleanVector, ConcreteDataType, Decimal128Vector, ListVector, NullVector, PrimitiveVector, StringVector, UInt32Vector, Vector, VectorRef, @@ -29,14 +27,6 @@ use crate::vectors::{ /// Vector compute operations. pub trait VectorOp { - /// Copies each element according `offsets` parameter. - /// - `i-th` element should be copied `offsets[i] - offsets[i - 1]` times - /// - `0-th` element would be copied `offsets[0]` times - /// - /// # Panics - /// Panics if `offsets.len() != self.len()`. - fn replicate(&self, offsets: &[usize]) -> VectorRef; - /// Filters the vector, returns elements matching the `filter` (i.e. where the values are true). /// /// Note that the nulls of `filter` are interpreted as `false` will lead to these elements being masked out. @@ -57,10 +47,6 @@ pub trait VectorOp { macro_rules! impl_scalar_vector_op { ($($VectorType: ident),+) => {$( impl VectorOp for $VectorType { - fn replicate(&self, offsets: &[usize]) -> VectorRef { - replicate::replicate_scalar(self, offsets) - } - fn filter(&self, filter: &BooleanVector) -> Result { filter::filter_non_constant!(self, $VectorType, filter) } @@ -92,10 +78,6 @@ macro_rules! impl_scalar_vector_op { impl_scalar_vector_op!(BinaryVector, BooleanVector, StringVector); impl VectorOp for ListVector { - fn replicate(&self, offsets: &[usize]) -> VectorRef { - replicate::replicate_list(self, offsets) - } - fn filter(&self, filter: &BooleanVector) -> Result { filter::filter_non_constant!(self, ListVector, filter) } @@ -110,10 +92,6 @@ impl VectorOp for ListVector { } impl VectorOp for Decimal128Vector { - fn replicate(&self, offsets: &[usize]) -> VectorRef { - std::sync::Arc::new(replicate::replicate_decimal128(self, offsets)) - } - fn filter(&self, filter: &BooleanVector) -> Result { filter::filter_non_constant!(self, Decimal128Vector, filter) } @@ -128,10 +106,6 @@ impl VectorOp for Decimal128Vector { } impl VectorOp for PrimitiveVector { - fn replicate(&self, offsets: &[usize]) -> VectorRef { - std::sync::Arc::new(replicate::replicate_primitive(self, offsets)) - } - fn filter(&self, filter: &BooleanVector) -> Result { filter::filter_non_constant!(self, PrimitiveVector, filter) } @@ -146,10 +120,6 @@ impl VectorOp for PrimitiveVector { } impl VectorOp for NullVector { - fn replicate(&self, offsets: &[usize]) -> VectorRef { - replicate::replicate_null(self, offsets) - } - fn filter(&self, filter: &BooleanVector) -> Result { filter::filter_non_constant!(self, NullVector, filter) } @@ -166,21 +136,3 @@ impl VectorOp for NullVector { take::take_indices!(self, NullVector, indices) } } - -impl VectorOp for ConstantVector { - fn replicate(&self, offsets: &[usize]) -> VectorRef { - self.replicate_vector(offsets) - } - - fn filter(&self, filter: &BooleanVector) -> Result { - self.filter_vector(filter) - } - - fn cast(&self, to_type: &ConcreteDataType) -> Result { - self.cast_vector(to_type) - } - - fn take(&self, indices: &UInt32Vector) -> Result { - self.take_vector(indices) - } -} diff --git a/src/datatypes/src/vectors/operations/filter.rs b/src/datatypes/src/vectors/operations/filter.rs index 7dde6c1f946..b27bbb5de48 100644 --- a/src/datatypes/src/vectors/operations/filter.rs +++ b/src/datatypes/src/vectors/operations/filter.rs @@ -39,7 +39,6 @@ mod tests { TimestampMicrosecond, TimestampMillisecond, TimestampNanosecond, TimestampSecond, }; use crate::types::WrapperType; - use crate::vectors::constant::ConstantVector; use crate::vectors::{ BooleanVector, Int32Vector, NullVector, StringVector, VectorOp, VectorRef, }; @@ -67,26 +66,6 @@ mod tests { check_filter_primitive(&[5, 7], &[5, 6, 7], &[true, false, true]); } - fn check_filter_constant(expect_length: usize, input_length: usize, filter: &[bool]) { - let v = ConstantVector::new(Arc::new(Int32Vector::from_slice([123])), input_length); - let filter = BooleanVector::from_slice(filter); - let out = v.filter(&filter).unwrap(); - - assert!(out.is_const()); - assert_eq!(expect_length, out.len()); - } - - #[test] - fn test_filter_constant() { - check_filter_constant(0, 0, &[]); - check_filter_constant(1, 1, &[true]); - check_filter_constant(0, 1, &[false]); - check_filter_constant(1, 2, &[false, true]); - check_filter_constant(2, 2, &[true, true]); - check_filter_constant(1, 4, &[false, false, false, true]); - check_filter_constant(2, 4, &[false, true, false, true]); - } - #[test] fn test_filter_scalar() { let v = StringVector::from_slice(&["0", "1", "2", "3"]); diff --git a/src/datatypes/src/vectors/operations/replicate.rs b/src/datatypes/src/vectors/operations/replicate.rs deleted file mode 100644 index dee0b3855b0..00000000000 --- a/src/datatypes/src/vectors/operations/replicate.rs +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::prelude::*; -pub(crate) use crate::vectors::decimal::replicate_decimal128; -pub(crate) use crate::vectors::null::replicate_null; -pub(crate) use crate::vectors::primitive::replicate_primitive; -use crate::vectors::{ListVector, ListVectorBuilder}; - -pub(crate) fn replicate_scalar(c: &C, offsets: &[usize]) -> VectorRef { - assert_eq!(offsets.len(), c.len()); - - if offsets.is_empty() { - return c.slice(0, 0); - } - let mut builder = <::Builder>::with_capacity(c.len()); - - let mut previous_offset = 0; - for (i, offset) in offsets.iter().enumerate() { - let data = c.get_data(i); - for _ in previous_offset..*offset { - builder.push(data.clone()); - } - previous_offset = *offset; - } - builder.to_vector() -} - -pub(crate) fn replicate_list(c: &ListVector, offsets: &[usize]) -> VectorRef { - assert_eq!(offsets.len(), c.len()); - - if offsets.is_empty() { - return c.slice(0, 0); - } - let mut builder = - ListVectorBuilder::with_type_capacity(c.item_type(), *offsets.last().unwrap()); - - let mut previous_offset = 0; - for (i, offset) in offsets.iter().enumerate() { - let data = c.get_data(i); - for _ in previous_offset..*offset { - builder.push(data.clone()); - } - previous_offset = *offset; - } - builder.to_vector() -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use common_time::timestamp::TimeUnit; - use common_time::{Date, Timestamp}; - use paste::paste; - - use super::*; - use crate::value::{ListValue, ListValueRef}; - use crate::vectors::constant::ConstantVector; - use crate::vectors::{ - Decimal128Vector, Int32Vector, ListVectorBuilder, NullVector, StringVector, VectorOp, - }; - - #[test] - fn test_replicate_primitive() { - let v = Int32Vector::from_iterator(0..5); - let offsets = [0, 1, 2, 3, 4]; - - let v = v.replicate(&offsets); - assert_eq!(4, v.len()); - - for i in 0..4 { - assert_eq!(Value::Int32(i as i32 + 1), v.get(i)); - } - } - - #[test] - fn test_replicate_nullable_primitive() { - let v = Int32Vector::from(vec![None, Some(1), None, Some(2)]); - let offsets = [2, 4, 6, 8]; - let v = v.replicate(&offsets); - assert_eq!(8, v.len()); - - let expect: VectorRef = Arc::new(Int32Vector::from(vec![ - None, - None, - Some(1), - Some(1), - None, - None, - Some(2), - Some(2), - ])); - assert_eq!(expect, v); - } - - #[test] - fn test_replicate_scalar() { - let v = StringVector::from_slice(&["0", "1", "2", "3"]); - let offsets = [1, 3, 5, 6]; - - let v = v.replicate(&offsets); - assert_eq!(6, v.len()); - - let expect: VectorRef = Arc::new(StringVector::from_slice(&["0", "1", "1", "2", "2", "3"])); - assert_eq!(expect, v); - } - - #[test] - fn test_replicate_list() { - let item_type = Arc::new(ConcreteDataType::int32_datatype()); - let first = ListValue::new(vec![Value::Int32(1), Value::Int32(2)], item_type.clone()); - let second = ListValue::new(vec![Value::Int32(3)], item_type.clone()); - let mut builder = ListVectorBuilder::with_type_capacity(item_type, 2); - builder.push(Some(ListValueRef::Ref { val: &first })); - builder.push(Some(ListValueRef::Ref { val: &second })); - let v = builder.finish(); - - let v = v.replicate(&[1, 3]); - let mut expect_builder = - ListVectorBuilder::with_type_capacity(Arc::new(ConcreteDataType::int32_datatype()), 3); - expect_builder.push(Some(ListValueRef::Ref { val: &first })); - expect_builder.push(Some(ListValueRef::Ref { val: &second })); - expect_builder.push(Some(ListValueRef::Ref { val: &second })); - let expect = expect_builder.to_vector(); - assert_eq!(expect, v); - } - - #[test] - fn test_replicate_constant() { - let v = Arc::new(StringVector::from_slice(&["hello"])); - let cv = ConstantVector::new(v.clone(), 2); - let offsets = [1, 4]; - - let cv = cv.replicate(&offsets); - assert_eq!(4, cv.len()); - - let expect: VectorRef = Arc::new(ConstantVector::new(v, 4)); - assert_eq!(expect, cv); - } - - #[test] - fn test_replicate_null() { - let v = NullVector::new(0); - let offsets = []; - let v = v.replicate(&offsets); - assert!(v.is_empty()); - - let v = NullVector::new(3); - let offsets = [1, 3, 5]; - - let v = v.replicate(&offsets); - assert_eq!(5, v.len()); - } - - macro_rules! impl_replicate_date_like_test { - ($VectorType: ident, $ValueType: ident, $method: ident) => {{ - use $crate::vectors::$VectorType; - - let v = $VectorType::from_iterator((0..5).map($ValueType::$method)); - let offsets = [0, 1, 2, 3, 4]; - - let v = v.replicate(&offsets); - assert_eq!(4, v.len()); - - for i in 0..4 { - assert_eq!( - Value::$ValueType($ValueType::$method((i as i32 + 1).into())), - v.get(i) - ); - } - }}; - } - - macro_rules! impl_replicate_timestamp_test { - ($unit: ident) => {{ - paste!{ - use $crate::vectors::[]; - use $crate::timestamp::[]; - let v = []::from_iterator((0..5).map([]::from)); - let offsets = [0, 1, 2, 3, 4]; - let v = v.replicate(&offsets); - assert_eq!(4, v.len()); - for i in 0..4 { - assert_eq!( - Value::Timestamp(Timestamp::new(i as i64 + 1, TimeUnit::$unit)), - v.get(i) - ); - } - } - }}; - } - - #[test] - fn test_replicate_date_like() { - impl_replicate_date_like_test!(DateVector, Date, new); - impl_replicate_timestamp_test!(Second); - impl_replicate_timestamp_test!(Millisecond); - impl_replicate_timestamp_test!(Microsecond); - impl_replicate_timestamp_test!(Nanosecond); - } - - #[test] - fn test_replicate_decimal() { - let data = vec![100]; - // create a decimal vector - let v = Decimal128Vector::from_values(data.clone()) - .with_precision_and_scale(10, 2) - .unwrap(); - let offsets = [5]; - let v = v.replicate(&offsets); - assert_eq!(5, v.len()); - - let expect: VectorRef = Arc::new( - Decimal128Vector::from_values(vec![100; 5]) - .with_precision_and_scale(10, 2) - .unwrap(), - ); - assert_eq!(expect, v); - } -} diff --git a/src/datatypes/src/vectors/operations/take.rs b/src/datatypes/src/vectors/operations/take.rs index 41947d3ef51..167f650c8ed 100644 --- a/src/datatypes/src/vectors/operations/take.rs +++ b/src/datatypes/src/vectors/operations/take.rs @@ -43,8 +43,7 @@ mod tests { use crate::types::{LogicalPrimitiveType, WrapperType}; use crate::vectors::operations::VectorOp; use crate::vectors::{ - BooleanVector, ConstantVector, Int32Vector, NullVector, PrimitiveVector, StringVector, - UInt32Vector, + BooleanVector, Int32Vector, NullVector, PrimitiveVector, StringVector, UInt32Vector, }; fn check_take_primitive( @@ -119,29 +118,6 @@ mod tests { take_time_like_test!(TimestampNanosecondVector, TimestampNanosecond, from_native); } - fn check_take_constant(expect_length: usize, input_length: usize, indices: &[u32]) { - let v = ConstantVector::new(Arc::new(Int32Vector::from_slice([111])), input_length); - let indices = UInt32Vector::from_slice(indices); - let out = v.take(&indices).unwrap(); - - assert!(out.is_const()); - assert_eq!(expect_length, out.len()); - } - - #[test] - fn test_take_constant() { - check_take_constant(2, 5, &[3, 4]); - check_take_constant(3, 10, &[1, 2, 3]); - check_take_constant(4, 10, &[1, 5, 3, 6]); - check_take_constant(5, 10, &[1, 9, 8, 7, 3]); - } - - #[test] - #[should_panic] - fn test_take_constant_out_of_index() { - check_take_constant(2, 5, &[3, 5]); - } - #[test] #[should_panic] fn test_take_out_of_index() { diff --git a/src/datatypes/src/vectors/primitive.rs b/src/datatypes/src/vectors/primitive.rs index dc85f94c4ff..92c8394e710 100644 --- a/src/datatypes/src/vectors/primitive.rs +++ b/src/datatypes/src/vectors/primitive.rs @@ -366,40 +366,6 @@ where } } -pub(crate) fn replicate_primitive( - vector: &PrimitiveVector, - offsets: &[usize], -) -> PrimitiveVector { - assert_eq!(offsets.len(), vector.len()); - - if offsets.is_empty() { - return vector.get_slice(0, 0); - } - - let mut builder = PrimitiveVectorBuilder::::with_capacity(*offsets.last().unwrap()); - - let mut previous_offset = 0; - - for (offset, value) in offsets.iter().zip(vector.array.iter()) { - let repeat_times = *offset - previous_offset; - match value { - Some(data) => { - unsafe { - // Safety: std::iter::Repeat and std::iter::Take implement TrustedLen. - builder - .mutable_array - .append_trusted_len_iter(std::iter::repeat_n(data, repeat_times)); - } - } - None => { - builder.mutable_array.append_nulls(repeat_times); - } - } - previous_offset = *offset; - } - builder.finish() -} - #[cfg(test)] mod tests { use std::vec; @@ -433,7 +399,6 @@ mod tests { fn check_vec(v: Int32Vector) { assert_eq!(4, v.len()); assert_eq!("Int32Vector", v.vector_type_name()); - assert!(!v.is_const()); assert!(v.validity().is_all_valid()); assert!(!v.only_null()); diff --git a/src/datatypes/src/vectors/string.rs b/src/datatypes/src/vectors/string.rs index 53324280c3f..0476ecc6440 100644 --- a/src/datatypes/src/vectors/string.rs +++ b/src/datatypes/src/vectors/string.rs @@ -604,7 +604,6 @@ mod tests { let v = StringVector::from(strs.clone()); assert_eq!(3, v.len()); assert_eq!("StringVector", v.vector_type_name()); - assert!(!v.is_const()); assert!(v.validity().is_all_valid()); assert!(!v.only_null()); assert_eq!(1040, v.memory_size()); diff --git a/src/datatypes/src/vectors/struct_vector.rs b/src/datatypes/src/vectors/struct_vector.rs index e6284351b4f..4798c8fddcd 100644 --- a/src/datatypes/src/vectors/struct_vector.rs +++ b/src/datatypes/src/vectors/struct_vector.rs @@ -15,7 +15,7 @@ use std::any::Any; use std::sync::Arc; -use arrow::array::{MutableArrayData, NullBufferBuilder}; +use arrow::array::NullBufferBuilder; use arrow::compute::TakeOptions; use arrow::datatypes::DataType as ArrowDataType; use arrow_array::{Array, ArrayRef, StructArray}; @@ -143,31 +143,6 @@ impl Vector for StructVector { } impl VectorOp for StructVector { - fn replicate(&self, offsets: &[usize]) -> VectorRef { - assert_eq!(offsets.len(), self.len()); - assert!(offsets.is_sorted(), "offsets must be non-decreasing"); - - let Some(&output_len) = offsets.last() else { - return self.slice(0, 0); - }; - - let source = self.array.to_data(); - let mut output = MutableArrayData::new(vec![&source], false, output_len); - let mut previous_offset = 0; - - for (index, &offset) in offsets.iter().enumerate() { - for _ in previous_offset..offset { - output.extend(0, index, index + 1); - } - previous_offset = offset; - } - - Arc::new(StructVector { - array: StructArray::from(output.freeze()), - fields: self.fields.clone(), - }) - } - fn cast(&self, _to_type: &ConcreteDataType) -> Result { UnsupportedOperationSnafu { op: "cast", @@ -466,10 +441,6 @@ impl ScalarVectorBuilder for StructVectorBuilder { #[cfg(test)] mod tests { - use arrow::array::{DictionaryArray, Int8Array, StringArray}; - use arrow::buffer::NullBuffer; - use arrow::datatypes::Int8Type; - use super::*; use crate::json::JsonSettings; use crate::schema::{ColumnDefaultConstraint, ColumnSchema}; @@ -555,7 +526,7 @@ mod tests { } #[test] - fn test_replicate_preserves_json2_identity() { + fn test_default_vector_preserves_json2_identity() { let json = JsonSettings::default() .encode(serde_json::json!({"answer": 42})) .unwrap(); @@ -576,39 +547,6 @@ mod tests { assert_eq!(replicated.len(), 2); } - #[test] - fn test_replicate_preserves_dictionary_and_nulls() { - let fields = StructType::new(Arc::new(vec![StructField::new( - "label", - ConcreteDataType::dictionary_datatype( - ConcreteDataType::int8_datatype(), - ConcreteDataType::string_datatype(), - ), - true, - )])); - let dictionary = DictionaryArray::::new( - Int8Array::from(vec![Some(0), Some(1)]), - Arc::new(StringArray::from(vec!["a", "b"])), - ); - let array = StructArray::new( - fields.as_arrow_fields(), - vec![Arc::new(dictionary)], - Some(NullBuffer::from(vec![true, false])), - ); - let vector = StructVector::try_new(fields.clone(), array).unwrap(); - - let replicated = vector.replicate(&[2, 3]); - - assert_eq!( - replicated.data_type(), - ConcreteDataType::struct_datatype(fields) - ); - assert_eq!(replicated.len(), 3); - assert_eq!(replicated.null_count(), 1); - assert_eq!(replicated.get(0), replicated.get(1)); - assert!(replicated.is_null(2)); - } - #[test] fn test_deep_nested_struct_list() { // level 1: struct diff --git a/src/file-engine/src/query/file_stream.rs b/src/file-engine/src/query/file_stream.rs index a480a503747..7c7717287fb 100644 --- a/src/file-engine/src/query/file_stream.rs +++ b/src/file-engine/src/query/file_stream.rs @@ -23,7 +23,8 @@ use datafusion::config::CsvOptions; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::object_store::ObjectStoreUrl; use datafusion::datasource::physical_plan::{ - CsvSource, FileGroup, FileScanConfigBuilder, FileSource, FileStream, JsonSource, ParquetSource, + CsvSource, FileGroup, FileScanConfigBuilder, FileSource, FileStreamBuilder, JsonSource, + ParquetSource, }; use datafusion::datasource::source::DataSourceExec; use datafusion::physical_expr::create_physical_expr; @@ -34,6 +35,7 @@ use datafusion::physical_plan::{ }; use datafusion::prelude::SessionContext; use datafusion_expr::expr::Expr; +use datafusion_expr::physical_planning_context::PhysicalPlanningContext; use datafusion_expr::utils::conjunction; use datatypes::schema::SchemaRef; use object_store::ObjectStore; @@ -66,13 +68,12 @@ fn build_record_batch_stream( )); let file_opener = config.file_source().create_file_opener(store, &config, 0)?; - let stream = FileStream::new( - &config, - 0, // partition: hard-code - file_opener, - &ExecutionPlanMetricsSet::new(), - ) - .context(error::BuildStreamSnafu)?; + let stream = FileStreamBuilder::new(&config) + .with_partition(0) // partition: hard-code + .with_file_opener(file_opener) + .with_metrics(&ExecutionPlanMetricsSet::new()) + .build() + .context(error::BuildStreamSnafu)?; Ok(Box::pin(stream)) } @@ -138,8 +139,13 @@ fn new_parquet_stream_with_exec_plan( .to_dfschema_ref() .context(error::ParquetScanPlanSnafu)?; - let filters = create_physical_expr(&expr, &df_schema, &ExecutionProps::new()) - .context(error::ParquetScanPlanSnafu)?; + let filters = create_physical_expr( + &expr, + &df_schema, + &ExecutionProps::new(), + &PhysicalPlanningContext::default(), + ) + .context(error::ParquetScanPlanSnafu)?; parquet_source = parquet_source.with_predicate(filters); }; diff --git a/src/flow/src/batching_mode/engine.rs b/src/flow/src/batching_mode/engine.rs index d05eec92518..3532b38d366 100644 --- a/src/flow/src/batching_mode/engine.rs +++ b/src/flow/src/batching_mode/engine.rs @@ -21,7 +21,10 @@ use std::time::Duration; use api::v1::flow::DirtyWindowRequests; use catalog::CatalogManagerRef; use common_error::ext::BoxedError; -use common_meta::ddl::create_flow::{FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType}; +use common_meta::ddl::create_flow::{ + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE, FlowType, +}; use common_meta::key::TableMetadataManagerRef; use common_meta::key::flow::FlowMetadataManagerRef; use common_meta::key::flow::flow_state::FlowStat; @@ -465,21 +468,24 @@ impl BatchingEngine { fn batch_opts_for_flow_options( &self, flow_options: &HashMap, + exact_sequence_range_required: bool, ) -> Result, Error> { let mut batch_opts = (*self.batch_opts).clone(); if let Some(enable_incremental_read) = flow_options.get(FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY) { - batch_opts.experimental_enable_incremental_read = enable_incremental_read - .parse::() - .map_err(|_| { + batch_opts.experimental_enable_incremental_read = if exact_sequence_range_required { + true + } else { + enable_incremental_read.parse::().map_err(|_| { InvalidQuerySnafu { reason: format!( "Invalid flow option {FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY}: {enable_incremental_read}" ), } .build() - })?; + })? + }; } Ok(Arc::new(batch_opts)) @@ -594,7 +600,14 @@ impl BatchingEngine { } ); - let batch_opts = self.batch_opts_for_flow_options(&flow_options)?; + // The meta layer validates this reserved sentinel before it reaches the + // flownode. Derive the requirement once and pass it directly to task + // config; query-context extensions are irrelevant. + let exact_sequence_range_required = flow_options + .get(FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY) + .is_some_and(|value| value == FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE); + let batch_opts = + self.batch_opts_for_flow_options(&flow_options, exact_sequence_range_required)?; let mut source_table_names = Vec::with_capacity(2); for src_id in source_table_ids { @@ -697,7 +710,21 @@ impl BatchingEngine { eval_schedule, }; - let task = BatchingTask::try_new(task_args)?; + let task = BatchingTask::try_new_with_exact_sequence_range_required( + task_args, + exact_sequence_range_required, + )?; + + if task.config.exact_sequence_range_required { + ensure!( + task.sequence_range_capable().await?, + UnsupportedSnafu { + reason: format!( + "Flow {flow_id} requires exact sequence-range reads, but a source table lacks the Mito preserve_row_sequence capability" + ), + } + ); + } let task_inner = task.clone(); let engine = self.query_engine.clone(); @@ -1087,16 +1114,32 @@ mod tests { async fn test_flow_option_overrides_incremental_read_switch() { let engine = new_test_engine().await; - let default_opts = engine.batch_opts_for_flow_options(&HashMap::new()).unwrap(); + let default_opts = engine + .batch_opts_for_flow_options(&HashMap::new(), false) + .unwrap(); assert!(!default_opts.experimental_enable_incremental_read); let enabled_opts = engine - .batch_opts_for_flow_options(&HashMap::from([( - FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(), - "true".to_string(), - )])) + .batch_opts_for_flow_options( + &HashMap::from([( + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(), + "true".to_string(), + )]), + false, + ) .unwrap(); assert!(enabled_opts.experimental_enable_incremental_read); + + let exact_opts = engine + .batch_opts_for_flow_options( + &HashMap::from([( + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(), + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE.to_string(), + )]), + true, + ) + .unwrap(); + assert!(exact_opts.experimental_enable_incremental_read); } #[test] diff --git a/src/flow/src/batching_mode/task.rs b/src/flow/src/batching_mode/task.rs index fb64dc9c20e..95b989851f7 100644 --- a/src/flow/src/batching_mode/task.rs +++ b/src/flow/src/batching_mode/task.rs @@ -149,6 +149,7 @@ pub struct TaskConfig { pub catalog_manager: CatalogManagerRef, pub query_type: QueryType, pub batch_opts: Arc, + pub exact_sequence_range_required: bool, pub flow_eval_interval: Option, /// Typed schedule configuration, pre-parsed at task creation time. pub eval_schedule: Option, @@ -292,7 +293,11 @@ struct ExecuteOnceOutcome { impl BatchingTask { #[allow(clippy::too_many_arguments)] - pub fn try_new( + pub fn try_new(args: TaskArgs<'_>) -> Result { + Self::try_new_with_exact_sequence_range_required(args, false) + } + + pub fn try_new_with_exact_sequence_range_required( TaskArgs { flow_id, query, @@ -308,6 +313,7 @@ impl BatchingTask { flow_eval_interval, eval_schedule, }: TaskArgs<'_>, + exact_sequence_range_required: bool, ) -> Result { let mut state = TaskState::with_dirty_time_windows( query_ctx.clone(), @@ -332,6 +338,7 @@ impl BatchingTask { catalog_manager, output_schema: plan.schema().clone(), query_type: determine_query_type(query, &query_ctx)?, + exact_sequence_range_required, batch_opts, flow_eval_interval, eval_schedule, diff --git a/src/flow/src/batching_mode/task/ckpt.rs b/src/flow/src/batching_mode/task/ckpt.rs index 960e541c73c..c736a0fdbdf 100644 --- a/src/flow/src/batching_mode/task/ckpt.rs +++ b/src/flow/src/batching_mode/task/ckpt.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::error::Error as StdError; use std::time::Duration; use client::OutputWithMetrics; @@ -31,42 +30,6 @@ use crate::metrics::{ }; use crate::{Error, FlowId}; -/// Liveness guard: when a fenced repair query fails with a wrapped error whose -/// text indicates a stale snapshot fence (even when `StatusCode::RequestOutdated` -/// was lost through client layers), classify it as `SnapshotFenceExpired` to -/// break the retry loop and force a rebind of the fence high `H`. -/// -/// Long-term the structured `StatusCode` / retry hint path should be preserved -/// end-to-end; this text fallback is a narrow safety measure. -fn matches_stale_snapshot_fence_text(err: &Error) -> bool { - let markers = [ - "STALE_SNAPSHOT_FENCE", - "REBIND_SNAPSHOT_FENCE", - "snapshot upper bound stale", - ]; - // Check the top-level error Display and Debug. - let debug_str = format!("{:?}", err); - let display_str = err.to_string(); - for marker in &markers { - if debug_str.contains(marker) || display_str.contains(marker) { - return true; - } - } - // Walk the error source chain. - let mut source = err.source(); - while let Some(s) = source { - let debug_str = format!("{:?}", s); - let display_str = s.to_string(); - for marker in &markers { - if debug_str.contains(marker) || display_str.contains(marker) { - return true; - } - } - source = s.source(); - } - false -} - impl BatchingTask { /// Classify execution errors into checkpoint fallback reasons. A stale /// snapshot fence is special only for fenced repair chunks. @@ -80,14 +43,6 @@ impl BatchingTask { } else { FlowQueryFallbackReason::StaleCursor } - } else if matches!(coverage, QueryCoverage::FencedRepairChunk { .. }) - && matches_stale_snapshot_fence_text(err) - { - // Narrow text-based fallback for wrapped errors where the - // structured StatusCode::RequestOutdated was lost through - // frontend/client layers. Without this fenced repair will - // retry the same stale `given_seq` every refresh tick. - FlowQueryFallbackReason::SnapshotFenceExpired } else if matches!(coverage, QueryCoverage::IncrementalDelta) { FlowQueryFallbackReason::IncrementalQueryFailure } else { diff --git a/src/flow/src/batching_mode/task/inc.rs b/src/flow/src/batching_mode/task/inc.rs index fbea3abd259..8c090d9814e 100644 --- a/src/flow/src/batching_mode/task/inc.rs +++ b/src/flow/src/batching_mode/task/inc.rs @@ -20,9 +20,10 @@ use common_telemetry::tracing::warn; use datafusion_expr::{DmlStatement, LogicalPlan}; use query::options::{ FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE, FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY, - FLOW_SINK_TABLE_ID, + FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE, FLOW_SINK_TABLE_ID, }; use snafu::ResultExt; +use store_api::mito_engine_options::PRESERVE_ROW_SEQUENCE; use table::metadata::TableId; use crate::Error; @@ -61,6 +62,50 @@ impl BatchingTask { Ok(table.table_info().table_id()) } + /// Whether every source table provably supports exact sequence-range + /// scans: it must be the canonical mito engine and declare the + /// `preserve_row_sequence` capability (a mito region option enforced to + /// require append-only mode). When the capability is unknown — a source + /// table that cannot be resolved, is not the mito engine, or lacks the + /// option — this returns `false` so the caller keeps the historical + /// `memtable_only` mode instead of upgrading. + pub async fn sequence_range_capable(&self) -> Result { + for name in &self.config.source_table_names { + let table = match self + .config + .catalog_manager + .table(&name[0], &name[1], &name[2], None) + .await + { + Ok(Some(table)) => table, + Ok(None) => { + return Err(UnexpectedSnafu { + reason: format!( + "Flow {} source table {} not found for sequence_range capability check", + self.config.flow_id, + name.join(".") + ), + } + .build()); + } + Err(err) => Err(BoxedError::new(err)).context(ExternalSnafu)?, + }; + + let info = table.table_info(); + let preserves = info.meta.engine == "mito" + && info + .meta + .options + .extra_options + .get(PRESERVE_ROW_SEQUENCE) + .is_some_and(|value| value.eq_ignore_ascii_case("true")); + if !preserves { + return Ok(false); + } + } + Ok(!self.config.source_table_names.is_empty()) + } + /// For incremental-mode SQL queries, attempt to prepare an executable plan /// that is safe for incremental scan extensions. /// @@ -109,6 +154,15 @@ impl BatchingTask { permanently disabling incremental for this flow", self.config.flow_id ); + if self.config.exact_sequence_range_required { + return Err(UnexpectedSnafu { + reason: format!( + "Flow {} requires exact sequence-range reads, but its incremental plan is unsupported", + self.config.flow_id + ), + } + .build()); + } self.state.write().unwrap().disable_incremental(); return Ok(None); }; @@ -119,6 +173,15 @@ impl BatchingTask { permanently disabling incremental for this flow", self.config.flow_id, analysis.unsupported_exprs ); + if self.config.exact_sequence_range_required { + return Err(UnexpectedSnafu { + reason: format!( + "Flow {} requires exact sequence-range reads, but its incremental aggregate is unsupported: {:?}", + self.config.flow_id, analysis.unsupported_exprs + ), + } + .build()); + } self.state.write().unwrap().disable_incremental(); return Ok(None); } @@ -219,12 +282,42 @@ impl BatchingTask { }; if let Some(checkpoints_json) = incremental_checkpoints_json { + // Select `sequence_range` only when every append-only source table + // proves the `preserve_row_sequence` capability; otherwise retain + // the historical `memtable_only` mode. The `sequence_range` scan + // keeps SSTs and reads the exact (checkpoint, scan-open snapshot] + // row-level delta; the engine fails closed when the capability + // does not hold at scan time. + let capable = match self.sequence_range_capable().await { + Ok(capable) => capable, + Err(err) => { + if self.config.exact_sequence_range_required { + return Err(err); + } + false + } + }; + if self.config.exact_sequence_range_required && !capable { + return Err(UnexpectedSnafu { + reason: format!( + "Flow {} requires exact sequence-range reads, but source capability was revoked", + self.config.flow_id + ), + } + .build()); + } let sink_table_id = self.sink_table_id().await?; + let incremental_mode = if capable { + debug!( + "Flow {} selected sequence_range incremental mode", + self.config.flow_id + ); + FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE + } else { + FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY + }; extensions.push((FLOW_SINK_TABLE_ID, sink_table_id.to_string())); - extensions.push(( - FLOW_INCREMENTAL_MODE, - FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY.to_string(), - )); + extensions.push((FLOW_INCREMENTAL_MODE, incremental_mode.to_string())); extensions.push((FLOW_INCREMENTAL_AFTER_SEQS, checkpoints_json)); } diff --git a/src/flow/src/batching_mode/task/test.rs b/src/flow/src/batching_mode/task/test.rs index 24cdbf883e0..db90cfbdcb3 100644 --- a/src/flow/src/batching_mode/task/test.rs +++ b/src/flow/src/batching_mode/task/test.rs @@ -13,9 +13,11 @@ // limitations under the License. use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; -use catalog::RegisterTableRequest; use catalog::memory::MemoryCatalogManager; +use catalog::{DeregisterTableRequest, RegisterTableRequest}; use client::OutputWithMetrics; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_error::ext::BoxedError; @@ -31,11 +33,13 @@ use datatypes::vectors::{ }; use pretty_assertions::assert_eq; use query::options::{ - FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY, FLOW_SCHEDULED_TIME_MILLIS, - QueryOptions, + FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE, FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY, + FLOW_SCHEDULED_TIME_MILLIS, FLOW_SINK_TABLE_ID, QueryOptions, }; use session::context::QueryContext; use snafu::ResultExt; +use table::Table; +use table::metadata::FilterPushDownType; use table::test_util::MemTable; use super::*; @@ -85,6 +89,18 @@ async fn new_test_task_engine_and_plan_with_query_and_opts( query: &str, sink_table: &str, batch_opts: Arc, +) -> TestTaskParts { + new_test_task_engine_and_plan_with_query_and_opts_and_required( + query, sink_table, batch_opts, false, + ) + .await +} + +async fn new_test_task_engine_and_plan_with_query_and_opts_and_required( + query: &str, + sink_table: &str, + batch_opts: Arc, + exact_sequence_range_required: bool, ) -> TestTaskParts { let query_engine = create_test_query_engine(); let ctx = QueryContext::arc(); @@ -98,29 +114,32 @@ async fn new_test_task_engine_and_plan_with_query_and_opts( .unwrap(); let (_tx, rx) = tokio::sync::oneshot::channel(); - let task = BatchingTask::try_new(TaskArgs { - flow_id: 1, - query, - plan: plan.clone(), - time_window_expr: None, - expire_after: None, - sink_table_name: [ - "greptime".to_string(), - "public".to_string(), - sink_table.to_string(), - ], - source_table_names: vec![[ - "greptime".to_string(), - "public".to_string(), - "numbers_with_ts".to_string(), - ]], - query_ctx: ctx, - catalog_manager: query_engine.engine_state().catalog_manager().clone(), - shutdown_rx: rx, - batch_opts, - flow_eval_interval: None, - eval_schedule: None, - }) + let task = BatchingTask::try_new_with_exact_sequence_range_required( + TaskArgs { + flow_id: 1, + query, + plan: plan.clone(), + time_window_expr: None, + expire_after: None, + sink_table_name: [ + "greptime".to_string(), + "public".to_string(), + sink_table.to_string(), + ], + source_table_names: vec![[ + "greptime".to_string(), + "public".to_string(), + "numbers_with_ts".to_string(), + ]], + query_ctx: ctx, + catalog_manager: query_engine.engine_state().catalog_manager().clone(), + shutdown_rx: rx, + batch_opts, + flow_eval_interval: None, + eval_schedule: None, + }, + exact_sequence_range_required, + ) .unwrap(); TestTaskParts { @@ -379,6 +398,62 @@ fn register_auto_created_aggregate_sink(query_engine: &QueryEngineRef, table_nam memory_catalog.register_table_sync(request).unwrap(); } +async fn configure_source_capability( + query_engine: &QueryEngineRef, + engine: &str, + preserve_row_sequence: bool, +) { + let catalog_manager = query_engine.engine_state().catalog_manager(); + let source = catalog_manager + .table( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "numbers_with_ts", + None, + ) + .await + .unwrap() + .unwrap(); + let mut info = (*source.table_info()).clone(); + info.meta.engine = engine.to_string(); + if preserve_row_sequence { + info.meta + .options + .extra_options + .insert("preserve_row_sequence".to_string(), "true".to_string()); + } else { + info.meta + .options + .extra_options + .remove("preserve_row_sequence"); + } + let source = Arc::new(Table::new( + Arc::new(info), + FilterPushDownType::Unsupported, + source.data_source(), + )); + let memory_catalog = catalog_manager + .as_any() + .downcast_ref::() + .unwrap(); + memory_catalog + .deregister_table_sync(DeregisterTableRequest { + catalog: DEFAULT_CATALOG_NAME.to_string(), + schema: DEFAULT_SCHEMA_NAME.to_string(), + table_name: "numbers_with_ts".to_string(), + }) + .unwrap(); + memory_catalog + .register_table_sync(RegisterTableRequest { + catalog: DEFAULT_CATALOG_NAME.to_string(), + schema: DEFAULT_SCHEMA_NAME.to_string(), + table_name: "numbers_with_ts".to_string(), + table_id: source.table_info().table_id(), + table: source, + }) + .unwrap(); +} + fn dirty_marker() -> DirtyTimeWindows { let mut dirty = DirtyTimeWindows::default(); dirty.set_dirty(); @@ -391,50 +466,6 @@ fn flow_error_with_status(status_code: StatusCode) -> Error { .unwrap_err() } -/// Test-only error that carries a non-RequestOutdated status code but -/// displays a stale-snapshot-fence marker string, simulating the real-world -/// scenario where the structured status code is lost through frontend/client -/// wrapping layers. -#[derive(Debug)] -struct StaleFenceTextError { - code: StatusCode, - message: String, -} - -impl std::fmt::Display for StaleFenceTextError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.message) - } -} - -impl std::error::Error for StaleFenceTextError {} - -impl common_error::ext::ErrorExt for StaleFenceTextError { - fn status_code(&self) -> StatusCode { - self.code - } - fn as_any(&self) -> &dyn std::any::Any { - self - } -} - -impl common_error::ext::StackError for StaleFenceTextError { - fn debug_fmt(&self, _: usize, _: &mut Vec) {} - fn next(&self) -> Option<&dyn common_error::ext::StackError> { - None - } -} - -fn flow_error_with_code_and_text(code: StatusCode, text: &str) -> Error { - let inner = StaleFenceTextError { - code, - message: text.to_string(), - }; - Err::<(), _>(BoxedError::new(inner)) - .context(crate::error::ExternalSnafu) - .unwrap_err() -} - fn dirty_range(start: i64, end: i64) -> DirtyTimeWindows { let mut dirty = DirtyTimeWindows::default(); dirty.add_window( @@ -547,6 +578,29 @@ fn register_scheduled_now_sink(query_engine: &QueryEngineRef, table_name: &str, memory_catalog.register_table_sync(request).unwrap(); } +struct ExactDeltaFailureHandler { + invoked: Arc, +} + +#[async_trait::async_trait] +impl crate::batching_mode::frontend_client::GrpcQueryHandlerWithBoxedError + for ExactDeltaFailureHandler +{ + async fn do_query( + &self, + _query: api::v1::greptime_request::Request, + ctx: QueryContextRef, + ) -> std::result::Result { + self.invoked.store(true, Ordering::SeqCst); + assert_eq!(ctx.extension(FLOW_INCREMENTAL_MODE), Some("sequence_range")); + assert_eq!( + ctx.extension(FLOW_INCREMENTAL_AFTER_SEQS), + Some("{\"1\":10}") + ); + Err(BoxedError::new(MockError::new(StatusCode::RequestOutdated))) + } +} + struct CaptureScheduledNowHandler { expected_extension: String, captured_sql: Arc>>, @@ -1447,78 +1501,6 @@ fn test_query_failure_reason_distinguishes_fenced_repair_stale_fence() { ); } -/// Wrapped errors carrying stale snapshot fence marker text in their -/// Display/Debug chain should be classified as `SnapshotFenceExpired` on -/// fenced repair coverage, even when the structured `StatusCode::RequestOutdated` -/// was lost through client layering. This prevents an infinite retry loop -/// where the fenced chunk re-sends the same stale `given_seq` every tick. -#[test] -fn test_query_failure_reason_text_fallback_stale_snapshot_fence() { - let high = BTreeMap::new(); - let fenced = QueryCoverage::FencedRepairChunk { high: high.clone() }; - - // STALE_SNAPSHOT_FENCE marker with a non-RequestOutdated status code - let err = flow_error_with_code_and_text( - StatusCode::Internal, - "gRPC error: STALE_SNAPSHOT_FENCE: snapshot upper bound stale, region: 1024/0", - ); - assert_eq!( - BatchingTask::query_failure_reason(&err, &fenced), - FlowQueryFallbackReason::SnapshotFenceExpired - ); - - // REBIND_SNAPSHOT_FENCE marker - let err = flow_error_with_code_and_text( - StatusCode::Internal, - "STALE_SNAPSHOT_FENCE ... retry_hint: REBIND_SNAPSHOT_FENCE", - ); - assert_eq!( - BatchingTask::query_failure_reason(&err, &fenced), - FlowQueryFallbackReason::SnapshotFenceExpired - ); - - // snapshot upper bound stale marker (the natural-language fragment) - let err = flow_error_with_code_and_text( - StatusCode::Internal, - "query failed: snapshot upper bound stale, consider rebinding", - ); - assert_eq!( - BatchingTask::query_failure_reason(&err, &fenced), - FlowQueryFallbackReason::SnapshotFenceExpired - ); - - // Fenced coverage with a generic wrapped error (no stale-fence marker) → - // still QueryFailure - let generic_err = - flow_error_with_code_and_text(StatusCode::Internal, "some transient network error"); - assert_eq!( - BatchingTask::query_failure_reason(&generic_err, &fenced), - FlowQueryFallbackReason::QueryFailure - ); - - // Non-fenced incremental coverage with stale-fence marker text must NOT - // classify as SnapshotFenceExpired; it should remain IncrementalQueryFailure. - let err = flow_error_with_code_and_text( - StatusCode::Internal, - "STALE_SNAPSHOT_FENCE blob in unexpected context", - ); - assert_eq!( - BatchingTask::query_failure_reason(&err, &QueryCoverage::IncrementalDelta), - FlowQueryFallbackReason::IncrementalQueryFailure - ); - - // Existing RequestOutdated behavior is unchanged. - let outdated_err = flow_error_with_status(StatusCode::RequestOutdated); - assert_eq!( - BatchingTask::query_failure_reason(&outdated_err, &fenced), - FlowQueryFallbackReason::SnapshotFenceExpired - ); - assert_eq!( - BatchingTask::query_failure_reason(&outdated_err, &QueryCoverage::IncrementalDelta), - FlowQueryFallbackReason::StaleCursor - ); -} - #[test] fn test_fenced_repair_coverage_produces_snapshot_seq_map_for_distributed_metadata_path() { // Covers the metadata boundary between QueryCoverage and the @@ -1563,11 +1545,28 @@ async fn test_fenced_repair_stale_fence_next_plan_is_scoped_base_repair() { { let mut state = task.state.write().unwrap(); + let error = Err::<(), _>(BoxedError::new( + common_recordbatch::error::Error::PollStream { + error: datafusion::error::DataFusionError::Shared(Arc::new( + datafusion::error::DataFusionError::External(Box::new(BoxedError::new( + MockError::new(StatusCode::RequestOutdated), + ))), + )), + location: snafu::Location::default(), + }, + )) + .context(crate::error::ExternalSnafu) + .unwrap_err(); + let reason = BatchingTask::query_failure_reason( + &error, + &QueryCoverage::FencedRepairChunk { high: high.clone() }, + ); + assert_eq!(reason, FlowQueryFallbackReason::SnapshotFenceExpired); let decision = BatchingTask::apply_query_failure_to_state( &mut state, std::time::Duration::from_millis(1), &QueryCoverage::FencedRepairChunk { high }, - FlowQueryFallbackReason::SnapshotFenceExpired, + reason, ); assert_eq!( decision, @@ -1577,9 +1576,11 @@ async fn test_fenced_repair_stale_fence_next_plan_is_scoped_base_repair() { }) ); assert!(state.pending_fenced_repair().is_none()); + assert_eq!(state.dirty_time_windows.len(), 1); // Simulate the outer execution failure restore for the in-flight chunk. state.restore_scoped_windows(&filter); + assert_eq!(state.dirty_time_windows.len(), 2); } let plan = task @@ -1666,82 +1667,6 @@ fn test_fenced_repair_transient_non_stale_failure_retries_same_high() { ); } -/// When `query_failure_reason` classifies a wrapped error as -/// `SnapshotFenceExpired` via the text-marker fallback (not via -/// `StatusCode::RequestOutdated`), the state machine must still -/// abandon the fenced repair and produce a `ScopedBaseRepair` plan -/// next, exactly like the structured-code path. -#[tokio::test] -async fn test_text_fallback_stale_fence_produces_scoped_base_repair() { - let TestTaskParts { - task, - query_engine, - .. - } = new_time_window_test_task_with_query( - "SELECT number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window, number", - ) - .await; - let high = BTreeMap::from([(1_u64, 10_u64), (2_u64, 20_u64)]); - let filter = { - let mut state = task.state.write().unwrap(); - state - .dirty_time_windows - .add_window(Timestamp::new_second(10), Some(Timestamp::new_second(15))); - state - .dirty_time_windows - .add_window(Timestamp::new_second(100), Some(Timestamp::new_second(105))); - state.start_fenced_repair(high.clone()).unwrap(); - next_fenced_repair_filter(&mut state, 1) - }; - - // Construct a wrapped error that hits the text fallback (non-RequestOutdated - // status code with STALE_SNAPSHOT_FENCE marker text). - let err = flow_error_with_code_and_text( - StatusCode::Internal, - "STALE_SNAPSHOT_FENCE: snapshot upper bound stale, retry_hint: REBIND_SNAPSHOT_FENCE", - ); - let coverage = QueryCoverage::FencedRepairChunk { high }; - let reason = BatchingTask::query_failure_reason(&err, &coverage); - assert_eq!(reason, FlowQueryFallbackReason::SnapshotFenceExpired); - - { - let mut state = task.state.write().unwrap(); - let decision = BatchingTask::apply_query_failure_to_state( - &mut state, - std::time::Duration::from_millis(1), - &coverage, - reason, - ); - assert_eq!( - decision, - Some(FlowCheckpointDecision::FallbackToFullSnapshot { - previous_mode: CheckpointMode::FullSnapshot, - reason: FlowQueryFallbackReason::SnapshotFenceExpired, - }) - ); - assert!(state.pending_fenced_repair().is_none()); - - // Simulate the outer execution failure restore for the in-flight chunk. - state.restore_scoped_windows(&filter); - } - - let plan = task - .gen_query_with_time_window( - query_engine, - &aggregate_time_window_sink_schema(), - &[], - false, - Some(1), - ) - .await - .unwrap() - .expect("text-fallback stale fence should restore dirty windows for a fresh scoped repair"); - assert!( - matches!(plan.coverage, QueryCoverage::ScopedBaseRepair), - "next plan after text-fallback stale fence should be ScopedBaseRepair" - ); -} - #[test] fn test_checkpoint_decision_labels_are_stable() { let advance = FlowCheckpointDecision::AdvancedIncremental { @@ -1773,6 +1698,101 @@ fn test_checkpoint_decision_labels_are_stable() { ); } +#[tokio::test] +async fn test_exact_required_attempt_rejects_revoked_capability_without_extensions() { + let task = new_test_task_engine_and_plan_with_query_and_opts_and_required( + "SELECT number, ts FROM numbers_with_ts", + "exact_required_revoked", + incremental_batch_opts(), + true, + ) + .await + .into_task_and_plan() + .0; + + task.state + .write() + .unwrap() + .advance_checkpoints(HashMap::from([(1_u64, 10_u64)])); + let checkpoints_before = task.state.read().unwrap().checkpoints().clone(); + + let err = task + .build_flow_query_extensions(true, true) + .await + .unwrap_err(); + assert!(err.to_string().contains("requires exact sequence-range")); + assert_eq!( + task.state.read().unwrap().checkpoints(), + &checkpoints_before + ); +} + +#[tokio::test] +async fn test_sequence_range_producer_emits_capable_source_extensions() { + let parts = new_test_task_engine_and_plan_with_query_and_opts_and_required( + "SELECT number, ts FROM numbers_with_ts", + "numbers_with_ts", + incremental_batch_opts(), + true, + ) + .await; + configure_source_capability(&parts.query_engine, "mito", true).await; + let task = parts.task; + task.state + .write() + .unwrap() + .advance_checkpoints(HashMap::from([(1024_u64, 10_u64), (2048_u64, 20_u64)])); + + let extensions = task.build_flow_query_extensions(true, true).await.unwrap(); + + assert_eq!( + extensions, + vec![ + ("flow.return_region_seq", "true".to_string()), + (FLOW_SINK_TABLE_ID, "1".to_string()), + (FLOW_INCREMENTAL_MODE, "sequence_range".to_string()), + ( + FLOW_INCREMENTAL_AFTER_SEQS, + serde_json::json!({"1024": 10, "2048": 20}).to_string(), + ), + ] + ); +} + +#[tokio::test] +async fn test_sequence_range_producer_keeps_memtable_only_for_non_mito_source() { + let parts = new_test_task_engine_and_plan_with_query_and_opts( + "SELECT number, ts FROM numbers_with_ts", + "numbers_with_ts", + incremental_batch_opts(), + ) + .await; + configure_source_capability(&parts.query_engine, "file", true).await; + let task = parts.task; + task.state + .write() + .unwrap() + .advance_checkpoints(HashMap::from([(1024_u64, 10_u64)])); + + let extensions = task.build_flow_query_extensions(true, true).await.unwrap(); + + assert_eq!( + extensions, + vec![ + ("flow.return_region_seq", "true".to_string()), + (FLOW_SINK_TABLE_ID, "1".to_string()), + ( + FLOW_INCREMENTAL_MODE, + FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY.to_string(), + ), + ( + FLOW_INCREMENTAL_AFTER_SEQS, + serde_json::json!({"1024": 10}).to_string(), + ), + ] + ); +} + #[tokio::test] async fn test_build_flow_query_extensions_switches_with_checkpoint_mode() { let (task, _) = new_test_task_engine_and_plan_with_query( @@ -2128,15 +2148,96 @@ async fn test_incremental_delta_applies_expire_after_retention_filter() { } #[tokio::test] -async fn test_successful_incremental_checkpoint_fallback_consumes_unscoped_dirty_signal() { +async fn test_exact_required_executed_failure_selects_full_snapshot_repair() { let TestTaskParts { - task, + mut task, query_engine, .. } = new_time_window_test_task_with_query( "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window", ) .await; + configure_source_capability(&query_engine, "mito", true).await; + register_twe_sink(&query_engine, "missing_sink", 9201); + Arc::get_mut(&mut task.config) + .expect("test task config should be uniquely owned") + .exact_sequence_range_required = true; + { + let mut state = task.state.write().unwrap(); + state.advance_checkpoints(HashMap::from([(1_u64, 10_u64)])); + state + .dirty_time_windows + .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5))); + } + let sink_schema = aggregate_time_window_sink_schema(); + let plan_info = task + .gen_insert_plan_unlocked(&query_engine, Some(1)) + .await + .unwrap() + .unwrap(); + assert!(matches!( + plan_info.coverage, + QueryCoverage::IncrementalDelta + )); + + let handler_invoked = Arc::new(AtomicBool::new(false)); + let handler: Arc = + Arc::new(ExactDeltaFailureHandler { + invoked: handler_invoked.clone(), + }); + let frontend_client = Arc::new(FrontendClient::from_grpc_handler( + Arc::downgrade(&handler), + QueryOptions::default(), + )); + task.execute_logical_plan_unlocked( + &frontend_client, + &plan_info.plan, + &plan_info.dirty_restore, + &plan_info.coverage, + ) + .await + .expect_err("the dispatched exact delta must fail through the injected frontend handler"); + assert!( + handler_invoked.load(Ordering::SeqCst), + "the injected frontend handler must receive the exact delta" + ); + task.handle_executed_query_failure(Some(&plan_info)); + + { + let state = task.state.read().unwrap(); + assert_eq!(state.checkpoint_mode(), CheckpointMode::FullSnapshot); + assert_eq!(state.checkpoints(), &BTreeMap::from([(1_u64, 10_u64)])); + assert_eq!(state.dirty_time_windows.len(), 1); + assert_eq!( + state.dirty_time_windows.window_size(), + std::time::Duration::from_secs(5) + ); + } + + let repair = task + .gen_query_with_time_window(query_engine, &sink_schema, &[], false, Some(1)) + .await + .unwrap() + .expect("failed exact delta must retry through existing base repair"); + assert!(matches!(repair.coverage, QueryCoverage::ScopedBaseRepair)); + assert!(repair.plan.to_string().contains("Filter:")); + assert!(!repair.plan.to_string().contains("Left Join")); +} + +#[tokio::test] +async fn test_exact_required_incomplete_proof_selects_base_recomputation() { + let TestTaskParts { + mut task, + query_engine, + .. + } = new_time_window_test_task_with_query( + "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window", + ) + .await; + configure_source_capability(&query_engine, "mito", true).await; + Arc::get_mut(&mut task.config) + .expect("test task config should be uniquely owned") + .exact_sequence_range_required = true; { let mut state = task.state.write().unwrap(); state.advance_checkpoints(HashMap::from([(1_u64, 10_u64), (2_u64, 20_u64)])); @@ -2186,13 +2287,24 @@ async fn test_successful_incremental_checkpoint_fallback_consumes_unscoped_dirty assert!(state.dirty_time_windows.is_empty()); } + task.state + .write() + .unwrap() + .dirty_time_windows + .add_window(Timestamp::new_second(5), Some(Timestamp::new_second(10))); let followup = task .gen_query_with_time_window(query_engine, &sink_schema, &[], false, Some(1)) .await - .unwrap(); + .unwrap() + .expect("new dirty work must use base recomputation after incomplete delta proof"); + assert!(matches!(followup.coverage, QueryCoverage::ScopedBaseRepair)); assert!( - followup.is_none(), - "successful fallback consumes the dirty signal instead of re-running it" + followup.plan.to_string().contains("Filter:"), + "base recomputation must retain its dirty-window filter" + ); + assert!( + !followup.plan.to_string().contains("Left Join"), + "base recomputation must not replay the additive incremental sink merge" ); } @@ -2273,6 +2385,96 @@ async fn test_executed_query_failure_restores_scoped_dirty_windows_for_flush_pat ); } +#[tokio::test] +async fn test_exact_required_unsupported_plan_keeps_exact_retry_state() { + let query_engine = create_test_query_engine(); + configure_source_capability(&query_engine, "mito", true).await; + let ctx = QueryContext::arc(); + let plan = sql_to_df_plan( + ctx.clone(), + query_engine.clone(), + "SELECT number, ts FROM numbers_with_ts", + true, + ) + .await + .unwrap(); + let (sink_table, _) = get_table_info_df_schema( + query_engine.engine_state().catalog_manager().clone(), + [ + "greptime".to_string(), + "public".to_string(), + "numbers_with_ts".to_string(), + ], + ) + .await + .unwrap(); + let dml_plan = LogicalPlan::Dml(DmlStatement::new( + datafusion_common::TableReference::bare("test"), + Arc::new(DefaultTableSource::new(Arc::new( + DfTableProviderAdapter::new(sink_table), + ))), + WriteOp::Insert(datafusion_expr::dml::InsertOp::Append), + Arc::new(plan), + )); + let (_tx, rx) = tokio::sync::oneshot::channel(); + let task = BatchingTask::try_new_with_exact_sequence_range_required( + TaskArgs { + flow_id: 1, + query: "SELECT number, ts FROM numbers_with_ts", + plan: dml_plan.clone(), + time_window_expr: None, + expire_after: None, + sink_table_name: [ + "greptime".to_string(), + "public".to_string(), + "numbers_with_ts".to_string(), + ], + source_table_names: vec![[ + "greptime".to_string(), + "public".to_string(), + "numbers_with_ts".to_string(), + ]], + query_ctx: ctx, + catalog_manager: query_engine.engine_state().catalog_manager().clone(), + shutdown_rx: rx, + batch_opts: incremental_batch_opts(), + flow_eval_interval: None, + eval_schedule: None, + }, + true, + ) + .unwrap(); + task.state + .write() + .unwrap() + .advance_checkpoints(HashMap::from([(1_u64, 10_u64)])); + task.state + .write() + .unwrap() + .dirty_time_windows + .add_window(Timestamp::new_second(10), Some(Timestamp::new_second(15))); + + let (frontend_client, _) = FrontendClient::from_empty_grpc_handler(QueryOptions::default()); + let frontend_client = Arc::new(frontend_client); + for _ in 0..2 { + let err = task + .execute_logical_plan_unlocked( + &frontend_client, + &dml_plan, + &DirtyRestore::Unscoped(dirty_range(10, 15)), + &QueryCoverage::IncrementalDelta, + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("requires exact sequence-range")); + + let state = task.state.read().unwrap(); + assert_eq!(state.checkpoint_mode(), CheckpointMode::Incremental); + assert_eq!(state.checkpoints(), &BTreeMap::from([(1_u64, 10_u64)])); + assert_eq!(state.dirty_time_windows.len(), 1); + } +} + #[tokio::test] async fn test_prepare_plan_for_incremental_disables_on_non_aggregate() { let query_engine = create_test_query_engine(); diff --git a/src/flow/src/batching_mode/time_window.rs b/src/flow/src/batching_mode/time_window.rs index 1c8ebea2a85..5dbd08150ac 100644 --- a/src/flow/src/batching_mode/time_window.rs +++ b/src/flow/src/batching_mode/time_window.rs @@ -37,6 +37,7 @@ use datafusion_common::tree_node::{ Transformed, TreeNode, TreeNodeRecursion, TreeNodeRewriter, TreeNodeVisitor, }; use datafusion_common::{DFSchema, TableReference}; +use datafusion_expr::physical_planning_context::PhysicalPlanningContext; use datafusion_expr::{ColumnarValue, LogicalPlan}; use datafusion_physical_expr::PhysicalExprRef; use datatypes::prelude::{ConcreteDataType, DataType}; @@ -830,7 +831,15 @@ fn to_phy_expr( let phy_planner = DefaultPhysicalPlanner::default(); let phy_expr: PhysicalExprRef = phy_planner - .create_physical_expr(expr, df_schema, session) + // Time-window expressions are standalone scalar expressions over the input + // time column, so they cannot contain scalar subqueries or lambda variables + // that would require a plan-level physical planning context. + .create_physical_expr( + expr, + df_schema, + session, + &PhysicalPlanningContext::default(), + ) .with_context(|_e| DatafusionSnafu { context: format!( "Failed to create physical expression from {expr:?} using {df_schema:?}" @@ -993,7 +1002,7 @@ mod test { Some(Timestamp::new(0, TimeUnit::Millisecond)), Some(Timestamp::new(300000, TimeUnit::Millisecond)), ), - "SELECT sum(numbers_with_ts.number), numbers_with_ts.number, date_bin('5 minutes', numbers_with_ts.ts) AS time_window, bucket_name FROM (SELECT numbers_with_ts.number, numbers_with_ts.ts, CASE WHEN (numbers_with_ts.number < 5) THEN 'bucket_0_5' WHEN (numbers_with_ts.number >= 5) THEN 'bucket_5_inf' END AS bucket_name FROM numbers_with_ts WHERE ((ts >= CAST('1970-01-01 00:00:00' AS TIMESTAMP)) AND (ts <= CAST('1970-01-01 00:05:00' AS TIMESTAMP)))) GROUP BY numbers_with_ts.number, date_bin('5 minutes', numbers_with_ts.ts), bucket_name", + "SELECT sum(number), number, date_bin('5 minutes', ts) AS time_window, bucket_name FROM (SELECT numbers_with_ts.number, numbers_with_ts.ts, CASE WHEN (numbers_with_ts.number < 5) THEN 'bucket_0_5' WHEN (numbers_with_ts.number >= 5) THEN 'bucket_5_inf' END AS bucket_name FROM numbers_with_ts WHERE ((ts >= CAST('1970-01-01 00:00:00' AS TIMESTAMP)) AND (ts <= CAST('1970-01-01 00:05:00' AS TIMESTAMP)))) GROUP BY number, date_bin('5 minutes', ts), bucket_name", ), // complex subquery alias ( diff --git a/src/flow/src/batching_mode/utils.rs b/src/flow/src/batching_mode/utils.rs index a707c2d9eb6..cb8e7e50634 100644 --- a/src/flow/src/batching_mode/utils.rs +++ b/src/flow/src/batching_mode/utils.rs @@ -31,7 +31,7 @@ use datafusion_common::tree_node::{ use datafusion_common::{ Column, DFSchema, DataFusionError, NullEquality, ScalarValue, TableReference, }; -use datafusion_expr::logical_plan::{Aggregate, TableScan}; +use datafusion_expr::logical_plan::{Aggregate, TableScanBuilder}; use datafusion_expr::{ Distinct, ExprSchemable, JoinType, LogicalPlan, LogicalPlanBuilder, Operator, Projection, and, binary_expr, bitwise_and, bitwise_or, bitwise_xor, is_null, or, when, @@ -657,17 +657,15 @@ pub async fn rewrite_incremental_aggregate_with_sink_merge( let table_provider = Arc::new(DfTableProviderAdapter::new(sink_table)); let table_source = Arc::new(DefaultTableSource::new(table_provider)); let sink_scan = LogicalPlan::TableScan( - TableScan::try_new( + TableScanBuilder::new( TableReference::Full { catalog: sink_table_name[0].clone().into(), schema: sink_table_name[1].clone().into(), table: sink_table_name[2].clone().into(), }, table_source, - None, - vec![], - None, ) + .build() .with_context(|_| DatafusionSnafu { context: "Failed to build sink table scan for incremental sink merge".to_string(), })?, diff --git a/src/flow/src/batching_mode/utils/test.rs b/src/flow/src/batching_mode/utils/test.rs index 1356f87e8f6..1c34caeedd7 100644 --- a/src/flow/src/batching_mode/utils/test.rs +++ b/src/flow/src/batching_mode/utils/test.rs @@ -18,7 +18,7 @@ use catalog::RegisterTableRequest; use common_recordbatch::RecordBatch; use common_time::Timestamp; use datafusion_common::tree_node::TreeNode as _; -use datafusion_expr::GroupingSet; +use datafusion_expr::{GroupingSet, TableScanBuilder}; use datatypes::prelude::{ConcreteDataType, MutableVector, Scalar, ScalarVectorBuilder, VectorRef}; use datatypes::schema::{ColumnSchema, Schema}; use datatypes::timestamp::TimestampMillisecond; @@ -92,17 +92,15 @@ fn test_sink_scan(sink_table: TableRef, sink_table_name: &TableName) -> LogicalP let table_provider = Arc::new(DfTableProviderAdapter::new(sink_table)); let table_source = Arc::new(DefaultTableSource::new(table_provider)); LogicalPlan::TableScan( - TableScan::try_new( + TableScanBuilder::new( TableReference::Full { catalog: sink_table_name[0].clone().into(), schema: sink_table_name[1].clone().into(), table: sink_table_name[2].clone().into(), }, table_source, - None, - vec![], - None, ) + .build() .unwrap(), ) } @@ -296,7 +294,7 @@ async fn test_add_filter() { // complex subquery without alias ( "SELECT sum(number), number, date_bin('5 minutes', ts) as time_window, bucket_name FROM (SELECT number, ts, case when number < 5 THEN 'bucket_0_5' when number >= 5 THEN 'bucket_5_inf' END as bucket_name FROM numbers_with_ts) GROUP BY number, time_window, bucket_name;", - "SELECT sum(numbers_with_ts.number), numbers_with_ts.number, date_bin('5 minutes', numbers_with_ts.ts) AS time_window, bucket_name FROM (SELECT numbers_with_ts.number, numbers_with_ts.ts, CASE WHEN (numbers_with_ts.number < 5) THEN 'bucket_0_5' WHEN (numbers_with_ts.number >= 5) THEN 'bucket_5_inf' END AS bucket_name FROM numbers_with_ts WHERE (number > 4)) GROUP BY numbers_with_ts.number, date_bin('5 minutes', numbers_with_ts.ts), bucket_name", + "SELECT sum(number), number, date_bin('5 minutes', ts) AS time_window, bucket_name FROM (SELECT numbers_with_ts.number, numbers_with_ts.ts, CASE WHEN (numbers_with_ts.number < 5) THEN 'bucket_0_5' WHEN (numbers_with_ts.number >= 5) THEN 'bucket_5_inf' END AS bucket_name FROM numbers_with_ts WHERE (number > 4)) GROUP BY number, date_bin('5 minutes', ts), bucket_name", ), // complex subquery alias ( diff --git a/src/flow/src/expr/scalar.rs b/src/flow/src/expr/scalar.rs index 18649bff123..af16381e4cf 100644 --- a/src/flow/src/expr/scalar.rs +++ b/src/flow/src/expr/scalar.rs @@ -216,6 +216,7 @@ impl ScalarExpr { msg: "Failed to convert literal to scalar value", })?, batch.row_count(), + None, ) .context(DataTypeSnafu { msg: "Failed to convert scalar value to vector ref when parsing literal", diff --git a/src/flow/src/transform/aggr.rs b/src/flow/src/transform/aggr.rs index 861ca8fe657..7b938524a4e 100644 --- a/src/flow/src/transform/aggr.rs +++ b/src/flow/src/transform/aggr.rs @@ -725,7 +725,7 @@ mod test { df_scalar_fn: DfScalarFunction::try_from_raw_fn( RawDfScalarFn { f: BytesMut::from( - b"\x08\x02\"\x0f\x1a\r\n\x0b\xa2\x02\x08\n\0\x12\x04\x10\x1e \t\"\n\x1a\x08\x12\x06\n\x04\x12\x02\x08\x01".as_ref(), + b"\x08\x02\x1a\x07\x8a\x02\x04\x08\x03\x18\x01\"\x0f\x1a\r\n\x0b\xa2\x02\x08\n\0\x12\x04\x10\x1e \t\"\n\x1a\x08\x12\x06\n\x04\x12\x02\x08\x01".as_ref(), ), input_schema: RelationType::new(vec![ColumnType::new( ConcreteDataType::interval_month_day_nano_datatype(), diff --git a/src/flow/src/transform/expr.rs b/src/flow/src/transform/expr.rs index 40dcaef8a64..a2150e2290b 100644 --- a/src/flow/src/transform/expr.rs +++ b/src/flow/src/transform/expr.rs @@ -20,6 +20,7 @@ use common_error::ext::BoxedError; use common_telemetry::debug; use datafusion::execution::SessionStateBuilder; use datafusion::functions::all_default_functions; +use datafusion_expr::physical_planning_context::PhysicalPlanningContext; use datafusion_physical_expr::PhysicalExpr; use datafusion_substrait::logical_plan::consumer::DefaultSubstraitConsumer; use datatypes::data_type::ConcreteDataType as CDT; @@ -101,11 +102,15 @@ pub(crate) async fn from_scalar_fn_to_df_fn_impl( context: "Failed to convert substrait scalar function to datafusion scalar function", } })?; - let phy_expr = - datafusion::physical_expr::create_physical_expr(&expr, &schema, &Default::default()) - .context(DatafusionSnafu { - context: "Failed to create physical expression from logical expression", - })?; + let phy_expr = datafusion::physical_expr::create_physical_expr( + &expr, + &schema, + &Default::default(), + &PhysicalPlanningContext::default(), + ) + .context(DatafusionSnafu { + context: "Failed to create physical expression from logical expression", + })?; Ok(phy_expr) } diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml index 7ad24a466e8..677dbbbe55f 100644 --- a/src/frontend/Cargo.toml +++ b/src/frontend/Cargo.toml @@ -29,6 +29,7 @@ cache.workspace = true catalog.workspace = true client.workspace = true common-base.workspace = true +common-batcher.workspace = true common-catalog.workspace = true common-config.workspace = true common-datasource.workspace = true diff --git a/src/frontend/src/instance/dashboard.rs b/src/frontend/src/instance/dashboard.rs index 33a9083cc8e..ee3ef6e5cc1 100644 --- a/src/frontend/src/instance/dashboard.rs +++ b/src/frontend/src/instance/dashboard.rs @@ -29,9 +29,9 @@ use common_query::OutputData; use common_recordbatch::util as record_util; use common_telemetry::info; use common_time::FOREVER; +use datafusion::common::TableReference; use datafusion::datasource::DefaultTableSource; use datafusion::logical_expr::col; -use datafusion::sql::TableReference; use datafusion_expr::{DmlStatement, LogicalPlan, lit}; use datatypes::arrow::array::{Array, AsArray}; use servers::error::{ diff --git a/src/frontend/src/instance/grpc.rs b/src/frontend/src/instance/grpc.rs index 7d4f0ee7959..2d89b5489df 100644 --- a/src/frontend/src/instance/grpc.rs +++ b/src/frontend/src/instance/grpc.rs @@ -453,6 +453,7 @@ impl Instance { request.flight_data, request.record_batch, request.schema_bytes, + ctx.skip_wal(), ) .await .context(TableOperationSnafu)?; diff --git a/src/frontend/src/server.rs b/src/frontend/src/server.rs index e112e5ae2a9..1812dc71a51 100644 --- a/src/frontend/src/server.rs +++ b/src/frontend/src/server.rs @@ -567,7 +567,23 @@ mod tests { // returns `None`; in these cases no request can wait for a pending-row // flush, so the timeout must not be raised. type KnobMutator = fn(&mut FrontendOptions); - let cases: [(&str, KnobMutator); 4] = [ + let cases: [(&str, KnobMutator); 9] = [ + ("oversized flush concurrency", |opts| { + opts.prom_store.max_concurrent_flushes = usize::MAX + }), + ("oversized worker channel", |opts| { + opts.prom_store.worker_channel_capacity = usize::MAX + }), + ("oversized inflight limit", |opts| { + opts.prom_store.max_inflight_requests = usize::MAX + }), + ("oversized flow queue", |opts| { + opts.prom_store.flow_notification_queue_capacity = + std::num::NonZeroUsize::new(usize::MAX).unwrap() + }), + ("unrepresentable deadline", |opts| { + opts.prom_store.pending_rows_flush_interval = Duration::MAX + }), ("zero max_batch_rows", |opts| { opts.prom_store.max_batch_rows = 0 }), diff --git a/src/frontend/src/service_config/prom_store.rs b/src/frontend/src/service_config/prom_store.rs index d2e386caae0..f8429a86925 100644 --- a/src/frontend/src/service_config/prom_store.rs +++ b/src/frontend/src/service_config/prom_store.rs @@ -15,8 +15,10 @@ use std::num::NonZeroUsize; use std::time::Duration; +use common_batcher::flush_policy::timing::TimingFlushPolicy; use serde::{Deserialize, Serialize}; use servers::prom_remote_write::validation::PromValidationMode; +use tokio::sync::Semaphore; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct PromStoreOptions { @@ -64,18 +66,20 @@ fn default_flow_notification_queue_capacity() -> NonZeroUsize { } impl PromStoreOptions { - /// Returns whether the pending rows batcher can be enabled with these - /// options. Mirrors the enablement conditions of - /// `PendingRowsBatcher::try_new` in the servers crate, which returns - /// `None` when any of these knobs is zero. + /// Returns whether batching is enabled and its controls pass construction validation. pub fn pending_rows_batching_enabled(&self) -> bool { self.enable && self.with_metric_engine - && !self.pending_rows_flush_interval.is_zero() + && TimingFlushPolicy::validate(self.pending_rows_flush_interval) && self.max_batch_rows > 0 - && self.max_concurrent_flushes > 0 - && self.worker_channel_capacity > 0 - && self.max_inflight_requests > 0 + && [ + self.max_concurrent_flushes, + self.worker_channel_capacity, + self.max_inflight_requests, + self.flow_notification_queue_capacity.get(), + ] + .into_iter() + .all(|capacity| (1..=Semaphore::MAX_PERMITS).contains(&capacity)) } } @@ -107,6 +111,42 @@ mod tests { default_worker_channel_capacity, }; + #[test] + fn test_batching_capacity_boundaries() { + let max = tokio::sync::Semaphore::MAX_PERMITS; + let options = PromStoreOptions { + pending_rows_flush_interval: Duration::from_secs(5), + max_batch_rows: usize::MAX, + max_concurrent_flushes: max, + worker_channel_capacity: max, + max_inflight_requests: max, + flow_notification_queue_capacity: std::num::NonZeroUsize::new(max).unwrap(), + ..Default::default() + }; + assert!(options.pending_rows_batching_enabled()); + let cases = [ + PromStoreOptions { + max_concurrent_flushes: max + 1, + ..options.clone() + }, + PromStoreOptions { + worker_channel_capacity: max + 1, + ..options.clone() + }, + PromStoreOptions { + max_inflight_requests: max + 1, + ..options.clone() + }, + PromStoreOptions { + flow_notification_queue_capacity: std::num::NonZeroUsize::new(max + 1).unwrap(), + ..options + }, + ]; + for options in cases { + assert!(!options.pending_rows_batching_enabled()); + } + } + #[test] fn test_prom_store_options() { let default = PromStoreOptions::default(); diff --git a/src/index/src/bloom_filter/creator.rs b/src/index/src/bloom_filter/creator.rs index 5a9b5a0b36a..8bbd4cf1c13 100644 --- a/src/index/src/bloom_filter/creator.rs +++ b/src/index/src/bloom_filter/creator.rs @@ -149,6 +149,35 @@ impl BloomFilterCreator { Ok(()) } + /// Adds `nrows` copies of a single borrowed value (or null), copying it only when + /// it is new to a segment. Row counts advance for nulls as well. + pub async fn push_n_row_elem(&mut self, mut nrows: usize, elem: Option<&[u8]>) -> Result<()> { + while nrows > 0 { + let rows_to_seg_end = + self.rows_per_segment - (self.accumulated_row_count % self.rows_per_segment); + let rows_to_push = nrows.min(rows_to_seg_end); + nrows -= rows_to_push; + self.accumulated_row_count += rows_to_push; + + if let Some(elem) = elem + && !self.cur_seg_distinct_elems.contains(elem) + { + self.cur_seg_distinct_elems.insert(elem.to_vec()); + self.cur_seg_distinct_elems_mem_usage += elem.len(); + self.global_memory_usage + .fetch_add(elem.len(), Ordering::Relaxed); + } + if self + .accumulated_row_count + .is_multiple_of(self.rows_per_segment) + { + self.finalize_segment().await?; + self.finalized_row_count = self.accumulated_row_count; + } + } + Ok(()) + } + /// Adds a row of elements to the bloom filter. If the number of accumulated rows /// reaches `rows_per_segment`, it finalizes the current segment. pub async fn push_row_elems(&mut self, elems: impl IntoIterator) -> Result<()> { @@ -420,6 +449,42 @@ mod tests { } } + #[tokio::test] + async fn borrowed_single_value_matches_owned_rows_across_segments() { + let make_creator = || { + BloomFilterCreator::new( + 3, + 0.01, + Arc::new(MockExternalTempFileProvider::new()), + Arc::new(AtomicUsize::new(0)), + None, + ) + }; + let mut borrowed = make_creator(); + let mut owned = make_creator(); + // Zero rows, nulls, empty values, duplicates and runs crossing segment boundaries. + for (rows, elem) in [ + (0, Some(b"ignored".as_slice())), + (1, None), + (5, Some(b"".as_slice())), + (1, Some(b"".as_slice())), + (8, Some(b"label".as_slice())), + (1, None), + ] { + borrowed.push_n_row_elem(rows, elem).await.unwrap(); + owned + .push_n_row_elems(rows, elem.map(<[u8]>::to_vec)) + .await + .unwrap(); + assert_eq!(borrowed.memory_usage(), owned.memory_usage()); + } + let mut borrowed_blob = Cursor::new(Vec::new()); + let mut owned_blob = Cursor::new(Vec::new()); + borrowed.finish(&mut borrowed_blob).await.unwrap(); + owned.finish(&mut owned_blob).await.unwrap(); + assert_eq!(borrowed_blob.into_inner(), owned_blob.into_inner()); + } + #[tokio::test] async fn test_final_seg_all_null() { let mut writer = Cursor::new(Vec::new()); diff --git a/src/meta-srv/src/error.rs b/src/meta-srv/src/error.rs index 53872d4817f..70997ebe7d1 100644 --- a/src/meta-srv/src/error.rs +++ b/src/meta-srv/src/error.rs @@ -368,9 +368,9 @@ pub enum Error { location: Location, }, - #[snafu(display("Failed to deserialize from json: {}", input))] + #[snafu(display("Failed to deserialize from json payload of length {}", input_len))] DeserializeFromJson { - input: String, + input_len: usize, #[snafu(source)] error: serde_json::error::Error, #[snafu(implicit)] diff --git a/src/meta-srv/src/event/gc.rs b/src/meta-srv/src/event/gc.rs index 7f679f1785b..94d671a176b 100644 --- a/src/meta-srv/src/event/gc.rs +++ b/src/meta-srv/src/event/gc.rs @@ -218,6 +218,7 @@ mod tests { use common_event_recorder::testing::assert_event_contract; use common_event_recorder::{EventTypeFilter, PersistentEventContext, TriggerReason}; use common_meta::key::TableMetadataManager; + use common_meta::key::runtime_switch::RuntimeSwitchManager; use common_meta::kv_backend::memory::MemoryKvBackend; use common_meta::sequence::SequenceBuilder; use common_procedure::{ @@ -588,11 +589,13 @@ mod tests { fn batch_gc_procedure() -> BatchGcProcedure { let kv_backend = Arc::new(MemoryKvBackend::new()); let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone())); + let runtime_switch_manager = Arc::new(RuntimeSwitchManager::new(kv_backend.clone())); let mailbox_sequence = SequenceBuilder::new("test_batch_gc_event", kv_backend).build(); let mailbox = MailboxContext::new(mailbox_sequence); BatchGcProcedure::new( mailbox.mailbox().clone(), table_metadata_manager, + runtime_switch_manager, "localhost".to_string(), vec![RegionId::new(1024, 1)], true, diff --git a/src/meta-srv/src/gc/ctx.rs b/src/meta-srv/src/gc/ctx.rs index 4b46a941e25..857f5e87cdc 100644 --- a/src/meta-srv/src/gc/ctx.rs +++ b/src/meta-srv/src/gc/ctx.rs @@ -27,6 +27,7 @@ use common_meta::ddl_manager::DdlManagerRef; #[cfg(feature = "enterprise")] use common_meta::key::DroppedTableName; use common_meta::key::TableMetadataManagerRef; +use common_meta::key::runtime_switch::RuntimeSwitchManagerRef; use common_meta::key::table_repart::TableRepartValue; use common_meta::key::table_route::PhysicalTableRouteValue; #[cfg(feature = "enterprise")] @@ -155,6 +156,8 @@ pub(crate) struct DefaultGcSchedulerCtx { pub(crate) table_metadata_manager: TableMetadataManagerRef, /// Procedure manager. pub(crate) procedure_manager: ProcedureManagerRef, + /// Runtime switch manager used by recovered and newly submitted GC procedures. + pub(crate) runtime_switch_manager: RuntimeSwitchManagerRef, /// DDL manager used to submit the existing purge procedure. #[cfg(feature = "enterprise")] pub(crate) ddl_manager: DdlManagerRef, @@ -176,6 +179,7 @@ impl DefaultGcSchedulerCtx { pub fn try_new( table_metadata_manager: TableMetadataManagerRef, procedure_manager: ProcedureManagerRef, + runtime_switch_manager: RuntimeSwitchManagerRef, #[cfg(feature = "enterprise")] ddl_manager: DdlManagerRef, meta_peer_client: MetaPeerClientRef, mailbox: MailboxRef, @@ -184,6 +188,7 @@ impl DefaultGcSchedulerCtx { Ok(Self { table_metadata_manager, procedure_manager, + runtime_switch_manager, #[cfg(feature = "enterprise")] ddl_manager, #[cfg(feature = "enterprise")] @@ -320,6 +325,7 @@ impl DefaultGcSchedulerCtx { let procedure = BatchGcProcedure::new( self.mailbox.clone(), self.table_metadata_manager.clone(), + self.runtime_switch_manager.clone(), self.server_addr.clone(), region_ids.to_vec(), full_file_listing, diff --git a/src/meta-srv/src/gc/procedure.rs b/src/meta-srv/src/gc/procedure.rs index f9f597577ae..db22f3ab00d 100644 --- a/src/meta-srv/src/gc/procedure.rs +++ b/src/meta-srv/src/gc/procedure.rs @@ -18,8 +18,14 @@ use std::sync::Arc; use std::time::Duration; use api::v1::meta::MailboxMessage; -use common_meta::instruction::{self, GcRegions, GetFileRefs, GetFileRefsReply, InstructionReply}; +use common_error::ext::PlainError; +use common_error::status_code::StatusCode; +use common_meta::instruction::{ + self, GetPackedFileRefs, GetPackedFileRefsReply, InstructionReply, PackedFileRefsManifest, + PackedGcRegions, PackedRegionFileRefs, +}; use common_meta::key::TableMetadataManagerRef; +use common_meta::key::runtime_switch::RuntimeSwitchManagerRef; use common_meta::key::table_repart::TableRepartValue; use common_meta::key::table_route::PhysicalTableRouteValue; use common_meta::lock_key::{RegionLock, TableLock}; @@ -34,6 +40,7 @@ use common_telemetry::tracing::Instrument as _; use common_telemetry::tracing_context::TracingContext; use common_telemetry::{debug, error, info, warn}; use futures::future::join_all; +use futures::stream::{FuturesUnordered, StreamExt}; use itertools::Itertools as _; use serde::{Deserialize, Serialize}; use snafu::ResultExt as _; @@ -49,14 +56,14 @@ use crate::metrics::{METRIC_META_GC_DATANODE_CALLS_TOTAL, METRIC_META_GC_FAILED_ use crate::procedure::utils::{instruction_error_result, instruction_to_error}; use crate::service::mailbox::{Channel, MailboxReceiver, MailboxRef}; -async fn send_get_file_refs_inner( +async fn send_get_packed_file_refs_inner( mailbox: &MailboxRef, server_addr: &str, peer: &Peer, - instruction: GetFileRefs, + instruction: GetPackedFileRefs, timeout: Duration, ) -> Result { - let instruction = instruction::Instruction::GetFileRefs(instruction); + let instruction = instruction::Instruction::GetPackedFileRefs(instruction); let tracing_ctx = TracingContext::from_current_span(); let msg = MailboxMessage::json_message( &format!("Get file references: {}", instruction), @@ -75,25 +82,25 @@ async fn send_get_file_refs_inner( .await } -async fn recv_get_file_refs_reply( +async fn recv_get_packed_file_refs_reply( peer: &Peer, mailbox_rx: MailboxReceiver, -) -> Result { +) -> Result { let reply = match mailbox_rx.await { Ok(reply_msg) => HeartbeatMailbox::json_reply(&reply_msg)?, Err(e) => { error!( - e; "Failed to receive reply from datanode {} for GetFileRefs instruction", + e; "Failed to receive reply from datanode {} for GetPackedFileRefs instruction", peer, ); return Err(e); } }; - let InstructionReply::GetFileRefs(reply) = reply else { + let InstructionReply::GetPackedFileRefs(reply) = reply else { return error::UnexpectedInstructionReplySnafu { - mailbox_message: format!("{:?}", reply), - reason: "Unexpected reply of the GetFileRefs instruction", + mailbox_message: "unexpected instruction reply for GetPackedFileRefs".to_string(), + reason: "Unexpected reply of the GetPackedFileRefs instruction", } .fail(); }; @@ -104,12 +111,11 @@ async fn recv_get_file_refs_reply( async fn send_gc_regions_inner( mailbox: &MailboxRef, peer: &Peer, - gc_regions: &GcRegions, + instruction: instruction::Instruction, server_addr: &str, timeout: Duration, description: &str, ) -> Result { - let instruction = instruction::Instruction::GcRegions(gc_regions.clone()); let tracing_ctx = TracingContext::from_current_span(); let msg = MailboxMessage::json_message( &format!("{}: {}", description, instruction), @@ -128,9 +134,43 @@ async fn send_gc_regions_inner( .await } +fn scoped_gc_instruction( + regions: Vec, + full_manifest: &FileRefsManifest, + full_file_listing: bool, +) -> instruction::Instruction { + let packed_file_refs_manifest = PackedFileRefsManifest { + file_refs: regions + .iter() + .filter_map(|region| { + full_manifest + .file_refs + .get(region) + .map(|refs| (*region, PackedRegionFileRefs::from_refs(refs))) + }) + .collect(), + manifest_version: regions + .iter() + .filter_map(|region| { + full_manifest + .manifest_version + .get(region) + .map(|version| (*region, *version)) + }) + .collect(), + cross_region_refs: HashMap::new(), + }; + + instruction::Instruction::PackedGcRegions(PackedGcRegions { + regions, + packed_file_refs_manifest, + full_file_listing, + }) +} + async fn recv_gc_regions_reply( peer: &Peer, - gc_regions: &GcRegions, + regions: &[RegionId], description: &str, mailbox_rx: MailboxReceiver, ) -> Result { @@ -147,7 +187,7 @@ async fn recv_gc_regions_reply( let InstructionReply::GcRegions(reply) = reply else { return error::UnexpectedInstructionReplySnafu { - mailbox_message: format!("{:?}", reply), + mailbox_message: "unexpected instruction reply for GcRegions".to_string(), reason: "Unexpected reply of the GcRegions instruction", } .fail(); @@ -158,14 +198,16 @@ async fn recv_gc_regions_reply( Ok(report) => Ok(report), Err(e) => { error!( - e; "Datanode {} reported error during GC for regions {:?}", - peer, gc_regions + e; "Datanode {} reported error during GC for {} regions", + peer, regions.len() ); instruction_error_result( &e, format!( - "Datanode {} reported error during GC for regions {:?}: {}", - peer, gc_regions, e + "Datanode {} reported error during GC for {} regions: {}", + peer, + regions.len(), + e ), ) } @@ -177,6 +219,7 @@ async fn recv_gc_regions_reply( pub struct BatchGcProcedure { mailbox: MailboxRef, table_metadata_manager: TableMetadataManagerRef, + runtime_switch_manager: RuntimeSwitchManagerRef, data: BatchGcData, } @@ -217,9 +260,11 @@ pub enum State { impl BatchGcProcedure { pub const TYPE_NAME: &'static str = "metasrv-procedure::BatchGcProcedure"; + #[allow(clippy::too_many_arguments)] pub fn new( mailbox: MailboxRef, table_metadata_manager: TableMetadataManagerRef, + runtime_switch_manager: RuntimeSwitchManagerRef, server_addr: String, regions: Vec, full_file_listing: bool, @@ -229,6 +274,7 @@ impl BatchGcProcedure { Self { mailbox, table_metadata_manager, + runtime_switch_manager, data: BatchGcData { state: State::Start, server_addr, @@ -251,6 +297,7 @@ impl BatchGcProcedure { pub fn new_update_repartition_for_test( mailbox: MailboxRef, table_metadata_manager: TableMetadataManagerRef, + runtime_switch_manager: RuntimeSwitchManagerRef, server_addr: String, regions: Vec, file_refs: FileRefsManifest, @@ -259,6 +306,7 @@ impl BatchGcProcedure { Self { mailbox, table_metadata_manager, + runtime_switch_manager, data: BatchGcData { state: State::UpdateRepartition, server_addr, @@ -286,6 +334,22 @@ impl BatchGcProcedure { }) } + async fn check_maintenance_mode(&self) -> ProcedureResult<()> { + let enabled = self + .runtime_switch_manager + .maintenance_mode() + .await + .context(error::RuntimeSwitchManagerSnafu) + .map_err(ProcedureError::retry_later)?; + if enabled { + return Err(ProcedureError::external(PlainError::new( + "maintenance mode is enabled".to_string(), + StatusCode::IllegalState, + ))); + } + Ok(()) + } + fn merge_gc_report(&mut self, report: GcReport) { let accumulated = self.data.gc_report.get_or_insert_default(); // Deleted objects are cumulative, while these sets describe the latest outcome @@ -644,10 +708,10 @@ impl BatchGcProcedure { } } - // Send GetFileRefs instructions to each datanode + // Send packed GetFileRefs instructions to each datanode let mut all_file_refs: HashMap> = HashMap::new(); let mut all_manifest_versions = HashMap::new(); - let mut all_cross_region_refs = HashMap::new(); + let mut all_cross_region_refs: HashMap> = HashMap::new(); let mut peers = HashSet::new(); peers.extend(datanode2query_regions.keys().cloned()); @@ -655,33 +719,34 @@ impl BatchGcProcedure { let mailbox = &self.mailbox; let server_addr = &self.data.server_addr; - let mut tasks = Vec::new(); - + // Each future owns both send and receive, so a completed reply is merged and + // dropped immediately rather than retained in a second join_all buffer. + let mut tasks = FuturesUnordered::new(); for peer in peers { let regions = datanode2query_regions.remove(&peer).unwrap_or_default(); let related_regions_for_peer = datanode2related_regions.remove(&peer).unwrap_or_default(); - if regions.is_empty() && related_regions_for_peer.is_empty() { continue; } - tasks.push(async move { - let instruction = GetFileRefs { - query_regions: regions.clone(), - related_regions: related_regions_for_peer.clone(), + let instruction = GetPackedFileRefs { + query_regions: regions, + related_regions: related_regions_for_peer, }; - - let reply = - send_get_file_refs_inner(mailbox, server_addr, &peer, instruction, timeout) - .await; - - (peer, regions, related_regions_for_peer, reply) + let rx = send_get_packed_file_refs_inner( + mailbox, + server_addr, + &peer, + instruction, + timeout, + ) + .await?; + let reply = recv_get_packed_file_refs_reply(&peer, rx).await?; + Ok::<_, crate::error::Error>((peer, reply)) }); } - let mut recv_tasks = Vec::new(); - // store error to make sure metrics doesn't ignore other peers let mut first_error = None; let mut record_get_file_refs_error = |e| { METRIC_META_GC_DATANODE_CALLS_TOTAL @@ -691,80 +756,56 @@ impl BatchGcProcedure { first_error = Some(e); } }; - for (peer, regions, related_regions_for_peer, reply) in join_all(tasks).await { - match reply { - Ok(mailbox_rx) => { - recv_tasks.push(async move { - let reply = recv_get_file_refs_reply(&peer, mailbox_rx).await; - (peer, regions, related_regions_for_peer, reply) - }); - } - Err(e) => record_get_file_refs_error(e), - } - } - - let replies = join_all(recv_tasks).await; - - for (peer, regions, related_regions_for_peer, reply) in replies { - let reply = match reply { + while let Some(result) = tasks.next().await { + let (peer, reply) = match result { Ok(reply) => reply, Err(e) => { record_get_file_refs_error(e); continue; } }; - debug!( - "Got file references from datanode: {:?}, query_regions: {:?}, related_regions: {:?}, reply: {:?}", - peer, regions, related_regions_for_peer, reply - ); - if !reply.success { - METRIC_META_GC_DATANODE_CALLS_TOTAL - .with_label_values(&["get_file_refs", "error"]) - .inc(); let err = if let Some(error) = &reply.error { instruction_to_error( error, - format!( - "Failed to get file references from datanode {}: {:?}", - peer, error - ), + format!("Failed to get file references from datanode {peer}"), ) } else { error::UnexpectedSnafu { - violated: format!( - "Failed to get file references from datanode {}: {:?}", - peer, reply.error - ), + violated: + "Datanode returned an unsuccessful GetPackedFileRefs reply without an error" + .to_string(), } .build() }; record_get_file_refs_error(err); continue; } + let manifest = match reply.packed_file_refs_manifest.into_manifest() { + Ok(manifest) => manifest, + Err(err) => { + record_get_file_refs_error(error::Error::Other { + source: common_error::ext::BoxedError::new(err), + location: snafu::Location::new(file!(), line!(), 0), + }); + continue; + } + }; METRIC_META_GC_DATANODE_CALLS_TOTAL .with_label_values(&["get_file_refs", "success"]) .inc(); - - // Merge the file references from this datanode - for (region_id, file_refs) in reply.file_refs_manifest.file_refs { - all_file_refs - .entry(region_id) - .or_default() - .extend(file_refs); + for (region_id, refs) in manifest.file_refs { + all_file_refs.entry(region_id).or_default().extend(refs); } - - // region manifest version should be the smallest one among all peers, so outdated region can be detected - for (region_id, version) in reply.file_refs_manifest.manifest_version { + for (region_id, version) in manifest.manifest_version { let entry = all_manifest_versions.entry(region_id).or_insert(version); *entry = (*entry).min(version); } - - for (region_id, related_region_ids) in reply.file_refs_manifest.cross_region_refs { - let entry = all_cross_region_refs + for (region_id, related) in manifest.cross_region_refs { + all_cross_region_refs .entry(region_id) - .or_insert_with(HashSet::new); - entry.extend(related_region_ids); + .or_default() + .extend(related); } } @@ -813,28 +854,23 @@ impl BatchGcProcedure { let tasks = datanode2regions .into_iter() .map(|(peer, regions_for_peer)| { - let gc_regions = GcRegions { - regions: regions_for_peer.clone(), - // file_refs_manifest could be somewhere large. But still intentionally clone per datanode here: - // this path is admin-triggered or scheduler-triggered, peer count is expected to be bounded, and - // and abnormal manifest growth should be addressed at the source - file_refs_manifest: file_refs.clone(), - full_file_listing, - }; - let region_count = gc_regions.regions.len() as u64; + let region_count = regions_for_peer.len() as u64; + let regions = regions_for_peer.clone(); + let instruction = + scoped_gc_instruction(regions_for_peer, file_refs, full_file_listing); async move { let report = send_gc_regions_inner( mailbox, &peer, - &gc_regions, + instruction, server_addr, timeout, "Batch GC", ) .await; - (peer, gc_regions, region_count, report) + (peer, regions, region_count, report) } }); @@ -851,12 +887,12 @@ impl BatchGcProcedure { first_error = Some(e); } }; - for (peer, gc_regions, region_count, report) in join_all(tasks).await { + for (peer, regions, region_count, report) in join_all(tasks).await { match report { Ok(mailbox_rx) => { recv_tasks.push(async move { let report = - recv_gc_regions_reply(&peer, &gc_regions, "Batch GC", mailbox_rx).await; + recv_gc_regions_reply(&peer, ®ions, "Batch GC", mailbox_rx).await; (peer, region_count, report) }); } @@ -887,12 +923,12 @@ impl BatchGcProcedure { if need_retry.is_empty() { info!( - "GC report from datanode {}: successfully deleted files for regions {:?}", + "GC report from datanode {}: successfully deleted files for region IDs {:?}", peer, success ); } else { warn!( - "GC report from datanode {}: successfully deleted files for regions {:?}, need retry for regions {:?}", + "GC report from datanode {}: successfully deleted files for region IDs {:?}, need retry for region IDs {:?}", peer, success, need_retry ); } @@ -921,6 +957,8 @@ impl Procedure for BatchGcProcedure { } async fn execute(&mut self, ctx: &ProcedureContext) -> ProcedureResult { + self.check_maintenance_mode().await?; + match self.data.state { State::Start => { let _regions_span = common_telemetry::tracing::debug_span!( @@ -1122,11 +1160,15 @@ mod tests { use api::v1::meta::MailboxMessage; use api::v1::meta::mailbox_message::Payload; - use common_meta::instruction::{GcRegionsReply, InstructionReply}; + use common_meta::instruction::{GcRegionsReply, Instruction, InstructionReply}; use common_meta::key::TableMetadataManager; + use common_meta::key::runtime_switch::RuntimeSwitchManager; use common_meta::kv_backend::memory::MemoryKvBackend; + use common_meta::kv_backend::test_util::MockKvBackendBuilder; use common_meta::peer::Peer; use common_meta::sequence::SequenceBuilder; + use common_procedure::Context as ProcedureContext; + use common_procedure_test::MockContextProvider; use common_time::util::current_time_millis; use store_api::storage::FileId; use tokio::sync::mpsc; @@ -1135,6 +1177,38 @@ mod tests { use crate::procedure::test_util::{MailboxContext, send_mock_reply}; use crate::service::mailbox::Channel; + #[test] + fn test_scoped_gc_instruction_selects_and_scopes_manifest() { + let region = RegionId::new(7, 3); + let other = RegionId::new(7, 4); + let mut manifest = FileRefsManifest::default(); + manifest.file_refs.insert(region, HashSet::new()); + manifest.file_refs.insert(other, HashSet::new()); + manifest.manifest_version.insert(region, 42); + + let packed = scoped_gc_instruction(vec![region], &manifest, true); + let Instruction::PackedGcRegions(packed) = packed else { + panic!("expected packed instruction"); + }; + assert_eq!(packed.regions, vec![region]); + assert_eq!( + packed.packed_file_refs_manifest.manifest_version[®ion], + 42 + ); + assert!( + !packed + .packed_file_refs_manifest + .file_refs + .contains_key(&other) + ); + assert!( + packed + .packed_file_refs_manifest + .cross_region_refs + .is_empty() + ); + } + #[test] fn test_done_with_gc_report_keeps_report() { let region_id = RegionId::new(1024, 1); @@ -1215,6 +1289,7 @@ mod tests { let kv_backend = Arc::new(MemoryKvBackend::new()); let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone())); + let runtime_switch_manager = Arc::new(RuntimeSwitchManager::new(kv_backend.clone())); let mailbox_sequence = SequenceBuilder::new("test_batch_gc_partial_report", kv_backend).build(); let mut mailbox = MailboxContext::new(mailbox_sequence); @@ -1230,6 +1305,7 @@ mod tests { let mut procedure = BatchGcProcedure::new( mailbox.mailbox().clone(), table_metadata_manager, + runtime_switch_manager, "localhost".to_string(), vec![first_region, second_region], true, @@ -1265,11 +1341,15 @@ mod tests { fn batch_gc_procedure() -> BatchGcProcedure { let kv_backend = Arc::new(MemoryKvBackend::new()); let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone())); - let mailbox_sequence = SequenceBuilder::new("test_batch_gc_procedure", kv_backend).build(); + let runtime_switch_manager = Arc::new(RuntimeSwitchManager::new(kv_backend)); + let mailbox_sequence = + SequenceBuilder::new("test_batch_gc_procedure", Arc::new(MemoryKvBackend::new())) + .build(); let mailbox = MailboxContext::new(mailbox_sequence); BatchGcProcedure::new( mailbox.mailbox().clone(), table_metadata_manager, + runtime_switch_manager, "localhost".to_string(), vec![RegionId::new(1024, 1)], true, @@ -1277,4 +1357,113 @@ mod tests { HashMap::new(), ) } + + #[tokio::test] + async fn test_maintenance_mode_gates_every_gc_state() { + let states = [ + State::Start, + State::Acquiring, + State::Gcing, + State::UpdateRepartition, + ]; + + for state in states { + let kv_backend = Arc::new(MemoryKvBackend::new()); + let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone())); + let runtime_switch_manager = Arc::new(RuntimeSwitchManager::new(kv_backend.clone())); + runtime_switch_manager.set_maintenance_mode().await.unwrap(); + let mailbox_sequence = + SequenceBuilder::new("test_batch_gc_maintenance_gate", kv_backend).build(); + let mut mailbox = MailboxContext::new(mailbox_sequence); + let (tx, mut rx) = mpsc::channel(1); + mailbox + .insert_heartbeat_response_receiver(Channel::Datanode(1), tx) + .await; + let mut procedure = BatchGcProcedure::new( + mailbox.mailbox().clone(), + table_metadata_manager, + runtime_switch_manager, + "localhost".to_string(), + vec![RegionId::new(1024, 1)], + true, + Duration::from_secs(10), + HashMap::new(), + ); + procedure.data.state = state.clone(); + let dump_before = procedure.dump().unwrap(); + + let ctx = ProcedureContext { + procedure_id: common_procedure::ProcedureId::random(), + provider: Arc::new(MockContextProvider::default()), + event_context: None, + }; + let err = procedure.execute(&ctx).await.unwrap_err(); + + assert!(!err.is_retry_later()); + assert_eq!( + common_error::ext::ErrorExt::retry_hint(&err), + common_error::ext::RetryHint::NonRetryable + ); + assert_eq!(procedure.data.state, state); + assert_eq!(procedure.dump().unwrap(), dump_before); + if matches!(state, State::Acquiring | State::Gcing) { + assert!(rx.try_recv().is_err()); + } + } + } + + #[tokio::test] + async fn test_maintenance_mode_read_error_retries_without_advancing_state() { + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let failing_once_calls = calls.clone(); + let kv_backend = Arc::new( + MockKvBackendBuilder::default() + .range_fn(Arc::new(move |_| { + if failing_once_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 { + common_meta::error::UnexpectedSnafu { + err_msg: "maintenance read failed", + } + .fail() + } else { + Ok(common_meta::rpc::store::RangeResponse { + kvs: vec![], + more: false, + }) + } + })) + .build() + .unwrap(), + ); + let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone())); + let runtime_switch_manager = Arc::new(RuntimeSwitchManager::new(kv_backend.clone())); + let mailbox_sequence = + SequenceBuilder::new("test_batch_gc_maintenance_read_error", kv_backend).build(); + let mailbox = MailboxContext::new(mailbox_sequence); + let mut procedure = BatchGcProcedure::new( + mailbox.mailbox().clone(), + table_metadata_manager, + runtime_switch_manager, + "localhost".to_string(), + vec![RegionId::new(1024, 1)], + true, + Duration::from_secs(10), + HashMap::new(), + ); + let ctx = ProcedureContext { + procedure_id: common_procedure::ProcedureId::random(), + provider: Arc::new(MockContextProvider::default()), + event_context: None, + }; + let dump_before = procedure.dump().unwrap(); + + assert!(procedure.execute(&ctx).await.unwrap_err().is_retry_later()); + assert_eq!(procedure.data.state, State::Start); + assert_eq!(procedure.dump().unwrap(), dump_before); + + assert!(matches!( + procedure.execute(&ctx).await.unwrap(), + Status::Executing { .. } + )); + assert_eq!(procedure.data.state, State::Acquiring); + } } diff --git a/src/meta-srv/src/handler.rs b/src/meta-srv/src/handler.rs index 50b53f4970a..320f26fae7c 100644 --- a/src/meta-srv/src/handler.rs +++ b/src/meta-srv/src/handler.rs @@ -428,9 +428,11 @@ impl HeartbeatMailbox { .as_ref() .with_context(|| UnexpectedInstructionReplySnafu { mailbox_message: msg.to_string(), - reason: format!("empty payload, msg: {msg:?}"), + reason: "empty JSON payload".to_string(), })?; - serde_json::from_str(payload).context(DeserializeFromJsonSnafu { input: payload }) + serde_json::from_str(payload).context(DeserializeFromJsonSnafu { + input_len: payload.len(), + }) } /// Parses the [Instruction] from [MailboxMessage]. @@ -443,9 +445,11 @@ impl HeartbeatMailbox { .as_ref() .with_context(|| UnexpectedInstructionReplySnafu { mailbox_message: msg.to_string(), - reason: format!("empty payload, msg: {msg:?}"), + reason: "empty JSON payload".to_string(), })?; - serde_json::from_str(payload).context(DeserializeFromJsonSnafu { input: payload }) + serde_json::from_str(payload).context(DeserializeFromJsonSnafu { + input_len: payload.len(), + }) } pub fn create(pushers: Pushers, sequence: Sequence) -> MailboxRef { @@ -526,7 +530,12 @@ impl Mailbox for HeartbeatMailbox { msg.id = message_id; let pusher_id = ch.pusher_id(); - debug!("Sending mailbox message {msg:?} to {pusher_id}"); + let payload_len = msg + .payload + .as_ref() + .map(|Payload::Json(payload)| payload.len()) + .unwrap_or_default(); + debug!(message_id, payload_len, %pusher_id, "Sending mailbox message"); let (tx, rx) = oneshot::channel(); let _ = self.senders.insert(message_id, tx); @@ -548,7 +557,12 @@ impl Mailbox for HeartbeatMailbox { msg.id = message_id; let pusher_id = ch.pusher_id(); - debug!("Sending mailbox message {msg:?} to {pusher_id}"); + let payload_len = msg + .payload + .as_ref() + .map(|Payload::Json(payload)| payload.len()) + .unwrap_or_default(); + debug!(message_id, payload_len, %pusher_id, "Sending one-way mailbox message"); self.pushers.push(pusher_id, msg).await?; @@ -560,7 +574,18 @@ impl Mailbox for HeartbeatMailbox { } async fn on_recv(&self, id: MessageId, maybe_msg: Result) -> Result<()> { - debug!("Received mailbox message {maybe_msg:?}"); + let payload_len = maybe_msg + .as_ref() + .ok() + .and_then(|msg| msg.payload.as_ref()) + .map(|Payload::Json(payload)| payload.len()) + .unwrap_or_default(); + debug!( + message_id = id, + payload_len, + success = maybe_msg.is_ok(), + "Received mailbox message" + ); let _ = self.timeouts.remove(&id); @@ -568,7 +593,15 @@ impl Mailbox for HeartbeatMailbox { tx.send(maybe_msg) .map_err(|_| error::MailboxClosedSnafu { id }.build())?; } else if let Ok(finally_msg) = maybe_msg { - warn!("The response arrived too late: {finally_msg:?}"); + let payload_len = finally_msg + .payload + .as_ref() + .map(|Payload::Json(payload)| payload.len()) + .unwrap_or_default(); + warn!( + message_id = id, + payload_len, "The mailbox response arrived too late" + ); } Ok(()) diff --git a/src/meta-srv/src/key.rs b/src/meta-srv/src/key.rs index 803ee9f0c6e..abdc92ff6d4 100644 --- a/src/meta-srv/src/key.rs +++ b/src/meta-srv/src/key.rs @@ -87,7 +87,9 @@ impl FromStr for LeaseValue { type Err = error::Error; fn from_str(value: &str) -> crate::Result { - serde_json::from_str(value).context(error::DeserializeFromJsonSnafu { input: value }) + serde_json::from_str(value).context(error::DeserializeFromJsonSnafu { + input_len: value.len(), + }) } } diff --git a/src/meta-srv/src/metasrv/builder.rs b/src/meta-srv/src/metasrv/builder.rs index 75b72f9a114..b40f06e3cfe 100644 --- a/src/meta-srv/src/metasrv/builder.rs +++ b/src/meta-srv/src/metasrv/builder.rs @@ -530,6 +530,7 @@ impl MetasrvBuilder { let gc_scheduler_ctx = DefaultGcSchedulerCtx::try_new( table_metadata_manager.clone(), procedure_manager.clone(), + runtime_switch_manager.clone(), #[cfg(feature = "enterprise")] ddl_manager.clone(), meta_peer_client.clone(), diff --git a/src/meta-srv/src/procedure/region_migration/manager.rs b/src/meta-srv/src/procedure/region_migration/manager.rs index c21de506243..8a115cd9f8e 100644 --- a/src/meta-srv/src/procedure/region_migration/manager.rs +++ b/src/meta-srv/src/procedure/region_migration/manager.rs @@ -510,17 +510,14 @@ impl RegionMigrationManager { ProcedureWithId::with_random_id(Box::new(procedure)).with_context(procedure_context); let procedure_id = procedure_with_id.id; info!("Starting region migration procedure {procedure_id} for {task}"); - let procedure_manager = self.procedure_manager.clone(); + let mut watcher = self + .procedure_manager + .submit(procedure_with_id) + .await + .context(error::SubmitProcedureSnafu)?; let num_region = task.region_ids.len(); common_runtime::spawn_global(async move { - let watcher = &mut match procedure_manager.submit(procedure_with_id).await { - Ok(watcher) => watcher, - Err(e) => { - error!(e; "Failed to submit region migration procedure {procedure_id} for {task}"); - return; - } - }; METRIC_META_REGION_MIGRATION_DATANODES .with_label_values(&["src", &task.from_peer.id.to_string()]) .inc_by(num_region as u64); @@ -528,7 +525,7 @@ impl RegionMigrationManager { .with_label_values(&["desc", &task.to_peer.id.to_string()]) .inc_by(num_region as u64); - if let Err(e) = watcher::wait(watcher).await { + if let Err(e) = watcher::wait(&mut watcher).await { error!(e; "Failed to wait region migration procedure {procedure_id} for {task}"); METRIC_META_REGION_MIGRATION_FAIL.inc(); return; @@ -613,15 +610,12 @@ impl RegionMigrationManager { ProcedureWithId::with_random_id(Box::new(procedure)).with_context(procedure_context); let procedure_id = procedure_with_id.id; info!("Starting region migration procedure {procedure_id} for {task}"); - let procedure_manager = self.procedure_manager.clone(); + let mut watcher = self + .procedure_manager + .submit(procedure_with_id) + .await + .context(error::SubmitProcedureSnafu)?; common_runtime::spawn_global(async move { - let watcher = &mut match procedure_manager.submit(procedure_with_id).await { - Ok(watcher) => watcher, - Err(e) => { - error!(e; "Failed to submit region migration procedure {procedure_id} for {task}"); - return; - } - }; METRIC_META_REGION_MIGRATION_DATANODES .with_label_values(&["src", &task.from_peer.id.to_string()]) .inc(); @@ -629,7 +623,7 @@ impl RegionMigrationManager { .with_label_values(&["desc", &task.to_peer.id.to_string()]) .inc(); - if let Err(e) = watcher::wait(watcher).await { + if let Err(e) = watcher::wait(&mut watcher).await { error!(e; "Failed to wait region migration procedure {procedure_id} for {task}"); METRIC_META_REGION_MIGRATION_FAIL.inc(); return; @@ -649,10 +643,148 @@ mod test { use common_meta::key::table_route::LogicalTableRouteValue; use common_meta::key::test_utils::new_test_table_info; use common_meta::rpc::router::Region; + use common_meta::state_store::KvStateStore; + use common_procedure::ProcedureManager; + use common_procedure::local::{LocalManager, ManagerConfig, PauseAware}; + use common_procedure::test_util::InMemoryPoisonStore; + use tokio::sync::Notify; use super::*; use crate::procedure::region_migration::test_util::TestingEnv; + #[derive(Default)] + struct SubmissionGate { + entered: Notify, + release: Notify, + } + + #[async_trait::async_trait] + impl PauseAware for SubmissionGate { + async fn is_paused(&self) -> std::result::Result { + self.entered.notify_one(); + self.release.notified().await; + Ok(false) + } + } + + async fn check_submission_registration(batch: bool) { + let env = TestingEnv::new(); + let gate = Arc::new(SubmissionGate::default()); + let procedure_manager = Arc::new(LocalManager::new( + ManagerConfig::default(), + Arc::new(KvStateStore::new(env.kv_backend())), + Arc::new(InMemoryPoisonStore::default()), + Some(gate.clone()), + None, + )); + let manager = RegionMigrationManager::new(procedure_manager.clone(), env.context_factory()); + let region_ids = if batch { + vec![RegionId::new(1024, 1), RegionId::new(1024, 2)] + } else { + vec![RegionId::new(1024, 1)] + }; + let region_routes = region_ids + .iter() + .map(|region_id| RegionRoute { + region: Region::new_test(*region_id), + leader_peer: Some(Peer::empty(1)), + ..Default::default() + }) + .collect(); + env.create_physical_table_metadata(new_test_table_info(1024), region_routes) + .await; + + // A stopped manager must reject submission and release all migration guards. + // Starting it then allows the same regions to be submitted successfully. + for started in [false, true] { + if started { + procedure_manager.start().await.unwrap(); + } + let submission = async { + if batch { + let result = manager + .submit_region_migration_task(RegionMigrationTaskBatch { + region_ids: region_ids.clone(), + from_peer: Peer::empty(1), + to_peer: Peer::empty(2), + timeout: Duration::from_secs(10), + trigger_reason: RegionMigrationTriggerReason::Manual, + }) + .await?; + assert_eq!(result.submitted, region_ids); + Ok(result.procedure_id) + } else { + manager + .submit_procedure( + ProcedureContext::default(), + RegionMigrationProcedureTask::new( + region_ids[0], + Peer::empty(1), + Peer::empty(2), + Duration::from_secs(10), + RegionMigrationTriggerReason::Manual, + ), + ) + .await + } + }; + tokio::pin!(submission); + tokio::select! { + biased; + result = &mut submission => panic!("Submission returned before registration: {result:?}"), + _ = gate.entered.notified() => {} + } + assert!(futures::poll!(&mut submission).is_pending()); + assert!( + procedure_manager + .list_procedures() + .await + .unwrap() + .is_empty() + ); + for region_id in ®ion_ids { + assert!(manager.tracker.contains(*region_id)); + } + + gate.release.notify_one(); + let result = tokio::time::timeout(Duration::from_secs(5), &mut submission) + .await + .expect("Submission should return after registration"); + if started { + let procedure_id = result.unwrap().unwrap(); + assert!( + procedure_manager + .procedure_state(procedure_id) + .await + .unwrap() + .is_some() + ); + } else { + assert_matches!( + result.unwrap_err(), + error::Error::SubmitProcedure { + source: common_procedure::Error::ManagerNotStart { .. }, + .. + } + ); + for region_id in ®ion_ids { + assert!(!manager.tracker.contains(*region_id)); + } + } + } + procedure_manager.stop().await.unwrap(); + } + + #[tokio::test] + async fn test_submit_procedure_registration() { + check_submission_registration(false).await; + } + + #[tokio::test] + async fn test_submit_region_migration_task_registration() { + check_submission_registration(true).await; + } + #[tokio::test] async fn test_insert_running_procedure() { let env = TestingEnv::new(); @@ -1039,6 +1171,7 @@ mod test { #[tokio::test] async fn test_running_regions() { let env = TestingEnv::new(); + env.procedure_manager().start().await.unwrap(); let context_factory = env.context_factory(); let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory); let region_id = RegionId::new(1024, 1); @@ -1072,5 +1205,6 @@ mod test { assert_eq!(result.migrating, vec![region_id]); assert_eq!(result.submitted, vec![RegionId::new(1024, 2)]); assert!(result.procedure_id.is_some()); + env.procedure_manager().stop().await.unwrap(); } } diff --git a/src/meta-srv/src/procedure/repartition.rs b/src/meta-srv/src/procedure/repartition.rs index ee8b3ee749b..f42d8aacd0c 100644 --- a/src/meta-srv/src/procedure/repartition.rs +++ b/src/meta-srv/src/procedure/repartition.rs @@ -34,6 +34,7 @@ use common_meta::cache_invalidator::CacheInvalidatorRef; use common_meta::ddl::DdlContext; use common_meta::ddl::allocator::region_routes::RegionRoutesAllocatorRef; use common_meta::ddl::allocator::wal_options::WalOptionsAllocatorRef; +use common_meta::ddl::utils::get_region_wal_options; use common_meta::ddl_manager::{RepartitionProcedureFactory, RepartitionSource}; use common_meta::instruction::CacheIdent; use common_meta::key::datanode_table::RegionInfo; @@ -402,16 +403,27 @@ impl Context { let datanode_table_value = get_datanode_table_value(&self.table_metadata_manager, table_id, datanode_id).await?; - let RegionInfo { - region_options, - region_wal_options, - .. - } = &datanode_table_value.region_info; + let RegionInfo { region_options, .. } = &datanode_table_value.region_info; + + let mut region_wal_options = get_region_wal_options( + &self.table_metadata_manager, + current_table_route_value, + table_id, + ) + .await + .context(error::TableMetadataManagerSnafu)?; + // Legacy regions without a persisted WAL option use RaftEngine. Only + // existing routes get this default; allocated regions must supply one. + for route in current_table_route_value.region_routes().unwrap() { + region_wal_options + .entry(route.region.id.region_number()) + .or_default(); + } // Merge and validate the new region wal options. let validated_region_wal_options = crate::procedure::repartition::utils::merge_and_validate_region_wal_options( - region_wal_options, + ®ion_wal_options, new_region_wal_options, &new_region_routes, table_id, diff --git a/src/meta-srv/src/procedure/repartition/allocate_region.rs b/src/meta-srv/src/procedure/repartition/allocate_region.rs index 857ac946708..3b64d7fd7ce 100644 --- a/src/meta-srv/src/procedure/repartition/allocate_region.rs +++ b/src/meta-srv/src/procedure/repartition/allocate_region.rs @@ -19,6 +19,7 @@ use common_meta::ddl::create_table::executor::CreateTableExecutor; use common_meta::ddl::create_table::template::{ CreateRequestBuilder, build_template_from_raw_table_info_for_physical_table, }; +use common_meta::ddl::utils::get_region_wal_options; use common_meta::lock_key::TableLock; use common_meta::node_manager::NodeManagerRef; use common_meta::peer::PeerAllocContext; @@ -28,6 +29,7 @@ use common_meta::wal_provider::{ }; use common_procedure::{Context as ProcedureContext, Status}; use common_telemetry::{debug, info}; +use common_wal::options::WalOptions; use serde::{Deserialize, Deserializer, Serialize}; use snafu::{OptionExt, ResultExt}; use store_api::region_request::RegionRequirements; @@ -165,6 +167,29 @@ impl ExecutePlan { ) .await .context(error::AllocateRegionRoutesSnafu { table_id })?; + let skip_wal = table_info_value.table_info.meta.options.skip_wal; + let allocate_noop = if skip_wal { + let table_route_value = ctx.get_table_route_value().await?; + let region_wal_options = + get_region_wal_options(&ctx.table_metadata_manager, &table_route_value, table_id) + .await + .context(error::AllocateWalOptionsSnafu { table_id })?; + // A table disabled after creation keeps its real providers. A table + // created with skip_wal has Noop providers and must keep them. + !table_route_value + .region_routes() + .unwrap() + .iter() + .all(|route| { + region_wal_options + .get(&route.region.id.region_number()) + .is_none_or(|option| { + matches!(option, WalOptions::RaftEngine | WalOptions::Kafka(_)) + }) + }) + } else { + false + }; let mut wal_options = ctx .wal_options_allocator .allocate( @@ -172,7 +197,7 @@ impl ExecutePlan { .iter() .map(|r| r.region_id.region_number()) .collect::>(), - table_info_value.table_info.meta.options.skip_wal, + allocate_noop, ) .await .context(error::AllocateWalOptionsSnafu { table_id })?; @@ -414,14 +439,19 @@ mod tests { use std::sync::Arc; use api::v1::region::region_request::Body; + use common_meta::ddl::allocator::wal_options::WalOptionsAllocator; use common_meta::ddl::test_util::datanode_handler::DatanodeWatcher; use common_meta::key::TableMetadataManagerRef; use common_meta::key::datanode_table::DatanodeTableKey; + use common_meta::key::table_route::TableRouteValue; + use common_meta::key::test_utils::new_test_table_info_with_name; use common_meta::peer::Peer; use common_meta::rpc::router::{Region, RegionRoute}; use common_meta::test_util::MockDatanodeManager; use common_procedure::{ContextProvider, ProcedureId, ProcedureState}; use common_procedure_test::MockContextProvider; + use common_wal::options::{KafkaWalOptions, WAL_OPTIONS_KEY, WalOptions}; + use store_api::mito_engine_options::SKIP_WAL_KEY; use store_api::storage::RegionId; use tokio::sync::{mpsc, watch}; use uuid::Uuid; @@ -510,6 +540,29 @@ mod tests { region_wal_options: RegionWalOptions, } + struct TestKafkaWalOptionsAllocator; + + #[async_trait::async_trait] + impl WalOptionsAllocator for TestKafkaWalOptionsAllocator { + async fn allocate( + &self, + region_numbers: &[RegionNumber], + skip_wal: bool, + ) -> common_meta::error::Result { + Ok(region_numbers + .iter() + .map(|®ion_number| { + let options = if skip_wal { + WalOptions::Noop + } else { + WalOptions::Kafka(KafkaWalOptions::new("new-topic".to_string())) + }; + (region_number, options) + }) + .collect()) + } + } + #[async_trait::async_trait] impl ContextProvider for ConcurrentTableRouteUpdateProvider { async fn procedure_state( @@ -804,6 +857,101 @@ mod tests { ); } + async fn check_execute_plan_skip_wal_provider(existing_wal_options: Option) { + let env = TestingEnv::new(); + let table_id = 1024; + let routes = create_current_region_routes(table_id, &[1]); + let mut table_info = new_test_table_info_with_name(table_id, "test_table"); + table_info.meta.column_ids = vec![0, 1, 2]; + table_info.meta.options.skip_wal = true; + let stored_wal_options = existing_wal_options + .clone() + .map(|options| HashMap::from([(1, options)])) + .unwrap_or_default(); + env.table_metadata_manager + .create_table_metadata( + table_info, + TableRouteValue::physical(routes), + stored_wal_options, + ) + .await + .unwrap(); + + let (sender, mut receiver) = mpsc::channel(1); + let node_manager = Arc::new(MockDatanodeManager::new(DatanodeWatcher::new(sender))); + let mut ctx = new_parent_context(&env, node_manager, table_id); + if matches!(&existing_wal_options, Some(WalOptions::Kafka(_))) { + ctx.wal_options_allocator = Arc::new(TestKafkaWalOptionsAllocator); + } + ctx.persistent_ctx.plans = vec![RepartitionPlanEntry { + group_id: Uuid::new_v4(), + source_regions: vec![], + target_regions: vec![create_target_region_descriptor(table_id, 2, "x", 0, 100)], + allocated_region_ids: vec![RegionId::new(table_id, 2)], + pending_deallocate_region_ids: vec![], + transition_map: vec![], + original_target_routes: vec![], + }]; + let mut state = ExecutePlan; + state + .next(&mut ctx, &TestingEnv::procedure_context()) + .await + .unwrap(); + + let (_, request) = receiver.recv().await.unwrap(); + let Some(Body::Create(create)) = request.body else { + unreachable!() + }; + assert_eq!( + Some("true"), + create.options.get(SKIP_WAL_KEY).map(String::as_str) + ); + let new_wal_options: WalOptions = + serde_json::from_str(create.options.get(WAL_OPTIONS_KEY).unwrap()).unwrap(); + let expected = match &existing_wal_options { + Some(WalOptions::Noop) => WalOptions::Noop, + Some(WalOptions::Kafka(_)) => WalOptions::Kafka(KafkaWalOptions { + topic: "new-topic".to_string(), + initial_pruned_entry_id: Some(0), + }), + None | Some(WalOptions::RaftEngine) => WalOptions::RaftEngine, + }; + assert_eq!(expected, new_wal_options); + let route = ctx.get_table_route_value().await.unwrap(); + let wal_options = get_region_wal_options(&ctx.table_metadata_manager, &route, table_id) + .await + .unwrap(); + assert_eq!(Some(&expected), wal_options.get(&2)); + let legacy_default = WalOptions::RaftEngine; + assert_eq!( + Some(existing_wal_options.as_ref().unwrap_or(&legacy_default)), + wal_options.get(&1) + ); + } + + #[tokio::test] + async fn test_execute_plan_keeps_real_wal_provider_for_skipped_table() { + check_execute_plan_skip_wal_provider(Some(WalOptions::RaftEngine)).await; + } + + #[tokio::test] + async fn test_execute_plan_keeps_legacy_raft_provider_for_skipped_table() { + check_execute_plan_skip_wal_provider(None).await; + } + + #[tokio::test] + async fn test_execute_plan_keeps_kafka_wal_provider_for_skipped_table() { + check_execute_plan_skip_wal_provider(Some(WalOptions::Kafka(KafkaWalOptions::new( + "existing-topic".to_string(), + )))) + .await; + } + + #[tokio::test] + async fn test_execute_plan_keeps_noop_for_table_created_without_wal() { + check_execute_plan_skip_wal_provider(Some(WalOptions::Noop)).await; + } + #[test] fn test_allocate_region_state_backward_compatibility() { // Arrange diff --git a/src/meta-srv/src/service/heartbeat.rs b/src/meta-srv/src/service/heartbeat.rs index 066d1560477..5cdc5b57894 100644 --- a/src/meta-srv/src/service/heartbeat.rs +++ b/src/meta-srv/src/service/heartbeat.rs @@ -206,7 +206,13 @@ where /// Handles the incoming heartbeat request, and returns whether to continue the session. async fn handle_request(&mut self, req: HeartbeatRequest, is_handshake: bool) -> bool { - debug!("Receiving heartbeat request: {:?}", req); + let mailbox_message_id = req.mailbox_message.as_ref().map(|msg| msg.id); + debug!( + sender_id = %self.sender_id, + ?mailbox_message_id, + is_handshake, + "Receiving heartbeat request" + ); let sender_id = self.sender_id.to_string(); METRIC_META_HEARTBEAT_RECV @@ -224,7 +230,12 @@ where let is_not_leader = res.as_ref().is_ok_and(|r| r.is_not_leader()); - debug!("Sending heartbeat response: {:?}", res); + debug!( + sender_id = %self.sender_id, + success = res.is_ok(), + is_not_leader, + "Sending heartbeat response" + ); if self.tx.send(res).await.is_err() { info!( diff --git a/src/metric-engine/src/engine/alter.rs b/src/metric-engine/src/engine/alter.rs index 8bc9ae00b7b..355e84cb322 100644 --- a/src/metric-engine/src/engine/alter.rs +++ b/src/metric-engine/src/engine/alter.rs @@ -284,6 +284,22 @@ mod test { use crate::test_util::{TestEnv, alter_logical_region_request, create_logical_region_request}; + async fn check_alter_physical_skip_wal( + engine: &super::MetricEngineInner, + region_id: RegionId, + skip_wal: bool, + ) { + let request = RegionAlterRequest { + kind: AlterKind::SetRegionOptions { + options: vec![SetRegionOption::SkipWal(skip_wal)], + }, + }; + engine + .alter_physical_region(region_id, request) + .await + .unwrap(); + } + #[tokio::test] async fn test_alter_region() { let env = TestEnv::new().await; @@ -304,16 +320,9 @@ mod test { "Alter request to physical region is forbidden".to_string() ); - // skip WAL on the physical region should be forwarded to the data region - let alter_region_option_request = RegionAlterRequest { - kind: AlterKind::SetRegionOptions { - options: vec![SetRegionOption::SkipWal], - }, - }; - engine_inner - .alter_physical_region(physical_region_id, alter_region_option_request) - .await - .unwrap(); + // skip-WAL changes on the physical region should be forwarded to the data region + check_alter_physical_skip_wal(&engine_inner, physical_region_id, true).await; + check_alter_physical_skip_wal(&engine_inner, physical_region_id, false).await; // alter logical region let metadata_region = env.metadata_region(); diff --git a/src/metric-engine/src/engine/bulk_insert.rs b/src/metric-engine/src/engine/bulk_insert.rs index 8472815b145..cb43a71406d 100644 --- a/src/metric-engine/src/engine/bulk_insert.rs +++ b/src/metric-engine/src/engine/bulk_insert.rs @@ -124,6 +124,7 @@ impl MetricEngineInner { let aligned_schema_version = Some(self.physical_schema_version(data_region_id).await?); let request = RegionBulkInsertsRequest { + skip_wal: request.skip_wal, region_id: data_region_id, payload: modified_batch, raw_data: ArrowIpc { @@ -241,18 +242,28 @@ mod tests { use common_error::ext::ErrorExt; use common_query::prelude::{greptime_timestamp, greptime_value}; use common_recordbatch::RecordBatches; + use datafusion::parquet::basic::Encoding; use datatypes::arrow::array::{Float64Array, StringArray, TimestampMillisecondArray}; use datatypes::arrow::datatypes::{DataType, Field, Schema as ArrowSchema, TimeUnit}; use datatypes::arrow::record_batch::RecordBatch; use mito2::config::MitoConfig; - use store_api::metric_engine_consts::PRIMARY_KEY_ENCODING; + use mito2::sst::parquet::metadata::MetadataLoader; + use mito2::sst::parquet::reader::MetadataCacheMetrics; + use store_api::metric_engine_consts::{ + METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY, PRIMARY_KEY_ENCODING, + }; + use store_api::mito_engine_options::EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING; use store_api::path_utils::table_dir; use store_api::region_engine::RegionEngine; - use store_api::region_request::{RegionBulkInsertsRequest, RegionPutRequest, RegionRequest}; + use store_api::region_request::{ + PathType, RegionBulkInsertsRequest, RegionCloseRequest, RegionFlushRequest, + RegionOpenRequest, RegionPutRequest, RegionRequest, + }; use store_api::storage::{RegionId, ScanRequest}; use super::record_batch_to_ipc; use crate::batch_modifier::{TagColumnInfo, modify_batch_sparse}; + use crate::engine::MetricEngine; use crate::error::Error; use crate::test_util::{self, TestEnv}; @@ -287,9 +298,14 @@ mod tests { .unwrap() } - fn build_bulk_request(logical_region_id: RegionId, batch: RecordBatch) -> RegionRequest { + fn build_bulk_request( + logical_region_id: RegionId, + batch: RecordBatch, + skip_wal: bool, + ) -> RegionRequest { let (schema, data_header, payload) = record_batch_to_ipc(&batch).unwrap(); RegionRequest::BulkInserts(RegionBulkInsertsRequest { + skip_wal, region_id: logical_region_id, payload: batch, raw_data: ArrowIpc { @@ -332,6 +348,7 @@ mod tests { let batch = build_logical_batch(0, 0); let request = RegionRequest::BulkInserts(RegionBulkInsertsRequest { + skip_wal: false, region_id: logical_region_id, payload: batch, raw_data: ArrowIpc::default(), @@ -348,6 +365,11 @@ mod tests { #[tokio::test] async fn test_bulk_insert_physical_region_passthrough() { + check_bulk_insert_physical_region_passthrough(false).await; + check_bulk_insert_physical_region_passthrough(true).await; + } + + async fn check_bulk_insert_physical_region_passthrough(skip_wal: bool) { // Use flat format so that BulkMemtable is used (supports write_bulk). let mito_config = MitoConfig { default_flat_format: true, @@ -355,12 +377,13 @@ mod tests { }; let env = TestEnv::with_mito_config("", mito_config, Default::default()).await; env.init_metric_region().await; + env.metric().inner.flush_task.stop().await.unwrap(); let physical_region_id = env.default_physical_region_id(); let logical_region_id = env.default_logical_region_id(); // First, do a normal logical bulk insert so we can compare results. let logical_batch = build_logical_batch(0, 3); - let logical_request = build_bulk_request(logical_region_id, logical_batch.clone()); + let logical_request = build_bulk_request(logical_region_id, logical_batch, skip_wal); let response = env .metric() .handle_request(logical_region_id, logical_request) @@ -385,7 +408,7 @@ mod tests { &non_tag_indices, ) .unwrap(); - let request = build_bulk_request(physical_region_id, physical_batch); + let request = build_bulk_request(physical_region_id, physical_batch, skip_wal); let response = env .metric() .handle_request(physical_region_id, request) @@ -401,6 +424,48 @@ mod tests { .unwrap(); let batches = RecordBatches::try_collect(stream).await.unwrap(); assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 6); + + // Closing without a flush must recover only WAL-backed data. Recreate + // the wrapper too, so its metadata cache cannot hide missing metadata. + let stat = env.mito().region_statistic(physical_region_id).unwrap(); + assert_eq!(stat.sst_num, 0); + env.metric() + .handle_request( + physical_region_id, + RegionRequest::Close(RegionCloseRequest { + flush_on_close: false, + }), + ) + .await + .unwrap(); + let reopened = MetricEngine::try_new(env.mito(), Default::default()).unwrap(); + reopened.inner.flush_task.stop().await.unwrap(); + reopened + .handle_request( + physical_region_id, + RegionRequest::Open(RegionOpenRequest { + engine: METRIC_ENGINE_NAME.to_string(), + table_dir: TestEnv::default_table_dir(), + path_type: PathType::Bare, + options: [(PHYSICAL_TABLE_METADATA_KEY.to_string(), String::new())] + .into_iter() + .collect(), + skip_wal_replay: false, + checkpoint: None, + requirements: Default::default(), + }), + ) + .await + .unwrap(); + let stream = reopened + .scan_to_stream(logical_region_id, ScanRequest::default()) + .await + .unwrap(); + let batches = RecordBatches::try_collect(stream).await.unwrap(); + assert_eq!( + batches.iter().map(|b| b.num_rows()).sum::(), + if skip_wal { 0 } else { 6 }, + ); } #[tokio::test] @@ -415,7 +480,7 @@ mod tests { let physical_region_id = env.default_physical_region_id(); let batch = build_logical_batch(0, 0); - let request = build_bulk_request(physical_region_id, batch); + let request = build_bulk_request(physical_region_id, batch, false); let response = env .metric() .handle_request(physical_region_id, request) @@ -449,7 +514,7 @@ mod tests { ) .unwrap(); - let request = build_bulk_request(logical_region_id, batch); + let request = build_bulk_request(logical_region_id, batch, false); let err = env .metric() .handle_request(logical_region_id, request) @@ -499,7 +564,7 @@ mod tests { ) .unwrap(); - let request = build_bulk_request(logical_region_id, batch); + let request = build_bulk_request(logical_region_id, batch, false); let response = env .metric() .handle_request(logical_region_id, request) @@ -522,7 +587,7 @@ mod tests { env.init_metric_region().await; let logical_region_id = env.default_logical_region_id(); - let request = build_bulk_request(logical_region_id, build_logical_batch(0, 3)); + let request = build_bulk_request(logical_region_id, build_logical_batch(0, 3), false); let response = env .metric() .handle_request(logical_region_id, request) @@ -530,7 +595,7 @@ mod tests { .unwrap(); assert_eq!(response.affected_rows, 3); - let request = build_bulk_request(logical_region_id, build_logical_batch(3, 5)); + let request = build_bulk_request(logical_region_id, build_logical_batch(3, 5), false); let response = env .metric() .handle_request(logical_region_id, request) @@ -550,16 +615,68 @@ mod tests { #[tokio::test] async fn test_bulk_insert_sparse_encoding() { let env = TestEnv::new().await; - env.init_metric_region().await; + let physical_region_id = env.default_physical_region_id(); + env.create_physical_region( + physical_region_id, + &TestEnv::default_table_dir(), + vec![( + EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING.to_string(), + "byte_stream_split".to_string(), + )], + ) + .await; let logical_region_id = env.default_logical_region_id(); + env.create_logical_region(physical_region_id, logical_region_id) + .await; - let request = build_bulk_request(logical_region_id, build_logical_batch(0, 4)); + let rows = 4; + let request = build_bulk_request(logical_region_id, build_logical_batch(0, rows), false); let response = env .metric() .handle_request(logical_region_id, request) .await .unwrap(); - assert_eq!(response.affected_rows, 4); + assert_eq!(response.affected_rows, rows); + + let data_region_id = crate::utils::to_data_region_id(physical_region_id); + env.mito() + .handle_request( + data_region_id, + RegionRequest::Flush(RegionFlushRequest::default()), + ) + .await + .unwrap(); + + let region = env.mito().find_region(data_region_id).unwrap(); + let entry = region + .manifest_sst_entries() + .await + .into_iter() + .find(|entry| entry.visible && entry.file_path.ends_with(".parquet")) + .unwrap(); + let mut cache_metrics = MetadataCacheMetrics::default(); + let footer = MetadataLoader::new( + region.access_layer().object_store().clone(), + &entry.file_path, + entry.file_size, + ) + .load(&mut cache_metrics) + .await + .unwrap(); + let field_column_index = footer + .file_metadata() + .schema_descr() + .columns() + .iter() + .position(|column| column.name() == greptime_value()) + .unwrap(); + assert!(!footer.row_groups().is_empty()); + assert!(footer.row_groups().iter().all(|row_group| { + row_group + .column(field_column_index) + .encodings() + .any(|encoding| encoding == Encoding::BYTE_STREAM_SPLIT) + })); let stream = env .metric() @@ -567,7 +684,7 @@ mod tests { .await .unwrap(); let batches = RecordBatches::try_collect(stream).await.unwrap(); - assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 4); + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), rows); } #[tokio::test] @@ -575,7 +692,7 @@ mod tests { let env = TestEnv::new().await; let logical_region_id = init_dense_metric_region(&env).await; - let request = build_bulk_request(logical_region_id, build_logical_batch(0, 2)); + let request = build_bulk_request(logical_region_id, build_logical_batch(0, 2), false); let err = env .metric() .handle_request(logical_region_id, request) @@ -617,7 +734,7 @@ mod tests { let env_bulk = TestEnv::new().await; env_bulk.init_metric_region().await; - let request = build_bulk_request(logical_region_id, build_logical_batch(0, 5)); + let request = build_bulk_request(logical_region_id, build_logical_batch(0, 5), false); env_bulk .metric() .handle_request(logical_region_id, request) diff --git a/src/metric-engine/src/engine/put.rs b/src/metric-engine/src/engine/put.rs index 8858251d082..56d4af2284f 100644 --- a/src/metric-engine/src/engine/put.rs +++ b/src/metric-engine/src/engine/put.rs @@ -576,7 +576,7 @@ impl MetricEngineInner { region_id: data_region_id, })?; ( - physical_state.physical_columns().clone(), + physical_state.physical_columns_snapshot(), physical_state.time_index_column_name().to_string(), ) }; diff --git a/src/metric-engine/src/engine/state.rs b/src/metric-engine/src/engine/state.rs index 06d5f19c4bd..e0f407b919b 100644 --- a/src/metric-engine/src/engine/state.rs +++ b/src/metric-engine/src/engine/state.rs @@ -15,6 +15,7 @@ //! Internal states of metric engine use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use api::v1::SemanticType; use common_time::timestamp::TimeUnit; @@ -30,7 +31,10 @@ use crate::utils::to_data_region_id; pub struct PhysicalRegionState { logical_regions: HashSet, - physical_columns: HashMap, + /// Columns of the physical region, wrapped in an [`Arc`] so that hot read + /// paths (e.g. write request verification) can hold a cheap snapshot + /// instead of deep-cloning the whole map on every row batch. + physical_columns: Arc>, /// Name of the time index column, cached at region load so that the write /// path doesn't have to scan `physical_columns` for the timestamp on every /// row batch. The time index is fixed at region creation and never @@ -58,7 +62,7 @@ impl PhysicalRegionState { .unwrap_or_default(); Self { logical_regions: HashSet::new(), - physical_columns, + physical_columns: Arc::new(physical_columns), time_index_column_name, primary_key_encoding, options, @@ -76,6 +80,12 @@ impl PhysicalRegionState { &self.physical_columns } + /// Returns a cheap snapshot of the physical columns that stays valid + /// after releasing the state lock. + pub fn physical_columns_snapshot(&self) -> Arc> { + self.physical_columns.clone() + } + /// Returns the cached name of the time index column. pub fn time_index_column_name(&self) -> &str { &self.time_index_column_name @@ -144,7 +154,7 @@ impl MetricEngineState { SemanticType::Timestamp, "unexpected time index column {col} added to an existing physical region" ); - state.physical_columns.insert(col, meta); + Arc::make_mut(&mut state.physical_columns).insert(col, meta); } } diff --git a/src/mito-codec/src/index.rs b/src/mito-codec/src/index.rs index 539a3adeddb..e0f2c819afe 100644 --- a/src/mito-codec/src/index.rs +++ b/src/mito-codec/src/index.rs @@ -24,13 +24,56 @@ use store_api::codec::PrimaryKeyEncoding; use store_api::metadata::ColumnMetadata; use store_api::storage::ColumnId; -use crate::error::{FieldTypeMismatchSnafu, IndexEncodeNullSnafu, Result}; +use crate::error::{ + FieldTypeMismatchSnafu, IndexEncodeNullSnafu, InvalidSparsePrimaryKeySnafu, Result, +}; +use crate::row_converter::sparse::{ + RESERVED_COLUMN_ID_TABLE_ID, RESERVED_COLUMN_ID_TSID, SparsePrimaryKeyView, +}; use crate::row_converter::{PrimaryKeyCodec, SortField, build_primary_key_codec_with_fields}; /// Encodes index values according to their data types for sorting and storage use. pub struct IndexValueCodec; impl IndexValueCodec { + /// Extracts one sparse PK column in index format, without constructing a Value. + /// Numeric reserved fields borrow the PK; strings are unchunked into the reusable buffer. + /// Missing and null labels return None, whereas an empty string returns Some(&[]). + pub fn encode_sparse_value<'a, 'pk: 'a>( + pk: &mut SparsePrimaryKeyView<'pk, '_>, + column_id: ColumnId, + buffer: &'a mut Vec, + ) -> Result> { + let Some(encoded) = pk.encoded_value(column_id)? else { + return Ok(None); + }; + if encoded[0] == 0 { + return Ok(None); + } + if matches!( + column_id, + RESERVED_COLUMN_ID_TABLE_ID | RESERVED_COLUMN_ID_TSID + ) { + return Ok(Some(encoded)); + } + + buffer.clear(); + // The view has checked the Option marker, bytes marker and every chunk length. + // Reserving once avoids growing the buffer for each 8-byte chunk. + buffer.reserve(encoded.len() - 2); + for chunk in encoded[2..].chunks_exact(9) { + let len = usize::from(chunk[8]).min(8); + buffer.extend_from_slice(&chunk[..len]); + } + std::str::from_utf8(buffer).map_err(|_| { + InvalidSparsePrimaryKeySnafu { + reason: "label is not valid UTF-8", + } + .build() + })?; + Ok(Some(buffer.as_slice())) + } + /// Serializes a non-null `ValueRef` using the data type defined in `SortField` and writes /// the result into a buffer. /// diff --git a/src/mito-codec/src/row_converter/sparse.rs b/src/mito-codec/src/row_converter/sparse.rs index c70293886f8..b2b3e0e3778 100644 --- a/src/mito-codec/src/row_converter/sparse.rs +++ b/src/mito-codec/src/row_converter/sparse.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod checked; + use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -34,6 +36,7 @@ use crate::error::{ use crate::key_values::KeyValue; use crate::primary_key_filter::SparsePrimaryKeyFilter; use crate::row_converter::dense::SortField; +pub use crate::row_converter::sparse::checked::SparsePrimaryKeyView; use crate::row_converter::{CompositeValues, PrimaryKeyCodec, PrimaryKeyFilter}; /// A codec for sparse key of metrics. diff --git a/src/mito-codec/src/row_converter/sparse/checked.rs b/src/mito-codec/src/row_converter/sparse/checked.rs new file mode 100644 index 00000000000..63666f91c4c --- /dev/null +++ b/src/mito-codec/src/row_converter/sparse/checked.rs @@ -0,0 +1,244 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use snafu::{OptionExt, ensure}; +use store_api::storage::ColumnId; + +use crate::error::{InvalidSparsePrimaryKeySnafu, Result}; +use crate::row_converter::sparse::{ + COLUMN_ID_ENCODE_SIZE, RESERVED_COLUMN_ID_TABLE_ID, RESERVED_COLUMN_ID_TSID, + SparseOffsetsCache, TABLE_ID_VALUE_OFFSET, TAGS_START_OFFSET, TSID_VALUE_OFFSET, +}; + +/// A borrowing view of one sparse primary key, with checked, lazy field lookup. +/// +/// The scratch cache is reset on construction and retains its capacity between keys. +/// Ordinary labels can be skipped without schema lookup: their encoding is always String. +pub struct SparsePrimaryKeyView<'a, 'b> { + pk: &'a [u8], + cache: &'b mut SparseOffsetsCache, +} + +impl<'a, 'b> SparsePrimaryKeyView<'a, 'b> { + /// Validates the fixed prefix and binds fresh offset discovery to this key. + pub fn new(pk: &'a [u8], cache: &'b mut SparseOffsetsCache) -> Result { + ensure!( + pk.len() >= TAGS_START_OFFSET + && pk[..COLUMN_ID_ENCODE_SIZE] == RESERVED_COLUMN_ID_TABLE_ID.to_be_bytes() + && pk[TSID_VALUE_OFFSET - COLUMN_ID_ENCODE_SIZE..TSID_VALUE_OFFSET] + == RESERVED_COLUMN_ID_TSID.to_be_bytes() + && pk[TABLE_ID_VALUE_OFFSET] == 1 + && pk[TSID_VALUE_OFFSET] == 1, + InvalidSparsePrimaryKeySnafu { + reason: "invalid table_id/tsid prefix" + } + ); + cache.clear(); + Ok(Self { pk, cache }) + } + + /// Returns the encoded value, including its null marker, or None for an absent label. + /// The result borrows the key, not the scratch cache. + pub fn encoded_value(&mut self, column_id: ColumnId) -> Result> { + match column_id { + RESERVED_COLUMN_ID_TABLE_ID => { + return Ok(Some( + &self.pk[TABLE_ID_VALUE_OFFSET..TSID_VALUE_OFFSET - COLUMN_ID_ENCODE_SIZE], + )); + } + RESERVED_COLUMN_ID_TSID => { + return Ok(Some(&self.pk[TSID_VALUE_OFFSET..TAGS_START_OFFSET])); + } + _ => {} + } + if let Some(offset) = self.cache.get(column_id) { + return encoded_label(&self.pk[offset..]).map(Some); + } + while !self.cache.finished && self.cache.cursor < self.pk.len() { + let bytes = &self.pk[self.cache.cursor..]; + let id = bytes.first_chunk::().context( + InvalidSparsePrimaryKeySnafu { + reason: "truncated label column id", + }, + )?; + let id = u32::from_be_bytes(*id); + let offset = self.cache.cursor + COLUMN_ID_ENCODE_SIZE; + let value = encoded_label(&self.pk[offset..])?; + self.cache.insert(id, offset); + self.cache.cursor = offset + value.len(); + if id == column_id { + return Ok(Some(value)); + } + } + self.cache.finished = true; + Ok(None) + } +} + +/// Finds the end of an Option without allocating or reading its payload. +/// Unlike memcomparable's skip_bytes, all advances are checked for truncated input. +fn encoded_label(bytes: &[u8]) -> Result<&[u8]> { + match bytes.first() { + Some(0) => return Ok(&bytes[..1]), + Some(1) => {} + _ => { + return InvalidSparsePrimaryKeySnafu { + reason: "invalid label null marker", + } + .fail(); + } + } + match bytes.get(1) { + Some(0) => return Ok(&bytes[..2]), + Some(1) => {} + _ => { + return InvalidSparsePrimaryKeySnafu { + reason: "invalid label bytes marker", + } + .fail(); + } + } + let mut end = 2; + loop { + let chunk = bytes + .get(end..end + 9) + .context(InvalidSparsePrimaryKeySnafu { + reason: "truncated label chunk", + })?; + end += 9; + match chunk[8] { + 1..=8 => return Ok(&bytes[..end]), + 9 => {} + _ => { + return InvalidSparsePrimaryKeySnafu { + reason: "invalid label chunk length", + } + .fail(); + } + } + } +} + +#[cfg(test)] +mod tests { + use datatypes::data_type::ConcreteDataType; + use datatypes::value::ValueRef; + + use super::*; + use crate::index::IndexValueCodec; + use crate::row_converter::{PrimaryKeyCodec, SortField, SparsePrimaryKeyCodec}; + + #[test] + fn index_bytes_match_full_decode_across_chunks_and_lookup_orders() { + let codec = SparsePrimaryKeyCodec::schemaless(); + let mut cache = SparseOffsetsCache::new(); + let mut buffer = Vec::new(); + // Reusing the cache across different keys must not reuse their field offsets. + for len in [0, 1, 7, 8, 9, 16, 24, 65] { + let label = "x".repeat(len); + let values = [ + (RESERVED_COLUMN_ID_TABLE_ID, ValueRef::UInt32(42)), + (RESERVED_COLUMN_ID_TSID, ValueRef::UInt64(u64::MAX)), + (90, ValueRef::String(&label)), + (7, ValueRef::String("")), + (1, ValueRef::Null), + (3, ValueRef::String("last中文")), + ]; + let mut pk = Vec::new(); + codec.encode_value_refs(&values, &mut pk).unwrap(); + // Exercise the offset cache's overflow storage as well as its inline entries. + codec + .encode_raw_tag_value((100..140).map(|id| (id, label.as_bytes())), &mut pk) + .unwrap(); + // Writers normally omit null labels; existing encoded nulls must remain null too. + pk.extend_from_slice(&2_u32.to_be_bytes()); + pk.push(0); + let decoded = codec.decode(&pk).unwrap().into_sparse(); + for order in [ + [ + RESERVED_COLUMN_ID_TABLE_ID, + RESERVED_COLUMN_ID_TSID, + 90, + 7, + 1, + 3, + ], + [ + 3, + 1, + 7, + 90, + RESERVED_COLUMN_ID_TSID, + RESERVED_COLUMN_ID_TABLE_ID, + ], + ] { + let mut view = SparsePrimaryKeyView::new(&pk, &mut cache).unwrap(); + for id in order.into_iter().chain([139, 100, 132, 2]) { + let expected = decoded.get(&id).filter(|v| !v.is_null()).map(|value| { + let field = SortField::new(match id { + RESERVED_COLUMN_ID_TABLE_ID => ConcreteDataType::uint32_datatype(), + RESERVED_COLUMN_ID_TSID => ConcreteDataType::uint64_datatype(), + _ => ConcreteDataType::string_datatype(), + }); + let mut bytes = Vec::new(); + IndexValueCodec::encode_nonnull_value( + value.as_value_ref(), + &field, + &mut bytes, + ) + .unwrap(); + bytes + }); + let actual = + IndexValueCodec::encode_sparse_value(&mut view, id, &mut buffer).unwrap(); + assert_eq!(actual, expected.as_deref(), "len={len}, column={id}"); + } + assert!(view.encoded_value(999).unwrap().is_none()); + } + } + } + + #[test] + fn malformed_sparse_values_return_errors() { + let codec = SparsePrimaryKeyCodec::schemaless(); + let mut pk = Vec::new(); + codec.encode_internal(1, 2, &mut pk).unwrap(); + codec + .encode_raw_tag_value([(1, b"0123456789".as_slice())].into_iter(), &mut pk) + .unwrap(); + let mut cache = SparseOffsetsCache::new(); + for end in 0..pk.len() { + // A complete reserved prefix is a valid key with no labels. + if end == TAGS_START_OFFSET { + continue; + } + let result = SparsePrimaryKeyView::new(&pk[..end], &mut cache) + .and_then(|mut view| view.encoded_value(1)); + assert!(result.is_err(), "truncation at {end}"); + } + for (offset, value) in [(0, 0), (4, 0), (13, 2), (26, 2), (27, 2), (36, 0), (45, 10)] { + let mut invalid = pk.clone(); + invalid[offset] = value; + assert!( + SparsePrimaryKeyView::new(&invalid, &mut cache) + .and_then(|mut view| view.encoded_value(1)) + .is_err() + ); + } + let mut invalid_utf8 = pk; + invalid_utf8[28] = 0xff; + let mut view = SparsePrimaryKeyView::new(&invalid_utf8, &mut cache).unwrap(); + assert!(IndexValueCodec::encode_sparse_value(&mut view, 1, &mut Vec::new()).is_err()); + } +} diff --git a/src/mito2/AGENTS.md b/src/mito2/AGENTS.md index faa581331b4..583ec31bd4a 100644 --- a/src/mito2/AGENTS.md +++ b/src/mito2/AGENTS.md @@ -26,7 +26,7 @@ snapshot isolation). It implements the `RegionEngine` trait from `store-api`. | `compaction` | `src/mito2/src/compaction/` | Compaction scheduler (`scheduler.rs` + `scheduler/`), TWCS picker, strict-window manual picker, compactor, memory control | | `access_layer` | `src/mito2/src/access_layer.rs` | SST read/write over the object store | | `sst` | `src/mito2/src/sst/` | Parquet format, file metadata, index layout | -| `series_index` | `src/mito2/src/series_index/` | Series index writer/searcher, bucket planning and SST builders, catalogs, immutable snapshots, file lifecycle, and background maintenance | +| `series_index` | `src/mito2/src/series_index/` | Series index writer/searcher, bucket planning, catalogs, worker-owned maintenance and purge tasks, and index lifecycle | | `read` | `src/mito2/src/read/` | `ScanRegion`, merge, dedup, projection, streaming | | `manifest` | `src/mito2/src/manifest/` | `RegionManifestManager`, manifest actions/edits | | `cache` | `src/mito2/src/cache.rs` | Write/file/page caches | diff --git a/src/mito2/Cargo.toml b/src/mito2/Cargo.toml index 100a6fb1023..ccc3fd6f176 100644 --- a/src/mito2/Cargo.toml +++ b/src/mito2/Cargo.toml @@ -145,5 +145,10 @@ required-features = ["test"] name = "bench_wal_encode" harness = false +[[bench]] +name = "bench_index_update" +harness = false +required-features = ["testing"] + [package.metadata.cargo-udeps.ignore] normal = ["aquamarine"] diff --git a/src/mito2/benches/bench_index_update.rs b/src/mito2/benches/bench_index_update.rs new file mode 100644 index 00000000000..eba89cf60ff --- /dev/null +++ b/src/mito2/benches/bench_index_update.rs @@ -0,0 +1,276 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Measures real index updates, excluding input generation, creator setup and cleanup. +//! +//! Run on the baseline revision with `--save-baseline before`, then on the candidate +//! with `--baseline before` (arguments after `--` in the command below): +//! `CARGO_PROFILE_BENCH_DEBUG=0 cargo bench -p mito2 --features testing --bench bench_index_update`. + +use std::collections::HashSet; +use std::num::NonZeroUsize; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use api::v1::SemanticType; +use common_test_util::temp_dir::create_temp_dir; +use criterion::{Criterion, Throughput, criterion_group, criterion_main}; +use datatypes::arrow::array::{ + ArrayRef, BinaryDictionaryBuilder, TimestampMillisecondArray, UInt8Array, UInt64Array, +}; +use datatypes::arrow::datatypes::UInt32Type; +use datatypes::arrow::record_batch::RecordBatch; +use datatypes::data_type::ConcreteDataType; +use datatypes::schema::{ColumnSchema, SkippingIndexOptions}; +use mito_codec::row_converter::SparsePrimaryKeyCodec; +use mito2::sst::index::intermediate::IntermediateManager; +use mito2::sst::{FlatSchemaOptions, to_flat_sst_arrow_schema}; +use mito2::test_util::bench_util::{BloomFilterIndexer, InvertedIndexer}; +use store_api::codec::PrimaryKeyEncoding; +use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder, RegionMetadataRef}; +use store_api::storage::consts::ReservedColumnId; +use store_api::storage::{ColumnId, FileId, RegionId}; + +const ROWS: usize = 4096; +const SEGMENT_ROWS: usize = 1024; + +struct Shape { + name: &'static str, + tags: u32, + indexed_tags: u32, + inverted: bool, + bloom: bool, + rows_per_key: usize, +} + +fn input(shape: &Shape) -> (RegionMetadataRef, HashSet, RecordBatch) { + let mut builder = RegionMetadataBuilder::new(RegionId::new(1, 1)); + let mut pk = vec![ReservedColumnId::table_id(), ReservedColumnId::tsid()]; + let mut inverted_columns = HashSet::new(); + let mut add_tag = |id, name: String, data_type, indexed| { + let mut schema = ColumnSchema::new(name, data_type, true).with_inverted_index(false); + if indexed && shape.inverted { + inverted_columns.insert(id); + } + if indexed && shape.bloom { + schema = schema + .with_skipping_options(SkippingIndexOptions { + granularity: SEGMENT_ROWS as _, + ..Default::default() + }) + .unwrap(); + } + builder.push_column_metadata(ColumnMetadata { + column_schema: schema, + semantic_type: SemanticType::Tag, + column_id: id, + }); + }; + add_tag( + pk[0], + "__table_id".into(), + ConcreteDataType::uint32_datatype(), + shape.indexed_tags == 0, + ); + add_tag( + pk[1], + "__tsid".into(), + ConcreteDataType::uint64_datatype(), + false, + ); + for id in 0..shape.tags { + // A single indexed label is deliberately last in the key. + add_tag( + id, + format!("tag_{id:03}"), + ConcreteDataType::string_datatype(), + id >= shape.tags - shape.indexed_tags, + ); + pk.push(id); + } + builder + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ), + semantic_type: SemanticType::Timestamp, + column_id: shape.tags, + }) + .primary_key(pk) + .primary_key_encoding(PrimaryKeyEncoding::Sparse); + let metadata = Arc::new(builder.build().unwrap()); + let codec = SparsePrimaryKeyCodec::schemaless(); + let mut keys = BinaryDictionaryBuilder::::new(); + for series in 0..ROWS / shape.rows_per_key { + let mut key = Vec::new(); + codec + .encode_internal((series / 128) as u32, series as u64, &mut key) + .unwrap(); + let labels: Vec<_> = (0..shape.tags) + .map(|id| (id, format!("tag-{id:03}-value-{series:010}"))) + .collect(); + codec + .encode_raw_tag_value( + labels.iter().map(|(id, value)| (*id, value.as_bytes())), + &mut key, + ) + .unwrap(); + for _ in 0..shape.rows_per_key { + keys.append(&key).unwrap(); + } + } + let schema = to_flat_sst_arrow_schema( + &metadata, + &FlatSchemaOptions::from_encoding(PrimaryKeyEncoding::Sparse), + ); + let columns: Vec = vec![ + Arc::new(TimestampMillisecondArray::from_iter_values(0..ROWS as i64)), + Arc::new(keys.finish()), + Arc::new(UInt64Array::from(vec![1; ROWS])), + Arc::new(UInt8Array::from(vec![1; ROWS])), + ]; + let batch = RecordBatch::try_new(schema, columns).unwrap(); + (metadata, inverted_columns, batch) +} + +fn bench_index_update(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let dir = create_temp_dir("bench_index_update"); + let intermediate = runtime + .block_on(IntermediateManager::init_fs(dir.path().to_str().unwrap())) + .unwrap(); + let mut group = c.benchmark_group("sparse_index_update"); + group.sample_size(30); + group.warm_up_time(Duration::from_secs(1)); + group.measurement_time(Duration::from_secs(3)); + group.throughput(Throughput::Elements(ROWS as u64)); + for shape in [ + Shape { + name: "table_id_bloom_40tags", + tags: 40, + indexed_tags: 0, + inverted: false, + bloom: true, + rows_per_key: 1, + }, + Shape { + name: "inverted_one_40tags", + tags: 40, + indexed_tags: 1, + inverted: true, + bloom: false, + rows_per_key: 1, + }, + Shape { + name: "bloom_one_40tags", + tags: 40, + indexed_tags: 1, + inverted: false, + bloom: true, + rows_per_key: 1, + }, + Shape { + name: "both_one_40tags", + tags: 40, + indexed_tags: 1, + inverted: true, + bloom: true, + rows_per_key: 1, + }, + Shape { + name: "both_all_40tags", + tags: 40, + indexed_tags: 40, + inverted: true, + bloom: true, + rows_per_key: 1, + }, + Shape { + name: "both_all_40tags_8rpk", + tags: 40, + indexed_tags: 40, + inverted: true, + bloom: true, + rows_per_key: 8, + }, + Shape { + name: "both_one_10tags", + tags: 10, + indexed_tags: 1, + inverted: true, + bloom: true, + rows_per_key: 1, + }, + Shape { + name: "both_all_10tags", + tags: 10, + indexed_tags: 10, + inverted: true, + bloom: true, + rows_per_key: 1, + }, + ] { + let (metadata, inverted_columns, batch) = input(&shape); + group.bench_function(shape.name, |b| { + b.iter_custom(|iterations| { + runtime.block_on(async { + let mut elapsed = Duration::ZERO; + for _ in 0..iterations { + let file_id = FileId::random(); + let mut inverted = shape.inverted.then(|| { + InvertedIndexer::new( + file_id, + &metadata, + intermediate.clone(), + None, + NonZeroUsize::new(SEGMENT_ROWS).unwrap(), + inverted_columns.clone(), + ) + }); + let mut bloom = if shape.bloom { + BloomFilterIndexer::new(file_id, &metadata, intermediate.clone(), None) + .unwrap() + } else { + None + }; + let start = Instant::now(); + if let Some(indexer) = &mut inverted { + indexer.update_flat(&batch).await.unwrap(); + } + if let Some(indexer) = &mut bloom { + indexer.update_flat(&batch).await.unwrap(); + } + elapsed += start.elapsed(); + if let Some(indexer) = &mut inverted { + indexer.abort().await.unwrap(); + } + if let Some(indexer) = &mut bloom { + indexer.abort().await.unwrap(); + } + } + elapsed + }) + }); + }); + } + group.finish(); +} + +criterion_group!(benches, bench_index_update); +criterion_main!(benches); diff --git a/src/mito2/src/access_layer.rs b/src/mito2/src/access_layer.rs index 85808842741..ac18f838db2 100644 --- a/src/mito2/src/access_layer.rs +++ b/src/mito2/src/access_layer.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::path::Path; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -567,6 +568,8 @@ pub struct SstWriteRequest { pub cache_manager: CacheManagerRef, #[allow(dead_code)] pub storage: Option, + /// Optional uniform row sequence for writes that do not preserve sequences. + /// Compaction passes `None` to retain the reader's effective input sequences. pub max_sequence: Option, pub sst_write_format: FormatType, @@ -655,14 +658,17 @@ impl TempFileCleaner { } pub(crate) async fn new_fs_cache_store(root: &str) -> Result { - let atomic_write_dir = join_dir(root, ATOMIC_WRITE_DIR); - clean_dir(&atomic_write_dir).await?; + // Preserve native filesystem prefixes such as Windows UNC shares. + let atomic_write_dir = Path::new(root).join(ATOMIC_WRITE_DIR); + clean_dir(&atomic_write_dir.to_string_lossy()).await?; // Compatible code. Remove this after a major release. let old_atomic_temp_dir = join_dir(root, OLD_ATOMIC_WRITE_DIR); clean_dir(&old_atomic_temp_dir).await?; - let builder = Fs::default().root(root).atomic_write_dir(&atomic_write_dir); + let builder = Fs::default() + .root(root) + .atomic_write_dir(&atomic_write_dir.to_string_lossy()); let store = ObjectStore::new(builder).context(OpenDalSnafu)?; Ok(with_instrument_layers(store, false)) @@ -759,3 +765,37 @@ impl FilePathProvider for RegionFilePathFactory { location::sst_file_path(&self.table_dir, file_id, self.path_type) } } + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::*; + + #[tokio::test] + async fn test_new_fs_cache_store() { + let root = common_test_util::temp_dir::create_temp_dir("fs-cache-store"); + let dirs = [ + root.path().join(ATOMIC_WRITE_DIR), + PathBuf::from(join_dir( + root.path().to_str().unwrap(), + OLD_ATOMIC_WRITE_DIR, + )), + ]; + for dir in &dirs { + tokio::fs::create_dir_all(dir).await.unwrap(); + tokio::fs::write(dir.join("stale"), b"stale").await.unwrap(); + } + let store = new_fs_cache_store(root.path().to_str().unwrap()) + .await + .unwrap(); + for dir in &dirs { + assert!(!dir.join("stale").exists()); + } + store.write("index", "contents").await.unwrap(); + assert_eq!( + tokio::fs::read(root.path().join("index")).await.unwrap(), + b"contents" + ); + } +} diff --git a/src/mito2/src/compaction/compactor.rs b/src/mito2/src/compaction/compactor.rs index 8ad1cdf0b22..2e59aae348d 100644 --- a/src/mito2/src/compaction/compactor.rs +++ b/src/mito2/src/compaction/compactor.rs @@ -404,10 +404,33 @@ pub trait SstMerger: Send + Sync + 'static { #[derive(Clone)] pub struct DefaultSstMerger; +/// File-level sequence metadata is independent of physical row encoding. +struct OutputSequenceMetadata { + file_sequence: Option, + exact_sequence_trusted: bool, +} + +/// Compaction does not admit new data, so it inherits the input sequence bounds +/// without allocating a new admission marker. Retaining physical row sequences +/// does not by itself restore exact-read capability for untrusted inputs. +fn output_sequence_metadata( + region: &CompactionRegion, + inputs: &[FileHandle], +) -> OutputSequenceMetadata { + let exact_sequence_trusted = region.region_options.preserve_row_sequence + && inputs + .iter() + .all(|f| f.is_effective_target_sequence_trusted(region.region_id)); + OutputSequenceMetadata { + file_sequence: known_max_input_sequence(inputs), + exact_sequence_trusted, + } +} + /// Computes the maximum target-domain sequence bound of the output of merging -/// `inputs`. Foreign files are described by their target-local barrier; local -/// trusted files use their physical sequence bound. Unknown bounds never become -/// trusted through a partial maximum. +/// `inputs`. A bound can be a row maximum or an admission marker; compaction +/// must preserve both for sequence-based pruning. Unknown bounds must remain +/// unknown rather than being replaced with a partial maximum. fn known_max_input_sequence(inputs: &[FileHandle]) -> Option { let mut max: Option = None; for input in inputs { @@ -452,40 +475,7 @@ impl SstMerger for DefaultSstMerger { .iter() .map(|f| f.file_id().to_string()) .join(","); - let input_max_sequence = known_max_input_sequence(&output.inputs); - // The output is trusted only when every input is trusted in the target - // domain. Foreign marker values describe the source and are ignored; - // their present FileMeta.sequence is the target-local barrier. - let output_preserves_sequence = compaction_region.region_options.preserve_row_sequence - && output - .inputs - .iter() - .all(|f| f.is_effective_target_sequence_trusted(region_id)); - let output_sequence = if output_preserves_sequence { - input_max_sequence - } else { - // The manifest records the region's latest accepted sequence for - // edit paths and its flushed frontier otherwise. Compaction only - // rewrites the immutable SST snapshot, so this is the same - // region-local admission barrier used when accepting files. - let manifest = compaction_region - .manifest_ctx - .manifest_manager - .read() - .await - .manifest(); - NonZeroU64::new( - manifest - .committed_sequence - .unwrap_or(manifest.flushed_sequence) - + 1, - ) - }; - // For untrusted output, keep the physical sequence override compatible - // with the main write path: use the known maximum sequence of the - // inputs. The FileMeta sequence remains the admission barrier, but it - // must not be written into the rows (or replaced with zero). - let write_max_sequence = input_max_sequence.map(NonZeroU64::get); + let sequence_metadata = output_sequence_metadata(&compaction_region, &output.inputs); let builder = CompactionSstReaderBuilder { metadata: compaction_region.region_metadata.clone(), sst_layer: compaction_region.access_layer.clone(), @@ -509,13 +499,16 @@ impl SstMerger for DefaultSstMerger { source, cache_manager: compaction_region.cache_manager.clone(), storage, - max_sequence: write_max_sequence, + // Readers resolve file overrides before merge/dedup. Replacing + // their effective sequences here could promote old rows above + // versions in SSTs that were not part of this merge. + max_sequence: None, sst_write_format: if flat_format { FormatType::Flat } else { FormatType::PrimaryKey }, - preserve_row_sequence: output_preserves_sequence, + preserve_row_sequence: sequence_metadata.exact_sequence_trusted, index_options, index_config, inverted_index_config, @@ -564,12 +557,12 @@ impl SstMerger for DefaultSstMerger { index_version: 0, num_rows: sst_info.num_rows as u64, num_row_groups: sst_info.num_row_groups, - sequence: output_sequence, + sequence: sequence_metadata.file_sequence, partition_expr: partition_expr.clone(), num_series: sst_info.num_series, primary_key_min, primary_key_max, - preserve_row_sequence: output_preserves_sequence, + preserve_row_sequence: sequence_metadata.exact_sequence_trusted, } }) .collect::>(); @@ -655,6 +648,7 @@ where write_buffer_size: compaction_region.engine_config.sst_write_buffer_size, max_file_size: picker_output.max_file_size, row_group_size: compaction_region.region_options.row_group_size(), + float_field_encoding: compaction_region.region_options.float_field_encoding, }; let merger = self.merger.clone(); let compaction_region = compaction_region.clone(); @@ -887,6 +881,268 @@ mod tests { assert_eq!(None, known_max_input_sequence(&[])); } + #[tokio::test] + async fn test_output_sequence_metadata_preserves_trust_and_unknown_bounds() { + let mut region = new_test_compaction_region().await; + let local = region.region_id; + let foreign = RegionId::new(1, 2); + let file = |region_id, sequence: Option, preserve_row_sequence| { + new_file_handle(FileMeta { + region_id, + sequence: sequence.and_then(NonZeroU64::new), + preserve_row_sequence, + ..dummy_file_meta() + }) + }; + // Bounds are independent of row trust: known, mixed trust, unknown in + // either sequence domain, empty inputs, and the largest possible bound. + let cases = [ + ( + vec![file(local, Some(3), true), file(foreign, Some(9), false)], + Some(9), + true, + ), + ( + vec![file(local, Some(3), true), file(local, Some(6), false)], + Some(6), + false, + ), + ( + vec![file(foreign, Some(9), false), file(foreign, Some(11), true)], + Some(11), + true, + ), + ( + vec![file(local, Some(3), true), file(local, None, true)], + None, + true, + ), + ( + vec![file(local, None, false), file(local, Some(6), true)], + None, + false, + ), + ( + vec![file(local, Some(3), true), file(foreign, None, true)], + None, + false, + ), + (vec![], None, true), + ( + vec![file(local, Some(u64::MAX), false)], + Some(u64::MAX), + false, + ), + ]; + for committed_sequence in [100, 101] { + region + .manifest_ctx + .manifest_manager + .write() + .await + .update( + RegionMetaActionList::with_action(RegionMetaAction::Edit(RegionEdit { + committed_sequence: Some(committed_sequence), + files_to_add: vec![], + files_to_remove: vec![], + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id: None, + flushed_sequence: None, + })), + false, + ) + .await + .unwrap(); + for preserve in [false, true] { + region.region_options.preserve_row_sequence = preserve; + for (inputs, known_bound, inputs_trusted) in &cases { + let metadata = output_sequence_metadata(®ion, inputs); + let trusted = preserve && *inputs_trusted; + assert_eq!(trusted, metadata.exact_sequence_trusted); + assert_eq!( + known_bound.and_then(NonZeroU64::new), + metadata.file_sequence + ); + } + } + } + } + + /// Independent outputs must be safe in either visibility order, including when + /// only one output succeeds. Merely committing both outputs together is not a + /// substitute for preserving version order across their separate merge streams. + #[rstest::rstest] + #[case(true)] + #[case(false)] + #[tokio::test] + async fn test_independent_outputs_preserve_delete_in_every_visibility_state( + #[case] flat_format: bool, + #[values(false, true)] unknown_put_bound: bool, + ) { + use datatypes::arrow::array::AsArray; + use datatypes::arrow::datatypes::TimestampMillisecondType; + use store_api::region_engine::RegionEngine; + use store_api::region_request::RegionRequest; + + use crate::compaction::CompactionOutput; + use crate::compaction::compactor::{CompactionRegion, Compactor, DefaultCompactor}; + use crate::compaction::picker::PickerOutput; + use crate::compaction::reader::CompactionSstReaderBuilder; + use crate::engine::compaction_test::{delete_and_flush, put_and_flush}; + use crate::region::options::MergeMode; + use crate::sst::file::FileHandle; + use crate::sst::file_purger::NoopFilePurger; + use crate::test_util::{CreateRequestBuilder, TestEnv, rows_schema}; + + let mut env = TestEnv::new().await; + let region_id = RegionId::new(1, 1); + let config = MitoConfig { + default_flat_format: flat_format, + min_compaction_interval: Duration::from_secs(3600), + ..Default::default() + }; + let engine = env.create_engine(config).await; + let request = CreateRequestBuilder::new().build(); + let columns = rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + put_and_flush(&engine, region_id, &columns, 0..1).await; + delete_and_flush(&engine, region_id, &columns, 0..1).await; + put_and_flush(&engine, region_id, &columns, 10..11).await; + let region = engine.get_region(region_id).unwrap(); + let version = region.version(); + let mut files: Vec<_> = version + .ssts + .levels() + .iter() + .flat_map(|level| level.files()) + .cloned() + .collect(); + assert_eq!(3, files.len()); + files.sort_unstable_by_key(|file| file.meta_ref().sequence); + if unknown_put_bound { + // Model a legacy SST whose physical rows are readable but whose + // file-level boundary is unknown. Merging cannot invent that bound. + let mut meta = files[0].meta_ref().clone(); + meta.sequence = None; + files[0] = FileHandle::new(meta, Arc::new(NoopFilePurger)); + } + let compaction_region = CompactionRegion { + region_id, + region_options: version.options.clone(), + engine_config: Arc::new(MitoConfig { + default_flat_format: flat_format, + ..Default::default() + }), + region_metadata: version.metadata.clone(), + cache_manager: engine.cache_manager(), + access_layer: region.access_layer.clone(), + manifest_ctx: region.manifest_ctx.clone(), + current_version: version.into(), + file_purger: None, + ttl: None, + max_parallelism: 2, + plugins: common_base::Plugins::new(), + }; + let puts = vec![files[0].clone(), files[2].clone()]; + let deletes = vec![files[1].clone()]; + let picker_output = PickerOutput { + outputs: [puts.clone(), deletes.clone()] + .into_iter() + .map(|inputs| CompactionOutput { + output_level: 1, + inputs, + filter_deleted: false, + output_time_range: None, + }) + .collect(), + expired_ssts: vec![], + time_window_size: 3600, + max_file_size: None, + }; + let merged = DefaultCompactor::with_merger(DefaultSstMerger) + .merge_ssts(&compaction_region, picker_output) + .await + .unwrap(); + assert_eq!(2, merged.files_to_add.len()); + let outputs: Vec<_> = merged + .files_to_add + .into_iter() + .map(|meta| FileHandle::new(meta, Arc::new(NoopFilePurger))) + .collect(); + let put_output = outputs + .iter() + .find(|file| file.meta_ref().num_rows == 2) + .unwrap(); + let delete_output = outputs + .iter() + .find(|file| file.meta_ref().num_rows == 1) + .unwrap(); + assert_eq!( + if unknown_put_bound { + None + } else { + NonZeroU64::new(3) + }, + put_output.meta_ref().sequence, + ); + assert_eq!(NonZeroU64::new(2), delete_output.meta_ref().sequence); + assert!( + outputs + .iter() + .all(|file| !file.meta_ref().preserve_row_sequence) + ); + + // Cover the original snapshot, each partial success/cancellation state, + // and both completed outputs using real persisted SSTs and merge readers. + for (put_done, delete_done) in [(false, false), (true, false), (false, true), (true, true)] + { + let mut visible = if put_done { + vec![put_output.clone()] + } else { + puts.clone() + }; + visible.extend(if delete_done { + vec![delete_output.clone()] + } else { + deletes.clone() + }); + let mut reader = CompactionSstReaderBuilder { + metadata: compaction_region.region_metadata.clone(), + sst_layer: region.access_layer.clone(), + cache: engine.cache_manager(), + inputs: &visible, + append_mode: false, + filter_deleted: true, + time_range: None, + merge_mode: MergeMode::LastRow, + } + .build_flat_sst_reader() + .await + .unwrap(); + let mut timestamps = Vec::new(); + while let Some(batch) = reader.next_batch().await.unwrap() { + timestamps.extend( + batch + .column_by_name("ts") + .unwrap() + .as_primitive::() + .values() + .iter() + .copied(), + ); + } + assert_eq!( + vec![10_000], + timestamps, + "put_done={put_done}, delete_done={delete_done}" + ); + } + } + /// Build a minimal [`CompactionRegion`] suitable for tests where the /// [`SstMerger`] is mocked and never touches the access layer. async fn new_test_compaction_region() -> CompactionRegion { diff --git a/src/mito2/src/compaction/json2.rs b/src/mito2/src/compaction/json2.rs index 71b571c586d..8eec0e11520 100644 --- a/src/mito2/src/compaction/json2.rs +++ b/src/mito2/src/compaction/json2.rs @@ -52,6 +52,8 @@ pub(crate) type Json2RewritePlans = HashMap; #[derive(Clone)] struct Json2LeafPathStats { rows: u64, + /// Number of input schemas that explicitly represent this leaf. + sources: usize, data_type: JsonNativeType, is_type_conflicted: bool, } @@ -59,8 +61,8 @@ struct Json2LeafPathStats { /// Builds the JSON2 rewrite plans for a compaction. /// /// Type hints from current region metadata are always retained. Existing explicit dynamic paths -/// from all input SST schemas are ranked once to produce a fixed layout; paths found only in a v2 -/// remainder are deliberately not promoted. [`rewrite_json2_batch`] decodes inputs and rewrites +/// shared by every input SST schema are ranked once to produce a fixed layout; paths that may +/// reside in a v2 remainder are deliberately not promoted. [`rewrite_json2_batch`] decodes inputs and rewrites /// them according to these plans. Non-JSON2 columns are omitted from the returned map. /// /// Returns an error when a JSON2 column has invalid or missing extension metadata, or when its @@ -144,6 +146,14 @@ pub(crate) fn collect_json2_rewrite_plans( collect_json2_path_stats(field, *rows, &hint_paths, &mut stats)?; } + // A leaf absent from an input's explicit schema may have arbitrary values + // (including scalar ancestors) in its remainder. Only common explicit leaves + // are safe to promote without inspecting rows. Keep unsafe paths opaque for + // the entire output SST so existing readers never miss remainder values when + // projecting an explicit leaf. + for stat in stats.values_mut() { + stat.is_type_conflicted |= stat.sources != schemas.len(); + } let mut hints = settings.type_hints().to_vec(); hints.extend(select_dynamic_hints(settings, &hint_paths, &stats)); let target_layout = JsonSettings::try_new(hints, Some(0)).context(DataTypeMismatchSnafu)?; @@ -187,12 +197,14 @@ fn collect_json2_path_stats<'a>( path, Json2LeafPathStats { rows, + sources: 1, data_type, is_type_conflicted: false, }, ); continue; }; + stat.sources += 1; if stat.data_type != data_type { stat.is_type_conflicted = true; } else { @@ -368,6 +380,7 @@ mod tests { )?; let stat = |rows, data_type, is_type_conflicted| Json2LeafPathStats { rows, + sources: 1, data_type, is_type_conflicted, }; diff --git a/src/mito2/src/compaction/window.rs b/src/mito2/src/compaction/window.rs index a9f3f021b84..b9053466444 100644 --- a/src/mito2/src/compaction/window.rs +++ b/src/mito2/src/compaction/window.rs @@ -337,6 +337,7 @@ mod tests { primary_key_encoding: None, write_buffer_size: None, preserve_row_sequence: false, + float_field_encoding: Default::default(), }, compaction_time_window: None, } diff --git a/src/mito2/src/config.rs b/src/mito2/src/config.rs index 6ec790c52b0..f525757a268 100644 --- a/src/mito2/src/config.rs +++ b/src/mito2/src/config.rs @@ -89,14 +89,16 @@ pub struct MitoConfig { // Background job configs: /// Max number of running background index build jobs (default: 1/8 of cpu cores). pub max_background_index_builds: usize, - // TODO: Document both series-index settings in the example configs and regenerate configuration - // docs before exposing the feature. - /// Under development; do not enable. Root directory for loading series indexes, currently stored - /// on the local filesystem. Empty disables the feature. Relative paths resolve under `data_home`. - pub experimental_series_index_root: String, + /// Under development; do not enable. Whether to enable series indexes (default false). + /// Indexes are stored on the local filesystem under `{data_home}/series_index`. + pub experimental_enable_series_index: bool, /// Interval between series-index maintenance runs (default 5 min). Zero uses the default. #[serde(with = "humantime_serde")] pub experimental_series_index_maintenance_interval: Duration, + /// Under development; do not enable. Requested minimum bucket width for series indexes. + /// It is rounded up to an exact multiple of each region's compaction time window. + #[serde(with = "humantime_serde")] + pub experimental_series_index_bucket_width: Duration, /// Max number of running background flush jobs (default: 1/2 of cpu cores). pub max_background_flushes: usize, /// Max number of running background compaction jobs (default: 1/4 of cpu cores). @@ -214,9 +216,10 @@ impl Default for MitoConfig { experimental_manifest_keep_removed_file_ttl: Duration::from_secs(60 * 60), compress_manifest: false, max_background_index_builds: divide_num_cpus(8), - experimental_series_index_root: String::new(), + experimental_enable_series_index: false, experimental_series_index_maintenance_interval: DEFAULT_SERIES_INDEX_MAINTENANCE_INTERVAL, + experimental_series_index_bucket_width: Duration::from_secs(5 * 24 * 60 * 60), max_background_flushes: divide_num_cpus(2), max_background_compactions: divide_num_cpus(4), max_background_purges: get_total_cpu_cores(), @@ -312,15 +315,6 @@ impl MitoConfig { DEFAULT_SERIES_INDEX_MAINTENANCE_INTERVAL; } - if !self.experimental_series_index_root.trim().is_empty() - && Path::new(&self.experimental_series_index_root).is_relative() - { - self.experimental_series_index_root = Path::new(data_home) - .join(&self.experimental_series_index_root) - .display() - .to_string(); - } - if self.global_write_buffer_reject_size <= self.global_write_buffer_size { self.global_write_buffer_reject_size = self.global_write_buffer_size * 2; warn!( @@ -434,22 +428,27 @@ mod tests { #[test] fn test_series_index_config() { - assert!( - MitoConfig::default() - .experimental_series_index_root - .is_empty() + assert!(!MitoConfig::default().experimental_enable_series_index); + assert_eq!( + MitoConfig::default().experimental_series_index_bucket_width, + Duration::from_secs(5 * 24 * 60 * 60) ); let mut config: MitoConfig = toml::from_str( - "experimental_series_index_root = 'indexes' - experimental_series_index_maintenance_interval = '30s'", + "experimental_enable_series_index = true + experimental_series_index_maintenance_interval = '30s' + experimental_series_index_bucket_width = '2days'", ) .unwrap(); config.sanitize("/data").unwrap(); - assert_eq!(config.experimental_series_index_root, "/data/indexes"); + assert!(config.experimental_enable_series_index); assert_eq!( config.experimental_series_index_maintenance_interval, Duration::from_secs(30) ); + assert_eq!( + config.experimental_series_index_bucket_width, + Duration::from_secs(2 * 24 * 60 * 60) + ); let restored: MitoConfig = toml::from_str(&toml::to_string(&config).unwrap()).unwrap(); assert_eq!(config, restored); diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs index 1c674abadbf..348ec4c1222 100644 --- a/src/mito2/src/engine.rs +++ b/src/mito2/src/engine.rs @@ -225,6 +225,7 @@ impl<'a, S: LogStore> MitoEngineBuilder<'a, S> { // so the engine (and thus the GC worker) can fire `on_region_gc`. let region_hook = self.plugins.get::(); let workers = WorkerGroup::start( + self.data_home, config.clone(), self.log_store.clone(), self.object_store_manager, @@ -1620,6 +1621,7 @@ impl MitoEngine { Ok(MitoEngine { inner: Arc::new(EngineInner { workers: WorkerGroup::start_for_test( + data_home, config.clone(), log_store, object_store_manager, diff --git a/src/mito2/src/engine/alter_test.rs b/src/mito2/src/engine/alter_test.rs index 916424622b9..fd74c4b498a 100644 --- a/src/mito2/src/engine/alter_test.rs +++ b/src/mito2/src/engine/alter_test.rs @@ -3003,6 +3003,7 @@ fn build_schema_v0_bulk_request(region_id: RegionId) -> RegionBulkInsertsRequest let raw_data = encode_arrow_ipc(&payload); RegionBulkInsertsRequest { + skip_wal: false, region_id, payload, raw_data, diff --git a/src/mito2/src/engine/basic_test.rs b/src/mito2/src/engine/basic_test.rs index 79c88f8a6c0..f4c843f1ce4 100644 --- a/src/mito2/src/engine/basic_test.rs +++ b/src/mito2/src/engine/basic_test.rs @@ -29,6 +29,7 @@ use datatypes::prelude::ConcreteDataType; use datatypes::schema::ColumnSchema; use futures::TryStreamExt; use itertools::Itertools; +use parquet::basic::{Encoding, Type as PhysicalType}; use rstest::rstest; use rstest_reuse::{self, apply}; use store_api::metadata::ColumnMetadata; @@ -826,7 +827,9 @@ async fn test_engine_with_write_cache_with_format(flat_format: bool) { let engine = env.create_engine(mito_config).await; let region_id = RegionId::new(1, 1); - let request = CreateRequestBuilder::new().build(); + let request = CreateRequestBuilder::new() + .insert_option("experimental_sst_float_field_encoding", "byte_stream_split") + .build(); let column_schemas = rows_schema(&request); engine @@ -842,6 +845,38 @@ async fn test_engine_with_write_cache_with_format(flat_format: bool) { flush_region(&engine, region_id, None).await; + let region = engine.get_region(region_id).unwrap(); + let file = region + .version() + .ssts + .levels() + .iter() + .flat_map(|level| level.files.values()) + .next() + .expect("write-cache flushed SST") + .clone(); + let reader = region + .access_layer + .read_sst(file) + .build() + .await + .unwrap() + .expect("write-cache flushed SST reader"); + assert!( + reader + .parquet_metadata() + .row_groups() + .iter() + .flat_map(|row_group| row_group.columns()) + .any(|column| { + column.column_path().string() == "field_0" + && column.column_type() == PhysicalType::DOUBLE + && column + .encodings() + .any(|encoding| encoding == Encoding::BYTE_STREAM_SPLIT) + }) + ); + let request = ScanRequest::default(); let scanner = engine.scanner(region_id, request).await.unwrap(); @@ -1306,6 +1341,7 @@ async fn test_all_index_metas_list_all_types_with_format(flat_format: bool, expe if let Some(inverted) = value.get_mut("inverted").and_then(|v| v.as_object_mut()) { inverted.insert("base_offset".to_string(), serde_json::Value::from(0)); } + value.sort_all_objects(); *meta_json = value.to_string(); } } diff --git a/src/mito2/src/engine/compaction_test.rs b/src/mito2/src/engine/compaction_test.rs index c67afbf8c6f..d2e03ac09a8 100644 --- a/src/mito2/src/engine/compaction_test.rs +++ b/src/mito2/src/engine/compaction_test.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::ops::Range; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -26,6 +26,7 @@ use common_recordbatch::{RecordBatches, SendableRecordBatchStream}; use common_time::Timestamp; use datatypes::arrow::array::AsArray; use datatypes::arrow::datatypes::TimestampMillisecondType; +use parquet::basic::{Encoding, Type as PhysicalType}; use store_api::region_engine::{RegionEngine, RegionRole}; use store_api::region_request::AlterKind::SetRegionOptions; use store_api::region_request::{ @@ -38,11 +39,11 @@ use tokio::sync::{Notify, Semaphore}; use crate::config::MitoConfig; use crate::engine::MitoEngine; -use crate::engine::flush_test::MockTimeProvider; use crate::engine::listener::{CompactionListener, EventListener}; use crate::test_util::{ CreateRequestBuilder, TestEnv, build_rows_for_key, column_metadata_to_column_schema, put_rows, }; +use crate::time_provider::mock::MockTimeProvider; pub(crate) async fn put_and_flush( engine: &MitoEngine, @@ -136,6 +137,295 @@ async fn collect_stream_ts(stream: SendableRecordBatchStream) -> Vec { res } +/// Flush may collapse versions within one file, but compaction must not promote +/// them, even with no external overlaps or after rewriting a previous output. +#[rstest::rstest] +#[case(false, false)] +#[case(true, false)] +#[case(true, true)] +#[tokio::test] +async fn test_compaction_preserves_flush_sequences_across_generations( + #[values(false, true)] flat_format: bool, + #[case] append_mode: bool, + #[case] preserve_row_sequence: bool, +) { + let mut env = TestEnv::new().await; + let config = MitoConfig { + default_flat_format: flat_format, + min_compaction_interval: Duration::from_secs(3600), + ..Default::default() + }; + let engine = env.create_engine(config.clone()).await; + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new() + .insert_option("append_mode", &append_mode.to_string()) + .insert_option("preserve_row_sequence", &preserve_row_sequence.to_string()) + .insert_option("compaction.type", "twcs") + .insert_option("compaction.twcs.time_window", "1h") + .build(); + let table_dir = request.table_dir.clone(); + let options = request.options.clone(); + let columns = crate::test_util::rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + + async fn read_sequences(engine: &MitoEngine, region_id: RegionId) -> Vec { + let region = engine.get_region(region_id).unwrap(); + let mut sequences = Vec::new(); + for file in region + .version() + .ssts + .levels() + .iter() + .flat_map(|l| l.files()) + { + let mut reader = region + .access_layer + .read_sst(file.clone()) + .build() + .await + .unwrap() + .unwrap(); + while let Some(batch) = reader.next_record_batch().await.unwrap() { + sequences.extend_from_slice( + batch + .column(batch.num_columns() - 2) + .as_primitive::() + .values(), + ); + } + } + sequences.sort_unstable(); + sequences + } + + let mut expected = Vec::new(); + for generation in 0..=2 { + // Interleaving time ranges exercise a merge without duplicate rows. + let rows = [generation, generation + 10] + .into_iter() + .flat_map(|ts| build_rows_for_key("a", ts, ts + 1, 0)) + .collect(); + put_rows( + &engine, + region_id, + Rows { + schema: columns.clone(), + rows, + }, + ) + .await; + flush(&engine, region_id).await; + let max = (generation * 2 + 2) as u64; + expected.extend(if preserve_row_sequence { + [max - 1, max] + } else { + [max, max] + }); + assert_eq!(expected, read_sequences(&engine, region_id).await); + if generation == 0 { + continue; + } + engine + .handle_request( + region_id, + RegionRequest::Compact(RegionCompactRequest { + options: api::v1::region::compact_request::Options::StrictWindow( + api::v1::region::StrictWindow { + window_seconds: 3600, + }, + ), + ..Default::default() + }), + ) + .await + .unwrap(); + let version = engine.get_region(region_id).unwrap().version(); + let files: Vec<_> = version + .ssts + .levels() + .iter() + .flat_map(|l| l.files()) + .collect(); + assert_eq!( + 1, + files.len(), + "generation {generation} must merge all inputs" + ); + assert_eq!( + preserve_row_sequence, + files[0].meta_ref().preserve_row_sequence + ); + assert_eq!(Some(max), files[0].meta_ref().sequence.map(|s| s.get())); + assert_eq!( + expected, + read_sequences(&engine, region_id).await, + "generation {generation}" + ); + } + let engine = env.reopen_engine(engine, config).await; + crate::test_util::reopen_region(&engine, region_id, table_dir, false, options).await; + assert_eq!(expected, read_sequences(&engine, region_id).await); +} + +#[tokio::test] +async fn test_partial_compaction_preserves_delete_order_flat() { + assert_partial_compaction_preserves_delete_order(true).await; +} + +#[tokio::test] +async fn test_partial_compaction_preserves_delete_order_primary_key() { + assert_partial_compaction_preserves_delete_order(false).await; +} + +/// A newer unrelated input must not promote an old Put above an unselected Delete. +/// All files fit in the same one-hour window; no cross-window SST is required. +async fn assert_partial_compaction_preserves_delete_order(flat_format: bool) { + let mut env = TestEnv::new().await; + let region_id = RegionId::new(1, 1); + let (engine, columns) = env_for_manual_compaction(&mut env, region_id, flat_format).await; + + put_and_flush(&engine, region_id, &columns, 10..16).await; + let mut deletes = build_rows_for_key("a", 10, 16, 0); + // Distinct keys keep the Delete SST large even after compression, so the + // byte-balance rule excludes it without fabricating FileMeta sizes. + let mut rng = ::seed_from_u64(0); + for _ in 0..3000 { + deletes.extend(build_rows_for_key( + &format!("{:032x}", rand::Rng::random::(&mut rng)), + 1, + 2, + 0, + )); + } + engine + .handle_request( + region_id, + RegionRequest::Delete(RegionDeleteRequest { + rows: Rows { + schema: columns.clone(), + rows: deletes, + }, + hint: None, + partition_expr_version: None, + }), + ) + .await + .unwrap(); + flush(&engine, region_id).await; + // These newer writes are legitimate, but must not change the version of 10..16. + for i in 10..25 { + put_and_flush(&engine, region_id, &columns, i * 10..i * 10 + 6).await; + } + + let region = engine.get_region(region_id).unwrap(); + let table_dir = region.table_dir().to_string(); + let version = region.version(); + let files = version + .ssts + .levels() + .iter() + .flat_map(|level| level.files()) + .collect::>(); + assert_eq!(17, files.len()); + assert!(files.iter().all(|file| { + file.time_range().0 >= Timestamp::new_millisecond(0) + && file.time_range().1 < Timestamp::new_millisecond(3_600_000) + })); + let delete_file = files + .iter() + .find(|file| file.meta_ref().num_rows == 3006) + .unwrap(); + let delete_id = delete_file.file_id(); + let delete_sequence = delete_file.meta_ref().sequence; + let old_ids = files + .iter() + .map(|file| file.file_id()) + .collect::>(); + let before = collect_stream_ts( + engine + .scanner(region_id, ScanRequest::default()) + .await + .unwrap() + .scan() + .await + .unwrap(), + ) + .await; + let expected = (10..25) + .flat_map(|i| (i * 10..i * 10 + 6).map(|ts| ts * 1000)) + .collect::>(); + assert_eq!( + expected, before, + "the Delete must hide old rows before compaction" + ); + + compact(&engine, region_id).await; + + let scanner = engine + .scanner(region_id, ScanRequest::default()) + .await + .unwrap(); + let current_ids = scanner.file_ids(); + assert_eq!( + 2, + current_ids.len(), + "the 16 small inputs must merge, leaving the large Delete SST" + ); + assert!(current_ids.contains(&delete_id)); + assert_eq!( + 1, + current_ids.iter().filter(|id| old_ids.contains(id)).count() + ); + let after = collect_stream_ts(scanner.scan().await.unwrap()).await; + + let current = region.version(); + let output = current + .ssts + .levels() + .iter() + .flat_map(|level| level.files()) + .find(|file| !old_ids.contains(&file.file_id())) + .unwrap(); + let mut reader = region + .access_layer + .read_sst(output.clone()) + .build() + .await + .unwrap() + .unwrap(); + let batch = reader.next_record_batch().await.unwrap().unwrap(); + let output_sequences = batch + .column(batch.num_columns() - 2) + .as_primitive::(); + let first_output_sequence = output_sequences.value(0); + + crate::test_util::reopen_region(&engine, region_id, table_dir, true, HashMap::new()).await; + let reopened = collect_stream_ts( + engine + .scanner(region_id, ScanRequest::default()) + .await + .unwrap() + .scan() + .await + .unwrap(), + ) + .await; + let resurrected = after + .iter() + .filter(|ts| (10_000..16_000).contains(*ts)) + .collect::>(); + assert!( + after == expected && reopened == expected, + "partial compaction changed delete ordering (flat_format={flat_format}, unselected Delete sequence={delete_sequence:?}, first output row sequence={first_output_sequence}); expected rows={}, after rows={}, reopened rows={}, resurrected timestamps={resurrected:?}", + expected.len(), + after.len(), + reopened.len() + ); +} + struct CompactionListenerGuard(Option>); impl CompactionListenerGuard { @@ -954,6 +1244,7 @@ async fn test_compaction_region_with_format(flat_format: bool) { let request = CreateRequestBuilder::new() .insert_option("compaction.type", "twcs") + .insert_option("experimental_sst_float_field_encoding", "byte_stream_split") .build(); let column_schemas = request @@ -974,6 +1265,35 @@ async fn test_compaction_region_with_format(flat_format: bool) { compact(&engine, region_id).await; + let region = engine.get_region(region_id).unwrap(); + let file = region.version().ssts.levels()[1] + .files + .values() + .next() + .expect("compaction output SST") + .clone(); + let reader = region + .access_layer + .read_sst(file) + .build() + .await + .unwrap() + .expect("compaction output SST reader"); + assert!( + reader + .parquet_metadata() + .row_groups() + .iter() + .flat_map(|row_group| row_group.columns()) + .any(|column| { + column.column_path().string() == "field_0" + && column.column_type() == PhysicalType::DOUBLE + && column + .encodings() + .any(|encoding| encoding == Encoding::BYTE_STREAM_SPLIT) + }) + ); + let scanner = engine .scanner(region_id, ScanRequest::default()) .await diff --git a/src/mito2/src/engine/edit_region_test.rs b/src/mito2/src/engine/edit_region_test.rs index de9611ee2df..ace17905eb4 100644 --- a/src/mito2/src/engine/edit_region_test.rs +++ b/src/mito2/src/engine/edit_region_test.rs @@ -35,12 +35,12 @@ use tokio::sync::{Barrier, mpsc, oneshot}; use crate::config::MitoConfig; use crate::engine::MitoEngine; -use crate::engine::flush_test::MockTimeProvider; use crate::engine::listener::EventListener; use crate::manifest::action::RegionEdit; use crate::region::{MitoRegionRef, RegionLeaderState, RegionRoleState}; use crate::sst::file::FileMeta; use crate::test_util::{CreateRequestBuilder, TestEnv, build_rows, rows_schema}; +use crate::time_provider::mock::MockTimeProvider; #[tokio::test] async fn test_edit_region_schedule_compaction() { @@ -700,6 +700,7 @@ fn build_bulk_insert_request( let (schema, record_batch) = encode_to_flight_data(payload.clone()); RegionBulkInsertsRequest { + skip_wal: false, region_id, payload, raw_data: ArrowIpc { diff --git a/src/mito2/src/engine/flush_test.rs b/src/mito2/src/engine/flush_test.rs index d7155e10520..b040ae5ec54 100644 --- a/src/mito2/src/engine/flush_test.rs +++ b/src/mito2/src/engine/flush_test.rs @@ -17,7 +17,7 @@ use std::assert_matches; use std::collections::HashMap; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicI64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; use api::v1::Rows; @@ -27,6 +27,7 @@ use common_base::readable_size::ReadableSize; use common_recordbatch::RecordBatches; use common_time::util::current_time_millis; use common_wal::options::{KafkaWalOptions, WAL_OPTIONS_KEY, WalOptions}; +use parquet::basic::{Encoding, Type as PhysicalType}; use rstest::rstest; use rstest_reuse::{self, apply}; use store_api::ManifestVersion; @@ -54,7 +55,7 @@ use crate::test_util::{ prepare_test_for_kafka_log_store, put_rows, raft_engine_log_store_factory, reopen_region, rows_schema, single_kafka_log_store_factory, }; -use crate::time_provider::TimeProvider; +use crate::time_provider::mock::MockTimeProvider; use crate::worker::MAX_INITIAL_CHECK_DELAY_SECS; async fn set_write_buffer_size_to_current_usage(engine: &MitoEngine, region_id: RegionId) { @@ -279,7 +280,9 @@ async fn test_manual_flush_with_format(flat_format: bool) { ) .await; - let request = CreateRequestBuilder::new().build(); + let request = CreateRequestBuilder::new() + .insert_option("experimental_sst_float_field_encoding", "byte_stream_split") + .build(); let column_schemas = rows_schema(&request); engine @@ -295,6 +298,38 @@ async fn test_manual_flush_with_format(flat_format: bool) { flush_region(&engine, region_id, None).await; + let region = engine.get_region(region_id).unwrap(); + let file = region + .version() + .ssts + .levels() + .iter() + .flat_map(|level| level.files.values()) + .next() + .expect("flushed SST") + .clone(); + let reader = region + .access_layer + .read_sst(file) + .build() + .await + .unwrap() + .expect("flushed SST reader"); + assert!( + reader + .parquet_metadata() + .row_groups() + .iter() + .flat_map(|row_group| row_group.columns()) + .any(|column| { + column.column_path().string() == "field_0" + && column.column_type() == PhysicalType::DOUBLE + && column + .encodings() + .any(|encoding| encoding == Encoding::BYTE_STREAM_SPLIT) + }) + ); + let request = ScanRequest::default(); let scanner = engine.scanner(region_id, request).await.unwrap(); assert_eq!(0, scanner.num_memtables()); @@ -1022,8 +1057,10 @@ async fn test_flush_empty_with_format(flat_format: bool) { let stream = scanner.scan().await.unwrap(); let batches = RecordBatches::try_collect(stream).await.unwrap(); let expected = "\ -++ -++"; ++-------+---------+----+ +| tag_0 | field_0 | ts | ++-------+---------+----+ ++-------+---------+----+"; assert_eq!(expected, batches.pretty_print().unwrap()); } @@ -1334,43 +1371,6 @@ fn kafka_wal_options(topic: &Option) -> HashMap { .unwrap_or_default() } -#[derive(Debug)] -pub(crate) struct MockTimeProvider { - now: AtomicI64, - elapsed: AtomicI64, -} - -impl TimeProvider for MockTimeProvider { - fn current_time_millis(&self) -> i64 { - self.now.load(Ordering::Relaxed) - } - - fn elapsed_since(&self, _current_millis: i64) -> i64 { - self.elapsed.load(Ordering::Relaxed) - } - - fn wait_duration(&self, _duration: Duration) -> Duration { - Duration::from_millis(20) - } -} - -impl MockTimeProvider { - pub(crate) fn new(now: i64) -> Self { - Self { - now: AtomicI64::new(now), - elapsed: AtomicI64::new(0), - } - } - - pub(crate) fn set_now(&self, now: i64) { - self.now.store(now, Ordering::Relaxed); - } - - fn set_elapsed(&self, elapsed: i64) { - self.elapsed.store(elapsed, Ordering::Relaxed); - } -} - #[tokio::test] async fn test_auto_flush_engine() { test_auto_flush_engine_with_format(false).await; diff --git a/src/mito2/src/engine/merge_mode_test.rs b/src/mito2/src/engine/merge_mode_test.rs index afb984e77bd..5430e6b246c 100644 --- a/src/mito2/src/engine/merge_mode_test.rs +++ b/src/mito2/src/engine/merge_mode_test.rs @@ -16,6 +16,9 @@ use api::v1::Rows; use common_recordbatch::RecordBatches; +use datafusion_expr::{col, lit}; +use datatypes::arrow::array::AsArray; +use datatypes::arrow::datatypes::UInt64Type; use store_api::region_engine::RegionEngine; use store_api::region_request::{RegionCompactRequest, RegionRequest}; use store_api::storage::{RegionId, ScanRequest}; @@ -27,6 +30,192 @@ use crate::test_util::{ delete_rows_schema, flush_region, put_rows, reopen_region, rows_schema, }; +#[rstest::rstest] +#[tokio::test] +async fn test_partial_compaction_preserves_last_row_put_versions( + #[values(false, true)] flat_format: bool, +) { + let mut env = TestEnv::new().await; + let config = MitoConfig { + default_flat_format: flat_format, + min_compaction_interval: std::time::Duration::from_secs(3600), + ..Default::default() + }; + let engine = env.create_engine(config.clone()).await; + let region_id = RegionId::new(1, 1); + env.get_schema_metadata_manager() + .register_region_table_info( + region_id.table_id(), + "test_table", + "test_catalog", + "test_schema", + None, + env.get_kv_backend(), + ) + .await; + let request = CreateRequestBuilder::new() + .field_num(2) + .insert_option("compaction.type", "twcs") + .insert_option("compaction.twcs.time_window", "1h") + .insert_option("merge_mode", "last_row") + .build(); + let table_dir = request.table_dir.clone(); + let region_opts = request.options.clone(); + let schema = rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + + // A(x=1), B(x=2) share a key and timestamp; C is unrelated. + // Promoting A when merging A+C must not hide the unselected B. + let a = build_rows_with_fields("a", &[10], &[(Some(1), None)]); + put_rows( + &engine, + region_id, + Rows { + schema: schema.clone(), + rows: a, + }, + ) + .await; + flush_region(&engine, region_id, None).await; + let mut b = build_rows_with_fields("a", &[10], &[(Some(2), None)]); + let mut rng = ::seed_from_u64(0); + for _ in 0..3000 { + b.extend(build_rows_with_fields( + &format!("{:032x}", rand::Rng::random::(&mut rng)), + &[1], + &[(Some(2), None)], + )); + } + put_rows( + &engine, + region_id, + Rows { + schema: schema.clone(), + rows: b, + }, + ) + .await; + flush_region(&engine, region_id, None).await; + let b_id = engine + .get_region(region_id) + .unwrap() + .version() + .ssts + .levels() + .iter() + .flat_map(|level| level.files()) + .find(|file| file.meta_ref().num_rows == 3001) + .unwrap() + .file_id(); + let c = build_rows_with_fields("z", &[10], &[(None, Some(3))]); + put_rows( + &engine, + region_id, + Rows { + schema: schema.clone(), + rows: c, + }, + ) + .await; + flush_region(&engine, region_id, None).await; + // Sixteen small inputs plus the large middle-version file B. The normal + // byte-balanced candidate excludes B. + for i in 0..14 { + let rows = build_rows_with_fields("z", &[100 + i], &[(Some(4), None)]); + put_rows( + &engine, + region_id, + Rows { + schema: schema.clone(), + rows, + }, + ) + .await; + flush_region(&engine, region_id, None).await; + } + let scan = || ScanRequest { + filters: vec![col("tag_0").eq(lit("a"))], + ..Default::default() + }; + let before = + RecordBatches::try_collect(engine.scan_to_stream(region_id, scan()).await.unwrap()) + .await + .unwrap() + .pretty_print() + .unwrap(); + let expected = "\ ++-------+---------+---------+---------------------+ +| tag_0 | field_0 | field_1 | ts | ++-------+---------+---------+---------------------+ +| a | 2.0 | | 1970-01-01T00:00:10 | ++-------+---------+---------+---------------------+"; + assert_eq!(expected, before); + engine + .handle_request( + region_id, + RegionRequest::Compact(RegionCompactRequest::default()), + ) + .await + .unwrap(); + let after = RecordBatches::try_collect(engine.scan_to_stream(region_id, scan()).await.unwrap()) + .await + .unwrap() + .pretty_print() + .unwrap(); + assert_eq!(expected, after, "flat_format={flat_format}"); + let version = engine.get_region(region_id).unwrap().version(); + let files: Vec<_> = version + .ssts + .levels() + .iter() + .flat_map(|level| level.files()) + .collect(); + assert_eq!(2, files.len()); + assert!(files.iter().any(|file| file.file_id() == b_id)); + assert!( + files + .iter() + .all(|file| !file.meta_ref().preserve_row_sequence) + ); + let region = engine.get_region(region_id).unwrap(); + let mut output_sequences = std::collections::HashSet::new(); + for file in files.iter().filter(|file| file.file_id() != b_id) { + let mut reader = region + .access_layer + .read_sst((*file).clone()) + .build() + .await + .unwrap() + .unwrap(); + while let Some(batch) = reader.next_record_batch().await.unwrap() { + output_sequences.extend( + batch + .column(batch.num_columns() - 2) + .as_primitive::() + .values() + .iter() + .copied(), + ); + } + } + assert!( + output_sequences.len() > 1, + "compaction must retain effective input sequences" + ); + let engine = env.reopen_engine(engine, config).await; + reopen_region(&engine, region_id, table_dir, false, region_opts).await; + let reopened = + RecordBatches::try_collect(engine.scan_to_stream(region_id, scan()).await.unwrap()) + .await + .unwrap() + .pretty_print() + .unwrap(); + assert_eq!(expected, reopened); +} + #[tokio::test] async fn test_merge_mode_write_query() { test_merge_mode_write_query_with_format(false).await; diff --git a/src/mito2/src/engine/scan_test.rs b/src/mito2/src/engine/scan_test.rs index 8c21e04541c..ab7d6d97237 100644 --- a/src/mito2/src/engine/scan_test.rs +++ b/src/mito2/src/engine/scan_test.rs @@ -391,6 +391,167 @@ async fn test_json2_v1_region_reopen_and_compaction() -> WhateverResult<()> { Ok(()) } +#[tokio::test] +async fn test_json2_mixed_subject_compaction_preserves_values() -> WhateverResult<()> { + let request = CreateRequestBuilder::new() + .field_datatype(ConcreteDataType::json2(JsonNativeType::Object( + JsonObjectType::new(), + ))) + .insert_option("append_mode", "true") + .insert_option("memtable.type", "bulk") + .insert_option("sst_format", "flat") + .build(); + let table_dir = request.table_dir.clone(); + let schema = test_util::rows_schema(&request); + let mut env = TestEnv::new().await; + let engine = env + .create_engine(MitoConfig { + min_compaction_interval: std::time::Duration::from_secs(3600), + ..Default::default() + }) + .await; + let region_id = RegionId::new(1027, 0); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await?; + // #9133: the second SST stores a heterogeneous subject in its remainder. + let values = [ + json!({"subject": {"cid": "first", "uri": "at://first"}}), + json!({"subject": "did:plc:x"}), + json!({"subject": {"cid": "last", "uri": "at://last"}}), + json!({"subject": {"cid": 42}}), + ]; + for (offset, batch) in [(0, &values[..1]), (1, &values[1..])] { + test_util::put_rows( + &engine, + region_id, + Rows { + schema: schema.clone(), + rows: batch + .iter() + .enumerate() + .map(|(i, value)| { + row(vec![ + ValueData::StringValue("tag".into()), + ValueData::JsonValue(encode_json_value(JsonValue::from(value.clone()))), + ValueData::TimestampMillisecondValue((offset + i) as i64 * 1000), + ]) + }) + .collect(), + }, + ) + .await; + test_util::flush_region(&engine, region_id, None).await; + } + let old_ids = engine + .scanner(region_id, ScanRequest::default()) + .await? + .file_ids(); + assert_eq!(2, old_ids.len()); + for _ in 0..2 { + engine + .handle_request( + region_id, + RegionRequest::Compact(RegionCompactRequest { + options: compact_request::Options::StrictWindow(StrictWindow { + window_seconds: 86400, + }), + ..Default::default() + }), + ) + .await?; + let scanner = engine + .scanner( + region_id, + ScanRequest { + projection: Some(vec![1]), + ..Default::default() + }, + ) + .await?; + assert_eq!( + 1, + scanner.num_files(), + "a swallowed merge failure must not pass" + ); + assert!(scanner.file_ids().iter().all(|id| !old_ids.contains(id))); + let batches = RecordBatches::try_collect(scanner.scan().await?).await?; + let mut actual = Vec::new(); + for batch in batches.iter() { + let array = batch.column_by_name("field_0").unwrap().clone(); + for i in 0..array.len() { + actual.push(JsonArray::from(&array).try_get_value(i)?); + } + } + assert_eq!(values.as_slice(), actual.as_slice()); + reopen_region(&engine, region_id, table_dir.clone(), true, HashMap::new()).await; + let scanner = engine + .scanner( + region_id, + ScanRequest { + projection: Some(vec![1]), + json_type_hint: HashMap::from([( + "field_0".into(), + JsonNativeType::Object(JsonObjectType::from([( + "subject".into(), + JsonNativeType::String, + )])), + )]), + ..Default::default() + }, + ) + .await?; + let batches = RecordBatches::try_collect(scanner.scan().await?).await?; + let mut subjects = Vec::new(); + for batch in batches.iter() { + let array = batch.column_by_name("field_0").unwrap().clone(); + for i in 0..array.len() { + subjects.push(JsonArray::from(&array).try_get_value(i)?); + } + } + let expected = values.iter().map(|value| { + let subject = &value["subject"]; + json!({"subject": subject.as_str().map(str::to_string).unwrap_or_else(|| subject.to_string())}) + }).collect::>(); + assert_eq!( + expected, subjects, + "projection must read values spilled to remainder" + ); + let scanner = engine + .scanner( + region_id, + ScanRequest { + projection: Some(vec![1]), + json_type_hint: HashMap::from([( + "field_0".into(), + JsonNativeType::Object(JsonObjectType::from([( + "subject".into(), + JsonNativeType::Object(JsonObjectType::from([( + "cid".into(), + JsonNativeType::String, + )])), + )])), + )]), + ..Default::default() + }, + ) + .await?; + let batches = RecordBatches::try_collect(scanner.scan().await?).await?; + let mut cids = Vec::new(); + for batch in batches.iter() { + let array = batch.column_by_name("field_0").unwrap().clone(); + for i in 0..array.len() { + cids.push(JsonArray::from(&array).try_get_value(i)?["subject"]["cid"].clone()); + } + } + assert_eq!( + vec![json!("first"), json!(null), json!("last"), json!("42")], + cids + ); + } + Ok(()) +} + #[tokio::test] async fn test_flush_aligns_different_json2_layouts() -> WhateverResult<()> { let mut request = CreateRequestBuilder::new() @@ -892,8 +1053,10 @@ async fn test_scan_with_min_sst_sequence_with_format(flat_format: bool) { Some(9), 0, "\ -++ -++", ++-------+---------+----+ +| tag_0 | field_0 | ts | ++-------+---------+----+ ++-------+---------+----+", ) .await; } @@ -2387,6 +2550,7 @@ fn build_bulk_insert_request( let (schema, record_batch) = encode_to_flight_data(payload.clone()); RegionBulkInsertsRequest { + skip_wal: false, region_id, payload, raw_data: ArrowIpc { @@ -2399,6 +2563,139 @@ fn build_bulk_insert_request( } } +#[tokio::test] +async fn test_bulk_skip_wal_recovery() { + check_bulk_skip_wal_recovery(false, false).await; + check_bulk_skip_wal_recovery(false, true).await; + check_bulk_skip_wal_recovery(true, false).await; + check_bulk_skip_wal_recovery(true, true).await; +} + +async fn check_bulk_skip_wal_recovery(skip_wal: bool, flush: bool) { + let mut env = TestEnv::new().await; + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new() + .insert_option("memtable.type", "bulk") + .build(); + let table_dir = request.table_dir.clone(); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + let mut request = build_bulk_insert_request(region_id, 0, 4); + request.skip_wal = skip_wal; + let affected = engine + .handle_request(region_id, RegionRequest::BulkInserts(request)) + .await + .unwrap(); + assert_eq!(affected.affected_rows, 4); + let region = engine.get_region(region_id).unwrap(); + let current = region.version_control.current(); + assert_eq!(current.committed_sequence, 4); + assert_eq!(current.last_entry_id, u64::from(!skip_wal)); + assert_eq!(current.version.flushed_entry_id, 0); + let stream = engine + .scan_to_stream(region_id, ScanRequest::default()) + .await + .unwrap(); + assert_eq!( + RecordBatches::try_collect(stream) + .await + .unwrap() + .iter() + .map(|b| b.num_rows()) + .sum::(), + 4 + ); + if flush { + test_util::flush_region(&engine, region_id, None).await; + let current = region.version_control.current(); + assert_eq!(current.last_entry_id, u64::from(!skip_wal)); + assert_eq!(current.version.flushed_sequence, 4); + assert_eq!( + engine + .region_statistic(region_id) + .unwrap() + .manifest + .data_flushed_entry_id(), + u64::from(!skip_wal) + ); + } + reopen_region(&engine, region_id, table_dir, false, HashMap::new()).await; + let stream = engine + .scan_to_stream(region_id, ScanRequest::default()) + .await + .unwrap(); + assert_eq!( + RecordBatches::try_collect(stream) + .await + .unwrap() + .iter() + .map(|b| b.num_rows()) + .sum::(), + if skip_wal && !flush { 0 } else { 4 } + ); +} + +#[tokio::test] +async fn test_bulk_skip_wal_flush_watermarks() { + let mut env = TestEnv::new().await; + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + engine + .handle_request( + region_id, + RegionRequest::Create( + CreateRequestBuilder::new() + .insert_option("memtable.type", "bulk") + .build(), + ), + ) + .await + .unwrap(); + // Establish a persisted baseline, skip one bulk write, then resume WAL. + for (round, skip_wal) in [false, true, false].into_iter().enumerate() { + let region = engine.get_region(region_id).unwrap(); + let before = region.version_control.current(); + let mut request = build_bulk_insert_request(region_id, round * 4, (round + 1) * 4); + request.skip_wal = skip_wal; + let affected = engine + .handle_request(region_id, RegionRequest::BulkInserts(request)) + .await + .unwrap(); + assert_eq!(affected.affected_rows, 4); + let expected_entry_id = before.last_entry_id + u64::from(!skip_wal); + let after = region.version_control.current(); + assert_eq!(after.last_entry_id, expected_entry_id); + assert_eq!(after.committed_sequence, before.committed_sequence + 4); + assert_eq!( + after.version.flushed_sequence, + before.version.flushed_sequence + ); + assert_eq!( + engine + .region_statistic(region_id) + .unwrap() + .manifest + .data_flushed_entry_id(), + before.version.flushed_entry_id + ); + test_util::flush_region(&engine, region_id, None).await; + let after = region.version_control.current(); + assert_eq!(after.last_entry_id, expected_entry_id); + assert_eq!(after.version.flushed_sequence, (round as u64 + 1) * 4); + assert_eq!( + engine + .region_statistic(region_id) + .unwrap() + .manifest + .data_flushed_entry_id(), + expected_entry_id + ); + } +} + /// Bulk parts fold per-part sequences (commit-unit granularity): an exact range /// keeps or drops each whole part. The same set must hold before and after flush. #[tokio::test] @@ -2612,9 +2909,8 @@ async fn test_bulk_write_sequence_not_committed_before_install() { } } -/// Non-preserving compaction must keep the physical input sequence below the -/// admission barrier. Otherwise a later row at the barrier can collide with -/// the compacted row when ordinary reads deduplicate overlapping SSTs. +/// Compaction must not allocate a new sequence that can collide with the next +/// write. Preserve the input's effective row sequence and file-level bound. #[tokio::test] async fn test_non_preserve_compaction_sequence_collision() { for flat_format in [false, true] { @@ -2684,8 +2980,7 @@ async fn test_non_preserve_compaction_sequence_collision_with_format(flat_format ); // Both current SSTs are compacted through the real picker, merger, writer, - // and manifest update. With committed/flushed sequence 2, the untrusted - // output's admission barrier is exactly 3. + // and manifest update. Both the row version and inherited file bound stay 2. engine .handle_request( region_id, @@ -2708,14 +3003,13 @@ async fn test_non_preserve_compaction_sequence_collision_with_format(flat_format assert!(!compacted.meta_ref().preserve_row_sequence); assert_eq!(1, compacted.num_rows()); assert_eq!( - Some(std::num::NonZeroU64::new(3).unwrap()), + Some(std::num::NonZeroU64::new(2).unwrap()), compacted.meta_ref().sequence, - "admission barrier is input max 2 plus one" + "compaction must inherit the input bound without allocating a sequence" ); // Hide FileMeta.sequence so this is a direct physical read, not a reader - // admission-barrier override. The output must contain physical sequence 2, - // not zero and not the output barrier 3. + // override. The output must contain physical sequence 2, not zero or 3. let mut physical_meta = compacted.meta_ref().clone(); physical_meta.sequence = None; let mut reader = region @@ -2742,10 +3036,8 @@ async fn test_non_preserve_compaction_sequence_collision_with_format(flat_format "after compaction: {after_compaction}" ); - // The next write receives physical sequence 3, equal to the compacted - // output's admission barrier. It must still win because the compacted row - // remains physically at sequence 2; an old zero/barrier encoding would - // collide here and incorrectly retain value 2. + // The next write receives sequence 3 and must win. The old zero/barrier + // encoding promoted the compacted row to 3 and could incorrectly retain 2. test_util::put_rows( &engine, region_id, @@ -2780,7 +3072,7 @@ async fn test_non_preserve_compaction_sequence_collision_with_format(flat_format .find(|file| file.meta_ref().level == 1) .expect("compacted L1 SST"); assert_eq!( - Some(std::num::NonZeroU64::new(3).unwrap()), + Some(std::num::NonZeroU64::new(2).unwrap()), compacted.meta_ref().sequence ); @@ -2791,24 +3083,27 @@ async fn test_non_preserve_compaction_sequence_collision_with_format(flat_format ); } -/// Compaction rewrites a legacy (unmarked) input as an untrusted output with -/// the region-local admission barrier. Its physical rows retain the known -/// input maximum, while exact scans skip it once C reaches that barrier and -/// fail closed while C is below it. +/// A legacy input must retain its admission boundary without gaining row trust. +/// A later flush can advance beyond manifest.committed_sequence, so compaction +/// must inherit the maximum input bound rather than sample that stale frontier. +#[rstest::rstest] #[tokio::test] -async fn test_compaction_output_non_preserve_not_laundered_from_legacy_input() { +async fn test_compaction_output_non_preserve_not_laundered_from_legacy_input( + #[values(false, true)] flat_format: bool, + #[values(false, true)] flush_after_edit: bool, +) { let mut env = TestEnv::with_prefix("test_compaction_output_not_laundered_from_legacy_input").await; // Suppress automatic edit-triggered compactions. The high TWCS trigger below // also prevents flush-triggered compaction from consuming the inputs, so the // explicit Compact below is the only compaction in flight (deterministic). - let engine = env - .create_engine(MitoConfig { - min_compaction_interval: std::time::Duration::from_secs(60 * 60), - schedule_compaction_after_edit: false, - ..Default::default() - }) - .await; + let config = MitoConfig { + default_flat_format: flat_format, + min_compaction_interval: std::time::Duration::from_secs(60 * 60), + schedule_compaction_after_edit: false, + ..Default::default() + }; + let engine = env.create_engine(config.clone()).await; let region_id = RegionId::new(1, 1); let request = CreateRequestBuilder::new() @@ -2818,6 +3113,8 @@ async fn test_compaction_output_non_preserve_not_laundered_from_legacy_input() { .insert_option("compaction.twcs.trigger_file_num", "100") .build(); let column_schemas = test_util::rows_schema(&request); + let table_dir = request.table_dir.clone(); + let region_options = request.options.clone(); engine .handle_request(region_id, RegionRequest::Create(request)) @@ -2877,6 +3174,30 @@ async fn test_compaction_output_non_preserve_not_laundered_from_legacy_input() { "seeded file should be unmarked" ); + let mut expected_sequences = vec![1, 2, 3, 4, 5, 6]; + let expected_bound = if flush_after_edit { + test_util::put_rows( + &engine, + region_id, + Rows { + schema: column_schemas, + rows: test_util::build_rows(6, 9), + }, + ) + .await; + test_util::flush_region(&engine, region_id, None).await; + expected_sequences.extend([8, 9, 10]); + 10 + } else { + 7 + }; + let manifest = region.manifest_ctx.manifest().await; + assert_eq!(Some(7), manifest.committed_sequence); + assert_eq!( + if flush_after_edit { 10 } else { 6 }, + manifest.flushed_sequence + ); + // Compact: the rewritten output must NOT be laundered back to marked. engine .handle_request( @@ -2903,25 +3224,24 @@ async fn test_compaction_output_non_preserve_not_laundered_from_legacy_input() { assert_eq!( 1, outputs.len(), - "two inputs should rewrite into one output file" + "all inputs should rewrite into one output file" ); assert!( !outputs[0].meta_ref().preserve_row_sequence, "legacy input must not be laundered into a marked output" ); - // The compacted output is sequence-less and carries the current region's - // admission barrier rather than any source-domain sequence. + // Inherit the admitted legacy bound and, if present, the newer flush bound. let barrier = outputs[0] .meta_ref() .sequence .expect("compaction output barrier") .get(); - assert_eq!(8, barrier); + assert_eq!(expected_bound, barrier); - // Reinstalling the legacy input assigns it sequence 7, so the physical - // parquet retains that known input maximum rather than encoding either - // zero or the output admission barrier 8. The manifest marker stays false. + // Reinstalling the legacy input assigns its metadata sequence 7, but the + // existing rows keep sequences 1..=6. Compaction must not rewrite them to + // the metadata bound, including when newer flushed rows join the merge. let mut sequence_meta = outputs[0].meta_ref().clone(); sequence_meta.sequence = None; let sequence_handle = FileHandle::new( @@ -2935,22 +3255,34 @@ async fn test_compaction_output_non_preserve_not_laundered_from_legacy_input() { .await .unwrap() .expect("compaction output reader"); - let batch = reader - .next_record_batch() - .await - .unwrap() - .expect("compaction output batch"); - let sequence = batch - .column(batch.num_columns() - 2) - .as_any() - .downcast_ref::() - .expect("sequence column"); - assert!( - sequence - .values() - .iter() - .all(|sequence| *sequence == barrier - 1) + let mut sequences = Vec::new(); + while let Some(batch) = reader.next_record_batch().await.unwrap() { + let sequence = batch + .column(batch.num_columns() - 2) + .as_any() + .downcast_ref::() + .expect("sequence column"); + sequences.extend_from_slice(sequence.values()); + } + sequences.sort_unstable(); + assert_eq!(expected_sequences, sequences); + + // The inherited bound and untrusted marker must survive manifest recovery. + let engine = env.reopen_engine(engine, config).await; + test_util::reopen_region(&engine, region_id, table_dir, false, region_options).await; + let version = engine.get_region(region_id).unwrap().version(); + let files: Vec<_> = version + .ssts + .levels() + .iter() + .flat_map(|level| level.files()) + .collect(); + assert_eq!(1, files.len()); + assert_eq!( + Some(expected_bound), + files[0].meta_ref().sequence.map(|s| s.get()) ); + assert!(!files[0].meta_ref().preserve_row_sequence); // A cursor before the barrier must fail closed. let err = engine @@ -2966,9 +3298,27 @@ async fn test_compaction_output_non_preserve_not_laundered_from_legacy_input() { .await .err() .expect("newer barrier must disable exact scanning"); - assert!(matches!(err, Error::SequenceRangeUnsupported { .. })); + if flush_after_edit { + // Without exact capability, the lower-bound fence is checked first. + assert!( + matches!( + err, + Error::IncrementalQueryStale { + given_seq: 9, + min_readable_seq: 10, + .. + } + ), + "unexpected error: {err:?}" + ); + } else { + assert!( + matches!(err, Error::SequenceRangeUnsupported { .. }), + "unexpected error: {err:?}" + ); + } - // Once C reaches the barrier the sequence-less file is skipped, so exact + // Once C reaches the barrier the untrusted file is skipped, so exact // capability is restored without attempting row-level filtering. let scanner = engine .scanner( diff --git a/src/mito2/src/engine/skip_wal_test.rs b/src/mito2/src/engine/skip_wal_test.rs index 88304777b2d..363318d1cf5 100644 --- a/src/mito2/src/engine/skip_wal_test.rs +++ b/src/mito2/src/engine/skip_wal_test.rs @@ -16,7 +16,10 @@ use std::sync::Arc; use std::time::Duration; use api::v1::Rows; +use common_error::ext::ErrorExt; use common_wal::options::{WAL_OPTIONS_KEY, WalOptions}; +use rstest::rstest; +use rstest_reuse::{self, apply}; use store_api::logstore::provider::Provider; use store_api::mito_engine_options::SKIP_WAL_KEY; use store_api::region_engine::{RegionEngine, RegionRole}; @@ -29,9 +32,19 @@ use store_api::storage::{RegionId, ScanRequest}; use crate::config::MitoConfig; use crate::engine::listener::AlterFlushListener; use crate::test_util::{ - CreateRequestBuilder, TestEnv, build_rows, flush_region, put_rows, rows_schema, + CreateRequestBuilder, LogStoreFactory, TestEnv, build_rows, flush_region, + kafka_log_store_factory, multiple_log_store_factories, prepare_test_for_kafka_log_store, + put_rows, raft_engine_log_store_factory, rows_schema, }; +fn set_skip_wal_request(skip_wal: bool) -> RegionRequest { + RegionRequest::Alter(RegionAlterRequest { + kind: AlterKind::SetRegionOptions { + options: vec![SetRegionOption::SkipWal(skip_wal)], + }, + }) +} + #[tokio::test] async fn test_close_region_skip_wal_with_pending_data() { test_close_region_skip_wal(true).await; @@ -76,25 +89,11 @@ async fn test_alter_skip_wal_stops_wal_and_flushes_on_close() { assert!(!before_alter.version.memtables.is_empty()); engine - .handle_request( - region_id, - RegionRequest::Alter(RegionAlterRequest { - kind: AlterKind::SetRegionOptions { - options: vec![SetRegionOption::SkipWal], - }, - }), - ) + .handle_request(region_id, set_skip_wal_request(true)) .await .unwrap(); engine - .handle_request( - region_id, - RegionRequest::Alter(RegionAlterRequest { - kind: AlterKind::SetRegionOptions { - options: vec![SetRegionOption::SkipWal], - }, - }), - ) + .handle_request(region_id, set_skip_wal_request(true)) .await .unwrap(); @@ -170,6 +169,176 @@ async fn test_alter_skip_wal_stops_wal_and_flushes_on_close() { ); } +#[apply(multiple_log_store_factories)] +async fn test_alter_skip_wal_round_trip_reuses_provider(factory: Option) { + let Some(factory) = factory else { + return; + }; + let mut env = TestEnv::with_prefix("alter-skip-wal-round-trip") + .await + .with_log_store_factory(factory.clone()); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let topic = prepare_test_for_kafka_log_store(&factory).await; + let request = CreateRequestBuilder::new().kafka_topic(topic).build(); + let schema = rows_schema(&request); + + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + let region = engine.get_region(region_id).unwrap(); + let provider = region.provider.clone(); + assert!(!matches!(&provider, Provider::Noop)); + + put_rows( + &engine, + region_id, + Rows { + schema: schema.clone(), + rows: build_rows(0, 1), + }, + ) + .await; + let last_entry_id = region.version_control.current().last_entry_id; + + engine + .handle_request(region_id, set_skip_wal_request(true)) + .await + .unwrap(); + put_rows( + &engine, + region_id, + Rows { + schema: schema.clone(), + rows: build_rows(1, 2), + }, + ) + .await; + assert_eq!( + last_entry_id, + region.version_control.current().last_entry_id + ); + + for _ in 0..2 { + engine + .handle_request(region_id, set_skip_wal_request(false)) + .await + .unwrap(); + } + assert!(!region.version().options.skip_wal); + assert_eq!(&provider, ®ion.provider); + + put_rows( + &engine, + region_id, + Rows { + schema, + rows: build_rows(2, 3), + }, + ) + .await; + assert!(region.version_control.current().last_entry_id > last_entry_id); +} + +#[apply(multiple_log_store_factories)] +async fn test_create_skipped_region_with_real_wal_can_reenable(factory: Option) { + let Some(factory) = factory else { + return; + }; + let mut env = TestEnv::with_prefix("create-skipped-with-real-wal") + .await + .with_log_store_factory(factory.clone()); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let topic = prepare_test_for_kafka_log_store(&factory).await; + let request = CreateRequestBuilder::new() + .kafka_topic(topic) + .insert_option(SKIP_WAL_KEY, "true") + .build(); + let schema = rows_schema(&request); + + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + let region = engine.get_region(region_id).unwrap(); + let provider = region.provider.clone(); + assert!(!matches!(&provider, Provider::Noop)); + assert!(region.version().options.skip_wal); + + let last_entry_id = region.version_control.current().last_entry_id; + put_rows( + &engine, + region_id, + Rows { + schema: schema.clone(), + rows: build_rows(0, 1), + }, + ) + .await; + assert_eq!( + last_entry_id, + region.version_control.current().last_entry_id + ); + + engine + .handle_request(region_id, set_skip_wal_request(false)) + .await + .unwrap(); + assert!(!region.version().options.skip_wal); + assert_eq!(&provider, ®ion.provider); + put_rows( + &engine, + region_id, + Rows { + schema, + rows: build_rows(1, 2), + }, + ) + .await; + assert!(region.version_control.current().last_entry_id > last_entry_id); +} + +#[tokio::test] +async fn test_alter_skip_wal_rejects_noop_provider() { + let mut env = TestEnv::with_prefix("alter-skip-wal-noop").await; + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let mut request = CreateRequestBuilder::new().build(); + request.options.insert( + WAL_OPTIONS_KEY.to_string(), + serde_json::to_string(&WalOptions::Noop).unwrap(), + ); + request + .options + .insert(SKIP_WAL_KEY.to_string(), "true".to_string()); + + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + let region = engine.get_region(region_id).unwrap(); + let provider = region.provider.clone(); + assert!(matches!(&provider, Provider::Noop)); + + engine + .handle_request(region_id, set_skip_wal_request(true)) + .await + .unwrap(); + assert!(region.version().options.skip_wal); + assert_eq!(&provider, ®ion.provider); + + let error = engine + .handle_request(region_id, set_skip_wal_request(false)) + .await + .unwrap_err(); + + assert!(error.output_msg().contains("uses the Noop WAL provider")); + assert!(region.version().options.skip_wal); + assert_eq!(&provider, ®ion.provider); +} + #[tokio::test] async fn test_alter_skip_wal_on_follower_survives_promotion() { let mut env = TestEnv::with_prefix("alter-skip-wal-follower").await; @@ -186,14 +355,7 @@ async fn test_alter_skip_wal_on_follower_survives_promotion() { .set_region_role(region_id, RegionRole::Follower) .unwrap(); engine - .handle_request( - region_id, - RegionRequest::Alter(RegionAlterRequest { - kind: AlterKind::SetRegionOptions { - options: vec![SetRegionOption::SkipWal], - }, - }), - ) + .handle_request(region_id, set_skip_wal_request(true)) .await .unwrap(); @@ -201,6 +363,12 @@ async fn test_alter_skip_wal_on_follower_survives_promotion() { assert!(region.is_follower()); assert!(region.version().options.skip_wal); + engine + .handle_request(region_id, set_skip_wal_request(false)) + .await + .unwrap(); + assert!(!region.version().options.skip_wal); + engine .set_region_role(region_id, RegionRole::Leader) .unwrap(); @@ -214,10 +382,7 @@ async fn test_alter_skip_wal_on_follower_survives_promotion() { }, ) .await; - assert_eq!( - last_entry_id, - region.version_control.current().last_entry_id - ); + assert!(region.version_control.current().last_entry_id > last_entry_id); } async fn test_close_region_skip_wal(insert: bool) { diff --git a/src/mito2/src/engine/sync_test.rs b/src/mito2/src/engine/sync_test.rs index 657ee868ce2..d75a9ef0313 100644 --- a/src/mito2/src/engine/sync_test.rs +++ b/src/mito2/src/engine/sync_test.rs @@ -148,7 +148,11 @@ async fn test_sync_after_flush_region_with_format(flat_format: bool) { common_telemetry::info!("Scan the region on the follower engine"); // Scan the region on the follower engine - let expected = "++\n++"; + let expected = "\ ++-------+---------+----+ +| tag_0 | field_0 | ts | ++-------+---------+----+ ++-------+---------+----+"; scan_check(&follower_engine, region_id, expected, 0, 0).await; // Returns error since the max manifest is 1 @@ -262,7 +266,11 @@ async fn test_sync_after_alter_region_with_format(flat_format: bool) { +-------+-------+---------+---------------------+"; scan_check(&engine, region_id, expected, 0, 1).await; - let expected = "++\n++"; + let expected = "\ ++-------+---------+----+ +| tag_0 | field_0 | ts | ++-------+---------+----+ ++-------+---------+----+"; scan_check(&follower_engine, region_id, expected, 0, 0).await; // Sync the region from the leader engine to the follower engine diff --git a/src/mito2/src/engine/truncate_test.rs b/src/mito2/src/engine/truncate_test.rs index c90bb960f87..61dbc062e78 100644 --- a/src/mito2/src/engine/truncate_test.rs +++ b/src/mito2/src/engine/truncate_test.rs @@ -155,7 +155,11 @@ async fn test_engine_truncate_region_basic_with_format(flat_format: bool) { let request = ScanRequest::default(); let stream = engine.scan_to_stream(region_id, request).await.unwrap(); let batches = RecordBatches::try_collect(stream).await.unwrap(); - let expected = "++\n++"; + let expected = "\ ++-------+---------+----+ +| tag_0 | field_0 | ts | ++-------+---------+----+ ++-------+---------+----+"; assert_eq!(expected, batches.pretty_print().unwrap()); } @@ -401,7 +405,11 @@ async fn test_engine_truncate_reopen_with_format(flat_format: bool) { let request = ScanRequest::default(); let stream = engine.scan_to_stream(region_id, request).await.unwrap(); let batches = RecordBatches::try_collect(stream).await.unwrap(); - let expected = "++\n++"; + let expected = "\ ++-------+---------+----+ +| tag_0 | field_0 | ts | ++-------+---------+----+ ++-------+---------+----+"; assert_eq!(expected, batches.pretty_print().unwrap()); } diff --git a/src/mito2/src/flush.rs b/src/mito2/src/flush.rs index b7c8513afc7..7b431c1b890 100644 --- a/src/mito2/src/flush.rs +++ b/src/mito2/src/flush.rs @@ -451,6 +451,7 @@ impl RegionFlushTask { let mut write_opts = WriteOptions { write_buffer_size: self.engine_config.sst_write_buffer_size, + float_field_encoding: version.options.float_field_encoding, ..Default::default() }; if let Some(row_group_size) = self.row_group_size { @@ -1728,6 +1729,7 @@ mod tests { ( SenderBulkRequest { + skip_wal: false, sender: OptionOutputTx::from(sender), region_id, request: converter.convert().unwrap(), diff --git a/src/mito2/src/memtable.rs b/src/mito2/src/memtable.rs index f6481fbcdce..bc085c3bd4e 100644 --- a/src/mito2/src/memtable.rs +++ b/src/mito2/src/memtable.rs @@ -491,6 +491,7 @@ impl MemtableBuilderProvider { BulkMemtableBuilder::new(self.write_buffer_manager.clone(), !dedup, merge_mode) .with_config(config.clone()) .with_row_group_size(options.row_group_size()) + .with_float_field_encoding(options.float_field_encoding) .with_compact_dispatcher(self.compact_dispatcher.clone()), ), Some(MemtableOptions::TimeSeries) => Arc::new(TimeSeriesMemtableBuilder::new( @@ -515,6 +516,7 @@ impl MemtableBuilderProvider { ) .with_config(self.default_bulk_memtable_config.clone()) .with_row_group_size(options.row_group_size()) + .with_float_field_encoding(options.float_field_encoding) .with_compact_dispatcher(self.compact_dispatcher.clone()); if let Some(MemtableOptions::Bulk(config)) = &options.memtable { diff --git a/src/mito2/src/memtable/bulk.rs b/src/mito2/src/memtable/bulk.rs index 8e1346b9921..b30349cc386 100644 --- a/src/mito2/src/memtable/bulk.rs +++ b/src/mito2/src/memtable/bulk.rs @@ -40,6 +40,7 @@ use rayon::prelude::*; use serde::{Deserialize, Serialize}; use serde_with::{DisplayFromStr, serde_as}; use store_api::metadata::RegionMetadataRef; +use store_api::mito_engine_options::FloatFieldEncoding; use store_api::storage::{ColumnId, FileId, RegionId, SequenceRange}; use tokio::sync::Semaphore; @@ -435,6 +436,7 @@ pub struct BulkMemtable { merge_mode: MergeMode, /// Max number of rows in a parquet row group for encoded parts. row_group_size: usize, + float_field_encoding: FloatFieldEncoding, } impl std::fmt::Debug for BulkMemtable { @@ -665,27 +667,7 @@ impl Memtable for BulkMemtable { } fn fork(&self, id: MemtableId, metadata: &RegionMetadataRef) -> MemtableRef { - Arc::new(Self { - id, - config: self.config.clone(), - parts: Arc::new(RwLock::new(BulkParts::default())), - metadata: metadata.clone(), - alloc_tracker: AllocTracker::new(self.alloc_tracker.write_buffer_manager()), - max_timestamp: AtomicI64::new(i64::MIN), - min_timestamp: AtomicI64::new(i64::MAX), - max_sequence: AtomicU64::new(0), - num_rows: AtomicUsize::new(0), - compactor: Arc::new(Mutex::new(MemtableCompactor::new( - metadata.region_id, - id, - self.config.clone(), - self.row_group_size, - ))), - compact_dispatcher: self.compact_dispatcher.clone(), - append_mode: self.append_mode, - merge_mode: self.merge_mode, - row_group_size: self.row_group_size, - }) + self.fork_inner(id, metadata) } fn compact(&self, for_flush: bool) -> Result<()> { @@ -714,6 +696,32 @@ impl Memtable for BulkMemtable { } impl BulkMemtable { + fn fork_inner(&self, id: MemtableId, metadata: &RegionMetadataRef) -> Arc { + Arc::new(Self { + id, + config: self.config.clone(), + parts: Arc::new(RwLock::new(BulkParts::default())), + metadata: metadata.clone(), + alloc_tracker: AllocTracker::new(self.alloc_tracker.write_buffer_manager()), + max_timestamp: AtomicI64::new(i64::MIN), + min_timestamp: AtomicI64::new(i64::MAX), + max_sequence: AtomicU64::new(0), + num_rows: AtomicUsize::new(0), + compactor: Arc::new(Mutex::new(MemtableCompactor::new( + metadata.region_id, + id, + self.config.clone(), + self.row_group_size, + self.float_field_encoding, + ))), + compact_dispatcher: self.compact_dispatcher.clone(), + append_mode: self.append_mode, + merge_mode: self.merge_mode, + row_group_size: self.row_group_size, + float_field_encoding: self.float_field_encoding, + }) + } + /// Creates a new BulkMemtable with the default row group size. pub fn new( id: MemtableId, @@ -747,6 +755,32 @@ impl BulkMemtable { append_mode: bool, merge_mode: MergeMode, row_group_size: usize, + ) -> Self { + Self::new_with_row_group_size_and_encoding( + id, + config, + metadata, + write_buffer_manager, + compact_dispatcher, + append_mode, + merge_mode, + row_group_size, + FloatFieldEncoding::default(), + ) + } + + /// Creates a new BulkMemtable with the given row group size and float encoding. + #[allow(clippy::too_many_arguments)] + fn new_with_row_group_size_and_encoding( + id: MemtableId, + config: BulkMemtableConfig, + metadata: RegionMetadataRef, + write_buffer_manager: Option, + compact_dispatcher: Option>, + append_mode: bool, + merge_mode: MergeMode, + row_group_size: usize, + float_field_encoding: FloatFieldEncoding, ) -> Self { let config = config.sanitize(); let region_id = metadata.region_id; @@ -765,11 +799,13 @@ impl BulkMemtable { id, config, row_group_size, + float_field_encoding, ))), compact_dispatcher, append_mode, merge_mode, row_group_size, + float_field_encoding, } } @@ -1179,6 +1215,7 @@ struct MemtableCompactor { config: BulkMemtableConfig, /// Max number of rows in a parquet row group for encoded parts. row_group_size: usize, + float_field_encoding: FloatFieldEncoding, } impl MemtableCompactor { @@ -1188,12 +1225,14 @@ impl MemtableCompactor { memtable_id: MemtableId, config: BulkMemtableConfig, row_group_size: usize, + float_field_encoding: FloatFieldEncoding, ) -> Self { Self { region_id, memtable_id, config, row_group_size, + float_field_encoding, } } @@ -1233,6 +1272,7 @@ impl MemtableCompactor { let encode_row_threshold = self.config.encode_row_threshold; let encode_bytes_threshold = self.config.encode_bytes_threshold; let row_group_size = self.row_group_size; + let float_field_encoding = self.float_field_encoding; // Merge all groups in parallel let merged_parts = collected @@ -1247,6 +1287,7 @@ impl MemtableCompactor { encode_row_threshold, encode_bytes_threshold, row_group_size, + float_field_encoding, ) }) .collect::>>>()?; @@ -1282,6 +1323,7 @@ impl MemtableCompactor { encode_row_threshold: usize, encode_bytes_threshold: usize, row_group_size: usize, + float_field_encoding: FloatFieldEncoding, ) -> Result> { if parts_to_merge.is_empty() { return Ok(None); @@ -1382,7 +1424,11 @@ impl MemtableCompactor { if estimated_total_rows > encode_row_threshold || estimated_total_bytes > encode_bytes_threshold { - let encoder = BulkPartEncoder::new(metadata.clone(), row_group_size)?; + let encoder = BulkPartEncoder::new_with_float_field_encoding( + metadata.clone(), + row_group_size, + float_field_encoding, + )?; let mut metrics = BulkPartEncodeMetrics::default(); let encoded_part = encoder.encode_record_batch_iter( boxed_iter, @@ -1508,6 +1554,7 @@ pub struct BulkMemtableBuilder { merge_mode: MergeMode, /// Max number of rows in a parquet row group for encoded parts. row_group_size: usize, + float_field_encoding: FloatFieldEncoding, } impl Default for BulkMemtableBuilder { @@ -1519,6 +1566,7 @@ impl Default for BulkMemtableBuilder { append_mode: false, merge_mode: MergeMode::default(), row_group_size: DEFAULT_ROW_GROUP_SIZE, + float_field_encoding: FloatFieldEncoding::default(), } } } @@ -1550,6 +1598,11 @@ impl BulkMemtableBuilder { self } + pub(super) fn with_float_field_encoding(mut self, encoding: FloatFieldEncoding) -> Self { + self.float_field_encoding = encoding; + self + } + /// Sets the compact dispatcher. pub fn with_compact_dispatcher(mut self, compact_dispatcher: Arc) -> Self { self.compact_dispatcher = Some(compact_dispatcher); @@ -1564,7 +1617,7 @@ impl BulkMemtableBuilder { impl MemtableBuilder for BulkMemtableBuilder { fn build(&self, id: MemtableId, metadata: &RegionMetadataRef) -> MemtableRef { - Arc::new(BulkMemtable::new_with_row_group_size( + Arc::new(BulkMemtable::new_with_row_group_size_and_encoding( id, self.config.clone(), metadata.clone(), @@ -1573,6 +1626,7 @@ impl MemtableBuilder for BulkMemtableBuilder { self.append_mode, self.merge_mode, self.row_group_size, + self.float_field_encoding, )) } @@ -1597,6 +1651,7 @@ mod tests { use datatypes::schema::ColumnSchema; use datatypes::types::json_type::{JsonNativeType, JsonObjectType}; use mito_codec::row_converter::build_primary_key_codec; + use parquet::basic::{Encoding, Type as PhysicalType}; use serde_json::json; use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder, RegionMetadataRef}; @@ -1807,7 +1862,7 @@ mod tests { let second = mock_bulk_part_with_json2_values( &metadata, vec![3000, 4000], - vec![json!({"b": 3}), json!({"b": 4})], + vec![json!({"a": 3, "b": 3}), json!({"a": 4, "b": 4})], 200, )?; let parts = vec![ @@ -1829,6 +1884,7 @@ mod tests { usize::MAX, usize::MAX, DEFAULT_ROW_GROUP_SIZE, + FloatFieldEncoding::default(), )? .unwrap(); let MergedPart::Multi(part) = merged else { @@ -1879,9 +1935,18 @@ mod tests { unreachable!() }; assert_eq!( - vec![JSON2_REMAINDER_FIELD_NAME, "a", "b"], + vec![JSON2_REMAINDER_FIELD_NAME], fields.iter().map(|x| x.name().as_str()).collect::>() ); + // Neither path is explicit in every source; keep both opaque without losing values. + assert_eq!(4, batch.num_rows()); + let json = datatypes::vectors::json::array::JsonArray::from(batch.column(0)); + let restored = json.project_to_v2(schema.field(0), &ArrowDataType::Binary)?; + let restored = datatypes::vectors::json::array::JsonArray::from(&restored); + assert_eq!(json!({"a": 1}), restored.try_get_value(0)?); + assert_eq!(json!({"a": 2}), restored.try_get_value(1)?); + assert_eq!(json!({"b": 3}), restored.try_get_value(2)?); + assert_eq!(json!({"b": 4}), restored.try_get_value(3)?); Ok(()) } @@ -2134,7 +2199,7 @@ mod tests { #[test] fn test_bulk_memtable_fork() { let metadata = metadata_for_test(); - let original_memtable = BulkMemtable::new( + let original_memtable = BulkMemtable::new_with_row_group_size_and_encoding( 333, BulkMemtableConfig::default(), metadata.clone(), @@ -2142,6 +2207,8 @@ mod tests { None, false, MergeMode::LastRow, + DEFAULT_ROW_GROUP_SIZE, + FloatFieldEncoding::ByteStreamSplit, ); let bulk_part = @@ -2150,9 +2217,25 @@ mod tests { original_memtable.write_bulk(bulk_part).unwrap(); - let forked_memtable = original_memtable.fork(444, &metadata); + let forked_memtable = original_memtable.fork_inner(444, &metadata); assert_eq!(forked_memtable.id(), 444); + assert_eq!( + FloatFieldEncoding::ByteStreamSplit, + original_memtable.float_field_encoding + ); + assert_eq!( + original_memtable.float_field_encoding, + forked_memtable.float_field_encoding + ); + assert_eq!( + original_memtable.float_field_encoding, + forked_memtable + .compactor + .lock() + .unwrap() + .float_field_encoding + ); assert!(forked_memtable.is_empty()); assert_eq!(0, forked_memtable.stats().num_rows); @@ -2771,6 +2854,42 @@ mod tests { assert!(!BulkParts::is_merge_candidate_by_size(true, 9, 8)); } + #[test] + fn test_bulk_part_encoder_preserves_byte_stream_split() -> WhateverResult<()> { + let metadata = metadata_for_test(); + let bulk_part = create_bulk_part_with_converter( + "byte_stream_split", + 0, + vec![1000, 2000], + vec![Some(10.0), Some(20.0)], + 100, + )?; + let encoder = BulkPartEncoder::new_with_float_field_encoding( + metadata, + DEFAULT_ROW_GROUP_SIZE, + FloatFieldEncoding::ByteStreamSplit, + )?; + let encoded_part = encoder.encode_part(&bulk_part)?.unwrap(); + let field_column = encoded_part + .metadata() + .parquet_metadata + .row_groups() + .iter() + .flat_map(|row_group| row_group.columns()) + .find(|column| { + column.column_path().string() == "v1" + && column.column_type() == PhysicalType::DOUBLE + }) + .expect("Float64 field should be present in encoded bulk part"); + + assert!( + field_column + .encodings() + .any(|encoding| encoding == Encoding::BYTE_STREAM_SPLIT) + ); + Ok(()) + } + #[test] fn test_encoded_part_batch_size_uses_largest_uncompressed_row_group() { const NUM_ROWS: usize = 14; diff --git a/src/mito2/src/memtable/bulk/part.rs b/src/mito2/src/memtable/bulk/part.rs index 2a0c42b7035..c3591cd4c08 100644 --- a/src/mito2/src/memtable/bulk/part.rs +++ b/src/mito2/src/memtable/bulk/part.rs @@ -53,6 +53,7 @@ use smallvec::SmallVec; use snafu::{OptionExt, ResultExt}; use store_api::codec::PrimaryKeyEncoding; use store_api::metadata::{RegionMetadata, RegionMetadataRef}; +use store_api::mito_engine_options::FloatFieldEncoding; use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME; use store_api::storage::{ColumnId, FileId, SequenceNumber, SequenceRange}; @@ -70,7 +71,7 @@ use crate::sst::SeriesEstimator; use crate::sst::index::IndexOutput; use crate::sst::parquet::flat_format::primary_key_column_index; use crate::sst::parquet::format::{PrimaryKeyArray, PrimaryKeyArrayBuilder}; -use crate::sst::parquet::{PARQUET_METADATA_KEY, SstInfo}; +use crate::sst::parquet::{PARQUET_METADATA_KEY, SstInfo, apply_float_field_encoding}; const INIT_DICT_VALUE_CAPACITY: usize = 8; @@ -1306,22 +1307,29 @@ pub struct BulkPartEncoder { impl BulkPartEncoder { pub fn new(metadata: RegionMetadataRef, row_group_size: usize) -> Result { + Self::new_with_float_field_encoding(metadata, row_group_size, FloatFieldEncoding::default()) + } + + pub(super) fn new_with_float_field_encoding( + metadata: RegionMetadataRef, + row_group_size: usize, + float_field_encoding: FloatFieldEncoding, + ) -> Result { // TODO(yingwen): Skip arrow schema if needed. let json = metadata.to_json().context(InvalidMetadataSnafu)?; let key_value_meta = parquet::file::metadata::KeyValue::new(PARQUET_METADATA_KEY.to_string(), json); // TODO(yingwen): Do we need compression? - let writer_props = Some( - WriterProperties::builder() - .set_key_value_metadata(Some(vec![key_value_meta])) - .set_write_batch_size(row_group_size) - .set_max_row_group_row_count(Some(row_group_size)) - .set_compression(Compression::ZSTD(ZstdLevel::default())) - .set_column_index_truncate_length(None) - .set_statistics_truncate_length(None) - .build(), - ); + let mut props = WriterProperties::builder() + .set_key_value_metadata(Some(vec![key_value_meta])) + .set_write_batch_size(row_group_size) + .set_max_row_group_row_count(Some(row_group_size)) + .set_compression(Compression::ZSTD(ZstdLevel::default())) + .set_column_index_truncate_length(None) + .set_statistics_truncate_length(None); + props = apply_float_field_encoding(props, &metadata, float_field_encoding); + let writer_props = Some(props.build()); Ok(Self { metadata, @@ -1558,7 +1566,7 @@ impl PruningStatistics for BatchPruningStats<'_> { None } - fn row_counts(&self, _column: &Column) -> Option { + fn row_counts(&self) -> Option { None } diff --git a/src/mito2/src/metrics.rs b/src/mito2/src/metrics.rs index 2d2decbde7c..49e91d93efd 100644 --- a/src/mito2/src/metrics.rs +++ b/src/mito2/src/metrics.rs @@ -319,6 +319,19 @@ lazy_static! { // Index metrics. lazy_static! { // Index metrics. + /// Outcomes of series-index reconciliation passes. + pub static ref SERIES_INDEX_RECONCILE_TOTAL: IntCounterVec = register_int_counter_vec!( + "greptime_mito_series_index_reconcile_total", + "series-index reconciliation passes", + &["result"], + ).unwrap(); + /// Elapsed time of series-index reconciliation phases. + pub static ref SERIES_INDEX_RECONCILE_ELAPSED: HistogramVec = register_histogram_vec!( + "greptime_mito_series_index_reconcile_elapsed", + "series-index reconciliation elapsed time", + &["phase"], + exponential_buckets(0.01, 10.0, 7).unwrap(), + ).unwrap(); /// Series and range index file operations. pub static ref SERIES_INDEX_FILE_OPERATION_TOTAL: IntCounterVec = register_int_counter_vec!( "greptime_mito_series_index_file_operation_total", diff --git a/src/mito2/src/read/flat_projection.rs b/src/mito2/src/read/flat_projection.rs index 19dc962c56e..2abf6aa48af 100644 --- a/src/mito2/src/read/flat_projection.rs +++ b/src/mito2/src/read/flat_projection.rs @@ -667,6 +667,77 @@ mod tests { ); } + #[test] + fn test_single_value_dictionary_preserves_string_type() { + use datatypes::arrow::array::{DictionaryArray, TimestampMillisecondArray, UInt32Array}; + use datatypes::arrow::datatypes::UInt32Type; + + for data_type in [ + ConcreteDataType::large_string_datatype(), + ConcreteDataType::utf8_view_datatype(), + ] { + let mut builder = RegionMetadataBuilder::new(RegionId::new(1024, 0)); + builder + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new("tag", data_type.clone(), true), + semantic_type: SemanticType::Tag, + column_id: 0, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ), + semantic_type: SemanticType::Timestamp, + column_id: 1, + }) + .primary_key(vec![0]); + let metadata = Arc::new(builder.build().unwrap()); + let mapper = FlatProjectionMapper::new(&metadata, [0, 1]).unwrap(); + + for value in [Value::from("greptime"), Value::Null] { + let mut values = data_type.create_mutable_vector(1); + values.try_push_value_ref(&value.as_value_ref()).unwrap(); + let dictionary = Arc::new( + DictionaryArray::::try_new( + UInt32Array::from(vec![0, 0, 0]), + values.to_vector().to_arrow_array(), + ) + .unwrap(), + ); + let arrays: Vec> = mapper + .batch_schema() + .iter() + .map(|(id, _)| match id { + 0 => dictionary.clone() as Arc, + 1 => Arc::new(TimestampMillisecondArray::from(vec![1, 2, 3])), + _ => unreachable!(), + }) + .collect(); + let fields = mapper + .batch_schema() + .iter() + .zip(&arrays) + .map(|((id, _), array)| { + Field::new(id.to_string(), array.data_type().clone(), true) + }) + .collect::>(); + let batch = DfRecordBatch::try_new( + Arc::new(datatypes::arrow::datatypes::Schema::new(fields)), + arrays, + ) + .unwrap(); + let output = mapper.convert(&batch, &CacheStrategy::Disabled).unwrap(); + let vector = Helper::try_into_vector(output.column(0)).unwrap(); + assert_eq!(data_type, vector.data_type()); + for row in 0..3 { + assert_eq!(value, vector.get(row)); + } + } + } + } + #[test] fn test_json_type_hint_does_not_concretize_legacy_json() { let metadata = metadata_with_legacy_json(); diff --git a/src/mito2/src/read/projection.rs b/src/mito2/src/read/projection.rs index c6d4baee3ac..fbf6a29b73d 100644 --- a/src/mito2/src/read/projection.rs +++ b/src/mito2/src/read/projection.rs @@ -19,7 +19,7 @@ use std::cmp::Ordering; use common_recordbatch::error::DataTypesSnafu; use datatypes::prelude::{ConcreteDataType, DataType}; use datatypes::value::Value; -use datatypes::vectors::VectorRef; +use datatypes::vectors::{Helper, VectorRef}; use snafu::{OptionExt, ResultExt}; use store_api::metadata::RegionMetadataRef; use store_api::storage::ColumnId; @@ -82,12 +82,21 @@ pub(crate) fn new_repeated_vector( value: &Value, num_rows: usize, ) -> common_recordbatch::error::Result { - let mut mutable_vector = data_type.create_mutable_vector(1); - mutable_vector - .try_push_value_ref(&value.as_value_ref()) - .context(DataTypesSnafu)?; - let base_vector = mutable_vector.to_vector(); - Ok(base_vector.replicate(&[num_rows])) + if let Ok(vector) = value + .try_to_scalar_value(data_type) + .and_then(|scalar| Helper::try_from_scalar_value(scalar, num_rows, Some(data_type))) + { + return Ok(vector); + } + + // Preserve extension types that cannot safely round-trip through ScalarValue. + let mut mutable_vector = data_type.create_mutable_vector(num_rows); + for _ in 0..num_rows { + mutable_vector + .try_push_value_ref(&value.as_value_ref()) + .context(DataTypesSnafu)?; + } + Ok(mutable_vector.to_vector()) } #[cfg(test)] @@ -110,6 +119,51 @@ mod tests { use crate::read::flat_projection::FlatProjectionMapper; use crate::read::read_columns::ReadColumns; + #[test] + fn test_repeated_struct_null_fields_and_json() { + use datatypes::types::{StructField, StructType}; + use datatypes::value::StructValue; + + let inner_type = StructType::from([StructField::new( + "x", + ConcreteDataType::int32_datatype(), + true, + )]); + let inner = Value::Struct(StructValue::new(vec![Value::Null], inner_type)); + let json = datatypes::json::JsonSettings::default() + .encode(serde_json::json!({"answer": 42})) + .unwrap(); + let nested_type = StructType::from([StructField::new("nested", inner.data_type(), true)]); + let json_type = StructType::from([StructField::new("json", json.data_type(), true)]); + let values = [ + inner.clone(), + Value::Struct(StructValue::new(vec![inner], nested_type)), + Value::Struct(StructValue::new(vec![], StructType::default())), + Value::Struct(StructValue::new(vec![json], json_type)), + ]; + for value in values { + let data_type = value.data_type(); + // JSON children are read back as their underlying struct values. + let expected = serde_json::Value::try_from(value.clone()).unwrap(); + for num_rows in [0, 1, 3] { + let vector = new_repeated_vector(&data_type, &value, num_rows).unwrap(); + assert_eq!(data_type, vector.data_type()); + assert_eq!( + data_type.as_arrow_type(), + *vector.to_arrow_array().data_type() + ); + assert_eq!(num_rows, vector.len()); + assert_eq!(0, vector.null_count()); + for row in 0..num_rows { + assert_eq!( + expected, + serde_json::Value::try_from(vector.get(row)).unwrap() + ); + } + } + } + } + fn print_record_batch(record_batch: RecordBatch) -> String { pretty::pretty_format_batches(&[record_batch.into_df_record_batch()]) .unwrap() diff --git a/src/mito2/src/read/scan_region.rs b/src/mito2/src/read/scan_region.rs index 073d1635cfa..ef4c44a96cc 100644 --- a/src/mito2/src/read/scan_region.rs +++ b/src/mito2/src/read/scan_region.rs @@ -1688,6 +1688,27 @@ impl ScanInput { ); } + /// Whether physical source row counts equal the rows visible to this region, + /// before applying query predicates. + pub(crate) fn total_rows_is_exact(&self) -> bool { + if self.region_partition_expr.is_none() { + return true; + } + + // Extension ranges do not expose their partition expressions. + #[cfg(feature = "enterprise")] + if !self.extension_ranges.is_empty() { + return false; + } + + // Repartition flushes existing memtables before entering staging. New + // writes are routed under the target partition rules, whereas historical + // SSTs can be shared by regions and still need partition filtering. + self.files + .iter() + .all(|file| self.should_skip_region_partition(file)) + } + pub(crate) fn total_rows(&self) -> usize { let rows_in_files: usize = self.files.iter().map(|f| f.num_rows()).sum(); let rows_in_memtables: usize = self.memtables.iter().map(|m| m.stats().num_rows()).sum(); @@ -1796,7 +1817,7 @@ impl PruningStatistics for FileLevelPruningStats { } } - fn row_counts(&self, _column: &Column) -> Option { + fn row_counts(&self) -> Option { None } @@ -1836,9 +1857,10 @@ fn pre_filter_mode(append_mode: bool, merge_mode: MergeMode) -> PreFilterMode { /// contribute a row to `(C, H]`. /// /// Unmarked local files use `FileMeta.sequence` as an admission barrier rather -/// than a row maximum. Compaction and edit assign it as `committed_sequence + 1`; -/// `C >= barrier` proves Flow has already consumed the entire file, so such a -/// file is excluded before the capability check. +/// than a row maximum. Region edits allocate this barrier; compaction inherits +/// the maximum input bound without assigning a new barrier. `C >= barrier` +/// proves Flow has already consumed the entire file, so such a file is excluded +/// before the capability check. An unknown bound cannot prove this exclusion. /// /// A foreign file is different: the parquet reader virtualizes every row to its /// target-local `FileMeta.sequence`. Consequently, a present sequence is the @@ -2324,6 +2346,43 @@ mod tests { .with_files(vec![file]) } + #[tokio::test] + async fn test_total_rows_is_exact_after_partition_filter() { + let expr = partition_col("k0").gt_eq(Value::String("foo".into())); + let other = partition_col("k0").gt_eq(Value::String("bar".into())); + for (region_expr, file_exprs, exact) in [ + (None, vec![None], true), + (None, vec![Some(expr.clone())], true), + (Some(expr.clone()), vec![], true), + (Some(expr.clone()), vec![Some(expr.clone())], true), + (Some(expr.clone()), vec![None], false), + (Some(expr.clone()), vec![Some(other)], false), + (Some(expr.clone()), vec![Some(expr), None], false), + ] { + let mut builder = + RegionMetadataBuilder::from_existing(metadata_with_primary_key(vec![0, 1], false)); + builder.partition_expr_json(region_expr.map(|expr| expr.as_json_str().unwrap())); + let metadata = Arc::new(builder.build_without_validation().unwrap()); + let files = file_exprs + .into_iter() + .map(|partition_expr| { + FileHandle::new( + FileMeta { + partition_expr, + ..Default::default() + }, + Arc::new(crate::sst::file_purger::NoopFilePurger), + ) + }) + .collect(); + let input = new_scan_input(metadata, vec![]) + .await + .with_files(files) + .build(); + assert_eq!(input.total_rows_is_exact(), exact); + } + } + /// Helper to create a timestamp millisecond literal. fn ts_lit(val: i64) -> datafusion_expr::Expr { lit(ScalarValue::TimestampMillisecond(Some(val), None)) diff --git a/src/mito2/src/read/seq_scan.rs b/src/mito2/src/read/seq_scan.rs index 29ea46f0f12..323704c31e2 100644 --- a/src/mito2/src/read/seq_scan.rs +++ b/src/mito2/src/read/seq_scan.rs @@ -76,7 +76,8 @@ impl SeqScan { pub(crate) fn new(input: ScanInput) -> Self { let mut properties = ScannerProperties::default() .with_append_mode(input.append_mode) - .with_total_rows(input.total_rows()); + .with_total_rows(input.total_rows()) + .with_total_rows_is_exact(input.append_mode && input.total_rows_is_exact()); if let Some(counters) = input.query_stat_counters.clone() { properties.set_query_stat_counters(counters); } diff --git a/src/mito2/src/read/series_scan.rs b/src/mito2/src/read/series_scan.rs index ca1d31b8804..bec48a14d8d 100644 --- a/src/mito2/src/read/series_scan.rs +++ b/src/mito2/src/read/series_scan.rs @@ -147,7 +147,8 @@ impl SeriesScan { }; let mut properties = ScannerProperties::default() .with_append_mode(input.append_mode) - .with_total_rows(input.total_rows()); + .with_total_rows(input.total_rows()) + .with_total_rows_is_exact(input.append_mode && input.total_rows_is_exact()); if let Some(counters) = input.query_stat_counters.clone() { properties.set_query_stat_counters(counters); } diff --git a/src/mito2/src/read/unordered_scan.rs b/src/mito2/src/read/unordered_scan.rs index 71e6b0a8acc..8bc4f2e0038 100644 --- a/src/mito2/src/read/unordered_scan.rs +++ b/src/mito2/src/read/unordered_scan.rs @@ -61,7 +61,8 @@ impl UnorderedScan { pub(crate) fn new(input: ScanInput) -> Self { let mut properties = ScannerProperties::default() .with_append_mode(input.append_mode) - .with_total_rows(input.total_rows()); + .with_total_rows(input.total_rows()) + .with_total_rows_is_exact(input.append_mode && input.total_rows_is_exact()); if let Some(counters) = input.query_stat_counters.clone() { properties.set_query_stat_counters(counters); } diff --git a/src/mito2/src/region.rs b/src/mito2/src/region.rs index 10195f56804..bdf18550cd6 100644 --- a/src/mito2/src/region.rs +++ b/src/mito2/src/region.rs @@ -242,7 +242,6 @@ impl StagingPartitionInfo { impl MitoRegion { /// Returns the current immutable series-index snapshot. - #[allow(dead_code)] // Used by the upcoming query integration. pub(crate) fn series_index_version(&self) -> Arc { self.series_index_version_control.current() } diff --git a/src/mito2/src/region/opener.rs b/src/mito2/src/region/opener.rs index c71b270054f..0d758f676ce 100644 --- a/src/mito2/src/region/opener.rs +++ b/src/mito2/src/region/opener.rs @@ -975,7 +975,8 @@ where region_write_ctx.push_bulk( OptionOutputTx::none(), part, - Some(bulk_sequence_from_wal) + Some(bulk_sequence_from_wal), + false ), RegionCorruptedSnafu { region_id, diff --git a/src/mito2/src/region/options.rs b/src/mito2/src/region/options.rs index 9a8d17c954c..fe603f9d1bc 100644 --- a/src/mito2/src/region/options.rs +++ b/src/mito2/src/region/options.rs @@ -32,7 +32,9 @@ use store_api::codec::PrimaryKeyEncoding; use store_api::metric_engine_consts::{ MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING, PRIMARY_KEY_ENCODING, }; -use store_api::mito_engine_options::{COMPACTION_OVERRIDE, MAX_ROW_GROUP_ROW_COUNT_LIMIT}; +use store_api::mito_engine_options::{ + COMPACTION_OVERRIDE, FloatFieldEncoding, MAX_ROW_GROUP_ROW_COUNT_LIMIT, +}; use store_api::storage::{ColumnId, RegionId}; use strum::EnumString; @@ -119,7 +121,8 @@ pub struct RegionOptions { /// the configured size; zero disables both limits. #[serde(skip_serializing_if = "Option::is_none")] pub write_buffer_size: Option, - /// Whether to preserve per-row sequence numbers through flush and compaction. + /// Whether to preserve original per-row sequence numbers when flushing. + /// Compaction always retains effective input sequences, regardless of this option. /// /// Only meaningful for append-only tables (`append_mode = true`): when enabled, /// every row keeps its exact sequence number in memtables, flushed SSTs and @@ -128,6 +131,9 @@ pub struct RegionOptions { /// sequence metadata. #[serde(default, skip_serializing_if = "is_false")] pub preserve_row_sequence: bool, + /// Encoding for direct floating-point field columns in Parquet SSTs. + #[serde(default)] + pub float_field_encoding: FloatFieldEncoding, } fn is_false(value: &bool) -> bool { @@ -287,6 +293,8 @@ impl RegionOptions { sst_format = Some(FormatType::Flat); } + let float_field_encoding = options.float_field_encoding.unwrap_or_default(); + let compaction_override_flag = options_map .get(COMPACTION_OVERRIDE) .map(|v| matches!(v.to_lowercase().as_str(), "true" | "1")) @@ -322,6 +330,7 @@ impl RegionOptions { primary_key_encoding, write_buffer_size: options.write_buffer_size, preserve_row_sequence: options.preserve_row_sequence, + float_field_encoding, }; opts.validate()?; @@ -469,6 +478,9 @@ struct RegionOptionsWithoutEnum { max_row_group_row_count: Option, #[serde_as(as = "DisplayFromStr")] preserve_row_sequence: bool, + #[serde(rename = "experimental_sst_float_field_encoding")] + #[serde_as(as = "NoneAsEmptyString")] + float_field_encoding: Option, } impl Default for RegionOptionsWithoutEnum { @@ -485,6 +497,7 @@ impl Default for RegionOptionsWithoutEnum { sst_format: options.sst_format, max_row_group_row_count: options.max_row_group_row_count, preserve_row_sequence: options.preserve_row_sequence, + float_field_encoding: Some(options.float_field_encoding), } } } @@ -628,7 +641,9 @@ mod tests { use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use common_wal::options::KafkaWalOptions; - use store_api::mito_engine_options::{SKIP_WAL_KEY, WRITE_BUFFER_SIZE_KEY}; + use store_api::mito_engine_options::{ + EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, SKIP_WAL_KEY, WRITE_BUFFER_SIZE_KEY, + }; use super::*; @@ -646,6 +661,26 @@ mod tests { assert_eq!(RegionOptions::default(), options); } + #[test] + fn test_float_field_encoding_defaults_and_parses() { + let options = RegionOptions::try_from_options(RegionId::new(0, 0), &make_map(&[])).unwrap(); + assert_eq!(FloatFieldEncoding::Default, options.float_field_encoding); + + let map = make_map(&[(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "default")]); + let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap(); + assert_eq!(FloatFieldEncoding::Default, options.float_field_encoding); + + let map = make_map(&[(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "byte_stream_split")]); + let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap(); + assert_eq!( + FloatFieldEncoding::ByteStreamSplit, + options.float_field_encoding + ); + + let map = make_map(&[(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "invalid")]); + assert!(RegionOptions::try_from_options(RegionId::new(0, 0), &map).is_err()); + } + #[test] fn test_with_ttl() { let map = make_map(&[("ttl", "7d")]); @@ -1103,6 +1138,7 @@ mod tests { primary_key_encoding: None, write_buffer_size: None, preserve_row_sequence: false, + float_field_encoding: FloatFieldEncoding::default(), }; assert_eq!(expect, options); } @@ -1161,6 +1197,7 @@ mod tests { primary_key_encoding: None, write_buffer_size: Some(ReadableSize::mb(128)), preserve_row_sequence: true, + float_field_encoding: FloatFieldEncoding::default(), }; let region_options_json_str = serde_json::to_string(&options).unwrap(); assert!(region_options_json_str.contains("preserve_row_sequence")); @@ -1172,6 +1209,7 @@ mod tests { let got: RegionOptions = serde_json::from_str(old_region_options_json_str).unwrap(); assert_eq!(None, got.write_buffer_size); assert!(!got.preserve_row_sequence); + assert_eq!(FloatFieldEncoding::Default, got.float_field_encoding); let CompactionOptions::Twcs(twcs) = got.compaction; assert_eq!(16, twcs.active_window_l1_merge_trigger); assert_eq!(8, twcs.inactive_window_l1_merge_trigger); @@ -1238,6 +1276,7 @@ mod tests { primary_key_encoding: None, write_buffer_size: None, preserve_row_sequence: false, + float_field_encoding: FloatFieldEncoding::default(), }; assert_eq!(options, got); } diff --git a/src/mito2/src/region_write_ctx.rs b/src/mito2/src/region_write_ctx.rs index f98d311a807..4bc0164cc01 100644 --- a/src/mito2/src/region_write_ctx.rs +++ b/src/mito2/src/region_write_ctx.rs @@ -318,24 +318,26 @@ impl RegionWriteCtx { sender: OptionOutputTx, mut bulk: BulkPart, sequence: Option, + skip_wal: bool, ) -> bool { if let Some(sequence) = sequence { self.next_sequence = sequence; } bulk.sequence = self.next_sequence; - let entry = match BulkWalEntry::try_from(&bulk) { - Ok(entry) => entry, - Err(e) => { - sender.send(Err(e)); - return false; - } - }; + if !skip_wal { + let entry = match BulkWalEntry::try_from(&bulk) { + Ok(entry) => entry, + Err(e) => { + sender.send(Err(e)); + return false; + } + }; + self.wal_entry.bulk_entries.push(entry); + } self.bulk_notifiers .push(WriteNotify::new(sender, bulk.num_rows())); - // Add bulk wal entry - self.wal_entry.bulk_entries.push(entry); self.next_sequence += bulk.num_rows() as u64; self.bulk_parts.push(bulk); true @@ -555,11 +557,16 @@ mod tests { #[test] fn test_request_skip_wal_preserves_sequences_and_other_writes() { // Ablate only the request flag: the workload and sequence allocation stay identical. - check_request_skip_wal_preserves_sequences_and_other_writes(false); - check_request_skip_wal_preserves_sequences_and_other_writes(true); + check_request_skip_wal_preserves_sequences_and_other_writes(false, false); + check_request_skip_wal_preserves_sequences_and_other_writes(false, true); + check_request_skip_wal_preserves_sequences_and_other_writes(true, false); + check_request_skip_wal_preserves_sequences_and_other_writes(true, true); } - fn check_request_skip_wal_preserves_sequences_and_other_writes(skip_wal: bool) { + fn check_request_skip_wal_preserves_sequences_and_other_writes( + skip_wal: bool, + bulk_skip_wal: bool, + ) { let builder = VersionControlBuilder::new(); let region_id = builder.region_id(); let version_control = Arc::new(builder.build()); @@ -600,10 +607,13 @@ mod tests { { assert_eq!(mutation.rows.as_ref().unwrap().rows.len(), notify.num_rows); } - assert!(ctx.push_bulk(OptionOutputTx::none(), new_bulk_part(), None)); + assert!(ctx.push_bulk(OptionOutputTx::none(), new_bulk_part(), None, bulk_skip_wal)); assert!(!ctx.skip_wal()); assert_eq!(ctx.next_sequence, 9); - assert_eq!(ctx.wal_entry.bulk_entries.len(), 1); + assert_eq!( + ctx.wal_entry.bulk_entries.len(), + usize::from(!bulk_skip_wal) + ); assert_eq!(ctx.bulk_parts[0].sequence, 7); let sequences: Vec<_> = ctx.wal_entry.mutations.iter().map(|m| m.sequence).collect(); assert_eq!(sequences, if skip_wal { vec![3, 6] } else { vec![1, 3, 6] }); @@ -709,8 +719,54 @@ mod tests { assert!(rx.blocking_recv().unwrap().is_err()); } + #[test] + fn test_bulk_skip_wal_preserves_sequences() { + check_bulk_skip_wal_preserves_sequences(false); + check_bulk_skip_wal_preserves_sequences(true); + } + + fn check_bulk_skip_wal_preserves_sequences(skip_wal: bool) { + let builder = VersionControlBuilder::new(); + let region_id = builder.region_id(); + let version_control = Arc::new(builder.build()); + let mut ctx = RegionWriteCtx::new( + region_id, + &version_control, + Provider::raft_engine_provider(region_id.as_u64()), + None, + ); + // Alternate policies in one context, retaining all parts for the memtable. + for skip in [skip_wal, false, skip_wal] { + assert!(ctx.push_bulk(OptionOutputTx::none(), new_bulk_part(), None, skip)); + } + assert_eq!(ctx.next_sequence, 7); + assert_eq!(ctx.bulk_notifiers.len(), 3); + assert_eq!( + ctx.bulk_parts + .iter() + .map(|p| p.sequence) + .collect::>(), + vec![1, 3, 5] + ); + let encoded = crate::wal::encoder::WalEntryEncoder::new().encode_to_vec(&ctx.wal_entry); + let decoded = WalEntry::decode(encoded.as_slice()).unwrap(); + assert_eq!( + decoded + .bulk_entries + .iter() + .map(|e| e.sequence) + .collect::>(), + if skip_wal { vec![3] } else { vec![1, 3, 5] } + ); + } + #[test] fn test_set_error_marks_bulk_notifiers_failed() { + check_set_error_marks_bulk_notifiers_failed(false); + check_set_error_marks_bulk_notifiers_failed(true); + } + + fn check_set_error_marks_bulk_notifiers_failed(skip_wal: bool) { let builder = VersionControlBuilder::new(); let region_id = builder.region_id(); let version_control = Arc::new(builder.build()); @@ -718,7 +774,9 @@ mod tests { RegionWriteCtx::new(region_id, &version_control, Provider::noop_provider(), None); let (tx, rx) = oneshot::channel(); - assert!(ctx.push_bulk(OptionOutputTx::from(tx), new_bulk_part(), None)); + assert!(ctx.push_bulk(OptionOutputTx::from(tx), new_bulk_part(), None, skip_wal)); + assert_eq!(ctx.wal_entry.bulk_entries.len(), usize::from(!skip_wal)); + assert_eq!(ctx.bulk_parts.len(), 1); ctx.set_error(Arc::new( UnexpectedSnafu { reason: "wal failed".to_string(), diff --git a/src/mito2/src/request.rs b/src/mito2/src/request.rs index 0d918e542bf..16b7eb4a6be 100644 --- a/src/mito2/src/request.rs +++ b/src/mito2/src/request.rs @@ -560,6 +560,7 @@ pub(crate) struct SenderWriteRequest { } pub(crate) struct SenderBulkRequest { + pub(crate) skip_wal: bool, pub(crate) sender: OptionOutputTx, pub(crate) region_id: RegionId, pub(crate) request: BulkPart, diff --git a/src/mito2/src/series_index.rs b/src/mito2/src/series_index.rs index 8173050f2ab..6982e4398f4 100644 --- a/src/mito2/src/series_index.rs +++ b/src/mito2/src/series_index.rs @@ -12,42 +12,37 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Series index writer and searcher. +//! Series index construction, search, and maintenance. +//! +//! Under development. Index files are currently stored on the local filesystem. -// These components are consumed by the upcoming query and maintenance integration. -#[allow(dead_code)] -mod catalog; -// Consumed by the follow-up background-maintenance integration. -#[allow(dead_code)] mod bucket; -// Consumed by the follow-up background-maintenance integration. -#[allow(dead_code)] mod builder; -#[allow(dead_code)] +mod catalog; +mod maintenance; mod purger; mod searcher; mod task; #[cfg(test)] mod tests; -#[allow(dead_code)] mod version; mod writer; use futures::stream::BoxStream; -pub use searcher::SeriesIndexSearcher; use store_api::metric_engine_consts::{ DATA_SCHEMA_TABLE_ID_COLUMN_NAME as TABLE_ID_COLUMN, DATA_SCHEMA_TSID_COLUMN_NAME as TSID_COLUMN, }; -pub use writer::{ - SeriesIndexWriter, SeriesIndexWriterMetrics, SeriesIndexWriterOptions, series_index_schema, -}; use crate::error::Result; pub(crate) use crate::series_index::catalog::{delete_catalogs, load_version_control}; pub(crate) use crate::series_index::purger::{IndexFilePurger, series_index_channel}; +pub use crate::series_index::searcher::SeriesIndexSearcher; pub(crate) use crate::series_index::task::{SeriesIndexTaskState, spawn_series_index_tasks}; pub(crate) use crate::series_index::version::{SeriesIndexVersion, SeriesIndexVersionControl}; +pub use crate::series_index::writer::{ + SeriesIndexWriter, SeriesIndexWriterMetrics, SeriesIndexWriterOptions, series_index_schema, +}; pub(crate) const MIN_TS_COLUMN: &str = "__series_min_ts"; pub(crate) const MAX_TS_COLUMN: &str = "__series_max_ts"; diff --git a/src/mito2/src/series_index/maintenance.rs b/src/mito2/src/series_index/maintenance.rs new file mode 100644 index 00000000000..4cf8ade65c0 --- /dev/null +++ b/src/mito2/src/series_index/maintenance.rs @@ -0,0 +1,319 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Worker-owned series-index reconciliation and publication. + +use std::collections::HashSet; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use common_telemetry::{debug, info}; +use object_store::ObjectStore; +use store_api::storage::RegionId; + +use crate::error::Result; +use crate::metrics::{SERIES_INDEX_RECONCILE_ELAPSED, SERIES_INDEX_RECONCILE_TOTAL}; +use crate::read::series_candidate::is_sparse_metric_metadata; +use crate::region::version::VersionRef; +use crate::region::{MitoRegionRef, RegionLeaderState, RegionRoleState}; +use crate::series_index::bucket::{ + group_files_into_series_buckets, plan_series_indexes, rounded_bucket_width, +}; +use crate::series_index::builder::{build_range_index, build_series_index}; +use crate::series_index::catalog::{ + RangeIndexCatalog, SeriesIndexCatalog, delete_catalogs, range_catalog_path, + series_catalog_path, store_catalog, +}; +use crate::series_index::purger::IndexFilePurger; +use crate::series_index::version::{SeriesIndexFileHandle, SeriesIndexVersion}; + +/// Retires newly completed series files unless their snapshot is published. +#[derive(Default)] +struct UnpublishedSeriesFiles(Vec); + +impl UnpublishedSeriesFiles { + fn disarm(&mut self) { + self.0.clear(); + } +} + +impl Drop for UnpublishedSeriesFiles { + fn drop(&mut self) { + for handle in &self.0 { + handle.mark_deleted(); + } + } +} + +#[derive(Debug, Default)] +pub(crate) struct ReconcileStats { + pub(crate) source_files: usize, + pub(crate) built_range: usize, + pub(crate) built_series: usize, + pub(crate) removed_range: usize, + pub(crate) removed_series: usize, + pub(crate) computed_buckets: usize, + pub(crate) skipped_buckets: usize, +} + +impl ReconcileStats { + fn changed(&self) -> bool { + self.built_range + self.built_series + self.removed_range + self.removed_series > 0 + } +} + +/// Reconciles indexes for one region snapshot, persists catalogs, then atomically publishes it. +pub(crate) async fn reconcile_series_indexes( + worker_id: u32, + store: ObjectStore, + region: MitoRegionRef, + requested_bucket_width: Duration, + now_ms: i64, + purger: IndexFilePurger, +) -> Result { + let total_start = Instant::now(); + // Use this snapshot throughout reconciliation, even if the region version advances. + let version = region.version_control.current().version; + if !is_sparse_metric_metadata(&version.metadata) { + SERIES_INDEX_RECONCILE_TOTAL + .with_label_values(&["noop"]) + .inc(); + return Ok(ReconcileStats::default()); + } + let build_start = Instant::now(); + let mut unpublished = UnpublishedSeriesFiles::default(); + let (next, mut stats) = build_index_version( + worker_id, + &store, + ®ion, + &version, + requested_bucket_width, + now_ms, + &purger, + &mut unpublished, + ) + .await?; + SERIES_INDEX_RECONCILE_ELAPSED + .with_label_values(&["build"]) + .observe(build_start.elapsed().as_secs_f64()); + // Persist both catalogs before making the new snapshot visible to readers. + let publish_result: Result<()> = async { + if let Some(next) = next { + persist_index_catalogs(&store, region.region_id, &next).await?; + publish_index_version(®ion, Arc::new(next)); + unpublished.disarm(); + } + Ok(()) + } + .await; + // Drop may have cleaned up while catalogs were being written, even if a write failed. + // If dropping starts after this check, the normal drop path cleans up our publication. + // This assumes the region ID is not reopened or replaced during cleanup. + if region.state() == RegionRoleState::Leader(RegionLeaderState::Dropping) { + delete_catalogs(&store, region.region_id).await; + region.series_index_version_control.mark_dropped(); + stats = ReconcileStats::default(); + } + publish_result?; + let result = if stats.changed() { "changed" } else { "noop" }; + SERIES_INDEX_RECONCILE_TOTAL + .with_label_values(&[result]) + .inc(); + SERIES_INDEX_RECONCILE_ELAPSED + .with_label_values(&["total"]) + .observe(total_start.elapsed().as_secs_f64()); + if stats.changed() { + info!( + "Reconciled series-index snapshot, worker: {worker_id}, region: {}, elapsed: {:?}, stats: {:?}", + region.region_id, + total_start.elapsed(), + stats + ); + } else { + debug!( + "Series-index reconciliation made no changes, worker: {worker_id}, region: {}", + region.region_id + ); + } + Ok(stats) +} + +/// Builds a changed snapshot, retaining reusable indexes and removing obsolete coverage. +/// Returns `None` when no range or series indexes were added or removed. +#[allow(clippy::too_many_arguments)] +async fn build_index_version( + worker_id: u32, + store: &ObjectStore, + region: &MitoRegionRef, + version: &VersionRef, + requested_bucket_width: Duration, + now_ms: i64, + purger: &IndexFilePurger, + unpublished: &mut UnpublishedSeriesFiles, +) -> Result<(Option, ReconcileStats)> { + let mut stats = ReconcileStats::default(); + let files = version + .ssts + .levels() + .iter() + .flat_map(|level| level.files()) + .cloned() + .collect::>(); + stats.source_files = files.len(); + let visible = files + .iter() + .map(|file| file.file_id().file_id()) + .collect::>(); + let current = region.series_index_version(); + // The SST purger deletes companion range files after final handle release. Prune + // metadata here using the captured SST snapshot, independently of physical deletion; + // a later region-version change is picked up by the next reconciliation. + stats.removed_range = current.range_indexes.difference(&visible).count(); + let buckets = match version.compaction_time_window { + Some(window) => rounded_bucket_width(requested_bucket_width, window) + // Successful rounding guarantees the window fits in i64; subsecond windows + // use the same one-second minimum as rounded_bucket_width. + .map(|width| { + group_files_into_series_buckets(&files, width, (window.as_secs() as i64).max(1)) + }) + .unwrap_or_default(), + None => { + debug!( + "Deferring series indexes without compaction window, worker: {worker_id}, region: {}", + region.region_id + ); + Vec::new() + } + }; + let plan = plan_series_indexes( + buckets, + current.index_buckets.clone(), + version.options.ttl, + now_ms, + ); + stats.computed_buckets = plan.computed_buckets; + stats.skipped_buckets = plan.skipped_buckets; + if plan.builds.is_empty() + && plan.expired_index_ids.is_empty() + && stats.removed_range == 0 + && current.range_indexes.len() == visible.len() + { + return Ok((None, stats)); + } + let mut range_indexes = current.range_indexes.clone(); + range_indexes.retain(|file_id| visible.contains(file_id)); + let mut series_indexes = current.series_indexes.clone(); + for id in plan + .expired_index_ids + .iter() + .chain(&plan.superseded_index_ids) + { + series_indexes.remove(id); + } + for (bucket, expected) in plan.builds { + // Complete companion indexes independently so a failed series build preserves them. + for file in &bucket.files { + if !range_indexes.contains(&file.file_id().file_id()) + && let Some(file_id) = + build_range_index(store, region, version, file.clone()).await? + { + stats.built_range += 1; + range_indexes.insert(file_id); + } + } + let series_handle = + build_series_index(store, region, version, &bucket, &expected, purger).await?; + unpublished.0.push(series_handle.clone()); + stats.built_series += 1; + series_indexes.insert(expected.index_uuid, series_handle); + } + // Cover SSTs outside planned aggregate builds, including skipped buckets. + for file in files { + let file_id = file.file_id().file_id(); + if range_indexes.contains(&file_id) { + continue; + } + if let Some(file_id) = build_range_index(store, region, version, file).await? { + stats.built_range += 1; + range_indexes.insert(file_id); + } + } + stats.removed_series = current + .series_indexes + .keys() + .filter(|id| !series_indexes.contains_key(id)) + .count(); + // Bucket reconciliation is speculative until indexes change. Recompute its coverage + // next time rather than replacing the published snapshot on a no-op pass. + if !stats.changed() { + return Ok((None, stats)); + } + let next = SeriesIndexVersion { + range_indexes, + series_indexes, + index_buckets: plan.index_buckets, + }; + Ok((Some(next), stats)) +} + +/// Writes catalogs in a stable order; the two writes are not atomic together. +async fn persist_index_catalogs( + store: &ObjectStore, + region_id: RegionId, + next: &SeriesIndexVersion, +) -> Result<()> { + let mut range_entries = next.range_indexes.iter().copied().collect::>(); + range_entries.sort_unstable_by(|left, right| left.as_bytes().cmp(right.as_bytes())); + let mut series_entries = next + .series_indexes + .values() + .map(|handle| handle.entry().clone()) + .collect::>(); + series_entries.sort_unstable_by_key(|entry| { + ( + entry.bucket_start, + entry.bucket_end, + entry.min_file_sequence, + entry.max_file_sequence, + ) + }); + store_catalog( + store, + &range_catalog_path(region_id), + &RangeIndexCatalog { + indexes: range_entries, + }, + ) + .await?; + store_catalog( + store, + &series_catalog_path(region_id), + &SeriesIndexCatalog { + indexes: series_entries, + }, + ) + .await?; + Ok(()) +} + +/// Publishes the snapshot and retires series files absent from the new version. +fn publish_index_version(region: &MitoRegionRef, next: Arc) { + let previous = region.series_index_version_control.publish(next.clone()); + for (id, handle) in &previous.series_indexes { + if !next.series_indexes.contains_key(id) { + // Purge only after readers release their retained handles. + handle.mark_deleted(); + } + } +} diff --git a/src/mito2/src/series_index/purger.rs b/src/mito2/src/series_index/purger.rs index 039ca66cccb..a3eb6d89e38 100644 --- a/src/mito2/src/series_index/purger.rs +++ b/src/mito2/src/series_index/purger.rs @@ -60,7 +60,7 @@ impl IndexFilePurger { pub(crate) fn purge(&self, request: PurgeRequest) { if let Err(error) = self.sender.send(request) { let store = self.store.clone(); - common_runtime::spawn_global(async move { + common_runtime::spawn_compact(async move { purge_file(&store, error.0).await; }); } @@ -108,3 +108,56 @@ pub(crate) fn series_index_channel( let (sender, receiver) = unbounded_channel(); (IndexFilePurger { store, sender }, receiver) } + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use object_store::ObjectStore; + use object_store::services::Memory; + use store_api::storage::{FileId, RegionId}; + + use super::*; + + #[tokio::test] + async fn test_purge_drains_queue_after_last_sender_drops() { + let store = ObjectStore::new(Memory::default()).unwrap(); + let (purger, receiver) = series_index_channel(store.clone()); + let mut paths = Vec::new(); + for _ in 0..3 { + let file_id = RegionFileId::new(RegionId::new(1, 1), FileId::random()); + let path = series_index_path(file_id.region_id(), file_id.file_id()); + store.write(&path, "index").await.unwrap(); + paths.push(path); + purger.purge(PurgeRequest { file_id }); + } + drop(purger); + tokio::time::timeout( + Duration::from_secs(10), + run_index_purge_task(0, store.clone(), receiver), + ) + .await + .unwrap(); + for path in paths { + assert!(!store.exists(&path).await.unwrap()); + } + } + + #[tokio::test] + async fn test_purge_falls_back_after_receiver_drops() { + let store = ObjectStore::new(Memory::default()).unwrap(); + let (purger, receiver) = series_index_channel(store.clone()); + drop(receiver); + let file_id = RegionFileId::new(RegionId::new(1, 1), FileId::random()); + let path = series_index_path(file_id.region_id(), file_id.file_id()); + store.write(&path, "index").await.unwrap(); + purger.purge(PurgeRequest { file_id }); + tokio::time::timeout(Duration::from_secs(10), async { + while store.exists(&path).await.unwrap() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + } +} diff --git a/src/mito2/src/series_index/task.rs b/src/mito2/src/series_index/task.rs index 76b42dd62f7..a7f263bef33 100644 --- a/src/mito2/src/series_index/task.rs +++ b/src/mito2/src/series_index/task.rs @@ -18,14 +18,18 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; -use common_telemetry::info; +use common_telemetry::{info, warn}; use object_store::ObjectStore; use tokio::sync::Notify; use tokio::sync::mpsc::UnboundedReceiver; use tokio::task::JoinHandle; use tokio::time::{Instant, MissedTickBehavior}; -use crate::series_index::purger::{PurgeRequest, run_index_purge_task}; +use crate::metrics::SERIES_INDEX_RECONCILE_TOTAL; +use crate::region::{RegionLeaderState, RegionMapRef, RegionRoleState}; +use crate::series_index::maintenance::reconcile_series_indexes; +use crate::series_index::purger::{IndexFilePurger, PurgeRequest, run_index_purge_task}; +use crate::time_provider::TimeProviderRef; /// Shared lifecycle state for a worker's series-index task. #[derive(Debug)] @@ -46,6 +50,10 @@ impl SeriesIndexTaskState { self.running.load(Ordering::Acquire) } + pub(crate) fn wake(&self) { + self.notify.notify_one(); + } + pub(crate) fn stop(&self) { self.running.store(false, Ordering::Release); // Retain a permit if maintenance has not started waiting yet. @@ -57,21 +65,35 @@ impl SeriesIndexTaskState { } } -/// Starts both tasks, detaching purge and returning the maintenance handle. +/// Starts both tasks on the compaction runtime, detaching purge and returning the maintenance handle. +#[allow(clippy::too_many_arguments)] pub(crate) fn spawn_series_index_tasks( worker_id: u32, store: ObjectStore, + regions: RegionMapRef, state: Arc, + bucket_width: Duration, + purger: IndexFilePurger, purge_receiver: UnboundedReceiver, interval: Duration, + time_provider: TimeProviderRef, ) -> JoinHandle<()> { // Snapshots may retain senders after the worker stops; purge until all senders drop. - common_runtime::spawn_global(run_index_purge_task(worker_id, store, purge_receiver)); - common_runtime::spawn_global(async move { + common_runtime::spawn_compact(run_index_purge_task( + worker_id, + store.clone(), + purge_receiver, + )); + common_runtime::spawn_compact(async move { SeriesIndexTask { worker_id, + store, + regions, + bucket_width, + purger, state, interval, + time_provider, } .run() .await; @@ -80,9 +102,14 @@ pub(crate) fn spawn_series_index_tasks( /// Periodic series-index maintenance for one region worker. struct SeriesIndexTask { + store: ObjectStore, + regions: RegionMapRef, + bucket_width: Duration, + purger: IndexFilePurger, worker_id: u32, state: Arc, interval: Duration, + time_provider: TimeProviderRef, } impl SeriesIndexTask { @@ -90,17 +117,17 @@ impl SeriesIndexTask { async fn run(mut self) { let worker_id = self.worker_id; info!("Start series-index background task, worker: {worker_id}"); - let interval = self.interval; + let interval = self.time_provider.wait_duration(self.interval); let mut timer = tokio::time::interval_at(Instant::now() + interval, interval); - timer.set_missed_tick_behavior(MissedTickBehavior::Skip); + // Schedule future ticks from a late tick rather than the original cadence. + timer.set_missed_tick_behavior(MissedTickBehavior::Delay); while self.state.is_running() { tokio::select! { _ = self.state.notified() => {} - _ = timer.tick() => { - if self.state.is_running() { - self.maintain().await; - } - } + _ = timer.tick() => {} + } + if self.state.is_running() { + self.maintain().await; } } info!("Stop series-index background task, worker: {worker_id}"); @@ -108,6 +135,94 @@ impl SeriesIndexTask { /// Runs periodic maintenance independently of incoming deletion requests. async fn maintain(&mut self) { - // TODO: Reconcile indexes and perform other periodic maintenance here. + for region in self.regions.list_regions() { + if !self.state.is_running() { + break; + } + // Best effort: the region can still change state during reconciliation. + // Local indexes can be built on followers as well as writable leaders. + if !matches!( + region.state(), + RegionRoleState::Follower | RegionRoleState::Leader(RegionLeaderState::Writable) + ) { + continue; + } + if let Err(error) = reconcile_series_indexes( + self.worker_id, + self.store.clone(), + region.clone(), + self.bucket_width, + self.time_provider.current_time_millis(), + self.purger.clone(), + ) + .await + { + SERIES_INDEX_RECONCILE_TOTAL + .with_label_values(&["failure"]) + .inc(); + warn!(error; "Failed to reconcile series indexes, worker: {}, region: {}", self.worker_id, region.region_id); + } + } + } +} + +#[cfg(test)] +mod tests { + use object_store::services::Memory; + use store_api::region_engine::RegionEngine; + + use super::*; + use crate::region::RegionMap; + use crate::series_index::catalog::{range_catalog_path, series_catalog_path}; + use crate::series_index::purger::series_index_channel; + use crate::series_index::tests::prepare_region; + use crate::test_util::TestEnv; + + #[rstest::rstest] + #[case::follower(RegionRoleState::Follower, true)] + #[case::writable(RegionRoleState::Leader(RegionLeaderState::Writable), true)] + #[case::staging(RegionRoleState::Leader(RegionLeaderState::Staging), false)] + #[case::entering_staging(RegionRoleState::Leader(RegionLeaderState::EnteringStaging), false)] + #[case::altering(RegionRoleState::Leader(RegionLeaderState::Altering), false)] + #[case::dropping(RegionRoleState::Leader(RegionLeaderState::Dropping), false)] + #[case::truncating(RegionRoleState::Leader(RegionLeaderState::Truncating), false)] + #[case::editing(RegionRoleState::Leader(RegionLeaderState::Editing), false)] + #[case::downgrading(RegionRoleState::Leader(RegionLeaderState::Downgrading), false)] + #[tokio::test] + async fn test_maintenance_region_states(#[case] role: RegionRoleState, #[case] builds: bool) { + let mut env = TestEnv::with_prefix("series-maintenance-state").await; + let (engine, region) = prepare_region(&mut env).await; + // Install the desired state without triggering the corresponding DDL. + region.switch_state_to_staging(RegionLeaderState::Writable); + region + .manifest_ctx + .exit_staging(region.region_id, role) + .unwrap(); + let store = ObjectStore::new(Memory::default()).unwrap(); + let (purger, _receiver) = series_index_channel(store.clone()); + let regions = Arc::new(RegionMap::default()); + regions.insert_region(region.clone()); + let mut task = SeriesIndexTask { + store: store.clone(), + regions, + bucket_width: Duration::from_secs(100), + purger, + worker_id: 0, + state: Arc::new(SeriesIndexTaskState::new()), + interval: Duration::from_secs(3600), + time_provider: Arc::new(crate::time_provider::StdTimeProvider), + }; + task.maintain().await; + assert_eq!( + builds, + !region.series_index_version().series_indexes.is_empty() + ); + for path in [ + range_catalog_path(region.region_id), + series_catalog_path(region.region_id), + ] { + assert_eq!(builds, store.exists(&path).await.unwrap()); + } + engine.stop().await.unwrap(); } } diff --git a/src/mito2/src/series_index/tests.rs b/src/mito2/src/series_index/tests.rs index fa65dadcddb..cbeb477e04a 100644 --- a/src/mito2/src/series_index/tests.rs +++ b/src/mito2/src/series_index/tests.rs @@ -12,25 +12,36 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Shared fixtures for series-index construction tests. +//! Behavioral coverage for series-index reconciliation. use std::sync::Arc; +use std::time::Duration; use api::v1::helper::row; use api::v1::value::ValueData; use api::v1::{ColumnDataType, Rows, SemanticType, WriteHint}; +use object_store::ObjectStore; +use object_store::layers::mock::{self, MockLayerBuilder, oio}; +use object_store::services::Memory; use store_api::codec::PrimaryKeyEncoding; use store_api::metric_engine_consts::PRIMARY_KEY_ENCODING; use store_api::region_engine::RegionEngine; use store_api::region_request::{RegionPutRequest, RegionRequest}; -use store_api::storage::RegionId; use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME; +use store_api::storage::{FileId, RegionId}; +use super::catalog::{ + delete_catalogs, load_version_control, range_catalog_path, range_index_path, + series_catalog_path, series_index_path, +}; +use super::maintenance::reconcile_series_indexes; +use super::purger::series_index_channel; use crate::config::MitoConfig; use crate::engine::MitoEngine; -use crate::region::MitoRegionRef; +use crate::region::{MitoRegionRef, RegionMap}; use crate::test_util::sst_util::{new_sparse_primary_key, sst_region_metadata_with_encoding}; use crate::test_util::{CreateRequestBuilder, TestEnv, flush_region, rows_schema}; +use crate::time_provider::StdTimeProvider; /// Builds real sparse SSTs; background maintenance is disabled so tests control publication. pub(super) async fn prepare_region(env: &mut TestEnv) -> (MitoEngine, MitoRegionRef) { @@ -66,10 +77,12 @@ async fn prepare_region_with_timestamps( "100s".to_string(), ); // Keep the source SSTs stable while tests reconcile multiple buckets. - request.options.insert( - "compaction.twcs.trigger_file_num".to_string(), - "100".to_string(), - ); + for window in ["active_window", "inactive_window"] { + request.options.insert( + format!("compaction.twcs.{window}.trigger_file_num"), + "100".to_string(), + ); + } let full_schema = rows_schema(&request); let mut pk_column = full_schema[0].clone(); pk_column.column_name = PRIMARY_KEY_COLUMN_NAME.to_string(); @@ -112,3 +125,676 @@ async fn prepare_region_with_timestamps( let region = engine.get_region(region_id).unwrap(); (engine, region) } + +struct IndexTest { + region: MitoRegionRef, + store: ObjectStore, + purger: super::purger::IndexFilePurger, +} + +impl IndexTest { + fn new( + region: MitoRegionRef, + ) -> ( + Self, + tokio::sync::mpsc::UnboundedReceiver, + ) { + let store = ObjectStore::new(Memory::default()).unwrap(); + let (purger, receiver) = series_index_channel(store.clone()); + ( + Self { + region, + store, + purger, + }, + receiver, + ) + } + + async fn reconcile( + &self, + store: ObjectStore, + ) -> crate::error::Result { + reconcile_series_indexes( + 0, + store, + self.region.clone(), + Duration::from_secs(100), + 0, + self.purger.clone(), + ) + .await + } +} + +#[tokio::test] +async fn test_reconcile_restores_and_reuses_indexes() { + let mut env = TestEnv::with_prefix("series-reconcile").await; + let (engine, region) = prepare_region(&mut env).await; + let (test, _receiver) = IndexTest::new(region.clone()); + let store = &test.store; + let purger = &test.purger; + let stats = test.reconcile(store.clone()).await.unwrap(); + assert_eq!((4, 1), (stats.built_range, stats.built_series)); + let first = region.series_index_version(); + let first_id = *first.series_indexes.keys().next().unwrap(); + + let missing_paths = [ + range_index_path( + region.region_id, + *first.range_indexes.iter().next().unwrap(), + ), + series_index_path(region.region_id, first_id), + ]; + for path in &missing_paths { + store.delete(path).await.unwrap(); + } + + // Restore catalog entries even when their index files are missing. + let restored = load_version_control(store, region.region_id, purger) + .await + .current(); + assert_eq!(first.range_indexes, restored.range_indexes); + assert_eq!(first.index_buckets, restored.index_buckets); + assert_eq!( + first.series_indexes[&first_id].entry(), + restored.series_indexes[&first_id].entry() + ); + + // Catalog changes are not reloaded, and missing index files are not repaired. + for path in [ + range_catalog_path(region.region_id), + series_catalog_path(region.region_id), + ] { + store.write(&path, "{}").await.unwrap(); + } + let stats = test.reconcile(store.clone()).await.unwrap(); + assert_eq!((0, 0), (stats.built_range, stats.built_series)); + assert!(Arc::ptr_eq(&first, ®ion.series_index_version())); + for path in [ + range_catalog_path(region.region_id), + series_catalog_path(region.region_id), + ] { + assert_eq!( + b"{}".as_slice(), + store.read(&path).await.unwrap().to_bytes().as_ref() + ); + } + for path in &missing_paths { + assert!(!store.exists(path).await.unwrap()); + } + engine.stop().await.unwrap(); +} + +#[tokio::test] +async fn test_reconcile_range_only_build_and_removal() { + let mut env = TestEnv::with_prefix("series-range-lifecycle").await; + let (engine, region) = prepare_region_with_timestamps(&mut env, &[1000]).await; + let (test, _receiver) = IndexTest::new(region.clone()); + let initial = region.series_index_version(); + let stats = test.reconcile(test.store.clone()).await.unwrap(); + assert_eq!((1, 0), (stats.built_range, stats.built_series)); + let built = region.series_index_version(); + assert!(!Arc::ptr_eq(&initial, &built)); + let files = region + .version() + .ssts + .levels() + .iter() + .flat_map(|level| level.files()) + .map(|file| file.meta_ref().clone()) + .collect(); + region.version_control.apply_edit( + Some(crate::manifest::action::RegionEdit { + files_to_remove: files, + files_to_add: Vec::new(), + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id: None, + flushed_sequence: None, + committed_sequence: None, + }), + &[], + crate::test_util::new_noop_file_purger(), + ); + let stats = test.reconcile(test.store.clone()).await.unwrap(); + assert_eq!( + (0, 0, 1), + (stats.built_range, stats.built_series, stats.removed_range) + ); + let removed = region.series_index_version(); + assert!(removed.range_indexes.is_empty()); + assert!(!Arc::ptr_eq(&built, &removed)); + let restored = load_version_control(&test.store, region.region_id, &test.purger).await; + assert!(restored.current().range_indexes.is_empty()); + test.reconcile(test.store.clone()).await.unwrap(); + assert!(Arc::ptr_eq(&removed, ®ion.series_index_version())); + engine.stop().await.unwrap(); +} + +struct FailingSeriesWriter { + inner: oio::Writer, + fail: bool, +} + +#[rstest::rstest] +#[case::series("series")] +#[case::range_catalog("range-index.json")] +#[case::series_catalog("series-index.json")] +#[tokio::test] +async fn test_reconcile_replaces_whole_bucket_after_failure(#[case] failure: &str) { + let mut env = TestEnv::with_prefix("series-replacement").await; + let (engine, region) = + prepare_region_with_timestamps(&mut env, &[1000, 2000, 3000, 4000, 5000]).await; + // Retain all source handles while hiding the newest SST for the initial build. + let sources = region.version(); + let newest = sources + .ssts + .levels() + .iter() + .flat_map(|level| level.files()) + .max_by_key(|file| file.meta_ref().sequence) + .unwrap() + .meta_ref() + .clone(); + let edit = |add| crate::manifest::action::RegionEdit { + files_to_add: if add { vec![newest.clone()] } else { vec![] }, + files_to_remove: if add { vec![] } else { vec![newest.clone()] }, + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id: None, + flushed_sequence: None, + committed_sequence: None, + }; + region.version_control.apply_edit( + Some(edit(false)), + &[], + crate::test_util::new_noop_file_purger(), + ); + let (test, mut receiver) = IndexTest::new(region.clone()); + test.reconcile(test.store.clone()).await.unwrap(); + let previous = region.series_index_version(); + let old_id = *previous.series_indexes.keys().next().unwrap(); + assert_eq!( + 4, + previous.series_indexes[&old_id] + .entry() + .source_file_ids + .len() + ); + region.version_control.apply_edit( + Some(edit(true)), + &[], + crate::test_util::new_noop_file_purger(), + ); + + let failure = failure.to_string(); + let layer = MockLayerBuilder::default() + .writer_factory(Arc::new(move |path, _, inner| { + Box::new(FailingSeriesWriter { + inner, + fail: if failure == "series" { + path.contains("/series/") + } else { + path.ends_with(&failure) + }, + }) + })) + .build() + .unwrap(); + assert!( + test.reconcile(test.store.clone().layer(layer)) + .await + .is_err() + ); + assert!(Arc::ptr_eq(&previous, ®ion.series_index_version())); + // Failed catalog publication may retire a completed replacement, never its predecessor. + while let Ok(request) = receiver.try_recv() { + assert_ne!(old_id, request.file_id.file_id()); + } + + let stats = test.reconcile(test.store.clone()).await.unwrap(); + assert_eq!((1, 1), (stats.built_series, stats.removed_series)); + let replacement = region.series_index_version(); + assert_eq!(1, replacement.series_indexes.len()); + assert!(!replacement.series_indexes.contains_key(&old_id)); + let entry = replacement.series_indexes.values().next().unwrap().entry(); + assert_eq!(5, entry.source_file_ids.len()); + assert!(entry.source_file_ids.contains(&newest.file_id)); + let restored = load_version_control(&test.store, region.region_id, &test.purger).await; + assert_eq!(replacement.index_buckets, restored.current().index_buckets); + assert_eq!(1, restored.current().series_indexes.len()); + test.reconcile(test.store.clone()).await.unwrap(); + assert!(Arc::ptr_eq(&replacement, ®ion.series_index_version())); + + // Publication retires the old file only after the last reader releases its snapshot. + let old_path = series_index_path(region.region_id, old_id); + assert!(test.store.exists(&old_path).await.unwrap()); + assert!(receiver.try_recv().is_err()); + drop(previous); + assert_eq!(old_id, receiver.try_recv().unwrap().file_id.file_id()); + assert!(receiver.try_recv().is_err()); + engine.stop().await.unwrap(); +} + +impl mock::Write for FailingSeriesWriter { + async fn write(&mut self, buffer: mock::Buffer) -> mock::Result<()> { + self.inner.write(buffer).await + } + + async fn close(&mut self) -> mock::Result { + if self.fail { + return Err(mock::Error::new( + mock::ErrorKind::Unexpected, + "injected series write failure", + )); + } + self.inner.close().await + } + + async fn abort(&mut self) -> mock::Result<()> { + self.inner.abort().await + } +} + +#[tokio::test] +async fn test_failed_series_build_keeps_completed_sst_range_indexes() { + let mut env = TestEnv::with_prefix("series-build-failure").await; + let (engine, region) = prepare_region(&mut env).await; + let (test, mut receiver) = IndexTest::new(region.clone()); + let store = &test.store; + let layer = MockLayerBuilder::default() + .writer_factory(Arc::new(|path, _, inner| { + Box::new(FailingSeriesWriter { + inner, + fail: path.contains("/series/"), + }) + })) + .build() + .unwrap(); + let failing_store = store.clone().layer(layer); + assert!(test.reconcile(failing_store).await.is_err()); + assert!(region.series_index_version().range_indexes.is_empty()); + assert!(receiver.try_recv().is_err()); + for file in region + .version() + .ssts + .levels() + .iter() + .flat_map(|level| level.files()) + { + let path = range_index_path(region.region_id, file.file_id().file_id()); + assert!(store.exists(&path).await.unwrap()); + } + // A retry can rebuild and publish the complete snapshot. + let stats = test.reconcile(store.clone()).await.unwrap(); + assert_eq!((4, 1), (stats.built_range, stats.built_series)); + engine.stop().await.unwrap(); +} + +#[tokio::test] +async fn test_reconcile_publishes_after_region_version_changes() { + for during_catalog_write in [false, true] { + let mut env = TestEnv::with_prefix("series-version-change").await; + let (engine, region) = prepare_region(&mut env).await; + let initial_version = region.version(); + let (test, mut receiver) = IndexTest::new(region.clone()); + let store = &test.store; + let purger = &test.purger; + let target_region = region.clone(); + let catalog_path = range_catalog_path(region.region_id); + let layer = MockLayerBuilder::default() + .writer_factory(Arc::new(move |path, _, inner| { + let should_advance = if during_catalog_write { + path == catalog_path + } else { + path.contains("/series/") + }; + if should_advance { + // Replace the version while reconciliation is writing its captured snapshot. + target_region + .version_control + .alter_options(target_region.version().options.clone()); + } + inner + })) + .build() + .unwrap(); + let stats = test.reconcile(store.clone().layer(layer)).await.unwrap(); + + assert!(!Arc::ptr_eq(&initial_version, ®ion.version())); + assert_eq!((4, 1), (stats.built_range, stats.built_series)); + let published = region.series_index_version(); + assert_eq!(4, published.range_indexes.len()); + assert_eq!(1, published.series_indexes.len()); + let restored = load_version_control(store, region.region_id, purger).await; + assert_eq!(published.range_indexes, restored.current().range_indexes); + let handle = published.series_indexes.values().next().unwrap(); + assert_eq!( + handle.entry(), + restored.current().series_indexes[&handle.entry().index_uuid].entry() + ); + assert!(receiver.try_recv().is_err()); + engine.stop().await.unwrap(); + } +} + +#[tokio::test] +async fn test_reconcile_cleans_up_after_region_drop() { + let mut env = TestEnv::with_prefix("series-reconcile-drop").await; + let (engine, region) = prepare_region(&mut env).await; + let (test, mut receiver) = IndexTest::new(region.clone()); + let store = test.store.clone(); + let target_region = region.clone(); + let catalog_path = range_catalog_path(region.region_id); + let layer = MockLayerBuilder::default() + .writer_factory(Arc::new(move |path, _, inner| { + if path == catalog_path { + // Simulate drop cleanup after building indexes but before persisting the + // first catalog. Neither catalog exists yet, so deletion is a no-op. + target_region + .set_dropping(crate::region::RegionLeaderState::Writable) + .unwrap(); + target_region.version_control.mark_dropped(); + target_region.series_index_version_control.mark_dropped(); + } + inner + })) + .build() + .unwrap(); + let stats = test.reconcile(store.clone().layer(layer)).await.unwrap(); + + assert_eq!((0, 0), (stats.built_range, stats.built_series)); + for path in [ + range_catalog_path(region.region_id), + series_catalog_path(region.region_id), + ] { + assert!(!store.exists(&path).await.unwrap()); + } + let published = region.series_index_version(); + assert!(published.range_indexes.is_empty()); + assert!(published.series_indexes.is_empty()); + let retired = receiver.try_recv().unwrap(); + assert!(receiver.try_recv().is_err()); + let series_path = series_index_path(region.region_id, retired.file_id.file_id()); + test.purger.purge(retired); + drop(test); + super::purger::run_index_purge_task(0, store.clone(), receiver).await; + assert!(!store.exists(&series_path).await.unwrap()); + engine.stop().await.unwrap(); +} + +#[tokio::test] +async fn test_failed_reconcile_retires_only_unpublished_series() { + use std::collections::{HashMap, HashSet}; + use std::sync::Mutex; + + use super::bucket::{group_files_into_series_buckets, plan_series_indexes}; + use super::builder::{build_range_index, build_series_index}; + use super::purger::run_index_purge_task; + use super::version::SeriesIndexVersion; + + for failure in ["series", "range", "range-index.json", "series-index.json"] { + let mut env = TestEnv::with_prefix("series-unpublished-cleanup").await; + let mut timestamps = vec![ + -99000, -98000, -97000, -96000, 1000, 2000, 3000, 4000, 101000, + ]; + if failure != "range" { + timestamps.extend([102000, 103000, 104000]); + } + let (engine, region) = prepare_region_with_timestamps(&mut env, ×tamps).await; + let (test, mut receiver) = IndexTest::new(region.clone()); + let store = &test.store; + let purger = &test.purger; + + // Publish the first bucket so failed attempts must preserve a reused handle. + let version = region.version(); + let files = version + .ssts + .levels() + .iter() + .flat_map(|level| level.files()) + .cloned() + .collect::>(); + let plan = plan_series_indexes( + group_files_into_series_buckets( + &files, + 100, + version.compaction_time_window.unwrap().as_secs() as i64, + ), + Default::default(), + None, + 0, + ); + assert_eq!(if failure == "range" { 2 } else { 3 }, plan.builds.len()); + let (bucket, entry) = &plan.builds[0]; + let mut ranges = HashSet::new(); + for file in &bucket.files { + let id = build_range_index(store, ®ion, &version, file.clone()) + .await + .unwrap() + .unwrap(); + ranges.insert(id); + } + let handle = build_series_index(store, ®ion, &version, bucket, entry, purger) + .await + .unwrap(); + let reused_path = series_index_path(region.region_id, entry.index_uuid); + region + .series_index_version_control + .publish(Arc::new(SeriesIndexVersion::new( + ranges, + HashMap::from([(entry.index_uuid, handle)]), + ))); + let previous = region.series_index_version(); + let standalone_path = files + .iter() + .find(|file| file.time_range().0 == common_time::Timestamp::new_millisecond(101000)) + .map(|file| range_index_path(region.region_id, file.file_id().file_id())) + .unwrap(); + + // Every completed unpublished output must be retired. + let attempted = Arc::new(Mutex::new(Vec::::new())); + let recorded = attempted.clone(); + let layer = MockLayerBuilder::default() + .writer_factory(Arc::new(move |path, _, inner| { + let mut paths = recorded.lock().unwrap(); + if path.contains("/series/") { + paths.push(path.to_string()); + } + let fail = match failure { + "series" => path.contains("/series/") && paths.len() == 2, + "range" => path == standalone_path, + catalog => path.ends_with(catalog), + }; + Box::new(FailingSeriesWriter { inner, fail }) + })) + .build() + .unwrap(); + let result = test.reconcile(store.clone().layer(layer)).await; + assert!( + result.is_err(), + "expected {failure} failure, got {result:?}; attempted series paths: {:?}", + attempted.lock().unwrap() + ); + assert!(Arc::ptr_eq(&previous, ®ion.series_index_version())); + + let completed = attempted + .lock() + .unwrap() + .iter() + .take(if failure.ends_with(".json") { 2 } else { 1 }) + .cloned() + .collect::>(); + assert_eq!( + if failure.ends_with(".json") { 2 } else { 1 }, + completed.len() + ); + let mut retired = HashSet::new(); + // Drain through the real purge task using a finite channel. + let (drain_purger, drain_receiver) = series_index_channel(store.clone()); + while let Ok(request) = receiver.try_recv() { + let path = series_index_path(request.file_id.region_id(), request.file_id.file_id()); + assert!(store.exists(&path).await.unwrap()); + assert!(retired.insert(path)); + drain_purger.purge(request); + } + assert_eq!(completed, retired); + drop(drain_purger); + run_index_purge_task(0, store.clone(), drain_receiver).await; + let attempted_paths = attempted.lock().unwrap().clone(); + for path in attempted_paths { + assert!(!store.exists(&path).await.unwrap()); + } + assert!(store.exists(&reused_path).await.unwrap()); + + let stats = test.reconcile(store.clone()).await.unwrap(); + assert_eq!(if failure == "range" { 1 } else { 2 }, stats.built_series); + let published = region.series_index_version(); + for handle in published.series_indexes.values() { + assert!( + store + .exists(&series_index_path( + region.region_id, + handle.entry().index_uuid + )) + .await + .unwrap() + ); + } + // Release all snapshots without retiring them: neither reused nor published files + // should have been marked deleted by the cleanup guard. + region.series_index_version_control.publish(Arc::default()); + drop(published); + drop(previous); + assert!(receiver.try_recv().is_err()); + engine.stop().await.unwrap(); + } +} + +/// Waits for a background state change without relying on exact scheduling delays. +async fn wait_for(mut ready: impl FnMut() -> bool) { + tokio::time::timeout(Duration::from_secs(10), async { + while !ready() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn test_maintenance_wakeup_and_timer() { + let mut env = TestEnv::with_prefix("series-task").await; + let (engine, region) = prepare_region(&mut env).await; + let mut options = region.version().options.clone(); + options.ttl = Some(common_time::TimeToLive::Duration(Duration::from_secs(100))); + region.version_control.alter_options(options); + let store = ObjectStore::new(Memory::default()).unwrap(); + let (purger, receiver) = series_index_channel(store.clone()); + let state = Arc::new(super::task::SeriesIndexTaskState::new()); + let clock = Arc::new(crate::time_provider::mock::MockTimeProvider::new(0)); + // A notification issued before the task starts must also trigger maintenance. + state.wake(); + let regions = Arc::new(RegionMap::default()); + regions.insert_region(region.clone()); + let task = super::task::spawn_series_index_tasks( + 0, + store, + regions, + state.clone(), + Duration::from_secs(100), + purger, + receiver, + Duration::from_secs(3600), + clock.clone(), + ); + wait_for(|| !region.series_index_version().series_indexes.is_empty()).await; + // Advance event time; periodic maintenance must expire coverage without a wakeup. + clock.set_now(201_000); + wait_for(|| region.series_index_version().series_indexes.is_empty()).await; + state.stop(); + task.await.unwrap(); + engine.stop().await.unwrap(); +} + +#[tokio::test] +async fn test_drop_catalogs_and_retained_snapshot_after_task_stop() { + use std::collections::BTreeMap; + + use common_time::Timestamp; + + use super::catalog::{SeriesIndexCatalog, SeriesIndexEntry, WindowSequence, store_catalog}; + + let store = ObjectStore::new(Memory::default()).unwrap(); + let region_id = RegionId::new(1, 1); + let entry = SeriesIndexEntry { + index_uuid: FileId::random(), + bucket_start: Timestamp::new_second(0), + bucket_end: Timestamp::new_second(100), + source_file_ids: vec![FileId::random()], + min_file_sequence: 1, + max_file_sequence: 2, + compaction_window_secs: 10, + window_sequences: BTreeMap::from([( + 0, + WindowSequence { + start: 0, + end: 100, + max_sequence: 2, + }, + )]), + }; + let path = series_index_path(region_id, entry.index_uuid); + store.write(&path, "index").await.unwrap(); + store_catalog( + &store, + &series_catalog_path(region_id), + &SeriesIndexCatalog { + indexes: vec![entry.clone()], + }, + ) + .await + .unwrap(); + store_catalog( + &store, + &range_catalog_path(region_id), + &super::catalog::RangeIndexCatalog::default(), + ) + .await + .unwrap(); + let (purger, receiver) = series_index_channel(store.clone()); + let control = load_version_control(&store, region_id, &purger).await; + let snapshot = control.current(); + assert_eq!(&entry, snapshot.series_indexes[&entry.index_uuid].entry()); + let state = Arc::new(super::task::SeriesIndexTaskState::new()); + state.stop(); + super::task::spawn_series_index_tasks( + 0, + store.clone(), + Arc::new(RegionMap::default()), + state, + Duration::from_secs(100), + purger, + receiver, + Duration::from_secs(3600), + Arc::new(StdTimeProvider), + ) + .await + .unwrap(); + delete_catalogs(&store, region_id).await; + // Removing already absent catalogs is harmless. + delete_catalogs(&store, region_id).await; + assert!(!store.exists(&range_catalog_path(region_id)).await.unwrap()); + assert!(!store.exists(&series_catalog_path(region_id)).await.unwrap()); + control.mark_dropped(); + assert!(store.exists(&path).await.unwrap()); + drop(snapshot); + tokio::time::timeout(Duration::from_secs(10), async { + while store.exists(&path).await.unwrap() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} diff --git a/src/mito2/src/sst.rs b/src/mito2/src/sst.rs index ba670ef7348..e53c28e54c7 100644 --- a/src/mito2/src/sst.rs +++ b/src/mito2/src/sst.rs @@ -1079,9 +1079,7 @@ mod tests { &builder.parquet_schema().root_schema().get_fields()[0].get_fields()[0]; assert_eq!( parquet_remainder.get_basic_info().logical_type_ref(), - Some(&LogicalType::Variant { - specification_version: None, - }) + Some(&LogicalType::variant(None)) ); let ArrowDataType::Struct(children) = builder.schema().field_with_name("data")?.data_type() diff --git a/src/mito2/src/sst/file.rs b/src/mito2/src/sst/file.rs index b604275ce93..9702477b07a 100644 --- a/src/mito2/src/sst/file.rs +++ b/src/mito2/src/sst/file.rs @@ -243,10 +243,13 @@ pub struct FileMeta { /// the default value `0` doesn't means the file doesn't contains any rows, /// but instead means the number of rows is unknown. pub num_row_groups: u64, - /// Sequence in this file. + /// File-level sequence bound or admission marker in the target region. /// - /// This sequence is the only sequence in this file. And it's retrieved from the max - /// sequence of the rows on generating this file. + /// Flush records the maximum input row sequence. Compaction outputs inherit + /// the maximum input bound, or remain unknown if any input bound is unknown, + /// independently of per-row sequence trust. This does not imply that every + /// physical row has this sequence. + /// Readers also use it to normalize foreign and legacy all-zero files. pub sequence: Option, /// Partition expression from the region metadata when the file is created. /// @@ -284,6 +287,8 @@ pub struct FileMeta { pub primary_key_max: Option, /// Whether the file preserves per-row sequence numbers usable for exact /// row-level sequence filtering. + /// Merely retaining physical input sequences during compaction does not + /// restore this capability for untrusted inputs. #[serde(default, skip_serializing_if = "is_false")] pub preserve_row_sequence: bool, } diff --git a/src/mito2/src/sst/file_purger.rs b/src/mito2/src/sst/file_purger.rs index 935c0272580..5ff88c0874b 100644 --- a/src/mito2/src/sst/file_purger.rs +++ b/src/mito2/src/sst/file_purger.rs @@ -288,8 +288,14 @@ mod tests { use crate::sst::index::puffin_manager::PuffinManagerFactory; use crate::sst::location; + #[rstest::rstest] + #[case(false)] + #[case(true)] #[tokio::test] - async fn test_file_purge() { + async fn test_range_index_purge_on_handle_release( + #[case] gc_enabled: bool, + #[values(false, true)] is_delete: bool, + ) { common_telemetry::init_default_ut_logging(); let dir = create_temp_dir("file-purge"); @@ -320,7 +326,19 @@ mod tests { let scheduler = Arc::new(LocalScheduler::new(3)); - let file_purger = Arc::new(LocalFilePurger::new(scheduler.clone(), layer, None)); + let index_store = ObjectStore::new(object_store::services::Memory::default()).unwrap(); + let owner = RegionId::new(9, 1); + let index_path = crate::sst::range_index::range_index_path(owner, sst_file_id.file_id()); + index_store.write(&index_path, "range index").await.unwrap(); + let file_purger = create_file_purger( + gc_enabled, + PathType::Bare, + scheduler.clone(), + layer, + None, + Arc::new(crate::sst::file_ref::FileReferenceManager::new(None)), + Some(RangeIndexDeleter::new(index_store.clone(), owner)), + ); { let handle = FileHandle::new( @@ -344,13 +362,22 @@ mod tests { }, file_purger, ); - // mark file as deleted and drop the handle, we expect the file is deleted. - handle.mark_deleted(); + if is_delete { + handle.mark_deleted(); + } + let reader = handle.clone(); + drop(handle); + assert!(index_store.exists(&index_path).await.unwrap()); + drop(reader); } scheduler.stop(true).await.unwrap(); - assert!(!object_store.exists(&path).await.unwrap()); + assert_eq!( + object_store.exists(&path).await.unwrap(), + gc_enabled || !is_delete + ); + assert_eq!(index_store.exists(&index_path).await.unwrap(), !is_delete); } #[tokio::test] diff --git a/src/mito2/src/sst/index.rs b/src/mito2/src/sst/index.rs index 268b8387397..c8fcd6fbc7e 100644 --- a/src/mito2/src/sst/index.rs +++ b/src/mito2/src/sst/index.rs @@ -17,7 +17,10 @@ pub(crate) mod fulltext_index; mod indexer; pub mod intermediate; pub(crate) mod inverted_index; +mod primary_key; pub mod puffin_manager; +#[cfg(test)] +mod sparse_test; mod statistics; pub(crate) mod store; #[cfg(feature = "vector_index")] @@ -30,14 +33,11 @@ use std::sync::Arc; use bloom_filter::creator::BloomFilterIndexer; use common_telemetry::{debug, error, info, warn}; -use datatypes::arrow::array::BinaryArray; use datatypes::arrow::record_batch::RecordBatch; -use mito_codec::index::IndexValuesCodec; -use mito_codec::row_converter::CompositeValues; use object_store::ObjectStore; use puffin_manager::SstPuffinManager; use smallvec::{SmallVec, smallvec}; -use snafu::{OptionExt, ResultExt}; +use snafu::ResultExt; use statistics::{ByteCount, RowCount}; use store_api::metadata::RegionMetadataRef; use store_api::storage::{ColumnId, FileId, RegionId}; @@ -54,8 +54,8 @@ use crate::cache::{CacheManagerRef, CacheStrategy}; use crate::config::VectorIndexConfig; use crate::config::{BloomFilterConfig, FulltextIndexConfig, InvertedIndexConfig}; use crate::error::{ - BuildIndexAsyncSnafu, DecodeSnafu, Error, InvalidRecordBatchSnafu, RegionClosedSnafu, - RegionDroppedSnafu, RegionTruncatedSnafu, Result, + BuildIndexAsyncSnafu, Error, RegionClosedSnafu, RegionDroppedSnafu, RegionTruncatedSnafu, + Result, }; use crate::metrics::{ INDEX_ARTIFACT_CLEANUP_FAILURE_TOTAL, INDEX_CREATE_MEMORY_USAGE, INDEX_PUBLICATION_STALE_TOTAL, @@ -79,8 +79,6 @@ use crate::sst::index::fulltext_index::creator::FulltextIndexer; use crate::sst::index::intermediate::IntermediateManager; use crate::sst::index::inverted_index::creator::InvertedIndexer; use crate::sst::parquet::SstInfo; -use crate::sst::parquet::flat_format::primary_key_column_index; -use crate::sst::parquet::format::PrimaryKeyArray; use crate::worker::WorkerListener; pub(crate) const TYPE_INVERTED_INDEX: &str = "inverted_index"; @@ -1450,55 +1448,6 @@ impl IndexBuildScheduler { } } -/// Decodes primary keys from a flat format RecordBatch. -/// Returns a list of (decoded_pk_value, count) tuples where count is the number of occurrences. -pub(crate) fn decode_primary_keys_with_counts( - batch: &RecordBatch, - codec: &IndexValuesCodec, -) -> Result> { - let primary_key_index = primary_key_column_index(batch.num_columns()); - let pk_dict_array = batch - .column(primary_key_index) - .as_any() - .downcast_ref::() - .context(InvalidRecordBatchSnafu { - reason: "Primary key column is not a dictionary array", - })?; - let pk_values_array = pk_dict_array - .values() - .as_any() - .downcast_ref::() - .context(InvalidRecordBatchSnafu { - reason: "Primary key values are not binary array", - })?; - let keys = pk_dict_array.keys(); - - // Decodes primary keys and count consecutive occurrences - let mut result: Vec<(CompositeValues, usize)> = Vec::new(); - let mut prev_key: Option = None; - - let pk_indices = keys.values(); - for ¤t_key in pk_indices.iter().take(keys.len()) { - // Checks if current key is the same as previous key - if let Some(prev) = prev_key - && prev == current_key - { - // Safety: We already have a key in the result vector. - result.last_mut().unwrap().1 += 1; - continue; - } - - // New key, decodes it. - let pk_bytes = pk_values_array.value(current_key as usize); - let decoded_value = codec.decoder().decode(pk_bytes).context(DecodeSnafu)?; - - result.push((decoded_value, 1)); - prev_key = Some(current_key); - } - - Ok(result) -} - #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/src/mito2/src/sst/index/bloom_filter/creator.rs b/src/mito2/src/sst/index/bloom_filter/creator.rs index 7828cdb9fdd..cfdd751bf7f 100644 --- a/src/mito2/src/sst/index/bloom_filter/creator.rs +++ b/src/mito2/src/sst/index/bloom_filter/creator.rs @@ -24,8 +24,10 @@ use datatypes::vectors::Helper; use index::bloom_filter::creator::BloomFilterCreator; use index::target::IndexTarget; use mito_codec::index::{IndexValueCodec, IndexValuesCodec}; -use mito_codec::row_converter::{CompositeValues, SortField}; +use mito_codec::row_converter::sparse::SparsePrimaryKeyView; +use mito_codec::row_converter::{SortField, SparseOffsetsCache}; use puffin::puffin_manager::{PuffinWriter, PutOptions}; +use smallvec::SmallVec; use snafu::{ResultExt, ensure}; use store_api::codec::PrimaryKeyEncoding; use store_api::metadata::RegionMetadataRef; @@ -33,17 +35,18 @@ use store_api::storage::{ColumnId, FileId}; use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt}; use crate::error::{ - BiErrorsSnafu, BloomFilterFinishSnafu, EncodeSnafu, IndexOptionsSnafu, + BiErrorsSnafu, BloomFilterFinishSnafu, DecodeSnafu, EncodeSnafu, IndexOptionsSnafu, OperateAbortedIndexSnafu, PuffinAddBlobSnafu, PushBloomFilterValueSnafu, Result, }; use crate::read::Batch; +use crate::sst::index::TYPE_BLOOM_FILTER_INDEX; use crate::sst::index::bloom_filter::INDEX_BLOB_TYPE; use crate::sst::index::intermediate::{ IntermediateLocation, IntermediateManager, TempFileProvider, }; +use crate::sst::index::primary_key::PrimaryKeyRuns; use crate::sst::index::puffin_manager::SstPuffinWriter; use crate::sst::index::statistics::{ByteCount, RowCount, Statistics}; -use crate::sst::index::{TYPE_BLOOM_FILTER_INDEX, decode_primary_keys_with_counts}; /// The buffer size for the pipe used to send index data to the puffin blob. const PIPE_BUFFER_SIZE_FOR_SENDING_BLOB: usize = 8192; @@ -58,6 +61,9 @@ pub struct BloomFilterIndexer { /// Codec for decoding primary keys. codec: IndexValuesCodec, + /// Scratch storage for extracting indexed tags from sparse primary keys. + pk_offsets: SparseOffsetsCache, + value_buf: Vec, /// Whether the indexing process has been aborted. aborted: bool, @@ -124,6 +130,8 @@ impl BloomFilterIndexer { creators, temp_file_provider, codec, + pk_offsets: SparseOffsetsCache::new(), + value_buf: Vec::new(), aborted: false, stats: Statistics::new(TYPE_BLOOM_FILTER_INDEX), global_memory_usage, @@ -185,7 +193,7 @@ impl BloomFilterIndexer { /// Returns the number of rows and bytes written. /// /// TODO(zhongzc): duplicate with `mito2::sst::index::inverted_index::creator::InvertedIndexCreator` - pub async fn finish( + pub(crate) async fn finish( &mut self, puffin_writer: &mut SstPuffinWriter, ) -> Result<(RowCount, ByteCount)> { @@ -292,7 +300,8 @@ impl BloomFilterIndexer { guard.inc_row_count(n); let is_sparse = self.metadata.primary_key_encoding == PrimaryKeyEncoding::Sparse; - let mut decoded_pks: Option> = None; + let mut sparse_columns: SmallVec<[(ColumnId, &mut BloomFilterCreator); 8]> = + SmallVec::new(); for (col_id, creator) in &mut self.creators { // Safety: `creators` are created from the metadata so it won't be None. @@ -321,45 +330,8 @@ impl BloomFilterIndexer { .context(PushBloomFilterValueSnafu)?; } } else if is_sparse && column_meta.semantic_type == SemanticType::Tag { - // Column not found in batch, tries to decode from primary keys for sparse encoding. - if decoded_pks.is_none() { - decoded_pks = Some(decode_primary_keys_with_counts(batch, &self.codec)?); - } - - let pk_values_with_counts = decoded_pks.as_ref().unwrap(); - let Some(col_info) = self.codec.pk_col_info(*col_id) else { - debug!( - "Column {} not found in primary key during building bloom filter index", - column_name - ); - continue; - }; - let pk_index = col_info.idx; - let field = &col_info.field; - for (decoded, count) in pk_values_with_counts { - let value = match decoded { - CompositeValues::Dense(dense) => dense.get(pk_index).map(|v| &v.1), - CompositeValues::Sparse(sparse) => sparse.get(col_id), - }; - - let elems = value - .filter(|v| !v.is_null()) - .map(|v| { - let mut buf = vec![]; - IndexValueCodec::encode_nonnull_value( - v.as_value_ref(), - field, - &mut buf, - ) - .context(EncodeSnafu)?; - Ok(buf) - }) - .transpose()?; - - creator - .push_n_row_elems(*count, elems) - .await - .context(PushBloomFilterValueSnafu)?; + if self.codec.pk_col_info(*col_id).is_some() { + sparse_columns.push((*col_id, creator)); } } else { debug!( @@ -369,6 +341,25 @@ impl BloomFilterIndexer { } } + if !sparse_columns.is_empty() { + for (pk, count) in PrimaryKeyRuns::try_new(batch)? { + let mut view = + SparsePrimaryKeyView::new(pk, &mut self.pk_offsets).context(DecodeSnafu)?; + for (col_id, creator) in &mut sparse_columns { + let value = IndexValueCodec::encode_sparse_value( + &mut view, + *col_id, + &mut self.value_buf, + ) + .context(DecodeSnafu)?; + creator + .push_n_row_elem(count, value) + .await + .context(PushBloomFilterValueSnafu)?; + } + } + } + Ok(()) } diff --git a/src/mito2/src/sst/index/inverted_index/creator.rs b/src/mito2/src/sst/index/inverted_index/creator.rs index 9cad6fa84ee..0243016fc7a 100644 --- a/src/mito2/src/sst/index/inverted_index/creator.rs +++ b/src/mito2/src/sst/index/inverted_index/creator.rs @@ -27,8 +27,10 @@ use index::inverted_index::create::sort_create::SortIndexCreator; use index::inverted_index::format::writer::InvertedIndexBlobWriter; use index::target::IndexTarget; use mito_codec::index::{IndexValueCodec, IndexValuesCodec}; -use mito_codec::row_converter::{CompositeValues, SortField}; +use mito_codec::row_converter::sparse::SparsePrimaryKeyView; +use mito_codec::row_converter::{SortField, SparseOffsetsCache}; use puffin::puffin_manager::{PuffinWriter, PutOptions}; +use smallvec::SmallVec; use snafu::{ResultExt, ensure}; use store_api::codec::PrimaryKeyEncoding; use store_api::metadata::RegionMetadataRef; @@ -37,17 +39,18 @@ use tokio::io::duplex; use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt}; use crate::error::{ - BiErrorsSnafu, EncodeSnafu, IndexFinishSnafu, OperateAbortedIndexSnafu, PuffinAddBlobSnafu, - PushIndexValueSnafu, Result, + BiErrorsSnafu, DecodeSnafu, EncodeSnafu, IndexFinishSnafu, OperateAbortedIndexSnafu, + PuffinAddBlobSnafu, PushIndexValueSnafu, Result, }; use crate::read::Batch; +use crate::sst::index::TYPE_INVERTED_INDEX; use crate::sst::index::intermediate::{ IntermediateLocation, IntermediateManager, TempFileProvider, }; use crate::sst::index::inverted_index::INDEX_BLOB_TYPE; +use crate::sst::index::primary_key::PrimaryKeyRuns; use crate::sst::index::puffin_manager::SstPuffinWriter; use crate::sst::index::statistics::{ByteCount, RowCount, Statistics}; -use crate::sst::index::{TYPE_INVERTED_INDEX, decode_primary_keys_with_counts}; /// The minimum memory usage threshold for one column. const MIN_MEMORY_USAGE_THRESHOLD_PER_COLUMN: usize = 1024 * 1024; // 1MB @@ -66,6 +69,8 @@ pub struct InvertedIndexer { codec: IndexValuesCodec, /// Reusable buffer for encoding index values. value_buf: Vec, + /// Scratch offsets shared by indexed tags of one sparse primary key. + pk_offsets: SparseOffsetsCache, /// Statistics of index creation. stats: Statistics, @@ -124,6 +129,7 @@ impl InvertedIndexer { index_creator, temp_file_provider, value_buf: vec![], + pk_offsets: SparseOffsetsCache::new(), stats: Statistics::new(TYPE_INVERTED_INDEX), aborted: false, memory_usage, @@ -173,7 +179,7 @@ impl InvertedIndexer { guard.inc_row_count(batch.num_rows()); let is_sparse = self.metadata.primary_key_encoding == PrimaryKeyEncoding::Sparse; - let mut decoded_pks: Option> = None; + let mut sparse_columns: SmallVec<[(ColumnId, &str); 8]> = SmallVec::new(); for (col_id, target_key) in &self.indexed_column_ids { let Some(column_meta) = self.metadata.column_by_id(*col_id) else { @@ -213,45 +219,8 @@ impl InvertedIndexer { } } } else if is_sparse && column_meta.semantic_type == SemanticType::Tag { - // Column not found in batch, tries to decode from primary keys for sparse encoding. - if decoded_pks.is_none() { - decoded_pks = Some(decode_primary_keys_with_counts(batch, &self.codec)?); - } - - let pk_values_with_counts = decoded_pks.as_ref().unwrap(); - let Some(col_info) = self.codec.pk_col_info(*col_id) else { - debug!( - "Column {} not found in primary key during building bloom filter index", - column_name - ); - continue; - }; - let pk_index = col_info.idx; - let field = &col_info.field; - for (decoded, count) in pk_values_with_counts { - let value = match decoded { - CompositeValues::Dense(dense) => dense.get(pk_index).map(|v| &v.1), - CompositeValues::Sparse(sparse) => sparse.get(col_id), - }; - - let elem = value - .filter(|v| !v.is_null()) - .map(|v| { - self.value_buf.clear(); - IndexValueCodec::encode_nonnull_value( - v.as_value_ref(), - field, - &mut self.value_buf, - ) - .context(EncodeSnafu)?; - Ok(self.value_buf.as_slice()) - }) - .transpose()?; - - self.index_creator - .push_with_name_n(target_key, elem, *count) - .await - .context(PushIndexValueSnafu)?; + if self.codec.pk_col_info(*col_id).is_some() { + sparse_columns.push((*col_id, target_key)); } } else { debug!( @@ -261,12 +230,32 @@ impl InvertedIndexer { } } + if !sparse_columns.is_empty() { + for (pk, count) in PrimaryKeyRuns::try_new(batch)? { + let mut view = + SparsePrimaryKeyView::new(pk, &mut self.pk_offsets).context(DecodeSnafu)?; + // Visit all needed tags before moving to the next PK so offset discovery is shared. + for &(col_id, target_key) in &sparse_columns { + let value = IndexValueCodec::encode_sparse_value( + &mut view, + col_id, + &mut self.value_buf, + ) + .context(DecodeSnafu)?; + self.index_creator + .push_with_name_n(target_key, value, count) + .await + .context(PushIndexValueSnafu)?; + } + } + } + Ok(()) } /// Finishes index creation and cleans up garbage. /// Returns the number of rows and bytes written. - pub async fn finish( + pub(crate) async fn finish( &mut self, puffin_writer: &mut SstPuffinWriter, ) -> Result<(RowCount, ByteCount)> { diff --git a/src/mito2/src/sst/index/primary_key.rs b/src/mito2/src/sst/index/primary_key.rs new file mode 100644 index 00000000000..4724ae44e6f --- /dev/null +++ b/src/mito2/src/sst/index/primary_key.rs @@ -0,0 +1,108 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use datatypes::arrow::array::{Array, BinaryArray}; +use datatypes::arrow::record_batch::RecordBatch; +use snafu::{OptionExt, ensure}; + +use crate::error::{InvalidRecordBatchSnafu, Result}; +use crate::sst::parquet::flat_format::primary_key_column_index; +use crate::sst::parquet::format::PrimaryKeyArray; + +/// Iterates consecutive dictionary-key runs, preserving their row order without decoding. +pub(crate) struct PrimaryKeyRuns<'a> { + keys: &'a [u32], + values: &'a BinaryArray, +} + +impl<'a> PrimaryKeyRuns<'a> { + pub(crate) fn try_new(batch: &'a RecordBatch) -> Result { + let pk = batch + .column(primary_key_column_index(batch.num_columns())) + .as_any() + .downcast_ref::() + .context(InvalidRecordBatchSnafu { + reason: "Primary key column is not a dictionary array", + })?; + let values = pk.values().as_any().downcast_ref::().context( + InvalidRecordBatchSnafu { + reason: "Primary key values are not binary array", + }, + )?; + ensure!( + pk.null_count() == 0 && values.null_count() == 0, + InvalidRecordBatchSnafu { + reason: "Primary keys must not be null" + } + ); + Ok(Self { + keys: pk.keys().values(), + values, + }) + } +} + +impl<'a> Iterator for PrimaryKeyRuns<'a> { + type Item = (&'a [u8], usize); + + fn next(&mut self) -> Option { + let &key = self.keys.first()?; + let count = self + .keys + .iter() + .take_while(|&¤t| current == key) + .count(); + self.keys = &self.keys[count..]; + Some((self.values.value(key as usize), count)) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use datatypes::arrow::array::{ArrayRef, BinaryDictionaryBuilder, UInt8Array}; + use datatypes::arrow::datatypes::UInt32Type; + + use super::*; + + #[test] + fn sliced_runs_preserve_order_and_nonconsecutive_keys() { + let mut keys = BinaryDictionaryBuilder::::new(); + for key in ["a", "a", "b", "b", "b", "a", "c"] { + keys.append(key).unwrap(); + } + let batch = RecordBatch::try_from_iter([ + ("pk", Arc::new(keys.finish()) as ArrayRef), + ("seq", Arc::new(UInt8Array::from(vec![0; 7])) as ArrayRef), + ("op", Arc::new(UInt8Array::from(vec![0; 7])) as ArrayRef), + ]) + .unwrap(); + let slice = batch.slice(1, 5); + assert_eq!( + PrimaryKeyRuns::try_new(&slice).unwrap().collect::>(), + vec![ + (b"a".as_slice(), 1), + (b"b".as_slice(), 3), + (b"a".as_slice(), 1) + ] + ); + assert!( + PrimaryKeyRuns::try_new(&batch.slice(0, 0)) + .unwrap() + .next() + .is_none() + ); + } +} diff --git a/src/mito2/src/sst/index/sparse_test.rs b/src/mito2/src/sst/index/sparse_test.rs new file mode 100644 index 00000000000..15d26bd1bc7 --- /dev/null +++ b/src/mito2/src/sst/index/sparse_test.rs @@ -0,0 +1,198 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use api::v1::SemanticType; +use datatypes::arrow::array::{ + ArrayRef, BinaryDictionaryBuilder, TimestampMillisecondArray, UInt8Array, UInt64Array, +}; +use datatypes::arrow::datatypes::UInt32Type; +use datatypes::arrow::record_batch::RecordBatch; +use datatypes::schema::SkippingIndexOptions; +use datatypes::value::ValueRef; +use index::bloom_filter::reader::{BloomFilterReader, BloomFilterReaderImpl}; +use index::inverted_index::format::reader::{InvertedIndexBlobReader, InvertedIndexReader}; +use mito_codec::row_converter::{PrimaryKeyCodec, SparsePrimaryKeyCodec}; +use object_store::ObjectStore; +use object_store::services::Memory; +use prost::Message; +use puffin::puffin_manager::{PuffinManager, PuffinReader}; +use store_api::codec::PrimaryKeyEncoding; +use store_api::metadata::{RegionMetadataBuilder, RegionMetadataRef}; +use store_api::storage::FileId; +use store_api::storage::consts::ReservedColumnId; + +use super::{IndexBuildType, IndexerBuilder, IndexerBuilderImpl}; +use crate::region::options::IndexOptions; +use crate::sst::file::{RegionFileId, RegionIndexId}; +use crate::sst::index::bloom_filter::creator::tests::TestPathProvider; +use crate::sst::index::intermediate::IntermediateManager; +use crate::sst::index::puffin_manager::PuffinManagerFactory; +use crate::sst::parquet::flat_format::FlatReadFormat; +use crate::sst::{FlatSchemaOptions, to_flat_sst_arrow_schema}; +use crate::test_util::sst_util::sst_region_metadata_with_encoding; + +fn sparse_input() -> (RegionMetadataRef, RecordBatch) { + let mut metadata = sst_region_metadata_with_encoding(PrimaryKeyEncoding::Sparse); + for column in &mut metadata.column_metadatas { + if column.semantic_type == SemanticType::Tag { + column.column_schema.set_inverted_index(true); + column + .column_schema + .set_skipping_options(&SkippingIndexOptions { + granularity: 3, + ..Default::default() + }) + .unwrap(); + } + } + let metadata = Arc::new( + RegionMetadataBuilder::from_existing(metadata) + .build() + .unwrap(), + ); + let codec = SparsePrimaryKeyCodec::new(&metadata); + let mut keys = BinaryDictionaryBuilder::::new(); + for (series, (first, second, count)) in [ + (Some("中文12345678"), Some(""), 5), + (None, Some("abcdefgh"), 2), + (Some(""), None, 4), + ] + .into_iter() + .enumerate() + { + let mut key = Vec::new(); + codec + .encode_value_refs( + &[ + (ReservedColumnId::table_id(), ValueRef::UInt32(42)), + (ReservedColumnId::tsid(), ValueRef::UInt64(series as u64)), + (0, first.map_or(ValueRef::Null, ValueRef::String)), + (1, second.map_or(ValueRef::Null, ValueRef::String)), + ], + &mut key, + ) + .unwrap(); + for _ in 0..count { + keys.append(&key).unwrap(); + } + } + let schema = to_flat_sst_arrow_schema( + &metadata, + &FlatSchemaOptions::from_encoding(PrimaryKeyEncoding::Sparse), + ); + let columns: Vec = vec![ + Arc::new(UInt64Array::from(vec![1; 11])), + Arc::new(TimestampMillisecondArray::from_iter_values(0..11)), + Arc::new(keys.finish()), + Arc::new(UInt64Array::from(vec![1; 11])), + Arc::new(UInt8Array::from(vec![1; 11])), + ]; + (metadata, RecordBatch::try_new(schema, columns).unwrap()) +} + +#[tokio::test] +async fn sparse_and_materialized_tags_produce_identical_indexes() { + let (metadata, sparse) = sparse_input(); + // Use the independent whole-PK read conversion as the oracle for both index types. + let materialized = FlatReadFormat::new_with_all_columns(metadata.clone()) + .convert_batch(sparse.clone(), None) + .unwrap(); + let projection = materialized + .schema() + .fields() + .iter() + .enumerate() + .filter_map(|(i, field)| { + (field.name() == "tag_0" || sparse.column_by_name(field.name()).is_some()).then_some(i) + }) + .collect::>(); + let mixed = materialized.project(&projection).unwrap(); + + let (dir, factory) = PuffinManagerFactory::new_for_test_async("sparse_index_bytes").await; + let puffin = factory.build( + ObjectStore::new(Memory::default()).unwrap(), + TestPathProvider, + ); + let mut options = IndexOptions::default(); + options.inverted_index.segment_row_count = 3; + let builder = IndexerBuilderImpl { + build_type: IndexBuildType::Flush, + metadata: metadata.clone(), + puffin_manager: puffin.clone(), + write_cache_enabled: false, + intermediate_manager: IntermediateManager::init_fs(dir.path().to_str().unwrap()) + .await + .unwrap(), + index_options: options, + inverted_index_config: Default::default(), + fulltext_index_config: Default::default(), + bloom_filter_index_config: Default::default(), + #[cfg(feature = "vector_index")] + vector_index_config: Default::default(), + }; + let mut expected = None; + for batch in [materialized, sparse, mixed] { + let file = RegionFileId::new(metadata.region_id, FileId::random()); + let mut indexer = builder.build(file, 0, None).await; + // Split a repeated PK across batches; runs also cross 3-row segment boundaries. + indexer.update_flat(&batch.slice(0, 2)).await; + indexer.update_flat(&batch.slice(2, 9)).await; + let output = indexer.finish().await; + assert_eq!(output.inverted_index.row_count, 11); + assert_eq!(output.bloom_filter.row_count, 11); + assert_eq!(output.inverted_index.columns.len(), 4); + assert_eq!(output.bloom_filter.columns.len(), 4); + + let reader = puffin.reader(&RegionIndexId::new(file, 0)).await.unwrap(); + let blob = reader.blob("greptime-inverted-index-v1").await.unwrap(); + let inverted = InvertedIndexBlobReader::new(blob.reader().await.unwrap()); + let metas = inverted.metadata(None).await.unwrap(); + let mut index_bytes = BTreeMap::new(); + // Columns can be emitted in a different order; compare each column's FST and bitmaps. + for (name, meta) in &metas.metas { + index_bytes.insert( + format!("inverted/{name}"), + inverted + .range_read(meta.base_offset, meta.inverted_index_size as u32, None) + .await + .unwrap(), + ); + } + for id in &metadata.primary_key { + let blob = reader + .blob(&format!("greptime-bloom-filter-v1-{id}")) + .await + .unwrap(); + let bloom = BloomFilterReaderImpl::new(blob.reader().await.unwrap()); + let meta = bloom.metadata(None).await.unwrap(); + assert_eq!(meta.row_count, 11); + assert_eq!(meta.segment_count, 4); + let mut bytes = bloom + .range_read(0, meta.bloom_filter_size as u32, None) + .await + .unwrap() + .to_vec(); + bytes.extend_from_slice(&meta.encode_to_vec()); + index_bytes.insert(format!("bloom/{id}"), bytes); + } + if let Some(expected) = &expected { + assert_eq!(&index_bytes, expected); + } else { + expected = Some(index_bytes); + } + } +} diff --git a/src/mito2/src/sst/parquet.rs b/src/mito2/src/sst/parquet.rs index 3d90a360925..87eeb7bae5d 100644 --- a/src/mito2/src/sst/parquet.rs +++ b/src/mito2/src/sst/parquet.rs @@ -17,9 +17,14 @@ use std::collections::BTreeMap; use std::sync::Arc; +use api::v1::SemanticType; use common_base::readable_size::ReadableSize; use datatypes::json::JsonSettings; use parquet::file::metadata::ParquetMetaData; +use parquet::file::properties::WriterPropertiesBuilder; +use parquet::schema::types::ColumnPath; +use store_api::metadata::RegionMetadataRef; +use store_api::mito_engine_options::FloatFieldEncoding; use store_api::storage::{ColumnId, FileId}; use crate::sst::DEFAULT_WRITE_BUFFER_SIZE; @@ -72,6 +77,27 @@ pub(crate) struct Json2TargetLayout { /// batching without changing the row group layout of newly written SSTs. pub const DEFAULT_ROW_GROUP_SIZE: usize = 100 * 1024; +/// Applies the configured encoding to direct floating-point field columns. +pub(crate) fn apply_float_field_encoding( + mut builder: WriterPropertiesBuilder, + metadata: &RegionMetadataRef, + encoding: FloatFieldEncoding, +) -> WriterPropertiesBuilder { + if encoding == FloatFieldEncoding::ByteStreamSplit { + for column in &metadata.column_metadatas { + if column.semantic_type == SemanticType::Field + && column.column_schema.data_type.is_float() + { + let path = ColumnPath::new(vec![column.column_schema.name.clone()]); + builder = builder + .set_column_encoding(path.clone(), parquet::basic::Encoding::BYTE_STREAM_SPLIT) + .set_column_dictionary_enabled(path, false); + } + } + } + builder +} + /// Parquet write options. #[derive(Debug, Clone)] pub struct WriteOptions { @@ -83,6 +109,8 @@ pub struct WriteOptions { /// Note: This is not a hard limit as we can only observe the file size when /// ArrowWrite writes to underlying writers. pub max_file_size: Option, + /// Encoding policy for direct floating-point field columns. + pub float_field_encoding: FloatFieldEncoding, } impl Default for WriteOptions { @@ -91,6 +119,7 @@ impl Default for WriteOptions { write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE, row_group_size: DEFAULT_ROW_GROUP_SIZE, max_file_size: None, + float_field_encoding: FloatFieldEncoding::default(), } } } @@ -125,6 +154,7 @@ mod tests { use std::sync::Arc; use api::v1::{OpType, SemanticType}; + use bytes::Bytes; use common_function::function::FunctionRef; use common_function::function_factory::ScalarFunctionFactory; use common_function::scalars::matches::MatchesFunction; @@ -135,20 +165,24 @@ mod tests { use datafusion_expr::{BinaryExpr, Expr, Literal, Operator, col, lit}; use datatypes::arrow; use datatypes::arrow::array::{ - ArrayRef, AsArray, BinaryDictionaryBuilder, RecordBatch, StringArray, - StringDictionaryBuilder, TimestampMillisecondArray, UInt8Array, UInt64Array, + Array, ArrayRef, AsArray, BinaryDictionaryBuilder, Float32Array, Float64Array, Int32Array, + RecordBatch, StringArray, StringDictionaryBuilder, TimestampMillisecondArray, UInt8Array, + UInt64Array, }; - use datatypes::arrow::datatypes::{DataType, Field, Schema, UInt32Type}; + use datatypes::arrow::datatypes::{DataType, Field, Schema, TimeUnit, UInt32Type}; use datatypes::arrow::util::pretty::pretty_format_batches; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{FulltextAnalyzer, FulltextBackend, FulltextOptions}; use object_store::ObjectStore; - use parquet::arrow::AsyncArrowWriter; + use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + use parquet::arrow::{ArrowWriter, AsyncArrowWriter}; use parquet::basic::{Compression, Encoding, ZstdLevel}; use parquet::file::metadata::{KeyValue, PageIndexPolicy}; use parquet::file::properties::WriterProperties; + use parquet::schema::types::ColumnPath; use store_api::codec::PrimaryKeyEncoding; use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataBuilder}; + use store_api::mito_engine_options::FloatFieldEncoding; use store_api::region_request::PathType; use store_api::storage::{ColumnSchema, RegionId}; use table::predicate::Predicate; @@ -187,6 +221,176 @@ mod tests { const FILE_DIR: &str = "/"; const REGION_ID: RegionId = RegionId::new(0, 0); + #[test] + fn test_float_field_encoding_properties_and_roundtrip() { + let mut metadata_builder = RegionMetadataBuilder::new(REGION_ID); + metadata_builder + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new("f32", ConcreteDataType::float32_datatype(), true), + semantic_type: SemanticType::Field, + column_id: 0, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new("f64", ConcreteDataType::float64_datatype(), true), + semantic_type: SemanticType::Field, + column_id: 1, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new("tag", ConcreteDataType::float32_datatype(), true), + semantic_type: SemanticType::Tag, + column_id: 2, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new("i32", ConcreteDataType::int32_datatype(), true), + semantic_type: SemanticType::Field, + column_id: 3, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ), + semantic_type: SemanticType::Timestamp, + column_id: 4, + }); + metadata_builder.primary_key(vec![2]); + let metadata = Arc::new(metadata_builder.build().unwrap()); + + let f32_values = [ + Some(1.5_f32), + Some(2.0), + Some(0.0), + Some(-0.0), + Some(f32::INFINITY), + Some(f32::from_bits(0x7fc0_1234)), + None, + ]; + let f64_values = [ + Some(1.5_f64), + Some(2.0), + Some(0.0), + Some(-0.0), + Some(f64::NEG_INFINITY), + Some(f64::from_bits(0x7ff8_0000_0000_1234)), + None, + ]; + let schema = Arc::new(Schema::new(vec![ + Field::new("f32", DataType::Float32, true), + Field::new("f64", DataType::Float64, true), + Field::new("tag", DataType::Float32, true), + Field::new("i32", DataType::Int32, true), + Field::new( + "ts", + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Float32Array::from(f32_values.to_vec())) as ArrayRef, + Arc::new(Float64Array::from(f64_values.to_vec())) as ArrayRef, + Arc::new(Float32Array::from(vec![ + Some(1.0), + None, + Some(-0.0), + Some(2.0), + Some(3.0), + Some(4.0), + None, + ])), + Arc::new(Int32Array::from(vec![ + Some(1), + None, + Some(3), + Some(4), + Some(5), + Some(6), + None, + ])), + Arc::new(TimestampMillisecondArray::from_iter_values(0..7)), + ], + ) + .unwrap(); + + let path = |name: &str| ColumnPath::new(vec![name.to_string()]); + let bss = apply_float_field_encoding( + WriterProperties::builder(), + &metadata, + FloatFieldEncoding::ByteStreamSplit, + ); + assert_eq!( + Some(Encoding::BYTE_STREAM_SPLIT), + bss.clone().build().encoding(&path("f32")) + ); + assert_eq!( + Some(Encoding::BYTE_STREAM_SPLIT), + bss.clone().build().encoding(&path("f64")) + ); + assert!(!bss.clone().build().dictionary_enabled(&path("f32"))); + assert!(!bss.clone().build().dictionary_enabled(&path("f64"))); + assert_eq!(None, bss.clone().build().encoding(&path("i32"))); + assert!(bss.clone().build().dictionary_enabled(&path("tag"))); + + let default = apply_float_field_encoding( + WriterProperties::builder().set_encoding(Encoding::PLAIN), + &metadata, + FloatFieldEncoding::Default, + ) + .build(); + assert_eq!(Some(Encoding::PLAIN), default.encoding(&path("f32"))); + assert!(default.dictionary_enabled(&path("f32"))); + assert!(default.dictionary_enabled(&path("f64"))); + + let mut bytes = Vec::new(); + let mut writer = ArrowWriter::try_new(&mut bytes, schema, Some(bss.build())).unwrap(); + writer.write(&batch).unwrap(); + let footer = writer.finish().unwrap(); + drop(writer); + for name in ["f32", "f64"] { + let column = footer.row_groups()[0] + .columns() + .iter() + .find(|column| column.column_path().string() == name) + .unwrap(); + assert!( + column + .encodings() + .any(|encoding| encoding == Encoding::BYTE_STREAM_SPLIT) + ); + } + let mut reader = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(bytes)) + .unwrap() + .build() + .unwrap(); + let actual = reader.next().unwrap().unwrap(); + let actual_f32 = actual + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let actual_f64 = actual + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + for (index, value) in f32_values.into_iter().enumerate() { + assert_eq!(value.is_none(), actual_f32.is_null(index)); + if let Some(value) = value { + assert_eq!(value.to_bits(), actual_f32.value(index).to_bits()); + } + } + for (index, value) in f64_values.into_iter().enumerate() { + assert_eq!(value.is_none(), actual_f64.is_null(index)); + if let Some(value) = value { + assert_eq!(value.to_bits(), actual_f64.value(index).to_bits()); + } + } + assert!(actual.column(2).is_null(1)); + assert!(actual.column(3).is_null(1)); + } + #[derive(Clone)] struct FixedPathProvider { region_file_id: RegionFileId, diff --git a/src/mito2/src/sst/parquet/flat_format.rs b/src/mito2/src/sst/parquet/flat_format.rs index 5432c112f49..b6dfdf0ec47 100644 --- a/src/mito2/src/sst/parquet/flat_format.rs +++ b/src/mito2/src/sst/parquet/flat_format.rs @@ -560,7 +560,8 @@ struct ParquetFlat { arrow_schema: SchemaRef, /// Projection computed for the flat format. format_projection: FormatProjection, - /// Column id to index in SST. + /// Column id to top-level SST index. Shared statistics helpers resolve + /// physical leaves from each file's actual Parquet schema. column_id_to_sst_index: HashMap, } @@ -618,7 +619,6 @@ impl ParquetFlat { // No such column in the SST. return StatValues::NoColumn; }; - let stats = column_null_counts(row_groups, *index); StatValues::from_stats_opt(stats) } @@ -633,8 +633,9 @@ impl ParquetFlat { // No such column in the SST. return StatValues::NoColumn; }; - // Safety: `column_id_to_sst_index` is built from `metadata`. - let index = self.column_id_to_sst_index.get(&column_id).unwrap(); + let Some(index) = self.column_id_to_sst_index.get(&column_id) else { + return StatValues::NoStats; + }; let stats = column_values(row_groups, column, *index, is_min); StatValues::from_stats_opt(stats) @@ -895,16 +896,25 @@ mod tests { use api::v1::SemanticType; use datatypes::arrow::array::{ - ArrayRef, BinaryArray, TimestampMillisecondArray, UInt8Array, UInt32Array, UInt64Array, + ArrayRef, BinaryArray, Int64Array, TimestampMillisecondArray, UInt8Array, UInt32Array, + UInt64Array, }; - use datatypes::arrow::datatypes::DataType as ArrowDataType; + use datatypes::arrow::datatypes::{DataType as ArrowDataType, Field, TimeUnit}; use datatypes::arrow::record_batch::RecordBatch; use datatypes::prelude::ConcreteDataType; use datatypes::schema::ColumnSchema; + use datatypes::types::json_type::{JsonNativeType, JsonObjectType}; + use parquet::arrow::ArrowSchemaConverter; + use parquet::basic::{Repetition, Type as PhysicalType}; + use parquet::file::metadata::{ColumnChunkMetaData, RowGroupMetaData}; + use parquet::file::statistics::Statistics; + use parquet::schema::types::{SchemaDescriptor, Type}; use store_api::codec::PrimaryKeyEncoding; use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataBuilder}; use store_api::storage::RegionId; - use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME; + use store_api::storage::consts::{ + OP_TYPE_COLUMN_NAME, PRIMARY_KEY_COLUMN_NAME, SEQUENCE_COLUMN_NAME, + }; use super::*; use crate::read::read_columns::ReadColumns; @@ -964,6 +974,331 @@ mod tests { builder.build().unwrap() } + /// Builds the metadata of a table with a JSON2 struct field column: + /// `[tag_0, field_0, payload, nullable_after_payload, ts]` with primary key `tag_0`. + fn metadata_with_struct_field() -> RegionMetadata { + let mut builder = RegionMetadataBuilder::new(RegionId::new(0, 0)); + builder + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "tag_0".to_string(), + ConcreteDataType::string_datatype(), + true, + ), + semantic_type: SemanticType::Tag, + column_id: 0, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "field_0".to_string(), + ConcreteDataType::int64_datatype(), + true, + ), + semantic_type: SemanticType::Field, + column_id: 1, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "payload".to_string(), + ConcreteDataType::json2(JsonNativeType::Object(JsonObjectType::new())), + false, + ), + semantic_type: SemanticType::Field, + column_id: 2, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "nullable_after_payload".to_string(), + ConcreteDataType::int64_datatype(), + true, + ), + semantic_type: SemanticType::Field, + column_id: 4, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "ts".to_string(), + ConcreteDataType::timestamp_nanosecond_datatype(), + false, + ), + semantic_type: SemanticType::Timestamp, + column_id: 3, + }); + builder.primary_key(vec![0]); + builder.primary_key_encoding(PrimaryKeyEncoding::Dense); + builder.build().unwrap() + } + + /// Builds a file schema and a row group in which the `payload` struct + /// column expands to three leaf columns. The `ns_edge` leaf carries small + /// Int64 statistics that must not be mistaken for the statistics of `ts`. + fn struct_column_file_and_row_group(raw_pk_columns: bool) -> (SchemaRef, RowGroupMetaData) { + let mut arrow_fields = vec![ + Field::new("tag_0", ArrowDataType::Utf8, true), + Field::new("field_0", ArrowDataType::Int64, true), + Field::new( + "payload", + ArrowDataType::Struct( + vec![ + Field::new("metadata", ArrowDataType::Binary, false), + Field::new("value", ArrowDataType::Binary, false), + Field::new("ns_edge", ArrowDataType::Int64, true), + ] + .into(), + ), + false, + ), + Field::new("nullable_after_payload", ArrowDataType::Int64, true), + Field::new( + "ts", + ArrowDataType::Timestamp(TimeUnit::Nanosecond, None), + false, + ), + Field::new(PRIMARY_KEY_COLUMN_NAME, ArrowDataType::Binary, false), + Field::new(SEQUENCE_COLUMN_NAME, ArrowDataType::UInt64, false), + Field::new(OP_TYPE_COLUMN_NAME, ArrowDataType::UInt8, false), + ]; + if !raw_pk_columns { + arrow_fields.remove(0); + } + let file_schema = Arc::new(Schema::new(arrow_fields)); + + let leaf = |name: &str, physical: PhysicalType| { + Arc::new( + Type::primitive_type_builder(name, physical) + .with_repetition(Repetition::OPTIONAL) + .build() + .unwrap(), + ) + }; + let payload = Arc::new( + Type::group_type_builder("payload") + .with_repetition(Repetition::OPTIONAL) + .with_fields(vec![ + leaf("metadata", PhysicalType::BYTE_ARRAY), + leaf("value", PhysicalType::BYTE_ARRAY), + leaf("ns_edge", PhysicalType::INT64), + ]) + .build() + .unwrap(), + ); + let mut parquet_fields = vec![ + leaf("tag_0", PhysicalType::BYTE_ARRAY), + leaf("field_0", PhysicalType::INT64), + payload, + leaf("nullable_after_payload", PhysicalType::INT64), + leaf("ts", PhysicalType::INT64), + leaf(PRIMARY_KEY_COLUMN_NAME, PhysicalType::BYTE_ARRAY), + leaf(SEQUENCE_COLUMN_NAME, PhysicalType::INT64), + leaf(OP_TYPE_COLUMN_NAME, PhysicalType::INT32), + ]; + if !raw_pk_columns { + parquet_fields.remove(0); + } + let schema_descr = Arc::new(SchemaDescriptor::new(Arc::new( + Type::group_type_builder("schema") + .with_fields(parquet_fields) + .build() + .unwrap(), + ))); + + // Omitted raw tags shift all subsequent leaves in primary-key SSTs. + let ns_edge_leaf = if raw_pk_columns { 4 } else { 3 }; + let nullable_leaf = ns_edge_leaf + 1; + let ts_leaf = nullable_leaf + 1; + let chunks: Vec<_> = (0..schema_descr.num_columns()) + .map(|i| { + let mut builder = ColumnChunkMetaData::builder(schema_descr.column(i)); + if i == ns_edge_leaf { + // Small values from the JSON payload, not timestamps. + builder = builder.set_statistics(Statistics::int64( + Some(0), + Some(86_400_000_000_000), + None, + Some(65), + true, + )); + } else if i == nullable_leaf { + builder = builder.set_statistics(Statistics::int64( + Some(100), + Some(200), + None, + Some(7), + true, + )); + } else if i == ts_leaf { + builder = builder.set_statistics(Statistics::int64( + Some(1_788_998_400_000_000_000), + Some(1_789_084_800_000_000_000), + None, + Some(0), + true, + )); + } + builder.build().unwrap() + }) + .collect(); + let row_group = RowGroupMetaData::builder(schema_descr) + .set_num_rows(69) + .set_total_byte_size(0) + .set_column_metadata(chunks) + .build() + .unwrap(); + + (file_schema, row_group) + } + + /// Regression test: row group statistics must be looked up by parquet leaf + /// column index. A struct field column (e.g. JSON2) expands to multiple + /// leaf columns, so statistics of columns after it must not be read from + /// the struct's leaves. Otherwise min-max pruning can drop a whole row + /// group by mistake (e.g. pruning `ts` with the small `ns_edge` stats), + /// which caused data loss during SWCS compaction. + #[test] + fn test_stats_with_struct_field_column() { + for (encoding, raw_pk_columns) in [ + (PrimaryKeyEncoding::Dense, true), + (PrimaryKeyEncoding::Dense, false), + (PrimaryKeyEncoding::Sparse, false), + ] { + let mut metadata = metadata_with_struct_field(); + metadata.primary_key_encoding = encoding; + let metadata = Arc::new(metadata); + let (file_schema, row_group) = struct_column_file_and_row_group(raw_pk_columns); + let read_format = FlatReadFormat::new( + metadata, + ReadColumns::new([0, 1, 2, 3, 4]), + Some(file_schema), + "test", + false, + ) + .unwrap(); + let row_groups = [&row_group]; + + // Statistics of `ts` come from the `ts` leaf column, not the leaves of + // the payload struct. + let StatValues::Values(min) = read_format.min_values(&row_groups, 3) else { + panic!("expected ts min values") + }; + let min = min.as_any().downcast_ref::().unwrap(); + assert_eq!(1_788_998_400_000_000_000, min.value(0)); + let StatValues::Values(max) = read_format.max_values(&row_groups, 3) else { + panic!("expected ts max values") + }; + let max = max.as_any().downcast_ref::().unwrap(); + assert_eq!(1_789_084_800_000_000_000, max.value(0)); + + let stats = crate::sst::parquet::stats::RowGroupPruningStats::new( + &row_groups, + &read_format, + None, + false, + ); + for (start, end, keep) in [ + (1_788_998_400_000_000_000, 1_789_084_800_000_000_000, true), + (1_789_084_800_000_000_001, 1_789_171_200_000_000_000, false), + ] { + let predicate = table::predicate::Predicate::new(vec![ + datafusion_expr::col("ts").gt_eq(datafusion_expr::lit( + datafusion_common::ScalarValue::TimestampNanosecond(Some(start), None), + )), + datafusion_expr::col("ts").lt(datafusion_expr::lit( + datafusion_common::ScalarValue::TimestampNanosecond(Some(end), None), + )), + ]); + assert_eq!( + vec![keep], + predicate + .prune_with_stats(&stats, read_format.metadata().schema.arrow_schema(),) + ); + } + + // Null counts of `ts` also read the correct leaf column. + let StatValues::Values(nulls) = read_format.null_counts(&row_groups, 3) else { + panic!("expected ts null counts") + }; + let nulls = nulls.as_any().downcast_ref::().unwrap(); + assert!(nulls.is_valid(0)); + assert_eq!(0, nulls.value(0)); + + // A null slot may contain an underlying zero. Check validity and + // a nonzero count to distinguish unknown or wrong-leaf statistics. + let StatValues::Values(nulls) = read_format.null_counts(&row_groups, 4) else { + panic!("expected nullable field null counts") + }; + let nulls = nulls.as_any().downcast_ref::().unwrap(); + assert!(nulls.is_valid(0)); + assert_eq!(7, nulls.value(0)); + + // A column that expands to multiple leaf columns has no single column + // statistics. + assert!(matches!( + read_format.min_values(&row_groups, 2), + StatValues::NoStats + )); + assert!(matches!( + read_format.max_values(&row_groups, 2), + StatValues::NoStats + )); + assert!(matches!( + read_format.null_counts(&row_groups, 2), + StatValues::NoStats + )); + } + } + + /// Even one leaf cannot supply the null count of a nested parent: + /// {"a": null} and [null] are non-null parents with null children. + #[test] + fn test_single_leaf_nested_columns_have_no_root_stats() { + let child = Arc::new(Field::new("a", ArrowDataType::Int64, true)); + for nested_type in [ + ArrowDataType::Struct(vec![child.clone()].into()), + ArrowDataType::List(child.clone()), + ArrowDataType::LargeList(child.clone()), + ArrowDataType::FixedSizeList(child, 1), + ] { + let metadata = Arc::new(metadata_with_struct_field()); + let (schema, _) = struct_column_file_and_row_group(true); + let mut fields = schema.fields().to_vec(); + fields[2] = Arc::new(Field::new("payload", nested_type.clone(), true)); + let schema = Arc::new(Schema::new(fields)); + let descriptor = Arc::new(ArrowSchemaConverter::new().convert(&schema).unwrap()); + let chunks = descriptor + .columns() + .iter() + .map(|column| { + ColumnChunkMetaData::builder(column.clone()) + .build() + .unwrap() + }) + .collect(); + let row_group = RowGroupMetaData::builder(descriptor) + .set_num_rows(1) + .set_total_byte_size(0) + .set_column_metadata(chunks) + .build() + .unwrap(); + let format = ParquetFlat::new(metadata, ReadColumns::new([0, 1, 2, 3]), schema); + + // The nested root has no usable statistics, independently of the + // values stored in any row group. Its leaf still occupies a slot. + let groups = &[row_group]; + assert!( + matches!(format.min_values(groups, 2), StatValues::NoStats), + "{nested_type:?}" + ); + assert!( + matches!(format.max_values(groups, 2), StatValues::NoStats), + "{nested_type:?}" + ); + assert!( + matches!(format.null_counts(groups, 2), StatValues::NoStats), + "{nested_type:?}" + ); + } + } + #[test] fn test_field_column_start() { // (num_tags, num_fields, encoding, expected) diff --git a/src/mito2/src/sst/parquet/format.rs b/src/mito2/src/sst/parquet/format.rs index 1a4325b7a7e..6ca7a486624 100644 --- a/src/mito2/src/sst/parquet/format.rs +++ b/src/mito2/src/sst/parquet/format.rs @@ -46,6 +46,7 @@ use mito_codec::row_converter::{ }; use parquet::file::metadata::{ParquetMetaData, RowGroupMetaData}; use parquet::file::statistics::Statistics; +use parquet::schema::types::SchemaDescriptor; use snafu::{OptionExt, ResultExt, ensure}; use store_api::metadata::{ColumnMetadata, RegionMetadataRef}; use store_api::storage::{ColumnId, NestedPath, SequenceNumber}; @@ -144,13 +145,16 @@ pub(crate) fn column_values( ) } -/// Returns min/max values of a parquet column with the given Arrow data type. +/// Returns min/max values for a top-level column with the given Arrow data type. +/// Resolves its leaf from the actual Parquet schema, not an inferred Arrow layout. pub(crate) fn column_values_by_type( row_groups: &[impl Borrow], data_type: &ArrowDataType, column_index: usize, is_min: bool, ) -> Option { + let column_index = + scalar_leaf_index(row_groups.first()?.borrow().schema_descr(), column_index)?; let null_scalar: ScalarValue = data_type.try_into().ok()?; let scalar_values = row_groups .iter() @@ -207,12 +211,14 @@ pub(crate) fn column_values_by_type( ScalarValue::iter_to_array(scalar_values).ok() } -/// Returns null counts of specific columns. +/// Returns null counts of a top-level column. /// The column should not be encoded as a part of a primary key. pub(crate) fn column_null_counts( row_groups: &[impl Borrow], column_index: usize, ) -> Option { + let column_index = + scalar_leaf_index(row_groups.first()?.borrow().schema_descr(), column_index)?; let values = row_groups.iter().map(|meta| { let col = meta.borrow().column(column_index); let stat = col.statistics()?; @@ -221,6 +227,21 @@ pub(crate) fn column_null_counts( Some(Arc::new(UInt64Array::from_iter(values))) } +/// Maps a scalar root to its physical leaf. Nested roots have no root-level +/// statistics, even with one leaf: child null counts do not describe parents. +/// All row groups of a file share the same schema. +fn scalar_leaf_index(schema: &SchemaDescriptor, root_index: usize) -> Option { + if !schema + .root_schema() + .get_fields() + .get(root_index)? + .is_primitive() + { + return None; + } + (0..schema.num_columns()).find(|&leaf| schema.get_column_root_idx(leaf) == root_index) +} + /// Helper for reading the SST format. pub struct PrimaryKeyReadFormat { /// The metadata stored in the SST. @@ -539,11 +560,12 @@ impl PrimaryKeyReadFormat { .into_iter(), ); + let primary_key_leaf = scalar_leaf_index( + row_groups.first()?.borrow().schema_descr(), + self.primary_key_position(), + )?; let values = row_groups.iter().map(|meta| { - let stats = meta - .borrow() - .column(self.primary_key_position()) - .statistics()?; + let stats = meta.borrow().column(primary_key_leaf).statistics()?; match stats { Statistics::Boolean(_) => None, Statistics::Int32(_) => None, diff --git a/src/mito2/src/sst/parquet/index_reader.rs b/src/mito2/src/sst/parquet/index_reader.rs index 0cf221986c4..d3a23e6b1c1 100644 --- a/src/mito2/src/sst/parquet/index_reader.rs +++ b/src/mito2/src/sst/parquet/index_reader.rs @@ -172,7 +172,7 @@ impl PruningStatistics for IndexRowGroupPruningStats<'_> { column_null_counts(self.row_groups, column_index) } - fn row_counts(&self, _column: &Column) -> Option { + fn row_counts(&self) -> Option { None } diff --git a/src/mito2/src/sst/parquet/json_align/stream.rs b/src/mito2/src/sst/parquet/json_align.rs similarity index 53% rename from src/mito2/src/sst/parquet/json_align/stream.rs rename to src/mito2/src/sst/parquet/json_align.rs index bfd0b82edc6..8af88f6ac0f 100644 --- a/src/mito2/src/sst/parquet/json_align/stream.rs +++ b/src/mito2/src/sst/parquet/json_align.rs @@ -19,12 +19,15 @@ use std::task::{Context, Poll}; use datafusion_common::cast_column; use datafusion_common::format::DEFAULT_CAST_OPTIONS; use datatypes::arrow::array::{ArrayRef, new_null_array}; -use datatypes::arrow::datatypes::{DataType, Field, FieldRef, SchemaRef}; +use datatypes::arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef}; use datatypes::arrow::record_batch::RecordBatch; use datatypes::extension::json::{JsonMetadata, is_json2_extension_type}; use datatypes::json::JsonSettings; use datatypes::vectors::json::array::JsonArray; +use datatypes::vectors::json::json2_physical_data_type; use futures::Stream; +use futures::stream::BoxStream; +use serde_json::from_str; use snafu::{ResultExt, ensure}; use crate::error::{ @@ -32,32 +35,44 @@ use crate::error::{ }; use crate::sst::parquet::Json2TargetLayout; +pub(crate) type ProjectedRecordBatchStream = BoxStream<'static, Result>; + +/// Specifies how JSON columns in a record batch are aligned. #[derive(Debug)] -struct Json2RewriteSettings { - logical_settings: JsonSettings, - target_layout: JsonSettings, +pub(crate) enum AlignMode { + /// Aligns JSON columns to the logical fields in the output schema. + AlignToSchema, + /// Rewrites JSON columns to physical layouts, typically for compaction. + Rewrite { + /// Target layouts keyed by root column name, not nested field path. + /// + /// Only listed columns are rewritten. Other existing arrays are reused + /// unchanged and must already match their output field types. + /// An empty map therefore only fills missing roots. + columns: HashMap, + }, } -/// Aligns projected batches to the expected output schema for nested projections. +/// Alignment mode with parsed rewrite metadata and validated target layouts. +#[derive(Debug)] +enum ResolvedAlignMode { + AlignToSchema, + Rewrite { + columns: HashMap, + }, +} + +/// Adapts Parquet record batches to the output schema expected by the reader. /// -/// Background -/// ---------- -/// Nested projection may ask parquet to read leaves under a root column. If none -/// of the requested leaves exists in the current parquet file, parquet decoding -/// omits the whole root from the physical [`RecordBatch`]. +/// Nested projection can return only part of a JSON2 column, or omit its root +/// entirely when no requested leaves are read. This stream restores missing +/// roots with null arrays of the expected types. /// -/// In addition, after nested-path filtering, returned struct arrays may contain -/// only a subset of fields. The current output schema is not pruned by nested -/// paths, so physical struct fields can be a subset of the expected struct -/// fields, and their nested schema can differ from the expected output schema. -/// -/// To keep projected batches schema-consistent before entering upper readers: -/// - Root-column presence alignment restores missing projected root columns by -/// inserting root-level null arrays. -/// - Nested struct alignment aligns struct arrays to the expected nested field -/// layout. +/// Existing JSON2 columns are aligned to the logical schema inferred from +/// type hints or rewritten to the specified JSON2 physical layout, as selected by +/// [`AlignMode`]. #[derive(derive_more::Debug)] -pub struct NestedSchemaAligner { +pub struct JsonSchemaAligner { #[debug(skip)] inner: S, /// Output schema expected by the upper reader. @@ -70,79 +85,55 @@ pub struct NestedSchemaAligner { /// Whether all projected roots are present and the stream can pass batches /// through. all_roots_present: bool, - /// JSON2 columns that require semantic source-to-target layout rewriting. - json2_rewrite_targets: HashMap, + /// Alignment mode with parsed and validated rewrite settings. + mode: ResolvedAlignMode, /// The cache for whether incoming batches already match output schema. is_schema_matched: Option, } -impl NestedSchemaAligner +impl JsonSchemaAligner where S: Stream>, { - pub fn new( + /// Creates an aligner with a shared output schema and an explicit operation. + /// Parses rewrite metadata once and validates layouts against output field types. + pub(crate) fn new( inner: S, projected_root_presence: Vec, output_schema: SchemaRef, - ) -> Result> { + mode: AlignMode, + ) -> Result> { ensure!( projected_root_presence.len() == output_schema.fields().len(), UnexpectedSnafu { reason: format!( - "NestedSchemaAligner projected root presence len {} does not match output schema columns {}", + "JsonSchemaAligner projected root presence len {} does not match output schema columns {}", projected_root_presence.len(), output_schema.fields().len() ), } ); + let mode = resolve_align_mode(mode, output_schema.as_ref())?; + let expected_input_col_num = projected_root_presence .iter() .filter(|matched| **matched) .count(); let all_roots_present = projected_root_presence.iter().all(|&m| m); - Ok(NestedSchemaAligner { + Ok(JsonSchemaAligner { inner, output_schema, projected_root_presence, expected_input_col_num, all_roots_present, - json2_rewrite_targets: HashMap::new(), + mode, is_schema_matched: None, }) } - - /// Sets JSON2 columns that must be rewritten into the output field layout. - pub(crate) fn with_json2_rewrite_targets( - mut self, - targets: &HashMap, - ) -> Result { - self.json2_rewrite_targets = targets - .iter() - .map(|(name, layout)| { - let metadata = serde_json::from_str::(&layout.extension_metadata) - .map_err(|e| { - UnexpectedSnafu { - reason: format!( - "invalid JSON2 extension metadata for column '{name}': {e}" - ), - } - .build() - })?; - Ok(( - name.clone(), - Json2RewriteSettings { - logical_settings: metadata.into_json_settings(), - target_layout: layout.target_layout.clone(), - }, - )) - }) - .collect::>()?; - Ok(self) - } } -impl Stream for NestedSchemaAligner +impl Stream for JsonSchemaAligner where S: Stream> + Unpin, { @@ -153,7 +144,8 @@ where match Pin::new(&mut this.inner).poll_next(cx) { Poll::Ready(Some(Ok(rb))) => { - let is_schema_matched = this.all_roots_present + let is_schema_matched = matches!(this.mode, ResolvedAlignMode::AlignToSchema) + && this.all_roots_present && *this .is_schema_matched .get_or_insert_with(|| rb.schema() == this.output_schema); @@ -166,7 +158,7 @@ where &this.output_schema, &this.projected_root_presence, this.expected_input_col_num, - &this.json2_rewrite_targets, + &this.mode, ))) } } @@ -182,13 +174,13 @@ fn align_projected_batch( output_schema: &SchemaRef, projected_root_presence: &[bool], expected_input_col_num: usize, - json2_rewrite_targets: &HashMap, + mode: &ResolvedAlignMode, ) -> Result { ensure!( rb.columns().len() == expected_input_col_num, UnexpectedSnafu { reason: format!( - "NestedSchemaAligner expected {} input columns but got {}", + "JsonSchemaAligner expected {} input columns but got {}", expected_input_col_num, rb.columns().len() ), @@ -205,12 +197,16 @@ fn align_projected_batch( continue; } - cols.push(align_array( - rb.column(idx), - input_schema.field(idx), - field, - json2_rewrite_targets.get(field.name()), - )?); + let array = match mode { + ResolvedAlignMode::AlignToSchema => { + align_array(rb.column(idx), input_schema.field(idx), field)? + } + ResolvedAlignMode::Rewrite { columns } => match columns.get(field.name()) { + Some(settings) => rewrite_array(rb.column(idx), input_schema.field(idx), settings)?, + None => rb.column(idx).clone(), + }, + }; + cols.push(array); idx += 1; } @@ -218,36 +214,111 @@ fn align_projected_batch( } fn align_array( - array: &ArrayRef, - source: &Field, - field: &FieldRef, - rewrite_settings: Option<&Json2RewriteSettings>, + source_array: &ArrayRef, + source_field: &Field, + target_field: &FieldRef, ) -> Result { - if let Some(settings) = rewrite_settings { - return JsonArray::from(array) - .rewrite_to_v2(source, &settings.logical_settings, &settings.target_layout) - .context(DataTypeMismatchSnafu); - } - if array.data_type() == field.data_type() { - return Ok(array.clone()); + if source_array.data_type() == target_field.data_type() { + return Ok(source_array.clone()); } - if is_json2_extension_type(field) { - if is_json2_extension_type(source) { - return JsonArray::from(array) - .project_to_v2(source, field.data_type()) + if is_json2_extension_type(target_field) { + if is_json2_extension_type(source_field) { + return JsonArray::from(source_array) + .project_to_v2(source_field, target_field.data_type()) .context(DataTypeMismatchSnafu); } - return JsonArray::from(array) - .project_to(field.data_type()) + return JsonArray::from(source_array) + .project_to(target_field.data_type()) .context(DataTypeMismatchSnafu); } - if !matches!(field.data_type(), DataType::Struct(_)) { - return Ok(array.clone()); + if !matches!(target_field.data_type(), DataType::Struct(_)) { + return Ok(source_array.clone()); } - cast_column(array, field.as_ref(), &DEFAULT_CAST_OPTIONS).context(CastColumnSnafu) + cast_column( + source_array, + target_field.data_type(), + &DEFAULT_CAST_OPTIONS, + ) + .context(CastColumnSnafu) +} + +fn rewrite_array( + source_array: &ArrayRef, + source_field: &Field, + settings: &RewriteSettings, +) -> Result { + JsonArray::from(source_array) + .rewrite_to_v2( + source_field, + &settings.logical_settings, + &settings.target_layout, + ) + .context(DataTypeMismatchSnafu) +} + +/// Resolved settings for rewriting one JSON2 column to a target physical layout. +/// +/// Created from [`Json2TargetLayout`] when resolving the alignment mode, so +/// extension metadata is parsed once and reused across batches. +#[derive(Debug)] +struct RewriteSettings { + /// Logical settings parsed from extension metadata and applied to JSON values + /// before encoding them into the target layout. + logical_settings: JsonSettings, + /// Settings defining the physical Arrow layout of the rewritten column. + target_layout: JsonSettings, +} + +impl TryFrom<&Json2TargetLayout> for RewriteSettings { + type Error = crate::error::Error; + + fn try_from(layout: &Json2TargetLayout) -> Result { + let metadata = from_str::(&layout.extension_metadata).map_err(|e| { + UnexpectedSnafu { + reason: format!("invalid JSON2 extension metadata: {e}"), + } + .build() + })?; + Ok(Self { + logical_settings: metadata.into_json_settings(), + target_layout: layout.target_layout.clone(), + }) + } +} + +/// Parses rewrite metadata and validates target layouts against the output schema. +fn resolve_align_mode(mode: AlignMode, output_schema: &Schema) -> Result { + let AlignMode::Rewrite { columns } = mode else { + return Ok(ResolvedAlignMode::AlignToSchema); + }; + + let mut rewrite_columns = HashMap::with_capacity(columns.len()); + for (name, layout) in columns { + let settings = RewriteSettings::try_from(&layout)?; + let field = output_schema.field_with_name(&name).map_err(|_| { + UnexpectedSnafu { + reason: format!("JSON2 rewrite column '{name}' is missing from output schema"), + } + .build() + })?; + ensure!( + is_json2_extension_type(field) + && field.data_type() == &json2_physical_data_type(&settings.target_layout), + UnexpectedSnafu { + reason: format!( + "JSON2 rewrite layout for column '{name}' does not match output field" + ), + } + ); + rewrite_columns.insert(name, settings); + } + + Ok(ResolvedAlignMode::Rewrite { + columns: rewrite_columns, + }) } #[cfg(test)] @@ -279,14 +350,21 @@ mod tests { target_layout: target_layout.clone(), }, )]); - let aligner = NestedSchemaAligner::new( + let aligner = JsonSchemaAligner::new( stream::empty::>(), - vec![], - schema(Vec::::new()), - )? - .with_json2_rewrite_targets(&rewrite_targets)?; - - let settings = &aligner.json2_rewrite_targets["j"]; + vec![false], + schema([ + Field::new("j", json2_physical_data_type(&target_layout), true) + .with_extension_type(Json2ExtensionType::default()), + ]), + AlignMode::Rewrite { + columns: rewrite_targets, + }, + )?; + let ResolvedAlignMode::Rewrite { columns } = &aligner.mode else { + panic!("expected rewrite mode"); + }; + let settings = &columns["j"]; assert_eq!(logical_settings, settings.logical_settings); assert_eq!(target_layout, settings.target_layout); Ok(()) @@ -305,8 +383,13 @@ mod tests { .unwrap(); let stream = stream::iter([Ok(input.clone())]); - let mut aligner = - NestedSchemaAligner::new(stream, vec![true, true], output_schema.clone()).unwrap(); + let mut aligner = JsonSchemaAligner::new( + stream, + vec![true, true], + output_schema.clone(), + AlignMode::AlignToSchema, + ) + .unwrap(); let output = aligner.next().await.unwrap().unwrap(); assert_eq!(input, output); @@ -324,9 +407,13 @@ mod tests { let input = RecordBatch::try_new(input_schema, vec![int_array([10, 20])]).unwrap(); let stream = stream::iter([Ok(input)]); - let mut aligner = - NestedSchemaAligner::new(stream, vec![true, false, false], output_schema.clone()) - .unwrap(); + let mut aligner = JsonSchemaAligner::new( + stream, + vec![true, false, false], + output_schema.clone(), + AlignMode::AlignToSchema, + ) + .unwrap(); let output = aligner.next().await.unwrap().unwrap(); assert_eq!(output_schema, output.schema()); @@ -362,8 +449,13 @@ mod tests { let input = RecordBatch::try_new(input_schema, vec![int_array([10, 20])]).unwrap(); let stream = stream::iter([Ok(input)]); - let mut aligner = - NestedSchemaAligner::new(stream, vec![true, false], output_schema.clone()).unwrap(); + let mut aligner = JsonSchemaAligner::new( + stream, + vec![true, false], + output_schema.clone(), + AlignMode::AlignToSchema, + ) + .unwrap(); let output = aligner.next().await.unwrap().unwrap(); assert_eq!(output_schema, output.schema()); @@ -377,8 +469,13 @@ mod tests { let output_schema = schema([Field::new("a", DataType::Int64, true)]); let stream = stream::iter([]); - let err = match NestedSchemaAligner::new(stream, vec![true, false], output_schema) { - Ok(_) => panic!("NestedSchemaAligner should reject projection length mismatch"), + let err = match JsonSchemaAligner::new( + stream, + vec![true, false], + output_schema, + AlignMode::AlignToSchema, + ) { + Ok(_) => panic!("JsonSchemaAligner should reject projection length mismatch"), Err(err) => err, }; @@ -399,8 +496,13 @@ mod tests { let input = RecordBatch::try_new(input_schema, vec![int_array([1, 2])]).unwrap(); let stream = stream::iter([Ok(input)]); - let mut aligner = - NestedSchemaAligner::new(stream, vec![true, true, false], output_schema).unwrap(); + let mut aligner = JsonSchemaAligner::new( + stream, + vec![true, true, false], + output_schema, + AlignMode::AlignToSchema, + ) + .unwrap(); let err = aligner.next().await.unwrap().unwrap_err(); assert!( @@ -410,7 +512,7 @@ mod tests { } #[tokio::test] - async fn test_nested_schema_aligner_aligns_struct_field() { + async fn test_json_schema_aligner_aligns_struct_field() { let output_schema = schema([Field::new( "nested", DataType::Struct(Fields::from(vec![ @@ -432,9 +534,13 @@ mod tests { ) .unwrap(); - let mut aligner = - NestedSchemaAligner::new(stream::iter([Ok(input)]), vec![true], output_schema.clone()) - .unwrap(); + let mut aligner = JsonSchemaAligner::new( + stream::iter([Ok(input)]), + vec![true], + output_schema.clone(), + AlignMode::AlignToSchema, + ) + .unwrap(); let output = aligner.next().await.unwrap().unwrap(); assert_eq!(output_schema, output.schema()); @@ -448,7 +554,7 @@ mod tests { } #[tokio::test] - async fn test_nested_schema_aligner_decodes_variant_to_struct() { + async fn test_json_schema_aligner_decodes_variant_to_struct() { let source_values = [ Some(parse_string_to_jsonb("1").unwrap()), Some(parse_string_to_jsonb(r#"{"b":2}"#).unwrap()), @@ -481,9 +587,13 @@ mod tests { true, ) .with_extension_type(Json2ExtensionType::default())]); - let mut aligner = - NestedSchemaAligner::new(stream::iter([Ok(input)]), vec![true], output_schema.clone()) - .unwrap(); + let mut aligner = JsonSchemaAligner::new( + stream::iter([Ok(input)]), + vec![true], + output_schema.clone(), + AlignMode::AlignToSchema, + ) + .unwrap(); let output = aligner.next().await.unwrap().unwrap(); assert_eq!(output_schema, output.schema()); @@ -516,7 +626,7 @@ mod tests { } #[tokio::test] - async fn test_nested_schema_aligner_preserves_struct_siblings() { + async fn test_json_schema_aligner_preserves_struct_siblings() { let source_values = [ Some(parse_string_to_jsonb(r#"{"x":1}"#).unwrap()), Some(parse_string_to_jsonb(r#"{"x":2}"#).unwrap()), @@ -570,9 +680,13 @@ mod tests { true, ) .with_extension_type(Json2ExtensionType::default())]); - let mut aligner = - NestedSchemaAligner::new(stream::iter([Ok(input)]), vec![true], output_schema.clone()) - .unwrap(); + let mut aligner = JsonSchemaAligner::new( + stream::iter([Ok(input)]), + vec![true], + output_schema.clone(), + AlignMode::AlignToSchema, + ) + .unwrap(); let output = aligner.next().await.unwrap().unwrap(); assert_eq!(output_schema, output.schema()); @@ -605,6 +719,159 @@ mod tests { ); } + #[tokio::test] + async fn test_rewrite_multiple_columns_and_fill_missing_roots() { + let logical_settings = JsonSettings::default(); + let target_layout = JsonSettings::try_new(vec![], Some(0)).unwrap(); + let target_type = json2_physical_data_type(&target_layout); + let output_schema = schema([ + Field::new("j", target_type.clone(), true) + .with_extension_type(Json2ExtensionType::default()), + Field::new("missing", target_type.clone(), true) + .with_extension_type(Json2ExtensionType::default()), + Field::new("k", target_type.clone(), true) + .with_extension_type(Json2ExtensionType::default()), + Field::new("a", DataType::Int64, true), + ]); + let values = [Some(parse_string_to_jsonb(r#"{"x":1}"#).unwrap()), None]; + let source = Arc::new(BinaryArray::from_iter( + values.iter().map(|value| value.as_deref()), + )) as ArrayRef; + let source_field = Field::new("j", DataType::Binary, true) + .with_extension_type(Json2ExtensionType::default()); + let expected = JsonArray::from(&source) + .rewrite_to_v2(&source_field, &logical_settings, &target_layout) + .unwrap(); + let input = RecordBatch::try_new( + schema([ + source_field, + Field::new("k", DataType::Binary, true) + .with_extension_type(Json2ExtensionType::default()), + Field::new("a", DataType::Int64, true), + ]), + vec![source.clone(), source, int_array([10, 20])], + ) + .unwrap(); + let columns = ["j", "missing", "k"] + .into_iter() + .map(|name| { + ( + name.to_string(), + Json2TargetLayout { + extension_metadata: serde_json::to_string(&JsonMetadata::new( + logical_settings.clone(), + )) + .unwrap(), + target_layout: target_layout.clone(), + }, + ) + }) + .collect(); + let mut aligner = JsonSchemaAligner::new( + stream::iter([Ok(input)]), + vec![true, false, true, true], + output_schema.clone(), + AlignMode::Rewrite { columns }, + ) + .unwrap(); + let output = aligner.next().await.unwrap().unwrap(); + assert_eq!(output_schema, output.schema()); + assert_eq!(expected.as_ref(), output.column(0).as_ref()); + assert_eq!(expected.as_ref(), output.column(2).as_ref()); + assert_eq!(&target_type, output.column(1).data_type()); + assert_eq!(2, output.column(1).null_count()); + assert_eq!(int_array([10, 20]).as_ref(), output.column(3).as_ref()); + } + + #[test] + fn test_rewrite_rejects_mismatched_output_layout() { + let columns = HashMap::from([( + "j".to_string(), + Json2TargetLayout { + extension_metadata: serde_json::to_string(&JsonMetadata::new( + JsonSettings::default(), + )) + .unwrap(), + target_layout: JsonSettings::try_new(vec![], Some(0)).unwrap(), + }, + )]); + let result = JsonSchemaAligner::new( + stream::empty::>(), + vec![false], + schema([Field::new("j", DataType::Binary, true) + .with_extension_type(Json2ExtensionType::default())]), + AlignMode::Rewrite { columns }, + ); + assert!( + result + .unwrap_err() + .to_string() + .contains("does not match output field") + ); + } + + #[tokio::test] + async fn test_empty_rewrite_only_fills_missing_roots() { + let source = int_array([10, 20]); + let input = RecordBatch::try_new( + schema([Field::new("a", DataType::Int64, true)]), + vec![source.clone()], + ) + .unwrap(); + let output_schema = schema([ + Field::new("missing", DataType::Utf8, true), + Field::new("a", DataType::Int64, true), + ]); + let mut aligner = JsonSchemaAligner::new( + stream::iter([Ok(input)]), + vec![false, true], + output_schema.clone(), + AlignMode::Rewrite { + columns: HashMap::new(), + }, + ) + .unwrap(); + let output = aligner.next().await.unwrap().unwrap(); + assert_eq!(output_schema, output.schema()); + assert_eq!(2, output.num_rows()); + assert_eq!(2, output.column(0).null_count()); + assert!(Arc::ptr_eq(&source, output.column(1))); + } + + #[tokio::test] + async fn test_empty_rewrite_does_not_align_existing_struct() { + let source = Arc::new(StructArray::from(vec![( + Arc::new(Field::new("x", DataType::Int64, true)), + int_array([1, 2]), + )])) as ArrayRef; + let input = RecordBatch::try_new( + schema([Field::new("j", source.data_type().clone(), true)]), + vec![source], + ) + .unwrap(); + let output_schema = schema([Field::new( + "j", + DataType::Struct(Fields::from(vec![ + Field::new("x", DataType::Int64, true), + Field::new("y", DataType::Utf8, true), + ])), + true, + )]); + let mut aligner = JsonSchemaAligner::new( + stream::iter([Ok(input)]), + vec![true], + output_schema, + AlignMode::Rewrite { + columns: HashMap::new(), + }, + ) + .unwrap(); + assert!(matches!( + aligner.next().await.unwrap(), + Err(crate::error::Error::NewRecordBatch { .. }) + )); + } + fn schema(fields: impl IntoIterator) -> SchemaRef { Arc::new(Schema::new(fields.into_iter().collect::>())) } diff --git a/src/mito2/src/sst/parquet/read_columns.rs b/src/mito2/src/sst/parquet/read_columns.rs index e6a5ab9fab3..07614375dcf 100644 --- a/src/mito2/src/sst/parquet/read_columns.rs +++ b/src/mito2/src/sst/parquet/read_columns.rs @@ -12,13 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; +use std::ops::Range; -use datatypes::extension::json::JSON2_REMAINDER_FIELD_NAME; +use datatypes::arrow::datatypes::Schema as ArrowSchema; +use datatypes::extension::json::{JSON2_REMAINDER_FIELD_NAME, is_json2_extension_type}; use parquet::arrow::ProjectionMask; use parquet::basic::{ConvertedType, Type as PhysicalType}; use parquet::schema::types::{ColumnDescriptor, SchemaDescriptor}; +use crate::error::Result as MitoResult; + /// A nested field access path inside one parquet root column. pub type ParquetNestedPath = Vec; @@ -147,6 +151,32 @@ impl ParquetReadColumn { } } +/// Nested leaf selection semantics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum NestedSelectionPolicy { + /// Also read the nearest JSONB Variant ancestor, or the remainder when it may + /// contain an unresolved path or children of a materialized object. + /// + /// For example, if `j.cold` has no matching field or Variant ancestor, read the + /// remainder because it may contain `j.cold`. When requesting a materialized + /// object such as `j.commit`, also read the remainder because it may contain + /// unmaterialized children of `j.commit`. + Json2, +} + +impl NestedSelectionPolicy { + /// Selects all leaves needed for one JSON2 root, including fallback data. + fn select_leaves( + self, + schema: &SchemaDescriptor, + leaf_range: Range, + col: &ParquetReadColumn, + selected: &mut HashSet, + ) { + select_json2_leaves(schema, leaf_range, col, selected); + } +} + /// Projection plan built for a parquet file. #[derive(Clone)] pub struct ProjectionMaskPlan { @@ -173,6 +203,9 @@ pub struct ProjectionMaskPlan { /// file. It is used to resolve requested nested paths to actual leaf /// column indices. /// +/// `source_schema` is the Arrow schema of the current parquet file. It is used +/// only for nested projections to identify JSON2 root fields. +/// /// See [`ProjectionMaskPlan`] for the returned value. /// /// For example, if the query requests `j.a` and `k`, but the current @@ -183,18 +216,20 @@ pub struct ProjectionMaskPlan { pub(crate) fn build_projection_plan( parquet_read_cols: &ParquetReadColumns, parquet_schema_desc: &SchemaDescriptor, -) -> ProjectionMaskPlan { + source_schema: &ArrowSchema, +) -> MitoResult { if !parquet_read_cols.has_nested() { let mask = ProjectionMask::roots(parquet_schema_desc, parquet_read_cols.root_indices_iter()); - return ProjectionMaskPlan { + + return Ok(ProjectionMaskPlan { mask, projected_root_presence: vec![true; parquet_read_cols.columns().len()], - }; + }); } let (matched_leaves, matched_roots) = - build_parquet_leaves_indices(parquet_schema_desc, parquet_read_cols); + build_parquet_leaves_indices(parquet_schema_desc, parquet_read_cols, source_schema)?; let projected_root_presence = parquet_read_cols .columns() @@ -203,10 +238,11 @@ pub(crate) fn build_projection_plan( .collect(); let mask = ProjectionMask::leaves(parquet_schema_desc, matched_leaves); - ProjectionMaskPlan { + + Ok(ProjectionMaskPlan { mask, projected_root_presence, - } + }) } /// Builds parquet leaf-column indices for reading a parquet file. @@ -217,89 +253,114 @@ pub(crate) fn build_projection_plan( fn build_parquet_leaves_indices( parquet_schema_desc: &SchemaDescriptor, projection: &ParquetReadColumns, -) -> (Vec, HashSet) { - let mut map = HashMap::with_capacity(projection.cols.len()); - for col in &projection.cols { - map.insert(col.root_index, col); - } + source_schema: &ArrowSchema, +) -> MitoResult<(Vec, HashSet)> { + let root_leaf_ranges = group_requested_leaf_ranges(parquet_schema_desc, projection); let mut matched_leaves = HashSet::new(); - let mut matched_roots = HashSet::with_capacity(projection.cols.len()); + let mut matched_roots = HashSet::with_capacity(projection.columns().len()); - // Tracks whether each requested nested path matched by prefix without fallback. - let mut prefix_matched = HashMap::>::new(); - for col in &projection.cols { - prefix_matched.insert(col.root_index, vec![false; col.nested_paths.len()]); - } + for col in projection.columns() { + let before = matched_leaves.len(); - // First select root/nested prefix matches. - for (leaf_idx, leaf_col) in parquet_schema_desc.columns().iter().enumerate() { - let root_idx = parquet_schema_desc.get_column_root_idx(leaf_idx); - let Some(col) = map.get(&root_idx) else { - continue; - }; - if col.nested_paths.is_empty() { - matched_leaves.insert(leaf_idx); - matched_roots.insert(root_idx); - continue; + let leaf_range = root_leaf_ranges[col.root_index()].clone(); + + if col.nested_paths().is_empty() { + matched_leaves.extend(leaf_range); + } else if is_json2_extension_type(&source_schema.fields()[col.root_index()]) { + NestedSelectionPolicy::Json2.select_leaves( + parquet_schema_desc, + leaf_range, + col, + &mut matched_leaves, + ); + } else { + select_prefix_leaves(parquet_schema_desc, leaf_range, col, &mut matched_leaves); } - let leaf_path = leaf_col.path().parts(); - let mut matched = false; - for (path_idx, _) in col - .nested_paths - .iter() - .enumerate() - .filter(|(_, nested_path)| leaf_path.starts_with(nested_path)) - { - prefix_matched.get_mut(&root_idx).unwrap()[path_idx] = true; - matched = true; - } - - if matched { - matched_leaves.insert(leaf_idx); - matched_roots.insert(root_idx); - } - } - - // Then include v2 remainder leaves or fallback prefix misses to their nearest variant parent. - // TODO(fys): Gate fallback planning on the root being JSON2. A raw Binary - // leaf is a JSONB variant only under a JSON2 root; plain struct Binary - // children should not enter this fallback path. - for col in &projection.cols { - let path_matches = &prefix_matched[&col.root_index]; - let mut needs_remainder = false; - for (matched, nested_path) in path_matches.iter().zip(&col.nested_paths) { - if *matched { - if !needs_remainder { - needs_remainder = - path_points_to_struct(parquet_schema_desc, col.root_index, nested_path); - } - continue; - } - - if let Some(leaf_idx) = - find_nearest_variant_parent(parquet_schema_desc, col.root_index, nested_path) - { - matched_leaves.insert(leaf_idx); - matched_roots.insert(col.root_index); - } else { - needs_remainder = true; - } - } - - if needs_remainder { - let remainder_leaves = find_remainder_leaves(parquet_schema_desc, col.root_index); - if !remainder_leaves.is_empty() { - matched_leaves.extend(remainder_leaves); - matched_roots.insert(col.root_index); - } + // Requested roots are unique, and leaves belong to exactly one root. + if matched_leaves.len() > before { + matched_roots.insert(col.root_index()); } } let mut matched_leaves = matched_leaves.into_iter().collect::>(); matched_leaves.sort_unstable(); - (matched_leaves, matched_roots) + + Ok((matched_leaves, matched_roots)) +} + +/// Groups file-level leaf ranges for requested roots. +fn group_requested_leaf_ranges( + schema: &SchemaDescriptor, + projection: &ParquetReadColumns, +) -> Vec> { + let root_count = schema.root_schema().get_fields().len(); + let mut requested = vec![false; root_count]; + for col in projection.columns() { + requested[col.root_index()] = true; + } + + let mut ranges = vec![0..0; root_count]; + let mut leaf_idx = 0; + while leaf_idx < schema.num_columns() { + let root_idx = schema.get_column_root_idx(leaf_idx); + let start = leaf_idx; + while leaf_idx < schema.num_columns() && schema.get_column_root_idx(leaf_idx) == root_idx { + leaf_idx += 1; + } + if requested[root_idx] { + ranges[root_idx] = start..leaf_idx; + } + } + ranges +} + +/// V2 can additionally store missing paths and object children in the remainder. +fn select_json2_leaves( + schema: &SchemaDescriptor, + leaf_range: Range, + col: &ParquetReadColumn, + selected: &mut HashSet, +) { + let prefix_matched = select_prefix_leaves(schema, leaf_range, col, selected); + let mut needs_remainder = false; + for (matched, path) in prefix_matched.iter().zip(&col.nested_paths) { + if *matched { + needs_remainder |= path_points_to_struct(schema, col.root_index, path); + } else if let Some(idx) = find_nearest_variant_parent(schema, col.root_index, path) { + selected.insert(idx); + } else { + needs_remainder = true; + } + } + if needs_remainder { + selected.extend(find_remainder_leaves(schema, col.root_index)); + } +} + +/// Selects prefix matches and returns a match flag for each requested nested path. +fn select_prefix_leaves( + schema: &SchemaDescriptor, + leaf_range: Range, + col: &ParquetReadColumn, + selected: &mut HashSet, +) -> Vec { + let mut prefix_matched = vec![false; col.nested_paths().len()]; + for leaf_idx in leaf_range { + let leaf_path = schema.columns()[leaf_idx].path().parts(); + let mut matched_leaf = false; + for (path, matched) in col.nested_paths().iter().zip(&mut prefix_matched) { + if leaf_path.starts_with(path) { + *matched = true; + matched_leaf = true; + } + } + if matched_leaf { + selected.insert(leaf_idx); + } + } + prefix_matched } /// Returns whether a nested path points to an explicitly materialized object. @@ -386,7 +447,10 @@ fn is_variant_leaf(leaf_col: &ColumnDescriptor) -> bool { mod tests { use std::sync::Arc; - use parquet::basic::{ConvertedType, LogicalType, Repetition}; + use datatypes::arrow::datatypes::{DataType, Field, Fields, Schema as ArrowSchema}; + use datatypes::extension::json::{Json2ExtensionType, JsonMetadata}; + use datatypes::json::JsonSettings; + use parquet::basic::{LogicalType, Repetition, VariantType}; use parquet::errors::ParquetError; use parquet::schema::types::Type; @@ -397,7 +461,7 @@ mod tests { let parquet_schema_desc = build_test_nested_parquet_schema(); let projection = ParquetReadColumns::from_deduped_root_indices([0, 1]); - let plan = build_projection_plan(&projection, &parquet_schema_desc); + let plan = build_projection_plan(&projection, &parquet_schema_desc, &[None; 2]); assert_eq!(vec![true, true], plan.projected_root_presence); assert_eq!( @@ -412,8 +476,11 @@ mod tests { let projection = ParquetReadColumns::from_deduped(vec![ParquetReadColumn::new(0)]); - let (matched_leaves, matched_roots) = - build_parquet_leaves_indices(&parquet_schema_desc, &projection); + let (matched_leaves, matched_roots) = build_parquet_leaves_indices_with_policies( + &parquet_schema_desc, + &projection, + &[None; 2], + ); assert_eq!(vec![0, 1, 2], matched_leaves); assert_eq!(HashSet::from([0]), matched_roots); } @@ -428,8 +495,11 @@ mod tests { ParquetReadColumn::new(1), ]); - let (matched_leaves, matched_roots) = - build_parquet_leaves_indices(&parquet_schema_desc, &projection); + let (matched_leaves, matched_roots) = build_parquet_leaves_indices_with_policies( + &parquet_schema_desc, + &projection, + &[None; 2], + ); assert_eq!(vec![1, 2, 3], matched_leaves); assert_eq!(HashSet::from([0, 1]), matched_roots); } @@ -443,8 +513,11 @@ mod tests { .with_nested_paths(vec![vec!["j".to_string(), "b".to_string()]]), ]); - let (matched_leaves, matched_roots) = - build_parquet_leaves_indices(&parquet_schema_desc, &projection); + let (matched_leaves, matched_roots) = build_parquet_leaves_indices_with_policies( + &parquet_schema_desc, + &projection, + &[None; 2], + ); assert_eq!(vec![1, 2], matched_leaves); assert_eq!(HashSet::from([0]), matched_roots); } @@ -460,8 +533,11 @@ mod tests { let read_column = ParquetReadColumn::new(0).with_nested_paths(nested_paths); let projection = ParquetReadColumns::from_deduped(vec![read_column]); - let (matched_leaves, matched_roots) = - build_parquet_leaves_indices(&parquet_schema_desc, &projection); + let (matched_leaves, matched_roots) = build_parquet_leaves_indices_with_policies( + &parquet_schema_desc, + &projection, + &[None; 2], + ); assert_eq!(vec![1, 2], matched_leaves); assert_eq!(HashSet::from([0]), matched_roots); } @@ -475,8 +551,11 @@ mod tests { vec![vec!["j".to_string(), "b".to_string(), "c".to_string()]], )]); - let (matched_leaves, matched_roots) = - build_parquet_leaves_indices(&parquet_schema_desc, &projection); + let (matched_leaves, matched_roots) = build_parquet_leaves_indices_with_policies( + &parquet_schema_desc, + &projection, + &[None; 2], + ); assert_eq!(vec![1], matched_leaves); assert_eq!(HashSet::from([0]), matched_roots); } @@ -491,7 +570,7 @@ mod tests { ParquetReadColumn::new(1), ]); - let plan = build_projection_plan(&projection, &parquet_schema_desc); + let plan = build_projection_plan(&projection, &parquet_schema_desc, &[None; 2]); assert_eq!(vec![false, true], plan.projected_root_presence); assert_eq!( @@ -511,7 +590,8 @@ mod tests { ], )]); - let plan = build_projection_plan(&projection, &parquet); + let plan = + build_projection_plan(&projection, &parquet, &[Some(NestedSelectionPolicy::Json2)]); assert_eq!(vec![true], plan.projected_root_presence); assert_eq!(ProjectionMask::leaves(&parquet, [0, 1]), plan.mask); @@ -526,7 +606,8 @@ mod tests { .with_nested_paths(vec![vec!["j".to_string(), "hot".to_string()]]), ]); - let plan = build_projection_plan(&projection, &parquet); + let plan = + build_projection_plan(&projection, &parquet, &[Some(NestedSelectionPolicy::Json2)]); assert_eq!(vec![true], plan.projected_root_presence); assert_eq!(ProjectionMask::leaves(&parquet, [3]), plan.mask); @@ -541,7 +622,8 @@ mod tests { .with_nested_paths(vec![vec!["j".to_string(), "commit".to_string()]]), ]); - let plan = build_projection_plan(&projection, &parquet); + let plan = + build_projection_plan(&projection, &parquet, &[Some(NestedSelectionPolicy::Json2)]); assert_eq!(vec![true], plan.projected_root_presence); assert_eq!(ProjectionMask::leaves(&parquet, [0, 1, 2]), plan.mask); @@ -563,7 +645,8 @@ mod tests { ]], )]); - let plan = build_projection_plan(&projection, &parquet); + let plan = + build_projection_plan(&projection, &parquet, &[Some(NestedSelectionPolicy::Json2)]); assert_eq!(vec![true], plan.projected_root_presence); assert_eq!(ProjectionMask::leaves(&parquet, [4]), plan.mask); @@ -582,8 +665,11 @@ mod tests { ], )]); - let (matched_leaves, matched_roots) = - build_parquet_leaves_indices(&parquet_schema_desc, &projection); + let (matched_leaves, matched_roots) = build_parquet_leaves_indices_with_policies( + &parquet_schema_desc, + &projection, + &[None; 2], + ); assert_eq!(vec![0, 2], matched_leaves); assert_eq!(HashSet::from([0]), matched_roots); } @@ -614,137 +700,6 @@ mod tests { assert!(col.nested_paths().is_empty()); } - #[test] - fn test_fallback_to_nearest_variant_parent() { - let parquet_schema_desc = build_test_variant_parent_schema(); - let projection = - ParquetReadColumns::from_deduped(vec![ParquetReadColumn::new(0).with_nested_paths( - vec![vec!["j".to_string(), "a".to_string(), "x".to_string()]], - )]); - - let plan = build_projection_plan(&projection, &parquet_schema_desc); - - assert_eq!(vec![true], plan.projected_root_presence); - assert_eq!( - ProjectionMask::leaves(&parquet_schema_desc, vec![0]), - plan.mask - ); - } - - #[test] - fn test_prefix_match_prevents_variant_parent_fallback() { - let parquet_schema_desc = build_test_variant_parent_schema(); - let projection = - ParquetReadColumns::from_deduped(vec![ParquetReadColumn::new(0).with_nested_paths( - vec![vec!["j".to_string(), "b".to_string(), "x".to_string()]], - )]); - - let plan = build_projection_plan(&projection, &parquet_schema_desc); - - assert_eq!(vec![true], plan.projected_root_presence); - assert_eq!( - ProjectionMask::leaves(&parquet_schema_desc, vec![1]), - plan.mask - ); - } - - #[test] - fn test_mixed_prefix_and_fallback_paths() { - let parquet_schema_desc = build_test_variant_parent_schema(); - let nested_paths = vec![ - vec!["j".to_string(), "a".to_string(), "x".to_string()], - vec!["j".to_string(), "b".to_string(), "x".to_string()], - ]; - let projection = ParquetReadColumns::from_deduped(vec![ - ParquetReadColumn::new(0).with_nested_paths(nested_paths), - ]); - - let plan = build_projection_plan(&projection, &parquet_schema_desc); - - assert_eq!(vec![true], plan.projected_root_presence); - assert_eq!( - ProjectionMask::leaves(&parquet_schema_desc, vec![0, 1]), - plan.mask - ); - } - - #[test] - fn test_fallback_selects_multiple_variant_parents() { - let parquet_schema_desc = build_test_two_variant_parents_schema(); - let nested_paths = vec![ - vec!["j".to_string(), "a".to_string(), "y".to_string()], - vec!["j".to_string(), "b".to_string(), "d".to_string()], - vec!["j".to_string(), "a".to_string(), "x".to_string()], - ]; - let projection = ParquetReadColumns::from_deduped(vec![ - ParquetReadColumn::new(0).with_nested_paths(nested_paths), - ]); - - let plan = build_projection_plan(&projection, &parquet_schema_desc); - - assert_eq!(vec![true], plan.projected_root_presence); - assert_eq!( - ProjectionMask::leaves(&parquet_schema_desc, vec![0, 1]), - plan.mask - ); - } - - #[test] - fn test_nested_paths_fallback_to_variant_parent_by_default() { - let parquet_schema_desc = build_test_variant_parent_schema(); - let projection = - ParquetReadColumns::from_deduped(vec![ParquetReadColumn::new(0).with_nested_paths( - vec![vec!["j".to_string(), "a".to_string(), "x".to_string()]], - )]); - - let plan = build_projection_plan(&projection, &parquet_schema_desc); - - assert_eq!(vec![true], plan.projected_root_presence); - assert_eq!( - ProjectionMask::leaves(&parquet_schema_desc, vec![0]), - plan.mask - ); - } - - #[test] - fn test_non_variant_parent_does_not_fallback() { - let parquet_schema_desc = build_test_nested_parquet_schema(); - let projection = - ParquetReadColumns::from_deduped(vec![ParquetReadColumn::new(0).with_nested_paths( - vec![vec!["j".to_string(), "a".to_string(), "x".to_string()]], - )]); - - let plan = build_projection_plan(&projection, &parquet_schema_desc); - - assert_eq!(vec![false], plan.projected_root_presence); - } - - #[test] - fn test_utf8_parent_does_not_fallback() { - let parquet_schema_desc = build_test_utf8_parent_schema(); - let projection = - ParquetReadColumns::from_deduped(vec![ParquetReadColumn::new(0).with_nested_paths( - vec![vec!["j".to_string(), "a".to_string(), "x".to_string()]], - )]); - - let plan = build_projection_plan(&projection, &parquet_schema_desc); - - assert_eq!(vec![false], plan.projected_root_presence); - } - - #[test] - fn test_root_variant_does_not_fallback() { - let parquet_schema_desc = build_test_root_variant_schema(); - let projection = ParquetReadColumns::from_deduped(vec![ - ParquetReadColumn::new(0) - .with_nested_paths(vec![vec!["j".to_string(), "a".to_string()]]), - ]); - - let plan = build_projection_plan(&projection, &parquet_schema_desc); - - assert_eq!(vec![false], plan.projected_root_presence); - } - // Test schema: // schema // |- j @@ -816,9 +771,9 @@ mod tests { let remainder = Arc::new( Type::group_type_builder(JSON2_REMAINDER_FIELD_NAME) .with_repetition(Repetition::OPTIONAL) - .with_logical_type(Some(LogicalType::Variant { + .with_logical_type(Some(LogicalType::Variant(VariantType { specification_version: None, - })) + }))) .with_fields(vec![metadata, value]) .build()?, ); @@ -859,127 +814,44 @@ mod tests { ))) } - // Test schema: - // schema - // `- j - // |- a: BYTE_ARRAY - // `- b - // `- x: INT64 - fn build_test_variant_parent_schema() -> SchemaDescriptor { - let leaf_a = Arc::new( - Type::primitive_type_builder("a", parquet::basic::Type::BYTE_ARRAY) - .with_repetition(Repetition::REQUIRED) - .build() - .unwrap(), - ); - let leaf_x = Arc::new( - Type::primitive_type_builder("x", parquet::basic::Type::INT64) - .with_repetition(Repetition::REQUIRED) - .build() - .unwrap(), - ); - let group_b = Arc::new( - Type::group_type_builder("b") - .with_repetition(Repetition::REQUIRED) - .with_fields(vec![leaf_x]) - .build() - .unwrap(), - ); - let root_j = Arc::new( - Type::group_type_builder("j") - .with_repetition(Repetition::REQUIRED) - .with_fields(vec![leaf_a, group_b]) - .build() - .unwrap(), - ); - let schema = Arc::new( - Type::group_type_builder("schema") - .with_fields(vec![root_j]) - .build() - .unwrap(), - ); - - SchemaDescriptor::new(schema) + fn source_schema(policies: &[Option]) -> ArrowSchema { + ArrowSchema::new( + policies + .iter() + .enumerate() + .map(|(index, policy)| { + let field = Field::new( + format!("root_{index}"), + DataType::Struct(Fields::empty()), + true, + ); + match policy { + None => field, + Some(NestedSelectionPolicy::Json2) => { + field.with_extension_type(Json2ExtensionType::new(Arc::new( + JsonMetadata::new(JsonSettings::default()), + ))) + } + } + }) + .collect::>(), + ) } - // Test schema: - // schema - // `- j - // |- a: BYTE_ARRAY - // `- b: BYTE_ARRAY - fn build_test_two_variant_parents_schema() -> SchemaDescriptor { - let leaf_a = Arc::new( - Type::primitive_type_builder("a", parquet::basic::Type::BYTE_ARRAY) - .with_repetition(Repetition::REQUIRED) - .build() - .unwrap(), - ); - let leaf_b = Arc::new( - Type::primitive_type_builder("b", parquet::basic::Type::BYTE_ARRAY) - .with_repetition(Repetition::REQUIRED) - .build() - .unwrap(), - ); - let root_j = Arc::new( - Type::group_type_builder("j") - .with_repetition(Repetition::REQUIRED) - .with_fields(vec![leaf_a, leaf_b]) - .build() - .unwrap(), - ); - let schema = Arc::new( - Type::group_type_builder("schema") - .with_fields(vec![root_j]) - .build() - .unwrap(), - ); - - SchemaDescriptor::new(schema) + fn build_projection_plan( + cols: &ParquetReadColumns, + parquet_schema: &SchemaDescriptor, + policies: &[Option], + ) -> ProjectionMaskPlan { + super::build_projection_plan(cols, parquet_schema, &source_schema(policies)).unwrap() } - fn build_test_utf8_parent_schema() -> SchemaDescriptor { - let leaf_a = Arc::new( - Type::primitive_type_builder("a", parquet::basic::Type::BYTE_ARRAY) - .with_repetition(Repetition::REQUIRED) - .with_logical_type(Some(LogicalType::String)) - .with_converted_type(ConvertedType::UTF8) - .build() - .unwrap(), - ); - let root_j = Arc::new( - Type::group_type_builder("j") - .with_repetition(Repetition::REQUIRED) - .with_fields(vec![leaf_a]) - .build() - .unwrap(), - ); - let schema = Arc::new( - Type::group_type_builder("schema") - .with_fields(vec![root_j]) - .build() - .unwrap(), - ); - - SchemaDescriptor::new(schema) - } - - // Test schema: - // schema - // `- j: BYTE_ARRAY - fn build_test_root_variant_schema() -> SchemaDescriptor { - let root_j = Arc::new( - Type::primitive_type_builder("j", parquet::basic::Type::BYTE_ARRAY) - .with_repetition(Repetition::REQUIRED) - .build() - .unwrap(), - ); - let schema = Arc::new( - Type::group_type_builder("schema") - .with_fields(vec![root_j]) - .build() - .unwrap(), - ); - - SchemaDescriptor::new(schema) + fn build_parquet_leaves_indices_with_policies( + parquet_schema: &SchemaDescriptor, + projection: &ParquetReadColumns, + policies: &[Option], + ) -> (Vec, HashSet) { + super::build_parquet_leaves_indices(parquet_schema, projection, &source_schema(policies)) + .unwrap() } } diff --git a/src/mito2/src/sst/parquet/reader.rs b/src/mito2/src/sst/parquet/reader.rs index b8588bd36cd..e3232c189d6 100644 --- a/src/mito2/src/sst/parquet/reader.rs +++ b/src/mito2/src/sst/parquet/reader.rs @@ -85,7 +85,7 @@ use crate::sst::parquet::file_range::{ }; use crate::sst::parquet::flat_format::{FlatReadFormat, primary_key_column_index}; use crate::sst::parquet::format::{INTERNAL_COLUMN_NUM, need_override_sequence}; -use crate::sst::parquet::json_align::{NestedSchemaAligner, ProjectedRecordBatchStream}; +use crate::sst::parquet::json_align::{AlignMode, JsonSchemaAligner, ProjectedRecordBatchStream}; use crate::sst::parquet::metadata::MetadataLoader; use crate::sst::parquet::prefilter::{ PrefilterContextBuilder, build_reader_filter_plan, execute_prefilter, @@ -522,13 +522,14 @@ impl ParquetReaderBuilder { let file_metadata = parquet_meta.file_metadata(); let parquet_schema_desc = file_metadata.schema_descr(); - let file_schema = + let file_schema = Arc::new( parquet_to_arrow_schema(parquet_schema_desc, file_metadata.key_value_metadata()) - .context(ParquetToArrowSchemaSnafu { file: &file_path })?; + .context(ParquetToArrowSchemaSnafu { file: &file_path })?, + ); let mut read_format = FlatReadFormat::new( region_meta.clone(), read_cols, - Some(Arc::new(file_schema)), + Some(file_schema.clone()), &file_path, skip_auto_convert, )?; @@ -565,7 +566,8 @@ impl ParquetReaderBuilder { // Computes the projection mask. let parquet_read_cols = read_format.parquet_read_columns(); - let projection_plan = build_projection_plan(parquet_read_cols, parquet_schema_desc); + let projection_plan = + build_projection_plan(parquet_read_cols, parquet_schema_desc, &file_schema)?; let has_nested_projection = parquet_read_cols.has_nested(); let selection = self .row_groups_to_read(&read_format, &parquet_meta, &mut metrics.filter_metrics) @@ -2070,12 +2072,20 @@ impl RowGroupReaderBuilder { return Ok(stream); } - Ok(NestedSchemaAligner::new( + let mode = if self.json2_rewrite_targets.is_empty() { + AlignMode::AlignToSchema + } else { + AlignMode::Rewrite { + columns: self.json2_rewrite_targets.clone(), + } + }; + + Ok(JsonSchemaAligner::new( stream, self.projection.projected_root_presence.clone(), self.output_schema.clone(), + mode, )? - .with_json2_rewrite_targets(&self.json2_rewrite_targets)? .boxed()) } @@ -2556,7 +2566,6 @@ impl FlatRowGroupReader { #[cfg(test)] mod tests { - use std::any::Any; use std::collections::HashMap; use std::fmt::{Debug, Formatter}; use std::sync::{Arc, LazyLock}; @@ -2694,7 +2703,17 @@ mod tests { let output_schema = read_format.arrow_schema().clone(); let parquet_schema = parquet_meta.file_metadata().schema_descr(); - let projection = build_projection_plan(read_format.parquet_read_columns(), parquet_schema); + let source_schema = parquet_to_arrow_schema( + parquet_schema, + parquet_meta.file_metadata().key_value_metadata(), + ) + .unwrap(); + let projection = build_projection_plan( + read_format.parquet_read_columns(), + parquet_schema, + &source_schema, + ) + .unwrap(); let arrow_metadata = ArrowReaderMetadata::try_new(parquet_meta.clone(), ArrowReaderOptions::new()).unwrap(); ( @@ -2990,7 +3009,8 @@ mod tests { ParquetReadColumns::from_deduped(vec![ParquetReadColumn::new(0).with_nested_paths( vec![vec!["j".to_string(), "a".to_string(), "x".to_string()]], )]); - let projection_plan = build_projection_plan(&projection, parquet_schema); + let projection_plan = + build_projection_plan(&projection, parquet_schema, batch.schema_ref()).unwrap(); assert_eq!(vec![true], projection_plan.projected_root_presence); assert_eq!( projection_plan.mask, @@ -3048,10 +3068,6 @@ mod tests { } impl ScalarUDFImpl for PanicDebugUdf { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "panic_debug_udf" } @@ -3150,10 +3166,6 @@ mod tests { } impl ScalarUDFImpl for TestVolatilityUdf { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { &self.name } diff --git a/src/mito2/src/sst/parquet/stats.rs b/src/mito2/src/sst/parquet/stats.rs index 368828d1f94..6961e57ab27 100644 --- a/src/mito2/src/sst/parquet/stats.rs +++ b/src/mito2/src/sst/parquet/stats.rs @@ -174,7 +174,7 @@ impl> PruningStatistics for RowGroupPruningStats<'_, } } - fn row_counts(&self, _column: &Column) -> Option { + fn row_counts(&self) -> Option { // TODO(LFC): Impl it. None } diff --git a/src/mito2/src/sst/parquet/writer.rs b/src/mito2/src/sst/parquet/writer.rs index b9d5016173e..9888b4ead53 100644 --- a/src/mito2/src/sst/parquet/writer.rs +++ b/src/mito2/src/sst/parquet/writer.rs @@ -38,7 +38,7 @@ use object_store::{FuturesAsyncWriter, ObjectStore}; use parquet::arrow::AsyncArrowWriter; use parquet::basic::{Compression, Encoding, ZstdLevel}; use parquet::file::metadata::KeyValue; -use parquet::file::properties::{WriterProperties, WriterPropertiesBuilder}; +use parquet::file::properties::WriterProperties; use parquet::schema::types::ColumnPath; use smallvec::smallvec; use snafu::{OptionExt, ResultExt}; @@ -61,7 +61,9 @@ use crate::sst::parquet::flat_format::{ FlatWriteFormat, primary_key_column_index, time_index_column_index, }; use crate::sst::parquet::format::{PrimaryKeyArray, PrimaryKeyWriteFormat}; -use crate::sst::parquet::{PARQUET_METADATA_KEY, SstInfo, WriteOptions}; +use crate::sst::parquet::{ + PARQUET_METADATA_KEY, SstInfo, WriteOptions, apply_float_field_encoding, +}; use crate::sst::{ DEFAULT_WRITE_BUFFER_SIZE, DEFAULT_WRITE_CONCURRENCY, FlatSchemaOptions, SeriesEstimator, maybe_wrap_schema, @@ -473,29 +475,6 @@ where Ok(results) } - /// Customizes per-column config according to schema and maybe column cardinality. - fn customize_column_config( - builder: WriterPropertiesBuilder, - region_metadata: &RegionMetadataRef, - ) -> WriterPropertiesBuilder { - let ts_col = ColumnPath::new(vec![ - region_metadata - .time_index_column() - .column_schema - .name - .clone(), - ]); - let seq_col = ColumnPath::new(vec![SEQUENCE_COLUMN_NAME.to_string()]); - let op_type_col = ColumnPath::new(vec![OP_TYPE_COLUMN_NAME.to_string()]); - - builder - .set_column_encoding(seq_col.clone(), Encoding::DELTA_BINARY_PACKED) - .set_column_dictionary_enabled(seq_col, false) - .set_column_encoding(ts_col.clone(), Encoding::DELTA_BINARY_PACKED) - .set_column_dictionary_enabled(ts_col, false) - .set_column_compression(op_type_col, Compression::UNCOMPRESSED) - } - async fn append_flat_batch( &mut self, batch: &RecordBatch, @@ -564,8 +543,22 @@ where .set_max_row_group_row_count(Some(opts.row_group_size)) .set_column_index_truncate_length(None) .set_statistics_truncate_length(None); - - let props_builder = Self::customize_column_config(props_builder, &self.metadata); + let ts_col = ColumnPath::new(vec![ + self.metadata.time_index_column().column_schema.name.clone(), + ]); + let seq_col = ColumnPath::new(vec![SEQUENCE_COLUMN_NAME.to_string()]); + let op_type_col = ColumnPath::new(vec![OP_TYPE_COLUMN_NAME.to_string()]); + let props_builder = props_builder + .set_column_encoding(seq_col.clone(), Encoding::DELTA_BINARY_PACKED) + .set_column_dictionary_enabled(seq_col, false) + .set_column_encoding(ts_col.clone(), Encoding::DELTA_BINARY_PACKED) + .set_column_dictionary_enabled(ts_col, false) + .set_column_compression(op_type_col, Compression::UNCOMPRESSED); + let props_builder = apply_float_field_encoding( + props_builder, + &self.metadata, + opts.float_field_encoding, + ); let writer_props = props_builder.build(); let sst_file_path = self.path_provider.build_sst_file_path(RegionFileId::new( diff --git a/src/mito2/src/test_util.rs b/src/mito2/src/test_util.rs index aa03e4bc847..dbfbc53db25 100644 --- a/src/mito2/src/test_util.rs +++ b/src/mito2/src/test_util.rs @@ -682,6 +682,7 @@ impl TestEnv { match log_store { LogStoreImpl::RaftEngine(log_store) => WorkerGroup::start( + &data_home, Arc::new(config), log_store, Arc::new(object_store_manager), @@ -693,6 +694,7 @@ impl TestEnv { .await .unwrap(), LogStoreImpl::Kafka(log_store) => WorkerGroup::start( + &data_home, Arc::new(config), log_store, Arc::new(object_store_manager), diff --git a/src/mito2/src/test_util/bench_util.rs b/src/mito2/src/test_util/bench_util.rs index 8f182e41578..1fa1a53a439 100644 --- a/src/mito2/src/test_util/bench_util.rs +++ b/src/mito2/src/test_util/bench_util.rs @@ -33,6 +33,8 @@ use store_api::storage::RegionId; use table::predicate::Predicate; use crate::memtable::KeyValues; +pub use crate::sst::index::bloom_filter::creator::BloomFilterIndexer; +pub use crate::sst::index::inverted_index::creator::InvertedIndexer; use crate::test_util::memtable_util::region_metadata_to_row_schema; pub struct Host { diff --git a/src/mito2/src/time_provider.rs b/src/mito2/src/time_provider.rs index 37cfd59e415..6acda19792b 100644 --- a/src/mito2/src/time_provider.rs +++ b/src/mito2/src/time_provider.rs @@ -50,3 +50,49 @@ impl TimeProvider for StdTimeProvider { current_time_millis() - current_millis } } + +/// Controllable event time with short waits for background-task tests. +#[cfg(test)] +pub(crate) mod mock { + use std::sync::atomic::{AtomicI64, Ordering}; + use std::time::Duration; + + use crate::time_provider::TimeProvider; + + #[derive(Debug)] + pub(crate) struct MockTimeProvider { + now: AtomicI64, + elapsed: AtomicI64, + } + + impl TimeProvider for MockTimeProvider { + fn current_time_millis(&self) -> i64 { + self.now.load(Ordering::Relaxed) + } + + fn elapsed_since(&self, _current_millis: i64) -> i64 { + self.elapsed.load(Ordering::Relaxed) + } + + fn wait_duration(&self, _duration: Duration) -> Duration { + Duration::from_millis(20) + } + } + + impl MockTimeProvider { + pub(crate) fn new(now: i64) -> Self { + Self { + now: AtomicI64::new(now), + elapsed: AtomicI64::new(0), + } + } + + pub(crate) fn set_now(&self, now: i64) { + self.now.store(now, Ordering::Relaxed); + } + + pub(crate) fn set_elapsed(&self, elapsed: i64) { + self.elapsed.store(elapsed, Ordering::Relaxed); + } + } +} diff --git a/src/mito2/src/wal/entry_distributor.rs b/src/mito2/src/wal/entry_distributor.rs index e48c21967a5..b2b774f67de 100644 --- a/src/mito2/src/wal/entry_distributor.rs +++ b/src/mito2/src/wal/entry_distributor.rs @@ -133,24 +133,7 @@ impl WalEntryReader for WalEntryReceiver { } let stream = stream! { - let mut buffered_entry: Option = None; - while let Some(next_entry) = entry_receiver.recv().await { - match buffered_entry.take() { - Some(entry) => { - if entry.is_complete() { - yield decode_raw_entry(entry); - } else { - warn!("Ignoring incomplete entry: {}", entry); - } - buffered_entry = Some(next_entry); - }, - None => { - buffered_entry = Some(next_entry); - } - }; - } - if let Some(entry) = buffered_entry { - // Ignores tail corrupted data. + while let Some(entry) = entry_receiver.recv().await { if entry.is_complete() { yield decode_raw_entry(entry); } else { @@ -220,8 +203,10 @@ pub fn build_wal_entry_distributor_and_receivers( #[cfg(test)] mod tests { + use std::time::Duration; + use api::v1::{Mutation, OpType, WalEntry}; - use futures::{TryStreamExt, stream}; + use futures::{StreamExt, TryStreamExt, stream}; use prost::Message; use store_api::logstore::entry::{Entry, MultiplePartEntry, MultiplePartHeader, NaiveEntry}; @@ -247,6 +232,34 @@ mod tests { } } + #[tokio::test] + async fn test_delivers_complete_entry_while_channel_is_alive() { + let provider = Provider::kafka_provider("my_topic".to_string()); + let region_id = RegionId::new(1024, 1); + let wal_entry = WalEntry::default(); + let (sender, receiver) = mpsc::channel(1); + let (arg_sender, arg_receiver) = oneshot::channel(); + let mut receiver = WalEntryReceiver::new(receiver, arg_sender); + let mut stream = receiver.read(&provider, 0).unwrap(); + assert_eq!(arg_receiver.await.unwrap(), 0); + sender + .send(Entry::Naive(NaiveEntry { + provider, + region_id, + entry_id: 1, + data: wal_entry.encode_to_vec(), + })) + .await + .unwrap(); + + let entry = tokio::time::timeout(Duration::from_secs(1), stream.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(entry, (1, wal_entry)); + } + #[tokio::test] async fn test_wal_entry_distributor_without_receivers() { let provider = Provider::kafka_provider("my_topic".to_string()); diff --git a/src/mito2/src/wal/entry_reader.rs b/src/mito2/src/wal/entry_reader.rs index 272aa251736..67944351cf7 100644 --- a/src/mito2/src/wal/entry_reader.rs +++ b/src/mito2/src/wal/entry_reader.rs @@ -79,24 +79,8 @@ impl WalEntryReader for LogStoreEntryReader { let mut stream = reader.read(ns, start_id)?; let stream = stream! { - let mut buffered_entry: Option = None; while let Some(next_entry) = stream.next().await { - match buffered_entry.take() { - Some(entry) => { - if entry.is_complete() { - yield decode_raw_entry(entry); - } else { - warn!("Ignoring incomplete entry: {}", entry); - } - buffered_entry = Some(next_entry?); - }, - None => { - buffered_entry = Some(next_entry?); - } - }; - } - if let Some(entry) = buffered_entry { - // Ignores tail corrupted data. + let entry = next_entry?; if entry.is_complete() { yield decode_raw_entry(entry); } else { @@ -112,15 +96,54 @@ impl WalEntryReader for LogStoreEntryReader { #[cfg(test)] mod tests { + use std::time::Duration; + use api::v1::{Mutation, OpType, WalEntry}; - use futures::TryStreamExt; + use futures::{StreamExt, TryStreamExt, stream}; use prost::Message; use store_api::logstore::entry::{Entry, MultiplePartEntry, MultiplePartHeader}; use store_api::logstore::provider::Provider; use store_api::storage::RegionId; + use crate::error::Result; use crate::test_util::wal_util::MockRawEntryStream; + use crate::wal::EntryId; use crate::wal::entry_reader::{LogStoreEntryReader, WalEntryReader}; + use crate::wal::raw_entry_reader::{EntryStream, RawEntryReader}; + + struct LiveRawEntryReader { + entry: Entry, + } + + impl RawEntryReader for LiveRawEntryReader { + fn read(&self, _ns: &Provider, _start_id: EntryId) -> Result> { + let stream = stream::iter([Ok(self.entry.clone())]).chain(stream::pending()); + Ok(stream.boxed()) + } + } + + #[tokio::test] + async fn test_delivers_complete_entry_while_stream_is_alive() { + let provider = Provider::kafka_provider("my_topic".to_string()); + let wal_entry = WalEntry::default(); + let raw_entry_reader = LiveRawEntryReader { + entry: Entry::Naive(store_api::logstore::entry::NaiveEntry { + provider: provider.clone(), + region_id: RegionId::new(1, 1), + entry_id: 1, + data: wal_entry.encode_to_vec(), + }), + }; + let mut reader = LogStoreEntryReader::new(raw_entry_reader); + let mut entries = reader.read(&provider, 0).unwrap(); + + let entry = tokio::time::timeout(Duration::from_secs(1), entries.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(entry, (1, wal_entry)); + } #[tokio::test] async fn test_tail_corrupted_stream() { diff --git a/src/mito2/src/worker.rs b/src/mito2/src/worker.rs index 8efcf47ab68..31fad8236c9 100644 --- a/src/mito2/src/worker.rs +++ b/src/mito2/src/worker.rs @@ -169,7 +169,9 @@ impl WorkerGroup { /// Starts a worker group. /// /// The number of workers should be power of two. + #[allow(clippy::too_many_arguments)] pub(crate) async fn start( + data_home: &str, config: Arc, log_store: Arc, object_store_manager: ObjectStoreManagerRef, @@ -195,11 +197,7 @@ impl WorkerGroup { .with_buffer_size(Some(config.index.write_buffer_size.as_bytes() as _)); let index_build_job_pool = Arc::new(LocalScheduler::new(config.max_background_index_builds)); - let series_index_store = if config.experimental_series_index_root.trim().is_empty() { - None - } else { - Some(new_fs_cache_store(&config.experimental_series_index_root).await?) - }; + let series_index_store = series_index_store_from_config(&config, data_home).await?; let flush_job_pool = Arc::new(LocalScheduler::new(config.max_background_flushes)); let compact_job_pool = Arc::new(LocalScheduler::new(config.max_background_compactions)); let flush_semaphore = Arc::new(Semaphore::new(config.max_background_flushes)); @@ -391,6 +389,7 @@ impl WorkerGroup { /// The number of workers should be power of two. #[allow(clippy::too_many_arguments)] pub(crate) async fn start_for_test( + data_home: &str, config: Arc, log_store: Arc, object_store_manager: ObjectStoreManagerRef, @@ -410,11 +409,7 @@ impl WorkerGroup { }); let index_build_job_pool = Arc::new(LocalScheduler::new(config.max_background_index_builds)); - let series_index_store = if config.experimental_series_index_root.trim().is_empty() { - None - } else { - Some(new_fs_cache_store(&config.experimental_series_index_root).await?) - }; + let series_index_store = series_index_store_from_config(&config, data_home).await?; let flush_job_pool = Arc::new(LocalScheduler::new(config.max_background_flushes)); let compact_job_pool = Arc::new(LocalScheduler::new(config.max_background_compactions)); let flush_semaphore = Arc::new(Semaphore::new(config.max_background_flushes)); @@ -517,6 +512,19 @@ fn region_id_to_index(id: RegionId, num_workers: usize) -> usize { % num_workers } +/// Opens the fixed local series-index store only when the feature is enabled. +async fn series_index_store_from_config( + config: &MitoConfig, + data_home: &str, +) -> Result> { + if !config.experimental_enable_series_index { + return Ok(None); + } + + let root = Path::new(data_home).join("series_index"); + new_fs_cache_store(&root.to_string_lossy()).await.map(Some) +} + pub async fn write_cache_from_config( config: &MitoConfig, puffin_manager_factory: PuffinManagerFactory, @@ -603,13 +611,17 @@ impl WorkerStarter { .zip(series_index_task_state.clone()) .map(|(store, state)| { let (purger, purge_receiver) = series_index_channel(store.clone()); - series_index_purger = Some(purger); + series_index_purger = Some(purger.clone()); spawn_series_index_tasks( self.id, store, + regions.clone(), state, + self.config.experimental_series_index_bucket_width, + purger, purge_receiver, self.config.experimental_series_index_maintenance_interval, + self.time_provider.clone(), ) }); let now = self.time_provider.current_time_millis(); @@ -636,6 +648,7 @@ impl WorkerStarter { self.index_build_job_pool, self.config.max_background_index_builds, ), + series_index_task_state: series_index_task_state.clone(), series_index_store: self.series_index_store, series_index_purger, flush_scheduler: FlushScheduler::new(self.flush_job_pool), @@ -700,9 +713,9 @@ pub(crate) struct RegionWorker { sender: Sender, /// Handle to the worker thread. handle: Mutex>>, - /// Handle to the series-index maintenance task. + /// Handle to the sequential series-index task. series_index_handle: Mutex>>, - /// Controls the worker-owned series-index maintenance task. + /// Controls the worker-owned series-index task. series_index_task_state: Option>, /// Whether to run the worker thread. running: Arc, @@ -935,6 +948,8 @@ struct RegionWorkerLoop { write_buffer_manager: WriteBufferManagerRef, /// Scheduler for index build task. index_build_scheduler: IndexBuildScheduler, + /// Controls the worker-owned series-index task. + series_index_task_state: Option>, /// Store for companion range indexes deleted by the region SST purger. series_index_store: Option, series_index_purger: Option, @@ -1620,18 +1635,41 @@ mod tests { #[tokio::test] async fn test_worker_group_start_stop() { - let env = TestEnv::with_prefix("group-stop").await; - let group = env - .create_worker_group(MitoConfig { - num_workers: 4, - experimental_series_index_root: "series-index".to_string(), - ..Default::default() - }) - .await; + for (enabled, existing_index) in [(false, false), (true, false), (false, true)] { + // Use a fresh WAL directory for each case: stopping workers does not stop the log store. + let env = TestEnv::with_prefix("group-stop").await; + let root = env.data_home().join("series_index"); + if existing_index { + tokio::fs::create_dir_all(&root).await.unwrap(); + tokio::fs::write(root.join("retained"), b"index") + .await + .unwrap(); + } + let group = env + .create_worker_group(MitoConfig { + num_workers: 4, + experimental_enable_series_index: enabled, + experimental_series_index_maintenance_interval: Duration::from_secs(3600), + ..Default::default() + }) + .await; - tokio::time::timeout(Duration::from_secs(5), group.stop()) - .await - .expect("series-index tasks should stop without waiting for their interval") - .unwrap(); + for worker in &group.workers { + assert_eq!(worker.series_index_task_state.is_some(), enabled); + assert_eq!(worker.series_index_handle.lock().await.is_some(), enabled); + } + assert_eq!(root.is_dir(), enabled || existing_index); + if existing_index { + assert_eq!( + tokio::fs::read(root.join("retained")).await.unwrap(), + b"index" + ); + } + + tokio::time::timeout(Duration::from_secs(5), group.stop()) + .await + .expect("series-index tasks should stop without waiting for their interval") + .unwrap(); + } } } diff --git a/src/mito2/src/worker/handle_alter.rs b/src/mito2/src/worker/handle_alter.rs index b13c1badf83..66cb2dd95bb 100644 --- a/src/mito2/src/worker/handle_alter.rs +++ b/src/mito2/src/worker/handle_alter.rs @@ -23,6 +23,7 @@ use common_telemetry::{error, info}; use humantime_serde::re::humantime; use snafu::{ResultExt, ensure}; use store_api::logstore::LogStore; +use store_api::logstore::provider::Provider; use store_api::metadata::{ InvalidSetRegionOptionRequestSnafu, MetadataError, RegionMetadata, RegionMetadataBuilder, RegionMetadataRef, @@ -50,16 +51,19 @@ impl RegionWorkerLoop { request: RegionAlterRequest, sender: OptionOutputTx, ) { - let skip_wal_only = only_enables_skip_wal(&request.kind); - let (region, is_follower) = match self.regions.writable_non_staging_region(region_id) { - Ok(region) => (region, false), - Err(_) if skip_wal_only => match self.regions.follower_region(region_id) { - Ok(region) => (region, true), - Err(e) => { - sender.send(Err(e)); - return; + let requested_skip_wal = skip_wal_value(&request.kind); + let (region, follower_skip_wal) = match self.regions.writable_non_staging_region(region_id) + { + Ok(region) => (region, None), + Err(_) if requested_skip_wal.is_some() => { + match self.regions.follower_region(region_id) { + Ok(region) => (region, requested_skip_wal), + Err(e) => { + sender.send(Err(e)); + return; + } } - }, + } Err(e) => { sender.send(Err(e)); return; @@ -68,13 +72,20 @@ impl RegionWorkerLoop { info!("Try to alter region: {}, request: {:?}", region_id, request); - // Followers only accept skip-WAL, which is an in-memory option change and must + // Followers only accept skip-WAL changes, which are in-memory option changes and must // never enter the leader path that may flush memtables. - if is_follower { + if let Some(skip_wal) = follower_skip_wal { + if let Err(e) = validate_skip_wal_change(®ion, skip_wal) { + sender.send(Err(e).context(InvalidMetadataSnafu)); + return; + } let mut options = region.version().options.clone(); - if !options.skip_wal { - info!("Stop writing WAL for follower region: {}", region_id); - options.skip_wal = true; + if options.skip_wal != skip_wal { + info!( + "Set skip_wal for follower region: {}, previous: {} new: {}", + region_id, options.skip_wal, skip_wal + ); + options.skip_wal = skip_wal; region.version_control.alter_options(options); } sender.send(Ok(0)); @@ -317,10 +328,14 @@ impl RegionWorkerLoop { current_options.preserve_row_sequence = new_preserve; } } - SetRegionOption::SkipWal => { - if !current_options.skip_wal { - info!("Stop writing WAL for region: {}", region.region_id); - current_options.skip_wal = true; + SetRegionOption::SkipWal(skip_wal) => { + validate_skip_wal_change(region, skip_wal)?; + if current_options.skip_wal != skip_wal { + info!( + "Set skip_wal for region: {}, previous: {} new: {}", + region.region_id, current_options.skip_wal, skip_wal + ); + current_options.skip_wal = skip_wal; } } } @@ -352,12 +367,28 @@ impl RegionWorkerLoop { } } -fn only_enables_skip_wal(kind: &AlterKind) -> bool { - matches!( - kind, - AlterKind::SetRegionOptions { options } - if matches!(options.as_slice(), [SetRegionOption::SkipWal]) - ) +fn skip_wal_value(kind: &AlterKind) -> Option { + let AlterKind::SetRegionOptions { options } = kind else { + return None; + }; + let [SetRegionOption::SkipWal(skip_wal)] = options.as_slice() else { + return None; + }; + Some(*skip_wal) +} + +fn validate_skip_wal_change( + region: &MitoRegionRef, + skip_wal: bool, +) -> std::result::Result<(), MetadataError> { + ensure!( + skip_wal || !matches!(®ion.provider, Provider::Noop), + store_api::metadata::InvalidRegionRequestSnafu { + region_id: region.region_id, + err: "cannot enable WAL because the region uses the Noop WAL provider".to_string(), + } + ); + Ok(()) } /// Returns the new region options if there are updates to the options. @@ -382,7 +413,7 @@ fn new_region_options_on_empty_memtable( | SetRegionOption::Ttl(_) | SetRegionOption::Twsc(_, _) | SetRegionOption::AutoFlushInterval(_) - | SetRegionOption::SkipWal => (), + | SetRegionOption::SkipWal(_) => (), SetRegionOption::Format(format_str) => { // Safety: handle_alter_region_options_fast() has validated this. let new_format = format_str.parse::().unwrap(); diff --git a/src/mito2/src/worker/handle_bulk_insert.rs b/src/mito2/src/worker/handle_bulk_insert.rs index b4a7ae4a182..c094c1fa6dc 100644 --- a/src/mito2/src/worker/handle_bulk_insert.rs +++ b/src/mito2/src/worker/handle_bulk_insert.rs @@ -96,6 +96,7 @@ impl RegionWorkerLoop { None }; pending_bulk_request.push(SenderBulkRequest { + skip_wal: request.skip_wal, sender, request: part, region_id: request.region_id, diff --git a/src/mito2/src/worker/handle_catchup.rs b/src/mito2/src/worker/handle_catchup.rs index 5ef45f82f70..576ffed7ba2 100644 --- a/src/mito2/src/worker/handle_catchup.rs +++ b/src/mito2/src/worker/handle_catchup.rs @@ -134,6 +134,9 @@ impl RegionWorkerLoop { .await?; debug_assert!(!reopened_region.is_writable()); self.regions.insert_region(reopened_region.clone()); + if let Some(state) = &self.series_index_task_state { + state.wake(); + } Ok(reopened_region) } diff --git a/src/mito2/src/worker/handle_create.rs b/src/mito2/src/worker/handle_create.rs index 3acb5da76d8..2263cfa2ffb 100644 --- a/src/mito2/src/worker/handle_create.rs +++ b/src/mito2/src/worker/handle_create.rs @@ -96,6 +96,9 @@ impl RegionWorkerLoop { // Insert the MitoRegion into the RegionMap. self.regions.insert_region(region); + if let Some(state) = &self.series_index_task_state { + state.wake(); + } Ok(0) } diff --git a/src/mito2/src/worker/handle_open.rs b/src/mito2/src/worker/handle_open.rs index 825a677b765..52b902ffe92 100644 --- a/src/mito2/src/worker/handle_open.rs +++ b/src/mito2/src/worker/handle_open.rs @@ -227,6 +227,7 @@ impl RegionWorkerLoop { let opening_regions = self.opening_regions.clone(); let region_count = self.region_count.clone(); let worker_id = self.id; + let series_index_task_state = self.series_index_task_state.clone(); opening_regions.insert_sender(region_id, sender); common_runtime::spawn_global(async move { match opener.open(&config, &wal).await { @@ -249,6 +250,9 @@ impl RegionWorkerLoop { // Insert the Region into the RegionMap. regions.insert_region(region); + if let Some(state) = &series_index_task_state { + state.wake(); + } let senders = opening_regions.remove_sender(region_id); for sender in senders { diff --git a/src/mito2/src/worker/handle_write.rs b/src/mito2/src/worker/handle_write.rs index ff5ae650229..6fee8fb4b2a 100644 --- a/src/mito2/src/worker/handle_write.rs +++ b/src/mito2/src/worker/handle_write.rs @@ -561,7 +561,7 @@ impl RegionWorkerLoop { } // Collect requests by region. - if !region_ctx.push_bulk(bulk_req.sender, bulk_req.request, None) { + if !region_ctx.push_bulk(bulk_req.sender, bulk_req.request, None, bulk_req.skip_wal) { return; } } @@ -926,17 +926,32 @@ mod tests { async fn test_request_skip_wal_does_not_append_empty_batch() { // Only change the request flag. A failing log store demonstrates that // the all-skipped path never invokes append_batch, including empty appends. - check_request_skip_wal_does_not_append_empty_batch(false).await; - check_request_skip_wal_does_not_append_empty_batch(true).await; + check_request_skip_wal_does_not_append_empty_batch(false, false).await; + check_request_skip_wal_does_not_append_empty_batch(false, true).await; + check_request_skip_wal_does_not_append_empty_batch(true, false).await; + check_request_skip_wal_does_not_append_empty_batch(true, true).await; } - async fn check_request_skip_wal_does_not_append_empty_batch(skip_wal: bool) { + async fn check_request_skip_wal_does_not_append_empty_batch(skip_wal: bool, bulk: bool) { let region_id = RegionId::new(1, 1); let wal = Wal::new(Arc::new(MockLogStore { fail_append: true, ..Default::default() })); - let (ctx, rx) = new_region_ctx(region_id, skip_wal); + let (ctx, rx) = if bulk { + let version_control = Arc::new(VersionControlBuilder::new().build()); + let mut ctx = RegionWriteCtx::new( + region_id, + &version_control, + Provider::raft_engine_provider(region_id.as_u64()), + None, + ); + let (tx, rx) = oneshot::channel(); + assert!(ctx.push_bulk(OptionOutputTx::from(tx), new_bulk_part(1), None, skip_wal)); + (ctx, rx) + } else { + new_region_ctx(region_id, skip_wal) + }; let version_control = ctx.version_control().clone(); let mut contexts = HashMap::from([(region_id, ctx)]); assert_eq!(write_wal(&wal, &mut contexts).await, skip_wal); @@ -944,6 +959,7 @@ mod tests { let ctx = contexts.get_mut(®ion_id).unwrap(); assert_eq!(ctx.next_entry_id(), 1); ctx.write_memtable().await; + ctx.write_bulk().await; ctx.publish_sequence_and_entry_id(); assert_eq!(version_control.committed_sequence(), 1); assert_eq!(version_control.current().last_entry_id, 0); @@ -1003,6 +1019,11 @@ mod tests { #[tokio::test] async fn test_bulk_write_sequence_not_committed_before_install_worker_level() { + check_bulk_write_sequence_not_committed_before_install_worker_level(false).await; + check_bulk_write_sequence_not_committed_before_install_worker_level(true).await; + } + + async fn check_bulk_write_sequence_not_committed_before_install_worker_level(skip_wal: bool) { let region_id = RegionId::new(1, 1); let version_control = Arc::new(VersionControlBuilder::new().build()); @@ -1014,7 +1035,7 @@ mod tests { None, ); let (tx, rx) = oneshot::channel(); - assert!(ctx.push_bulk(OptionOutputTx::from(tx), new_bulk_part(3), None)); + assert!(ctx.push_bulk(OptionOutputTx::from(tx), new_bulk_part(3), None, skip_wal)); region_ctxs.insert(region_id, ctx); let wal = Wal::new(Arc::new(MockLogStore::default())); diff --git a/src/object-store/src/secure_fs.rs b/src/object-store/src/secure_fs.rs index 581cc2837c0..b940755c5b3 100644 --- a/src/object-store/src/secure_fs.rs +++ b/src/object-store/src/secure_fs.rs @@ -73,6 +73,20 @@ impl SecureFsRoot { &self.path } + /// Checks for a regular file without following the final symbolic link. + pub async fn is_file(&self, path: &str) -> Result { + let path = backend_path(path).map_err(new_std_io_error)?; + let root = self.clone(); + common_runtime::spawn_blocking_global(move || { + root.dir + .symlink_metadata(path) + .map(|metadata| metadata.is_file()) + }) + .await + .map_err(new_task_join_error)? + .map_err(new_std_io_error) + } + /// Opens a descendant directory without leaving this capability root. pub fn open_subdir(&self, path: impl AsRef) -> io::Result { let path = normalize_relative_path(path.as_ref())?; diff --git a/src/operator/Cargo.toml b/src/operator/Cargo.toml index c9258df2c0f..4ed4927b41f 100644 --- a/src/operator/Cargo.toml +++ b/src/operator/Cargo.toml @@ -85,7 +85,10 @@ tonic.workspace = true tracing.workspace = true [dev-dependencies] +axum.workspace = true catalog = { workspace = true, features = ["testing"] } common-meta = { workspace = true, features = ["testing"] } common-test-util.workspace = true +object-store = { workspace = true, features = ["testing"] } path-slash = "0.2" +url.workspace = true diff --git a/src/operator/src/bulk_insert.rs b/src/operator/src/bulk_insert.rs index 22fae236196..c47236d7dbc 100644 --- a/src/operator/src/bulk_insert.rs +++ b/src/operator/src/bulk_insert.rs @@ -20,7 +20,6 @@ use api::v1::region::{ BulkInsertRequest, RegionRequest, RegionRequestHeader, bulk_insert_request, region_request, }; use api::v1::{ArrowIpc, PartitionExprVersion}; -use arrow::array::Array; use arrow::record_batch::RecordBatch; use bytes::Bytes; use common_base::AffectedRows; @@ -28,12 +27,13 @@ use common_grpc::FlightData; use common_grpc::flight::{FlightEncoder, FlightMessage}; use common_telemetry::error; use common_telemetry::tracing_context::TracingContext; -use snafu::{OptionExt, ResultExt, ensure}; +use snafu::{ResultExt, ensure}; use store_api::storage::RegionId; use table::TableRef; use table::metadata::TableInfoRef; use crate::insert::Inserter; +use crate::req_convert::insert::extract_timestamps; use crate::{error, metrics}; impl Inserter { @@ -44,6 +44,7 @@ impl Inserter { raw_flight_data: FlightData, record_batch: RecordBatch, schema_bytes: Bytes, + skip_wal: bool, ) -> error::Result { let table_info = table.table_info(); let table_id = table_info.table_id(); @@ -103,6 +104,7 @@ impl Inserter { ..Default::default() }), body: Some(region_request::Body::BulkInsert(BulkInsertRequest { + skip_wal, region_id: region_id.as_u64(), partition_expr_version: partition_expr_version .map(|value| PartitionExprVersion { value }), @@ -218,6 +220,7 @@ impl Inserter { ..Default::default() }), body: Some(region_request::Body::BulkInsert(BulkInsertRequest { + skip_wal, region_id: region_id.as_u64(), partition_expr_version: partition_expr_version .map(|value| PartitionExprVersion { value }), @@ -316,22 +319,3 @@ impl Inserter { }); } } - -/// Calculate the timestamp range of record batch. Return `None` if record batch is empty. -fn extract_timestamps(rb: &RecordBatch, timestamp_index_name: &str) -> error::Result> { - let ts_col = rb - .column_by_name(timestamp_index_name) - .context(error::ColumnNotFoundSnafu { - msg: timestamp_index_name, - })?; - if rb.num_rows() == 0 { - return Ok(vec![]); - } - let (primitive, _) = - datatypes::timestamp::timestamp_array_to_primitive(ts_col).with_context(|| { - error::InvalidTimeIndexTypeSnafu { - ty: ts_col.data_type().clone(), - } - })?; - Ok(primitive.iter().flatten().collect()) -} diff --git a/src/operator/src/error.rs b/src/operator/src/error.rs index ae680be33ba..3c1a6ae3604 100644 --- a/src/operator/src/error.rs +++ b/src/operator/src/error.rs @@ -85,6 +85,15 @@ pub enum Error { source: common_meta::error::Error, }, + #[snafu(display("Invalid logical table export: {reason}"))] + InvalidLogicalTableExport { reason: String }, + + #[snafu(display("Logical table export resource limit exceeded: {reason}"))] + LogicalTableExportResource { reason: String }, + + #[snafu(display("Logical table export cancelled"))] + LogicalTableExportCancelled {}, + #[snafu(display("Unexpected, violated: {}", violated))] Unexpected { violated: String, @@ -1071,6 +1080,9 @@ impl ErrorExt for Error { Error::InvalidTimeIndexType { .. } | Error::InvalidTimezone { .. } => { StatusCode::InvalidArguments } + Error::InvalidLogicalTableExport { .. } => StatusCode::InvalidArguments, + Error::LogicalTableExportResource { .. } => StatusCode::Suspended, + Error::LogicalTableExportCancelled { .. } => StatusCode::Cancelled, Error::InvalidProcessId { .. } => StatusCode::InvalidArguments, Error::ProcessManagerMissing { .. } => StatusCode::Unexpected, Error::TimestampFormatNotSupported { .. } => StatusCode::InvalidArguments, diff --git a/src/operator/src/req_convert/common.rs b/src/operator/src/req_convert/common.rs index 63226ef8e48..e41c3505be2 100644 --- a/src/operator/src/req_convert/common.rs +++ b/src/operator/src/req_convert/common.rs @@ -51,7 +51,7 @@ fn encode_string_to_jsonb_binary(value_data: ValueData) -> Result { } } -/// Prepares row insertion requests by converting any JSON values to binary JSONB format. +/// Converts legacy JSON strings to binary JSONB and preserves native JSON2 values. pub fn preprocess_row_insert_requests(requests: &mut Vec) -> Result<()> { for request in requests { validate_rows(&request.rows)?; @@ -61,7 +61,7 @@ pub fn preprocess_row_insert_requests(requests: &mut Vec) -> R Ok(()) } -/// Prepares row deletion requests by converting any JSON values to binary JSONB format. +/// Converts legacy JSON strings to binary JSONB and preserves native JSON2 values. pub fn preprocess_row_delete_requests(requests: &mut Vec) -> Result<()> { for request in requests { validate_rows(&request.rows)?; @@ -79,12 +79,28 @@ fn prepare_rows(rows: &mut Option) -> Result<()> { .enumerate() .filter_map(|(idx, schema)| { if schema.datatype() == ColumnDataType::Json { - Some(idx) + match schema + .datatype_extension + .as_ref() + .and_then(|x| x.type_ext.as_ref()) + { + None | Some(TypeExt::JsonType(_)) => Some(Ok(idx)), + Some(TypeExt::JsonNativeType(_)) => None, + Some(_) => Some( + InvalidInsertRequestSnafu { + reason: format!( + "Invalid type extension for JSON column '{}'", + schema.column_name + ), + } + .fail(), + ), + } } else { None } }) - .collect::>(); + .collect::>>()?; for idx in &indexes { let column = &mut rows.schema[*idx]; column.datatype_extension = Some(ColumnDataTypeExtension { @@ -390,9 +406,65 @@ mod tests { use api::v1::column::Values; use api::v1::{SemanticType, VectorTypeExtension}; use common_base::bit_vec::prelude::*; + use common_error::ext::WhateverResult; use super::*; + #[test] + fn test_prepare_jsonb_rows() -> WhateverResult<()> { + let binary = Some(ColumnDataTypeExtension { + type_ext: Some(TypeExt::JsonType(JsonTypeExtension::JsonBinary.into())), + }); + for extension in [ + None, + Some(ColumnDataTypeExtension::default()), + binary.clone(), + ] { + for value in [Some(ValueData::StringValue(r#"{"key":1}"#.into())), None] { + let expected = value.as_ref().map(|_| { + ValueData::BinaryValue(jsonb::parse_value(br#"{"key":1}"#).unwrap().to_vec()) + }); + let mut rows = Some(Rows { + schema: vec![ColumnSchema { + datatype: ColumnDataType::Json as i32, + datatype_extension: extension.clone(), + ..Default::default() + }], + rows: vec![Row { + values: vec![Value { value_data: value }], + }], + }); + prepare_rows(&mut rows)?; + let rows = rows.unwrap(); + assert_eq!(rows.schema[0].datatype_extension, binary); + assert_eq!(rows.rows[0].values[0].value_data, expected); + } + } + Ok(()) + } + + #[test] + fn test_preprocess_json_invalid_extension() { + let mut rows = Some(Rows { + schema: vec![ColumnSchema { + datatype: ColumnDataType::Json as i32, + datatype_extension: Some(ColumnDataTypeExtension { + type_ext: Some(TypeExt::VectorType(VectorTypeExtension { dim: 3 })), + }), + ..Default::default() + }], + rows: vec![Row { + values: vec![Value { value_data: None }], + }], + }); + assert!( + prepare_rows(&mut rows) + .unwrap_err() + .to_string() + .contains("Invalid type extension") + ); + } + #[test] fn test_request_column_to_row() { let columns = vec![ diff --git a/src/operator/src/req_convert/insert.rs b/src/operator/src/req_convert/insert.rs index f9bf83d926e..72d6693c6be 100644 --- a/src/operator/src/req_convert/insert.rs +++ b/src/operator/src/req_convert/insert.rs @@ -17,6 +17,7 @@ mod fill_impure_default; mod row_to_region; mod stmt_to_region; mod table_to_region; +mod timestamps; use api::v1::SemanticType; pub use column_to_row::ColumnToRow; @@ -26,6 +27,7 @@ use snafu::{OptionExt, ResultExt}; pub use stmt_to_region::StatementToRegion; use table::metadata::TableInfo; pub use table_to_region::TableToRegion; +pub use timestamps::extract_timestamps; use crate::error::{ColumnNotFoundSnafu, MissingTimeIndexColumnSnafu, Result}; diff --git a/src/operator/src/req_convert/insert/timestamps.rs b/src/operator/src/req_convert/insert/timestamps.rs new file mode 100644 index 00000000000..f0315135f52 --- /dev/null +++ b/src/operator/src/req_convert/insert/timestamps.rs @@ -0,0 +1,38 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use arrow::record_batch::RecordBatch; +use datatypes::timestamp::append_timestamps; +use snafu::OptionExt; + +use crate::error::{self, Result}; + +/// Extracts non-null timestamps in the source column's native time unit. +pub fn extract_timestamps(rb: &RecordBatch, timestamp_index_name: &str) -> Result> { + let ts_col = rb + .column_by_name(timestamp_index_name) + .context(error::ColumnNotFoundSnafu { + msg: timestamp_index_name, + })?; + if rb.num_rows() == 0 { + return Ok(vec![]); + } + let mut timestamps = Vec::with_capacity(rb.num_rows()); + append_timestamps(ts_col, &mut timestamps).with_context(|| { + error::InvalidTimeIndexTypeSnafu { + ty: ts_col.data_type().clone(), + } + })?; + Ok(timestamps) +} diff --git a/src/operator/src/statement.rs b/src/operator/src/statement.rs index 150020bc9f3..69d387de206 100644 --- a/src/operator/src/statement.rs +++ b/src/operator/src/statement.rs @@ -22,6 +22,7 @@ mod cursor; pub mod ddl; mod describe; mod dml; +pub mod export_logical_tables; mod kill; pub mod semantic_graph; mod set; diff --git a/src/operator/src/statement/copy_table_from.rs b/src/operator/src/statement/copy_table_from.rs index ac5b5f4fb6b..c69d0477dbd 100644 --- a/src/operator/src/statement/copy_table_from.rs +++ b/src/operator/src/statement/copy_table_from.rs @@ -27,7 +27,7 @@ use common_datasource::file_format::json::JsonFormat; use common_datasource::file_format::orc::{ReaderAdapter, infer_orc_schema, new_orc_stream_reader}; use common_datasource::file_format::{FileFormat, Format, file_to_stream}; use common_datasource::lister::{Lister, Source}; -use common_datasource::object_store::build_backend_with_path; +use common_datasource::object_store::{LocalFileAccess, build_backend_with_path}; use common_query::{OutputCost, OutputRows}; use common_recordbatch::DfSendableRecordBatchStream; use common_recordbatch::adapter::RecordBatchStreamTypeAdapter; @@ -44,7 +44,7 @@ use datatypes::arrow::datatypes::{DataType as ArrowDataType, Schema, SchemaRef}; use datatypes::arrow::record_batch::RecordBatch; use datatypes::vectors::Helper; use futures_util::StreamExt; -use object_store::{Entry, EntryMode, ObjectStore}; +use object_store::{EntryMode, ObjectStore}; use regex::Regex; use session::context::QueryContextRef; use snafu::{ResultExt, ensure}; @@ -92,43 +92,59 @@ impl FileMetadata { } } -impl StatementExecutor { - async fn list_copy_from_entries( - &self, - req: &CopyTableRequest, - ) -> Result<(ObjectStore, Vec)> { - let backend = - build_backend_with_path(&req.location, &req.connection, &self.local_file_access) - .await - .context(error::BuildBackendSnafu)?; - let regex = req - .pattern - .as_ref() - .map(|x| Regex::new(x)) - .transpose() - .context(error::BuildRegexSnafu)?; +async fn list_copy_from_paths( + req: &CopyTableRequest, + local_file_access: &LocalFileAccess, +) -> Result<(ObjectStore, Vec)> { + let backend = build_backend_with_path(&req.location, &req.connection, local_file_access) + .await + .context(error::BuildBackendSnafu)?; + let regex = req + .pattern + .as_ref() + .map(|x| Regex::new(x)) + .transpose() + .context(error::BuildRegexSnafu)?; - let source = if let Some(filename) = backend.object_path { - Source::Filename(filename) + // Listing a known file's parent for every table makes COPY DATABASE do + // quadratic directory work. + if let Some(filename) = &backend.object_path { + let is_file = backend + .is_file(filename) + .await + .with_context(|_| common_datasource::error::ListObjectsSnafu { + path: req.location.clone(), + }) + .context(error::ListObjectsSnafu)?; + let paths = if is_file { + vec![filename.clone()] } else { - Source::Dir + vec![] }; - - let lister = Lister::new( - backend.object_store.clone(), - source.clone(), - req.location.clone(), - regex, - ); - - let entries = lister.list().await.context(error::ListObjectsSnafu)?; - debug!( - "Copy from location: {:?}, {source:?}, entries: {entries:?}", - req.location - ); - Ok((backend.object_store, entries)) + return Ok((backend.object_store, paths)); } + let lister = Lister::new( + backend.object_store.clone(), + Source::Dir, + req.location.clone(), + regex, + ); + + let entries = lister.list().await.context(error::ListObjectsSnafu)?; + debug!( + "Copy from location: {:?}, entries: {entries:?}", + req.location + ); + let paths = entries + .into_iter() + .filter(|entry| entry.metadata().mode() == EntryMode::FILE) + .map(|entry| entry.path().to_string()) + .collect(); + Ok((backend.object_store, paths)) +} + +impl StatementExecutor { async fn collect_metadata( &self, object_store: &ObjectStore, @@ -371,8 +387,8 @@ impl StatementExecutor { }; let table = self.get_table(&table_ref).await?; let format = Format::try_from(&req.with).context(error::ParseFileFormatSnafu)?; - let (object_store, entries) = self.list_copy_from_entries(&req).await?; - let mut files = Vec::with_capacity(entries.len()); + let (object_store, paths) = list_copy_from_paths(&req, &self.local_file_access).await?; + let mut files = Vec::with_capacity(paths.len()); let table_schema = table.schema().arrow_schema().clone(); let filters = table .schema() @@ -383,13 +399,9 @@ impl StatementExecutor { .into_iter() .collect::>(); - for entry in entries.iter() { - if entry.metadata().mode() != EntryMode::FILE { - continue; - } - let path = entry.path(); + for path in paths { let file_metadata = self - .collect_metadata(&object_store, format.clone(), path.to_string()) + .collect_metadata(&object_store, format.clone(), path) .await?; validate_csv_headers_if_required(&file_metadata, &table_schema)?; @@ -826,6 +838,236 @@ mod tests { use super::*; + fn copy_from_request(location: &str, pattern: Option<&str>) -> CopyTableRequest { + CopyTableRequest { + catalog_name: "greptime".into(), + schema_name: "public".into(), + table_name: "test".into(), + location: location.into(), + with: HashMap::new(), + connection: HashMap::new(), + pattern: pattern.map(String::from), + direction: table::requests::CopyDirection::Import, + timestamp_range: None, + limit: None, + } + } + + #[tokio::test] + async fn test_copy_from_local_paths() { + use common_test_util::temp_dir::create_temp_dir; + + let dir = create_temp_dir("copy_from_paths"); + let nested = dir.path().join("nested"); + std::fs::create_dir_all(nested.join("data.dir")).unwrap(); + std::fs::write(nested.join("data file.parquet"), b"data").unwrap(); + std::fs::write(nested.join("other.csv"), b"other").unwrap(); + let access = LocalFileAccess::sandboxed(dir.path()).unwrap(); + for location in [ + "nested/data file.parquet".to_string(), + url::Url::from_file_path(nested.join("data file.parquet")) + .unwrap() + .to_string(), + ] { + // PATTERN applies to directories, not an explicitly selected file. + let req = copy_from_request(&location, Some("does-not-match")); + let (store, paths) = list_copy_from_paths(&req, &access).await.unwrap(); + assert_eq!(paths, ["data file.parquet"]); + assert_eq!(store.read(&paths[0]).await.unwrap().to_vec(), b"data"); + } + + let req = copy_from_request("nested/", Some("^data")); + let (_, paths) = list_copy_from_paths(&req, &access).await.unwrap(); + assert_eq!(paths, ["data file.parquet"]); + let req = copy_from_request("nested/", None); + let (_, mut paths) = list_copy_from_paths(&req, &access).await.unwrap(); + paths.sort(); + assert_eq!(paths, ["data file.parquet", "other.csv"]); + let req = copy_from_request("nested/data.dir", None); + assert!( + list_copy_from_paths(&req, &access) + .await + .unwrap() + .1 + .is_empty() + ); + let req = copy_from_request("nested/", Some("does-not-match")); + assert!( + list_copy_from_paths(&req, &access) + .await + .unwrap() + .1 + .is_empty() + ); + + let req = copy_from_request("nested/missing.parquet", None); + assert!(matches!( + list_copy_from_paths(&req, &access).await, + Err(error::Error::ListObjects { + source: common_datasource::error::Error::ListObjects { error, .. }, .. + }) if error.kind() == object_store::ErrorKind::NotFound + )); + let req = copy_from_request("nested/data file.parquet", Some("[")); + assert!(matches!( + list_copy_from_paths(&req, &access).await, + Err(error::Error::BuildRegex { .. }) + )); + let req = copy_from_request("nested/data file.parquet", None); + assert!(matches!( + list_copy_from_paths(&req, &LocalFileAccess::Disabled).await, + Err(error::Error::BuildBackend { + source: common_datasource::error::Error::LocalFileAccessDisabled { .. }, + .. + }) + )); + let req = copy_from_request("../outside.parquet", None); + assert!(matches!( + list_copy_from_paths(&req, &access).await, + Err(error::Error::BuildBackend { + source: common_datasource::error::Error::LocalFileAccessDenied { .. }, + .. + }) + )); + } + + #[cfg(unix)] + #[tokio::test] + async fn test_copy_from_local_symlinks() { + use common_test_util::temp_dir::create_temp_dir; + + let dir = create_temp_dir("copy_from_symlinks"); + std::fs::write(dir.path().join("data.parquet"), b"data").unwrap(); + std::os::unix::fs::symlink("data.parquet", dir.path().join("link.parquet")).unwrap(); + std::os::unix::fs::symlink("missing.parquet", dir.path().join("dangling.parquet")).unwrap(); + let access = LocalFileAccess::sandboxed(dir.path()).unwrap(); + + let req = copy_from_request("link.parquet", None); + assert!( + list_copy_from_paths(&req, &access) + .await + .unwrap() + .1 + .is_empty() + ); + let req = copy_from_request("./", None); + let (_, paths) = list_copy_from_paths(&req, &access).await.unwrap(); + assert_eq!(paths, ["data.parquet"]); + let req = copy_from_request("dangling.parquet", None); + assert!(matches!( + list_copy_from_paths(&req, &access).await, + Err(error::Error::ListObjects { + source: common_datasource::error::Error::ListObjects { error, .. }, .. + }) if error.kind() == object_store::ErrorKind::NotFound + )); + } + + #[tokio::test] + async fn test_copy_from_s3_paths_without_relisting() { + use std::sync::Mutex; + + use axum::Router; + use axum::http::{Method, StatusCode, Uri}; + use axum::response::IntoResponse; + + let requests = Arc::new(Mutex::new(Vec::new())); + let recorded = requests.clone(); + let app = Router::new().fallback(move |method: Method, uri: Uri| { + let recorded = recorded.clone(); + async move { + recorded.lock().unwrap().push((method.clone(), uri.clone())); + if method == Method::HEAD { + let status = if uri.path().ends_with("missing.parquet") { + StatusCode::NOT_FOUND + } else if uri.path().ends_with("denied.parquet") { + StatusCode::FORBIDDEN + } else { + StatusCode::OK + }; + return (status, [("content-length", "0")]).into_response(); + } + assert_eq!(method, Method::GET); + let query = + axum::extract::Query::>::try_from_uri(&uri).unwrap(); + assert_eq!(query.get("prefix").unwrap(), "backup/schema/1/"); + assert_eq!(query.get("list-type").unwrap(), "2"); + let files = (0..16).map(|i| format!( + "backup/schema/1/data.{i}.parquet2026-09-01T00:00:00Z0" + )).collect::(); + format!( + "false{files}\ + backup/schema/1/other.csv2026-09-01T00:00:00Z0\ + backup/schema/1/data.dir/\ + " + ) + .into_response() + } + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let connection = HashMap::from([ + ("endpoint".into(), endpoint), + ("region".into(), "us-east-1".into()), + ("access_key_id".into(), "test".into()), + ("secret_access_key".into(), "test".into()), + ("disable_ec2_metadata".into(), "true".into()), + ]); + for i in 0..16 { + let filename = format!("data.{i}.parquet"); + let mut req = copy_from_request( + &format!("s3://bucket/backup/schema/1/{filename}"), + Some("does-not-match"), + ); + req.connection = connection.clone(); + let (_, paths) = list_copy_from_paths(&req, &LocalFileAccess::Disabled) + .await + .unwrap(); + assert_eq!(paths, [filename]); + } + { + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 16); + assert!(requests.iter().all(|(method, uri)| method == Method::HEAD + && uri.path().starts_with("/bucket/backup/schema/1/"))); + } + let mut req = copy_from_request("s3://bucket/backup/schema/1/", Some(r"^data\.")); + req.connection = connection.clone(); + let (_, mut paths) = list_copy_from_paths(&req, &LocalFileAccess::Disabled) + .await + .unwrap(); + paths.sort(); + let mut expected = (0..16) + .map(|i| format!("data.{i}.parquet")) + .collect::>(); + expected.sort(); + assert_eq!(paths, expected); + assert_eq!( + requests + .lock() + .unwrap() + .iter() + .filter(|(method, _)| *method == Method::GET) + .count(), + 1 + ); + + for (name, kind) in [ + ("missing", object_store::ErrorKind::NotFound), + ("denied", object_store::ErrorKind::PermissionDenied), + ] { + let mut req = + copy_from_request(&format!("s3://bucket/backup/schema/1/{name}.parquet"), None); + req.connection = connection.clone(); + assert!(matches!( + list_copy_from_paths(&req, &LocalFileAccess::Disabled).await, + Err(error::Error::ListObjects { + source: common_datasource::error::Error::ListObjects { error, .. }, .. + }) if error.kind() == kind + )); + } + server.abort(); + } + fn test_schema_matches(from: (DataType, bool), to: (DataType, bool), matches: bool) { let s1 = Arc::new(Schema::new(vec![Field::new("col", from.0.clone(), from.1)])); let s2 = Arc::new(Schema::new(vec![Field::new("col", to.0.clone(), to.1)])); diff --git a/src/operator/src/statement/copy_table_to.rs b/src/operator/src/statement/copy_table_to.rs index 4cf9448d8b6..faa06b88216 100644 --- a/src/operator/src/statement/copy_table_to.rs +++ b/src/operator/src/statement/copy_table_to.rs @@ -25,8 +25,8 @@ use common_datasource::object_store::build_backend_for_write_with_path; use common_query::Output; use common_recordbatch::adapter::DfRecordBatchStreamAdapter; use common_recordbatch::{ - RecordBatchStream, SendableRecordBatchMapper, SendableRecordBatchStream, - map_json_type_to_string, map_json_type_to_string_schema, + SendableRecordBatchMapper, SendableRecordBatchStream, map_json_type_to_string, + map_json_type_to_string_schema, }; use common_telemetry::{debug, tracing}; use datafusion::datasource::DefaultTableSource; @@ -85,18 +85,14 @@ impl StatementExecutor { ) .await .context(error::WriteStreamToFileSnafu { path }), - Format::Parquet(_) => { - let schema = stream.schema(); - stream_to_parquet( - Box::pin(DfRecordBatchStreamAdapter::new(stream)), - schema, - object_store, - path, - WRITE_CONCURRENCY, - ) - .await - .context(error::WriteStreamToFileSnafu { path }) - } + Format::Parquet(_) => stream_to_parquet( + Box::pin(DfRecordBatchStreamAdapter::new(stream)), + object_store, + path, + WRITE_CONCURRENCY, + ) + .await + .context(error::WriteStreamToFileSnafu { path }), _ => error::UnsupportedFormatSnafu { format: format.clone(), } diff --git a/src/operator/src/statement/export_logical_tables.rs b/src/operator/src/statement/export_logical_tables.rs new file mode 100644 index 00000000000..7c38af14ba0 --- /dev/null +++ b/src/operator/src/statement/export_logical_tables.rs @@ -0,0 +1,653 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Internal physical-table export into logical-table Parquet files. +//! +//! Callers must authorize the captured tables and destination before invoking +//! this component, and keep the source stable. Each destination must be owned +//! exclusively by the export attempt. Completion and cleanup of closed files +//! belong to the caller's chunk protocol, not individual file completion. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::Arc; + +use arrow::array::{Array, AsArray, UInt32Array}; +use arrow::compute::cast; +use arrow::datatypes::{DataType, SchemaRef}; +use arrow::downcast_dictionary_array; +use arrow::record_batch::RecordBatch; +use common_datasource::object_store::build_backend_for_write; +use common_datasource::parquet_writer::{ParquetFileWriter, ParquetWriterLimits}; +use common_meta::key::table_route::{TableRouteManager, TableRouteValue}; +use common_query::OutputData; +use common_recordbatch::SendableRecordBatchStream; +use common_time::range::TimestampRange; +use datafusion::datasource::DefaultTableSource; +use datafusion_common::TableReference as DfTableReference; +use datafusion_expr::{LogicalPlan, LogicalPlanBuilder, col}; +use futures::StreamExt; +use object_store::ObjectStore; +use session::context::QueryContextRef; +use snafu::{IntoError, OptionExt, ResultExt, ensure}; +use store_api::metric_engine_consts::{ + DATA_SCHEMA_TABLE_ID_COLUMN_NAME as TABLE_ID, DATA_SCHEMA_TSID_COLUMN_NAME as TSID, + LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY, +}; +use table::TableRef; +use table::metadata::TableId; +use table::table::adapter::DfTableProviderAdapter; +use tokio_util::sync::CancellationToken; + +use crate::error::{self, InvalidLogicalTableExportSnafu, LogicalTableExportResourceSnafu, Result}; +use crate::statement::StatementExecutor; + +/// Export preprocessing and per-file limits. Query memory and spill remain +/// governed by the query engine. +/// Batch checks happen after allocation; flush thresholds are not hard RSS caps. +#[derive(Clone, Copy, Debug)] +pub struct LogicalTableExportLimits { + /// Maximum retained scan batch before routing. + pub input_batch_bytes: usize, + /// Maximum estimated expanded logical values in one conversion slice. + pub conversion_bytes: usize, + /// Limits owned by the single-file Parquet writer. + pub writer: ParquetWriterLimits, +} + +impl Default for LogicalTableExportLimits { + fn default() -> Self { + Self { + input_batch_bytes: 64 * 1024 * 1024, + conversion_bytes: 1024 * 1024, + writer: ParquetWriterLimits { + row_group_rows: 8192, + flush_threshold_bytes: 8 * 1024 * 1024, + max_row_groups: 4096, + }, + } + } +} + +impl LogicalTableExportLimits { + fn validate(self) -> Result<()> { + ensure!( + self.input_batch_bytes > 0 && self.conversion_bytes > 0, + InvalidLogicalTableExportSnafu { + reason: "export limits must be positive" + } + ); + Ok(()) + } +} + +/// Metadata captured for one physical table and its selected logical tables. +/// Construct one unit per physical table, schema and time chunk. Capturing these +/// references supplies no snapshot isolation or locking guarantee. +pub struct LogicalTableExport { + physical_table: TableRef, + scan_projection: Vec, + logical_tables: BTreeMap, +} + +struct LogicalTableProjection { + name: String, + schema: SchemaRef, + projection: Vec, +} + +impl LogicalTableExport { + /// Capture schemas from selected Metric table references. + /// Export validates their physical-table association against table routes. + pub fn try_new(physical: TableRef, tables: &[TableRef]) -> Result { + let physical_info = physical.table_info(); + ensure!( + physical_info.meta.engine == METRIC_ENGINE_NAME + && physical_info + .meta + .options + .extra_options + .contains_key(PHYSICAL_TABLE_METADATA_KEY) + && !tables.is_empty(), + InvalidLogicalTableExportSnafu { + reason: "expected a Metric physical table and selected logical tables" + } + ); + let physical_schema = physical.schema(); + let id_index = physical_schema.column_index_by_name(TABLE_ID).context( + InvalidLogicalTableExportSnafu { + reason: "physical schema has no __table_id", + }, + )?; + let mut scan_projection = BTreeSet::from([id_index]); + let mut logical_tables = BTreeMap::new(); + for table in tables { + let info = table.table_info(); + let name = &info.name; + ensure!( + !name.contains('/') && !name.contains('\\'), + InvalidLogicalTableExportSnafu { + reason: "logical table names must not contain path separators" + } + ); + ensure!( + info.catalog_name == physical_info.catalog_name + && info.schema_name == physical_info.schema_name + && info.meta.engine == METRIC_ENGINE_NAME + && info + .meta + .options + .extra_options + .contains_key(LOGICAL_TABLE_METADATA_KEY), + InvalidLogicalTableExportSnafu { + reason: format!( + "{name} is not a Metric logical table in the physical table schema" + ) + } + ); + let schema = table.schema().arrow_schema().clone(); + let indices = schema + .fields() + .iter() + .map(|field| { + ensure!( + field.name() != TABLE_ID && field.name() != TSID, + InvalidLogicalTableExportSnafu { + reason: "logical schema contains internal Metric columns" + } + ); + let index = physical_schema + .column_index_by_name(field.name()) + .with_context(|| InvalidLogicalTableExportSnafu { + reason: format!("physical schema lacks {}", field.name()), + })?; + ensure!( + physical_schema.arrow_schema().field(index).data_type() + == field.data_type(), + InvalidLogicalTableExportSnafu { + reason: format!("physical/logical type mismatch for {}", field.name()) + } + ); + Ok(index) + }) + .collect::>>()?; + scan_projection.extend(indices.iter().copied()); + ensure!( + logical_tables + .insert( + info.table_id(), + LogicalTableProjection { + name: name.clone(), + schema, + projection: indices, + } + ) + .is_none(), + InvalidLogicalTableExportSnafu { + reason: "duplicate logical table" + } + ); + } + let scan_projection = scan_projection.into_iter().collect::>(); + for file in logical_tables.values_mut() { + for index in &mut file.projection { + *index = scan_projection.binary_search(index).map_err(|_| { + error::UnexpectedSnafu { + violated: "logical column missing from physical projection", + } + .build() + })?; + } + } + Ok(Self { + physical_table: physical, + scan_projection, + logical_tables, + }) + } + + async fn validate_table_routes(&self, manager: &TableRouteManager) -> Result<()> { + let table_ids = self.logical_tables.keys().copied().collect::>(); + let routes = manager + .table_route_storage() + .batch_get(&table_ids) + .await + .context(error::TableMetadataManagerSnafu)?; + let physical_table_id = self.physical_table.table_info().table_id(); + for (table_id, route) in table_ids.into_iter().zip(routes) { + ensure!( + matches!(route, Some(TableRouteValue::Logical(route)) if route.physical_table_id() == physical_table_id), + InvalidLogicalTableExportSnafu { + reason: format!( + "logical table {table_id} does not belong to physical table {physical_table_id}" + ) + } + ); + } + Ok(()) + } + + fn build_plan(&self, time_range: Option<&TimestampRange>) -> Result { + let info = self.physical_table.table_info(); + let filters = self + .physical_table + .schema() + .timestamp_column() + .and_then(|column| { + common_query::logical_plan::build_filter_from_timestamp(&column.name, time_range) + }) + .into_iter() + .collect::>(); + let source = Arc::new(DefaultTableSource::new(Arc::new( + DfTableProviderAdapter::new(self.physical_table.clone()), + ))); + let mut builder = LogicalPlanBuilder::scan_with_filters( + DfTableReference::full( + info.catalog_name.clone(), + info.schema_name.clone(), + info.name.clone(), + ), + source, + Some(self.scan_projection.clone()), + filters.clone(), + ) + .context(error::BuildDfLogicalPlanSnafu)?; + for filter in filters { + builder = builder + .filter(filter) + .context(error::BuildDfLogicalPlanSnafu)?; + } + builder + .sort(vec![col(TABLE_ID).sort(true, false)]) + .context(error::BuildDfLogicalPlanSnafu)? + .build() + .context(error::BuildDfLogicalPlanSnafu) + } +} + +/// Counts returned only after every selected logical file has been closed. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct LogicalTableExportSummary { + pub rows: usize, + pub skipped_rows: usize, + pub files: usize, +} + +impl StatementExecutor { + /// Export selected logical tables to a fresh directory using one physical query. + /// The caller bounds concurrent units and owns retry/cleanup of this directory. + /// Cancellation waits for in-flight I/O before aborting the active upload. + /// Closed files remain an incomplete chunk until the caller publishes its + /// completion state. + #[allow(clippy::too_many_arguments)] + pub async fn export_logical_tables( + &self, + unit: &LogicalTableExport, + directory: &str, + connection: &HashMap, + time_range: Option<&TimestampRange>, + limits: LogicalTableExportLimits, + cancellation: &CancellationToken, + query_ctx: QueryContextRef, + ) -> Result { + limits.validate()?; + let (store, stream) = tokio::select! { + biased; + _ = cancellation.cancelled() => return error::LogicalTableExportCancelledSnafu.fail(), + result = async { + unit.validate_table_routes(self.table_metadata_manager.table_route_manager()).await?; + let store = build_backend_for_write(&format!("{}/", directory.trim_end_matches('/')), connection, &self.local_file_access) + .await.context(error::BuildBackendSnafu)?; + let output = self.query_engine.execute(unit.build_plan(time_range)?, query_ctx) + .await.context(error::ExecLogicalPlanSnafu)?; + let stream = match output.data { + OutputData::Stream(stream) => stream, + OutputData::RecordBatches(batches) => batches.as_stream(), + _ => return error::UnexpectedSnafu { violated: "expected physical query rows" }.fail(), + }; + Ok((store, stream)) + } => result?, + }; + export_stream(unit, stream, &store, limits, cancellation).await + } +} + +async fn export_stream( + unit: &LogicalTableExport, + stream: SendableRecordBatchStream, + store: &ObjectStore, + limits: LogicalTableExportLimits, + cancellation: &CancellationToken, +) -> Result { + let mut active = None; + let result = write_tables(unit, stream, store, limits, cancellation, &mut active).await; + if result.is_err() + && let Some(writer) = active + && let Err(cleanup_error) = writer + .writer + .abort() + .await + .map_err(|error| map_writer_error(error, &writer.path)) + { + common_telemetry::warn!(cleanup_error; "Failed to clean up incomplete Metric export file"); + } + result +} + +fn check_cancelled(cancellation: &CancellationToken) -> Result<()> { + ensure!( + !cancellation.is_cancelled(), + error::LogicalTableExportCancelledSnafu + ); + Ok(()) +} + +async fn write_tables( + unit: &LogicalTableExport, + mut stream: SendableRecordBatchStream, + store: &ObjectStore, + limits: LogicalTableExportLimits, + cancellation: &CancellationToken, + active: &mut Option, +) -> Result { + let id_index = + stream + .schema() + .column_index_by_name(TABLE_ID) + .context(error::UnexpectedSnafu { + violated: "physical query omitted __table_id", + })?; + let mut summary = LogicalTableExportSummary::default(); + let mut previous = None; + let mut written = BTreeSet::new(); + loop { + let batch = tokio::select! { + biased; + _ = cancellation.cancelled() => return error::LogicalTableExportCancelledSnafu.fail(), + batch = stream.next() => batch, + }; + let Some(batch) = batch else { + break; + }; + let batch = batch + .context(error::BuildRecordBatchSnafu)? + .into_df_record_batch(); + ensure!( + batch.get_array_memory_size() <= limits.input_batch_bytes, + LogicalTableExportResourceSnafu { + reason: "scan batch exceeds input byte budget" + } + ); + let ids = batch + .column(id_index) + .as_any() + .downcast_ref::() + .context(error::UnexpectedSnafu { + violated: "__table_id must be UInt32", + })?; + ensure!( + ids.null_count() == 0, + error::UnexpectedSnafu { + violated: "null __table_id" + } + ); + let mut start = 0; + while start < batch.num_rows() { + check_cancelled(cancellation)?; + let id = ids.value(start); + ensure!( + previous.is_none_or(|last| last <= id), + error::UnexpectedSnafu { + violated: "physical query is not ordered by __table_id" + } + ); + previous = Some(id); + let mut end = start + 1; + while end < batch.num_rows() && ids.value(end) == id { + end += 1; + } + if active.as_ref().is_some_and(|writer| writer.table_id != id) { + finish_active(active, cancellation).await?; + } + check_cancelled(cancellation)?; + if let Some(file) = unit.logical_tables.get(&id) { + if active.is_none() { + *active = Some(ActiveWriter::open(id, file, store, limits).await?); + written.insert(id); + summary.files += 1; + } + let projected = batch + .project(&file.projection) + .context(error::ProjectSchemaSnafu)?; + let writer = active.as_mut().context(error::UnexpectedSnafu { + violated: "missing logical writer", + })?; + let mut offset = start; + while offset < end { + let (expanded, consumed) = expand_bounded_slice( + projected.clone(), + file.schema.clone(), + offset, + end, + limits.conversion_bytes, + ) + .await?; + check_cancelled(cancellation)?; + // Do not drop an in-flight file operation before cleanup. + writer + .writer + .write(expanded, Some(cancellation)) + .await + .map_err(|error| map_writer_error(error, &writer.path))?; + check_cancelled(cancellation)?; + offset += consumed; + summary.rows += consumed; + } + } else { + summary.skipped_rows += end - start; + } + start = end; + } + } + finish_active(active, cancellation).await?; + for (&id, file) in &unit.logical_tables { + if !written.contains(&id) { + check_cancelled(cancellation)?; + *active = Some(ActiveWriter::open(id, file, store, limits).await?); + finish_active(active, cancellation).await?; + summary.files += 1; + } + } + check_cancelled(cancellation)?; + Ok(summary) +} + +struct ActiveWriter { + table_id: u32, + path: String, + writer: ParquetFileWriter, +} + +impl ActiveWriter { + async fn open( + id: u32, + table: &LogicalTableProjection, + store: &ObjectStore, + limits: LogicalTableExportLimits, + ) -> Result { + let path = format!("{}.parquet", table.name); + ensure!( + !store + .exists(&path) + .await + .context(error::ReadObjectSnafu { path: &path })?, + InvalidLogicalTableExportSnafu { + reason: format!("output already exists: {path}") + } + ); + let writer = ParquetFileWriter::open( + table.schema.clone(), + store.clone(), + &path, + 1, + Some(limits.writer), + ) + .await + .map_err(|error| map_writer_error(error, &path))?; + Ok(Self { + table_id: id, + path, + writer, + }) + } +} + +async fn finish_active( + active: &mut Option, + cancellation: &CancellationToken, +) -> Result<()> { + if let Some(writer) = active.as_mut() { + writer + .writer + .finish(Some(cancellation)) + .await + .map_err(|error| map_writer_error(error, &writer.path))?; + check_cancelled(cancellation)?; + *active = None; + } + Ok(()) +} + +async fn expand_bounded_slice( + batch: RecordBatch, + schema: SchemaRef, + start: usize, + end: usize, + budget: usize, +) -> Result<(RecordBatch, usize)> { + common_runtime::spawn_blocking_global(move || { + let len = rows_within_budget(&batch, start, end, budget)?; + let slice = batch.slice(start, len); + let arrays = slice + .columns() + .iter() + .zip(schema.fields()) + .map(|(array, field)| cast(array, field.data_type()).context(error::ComputeArrowSnafu)) + .collect::>>()?; + let expanded = RecordBatch::try_new(schema, arrays).context(error::ComputeArrowSnafu)?; + Ok((expanded, len)) + }) + .await + .context(error::JoinTaskSnafu)? +} + +fn map_writer_error(source: common_datasource::error::Error, path: &str) -> error::Error { + match source { + common_datasource::error::Error::ParquetWriteCancelled {} => { + error::LogicalTableExportCancelledSnafu.build() + } + common_datasource::error::Error::InvalidParquetWriterLimits {} => { + InvalidLogicalTableExportSnafu { + reason: "Parquet writer limits must be positive", + } + .build() + } + common_datasource::error::Error::ParquetWriterResource { reason } => { + LogicalTableExportResourceSnafu { reason }.build() + } + source => error::WriteStreamToFileSnafu { path }.into_error(source), + } +} + +// Count only selected logical values before dictionary expansion, including nested +// histogram lists/structs. Offset and validity overhead is charged per value. +fn estimate_value_size(array: &dyn Array, row: usize) -> Result { + if array.is_null(row) { + return Ok(32); + } + let bytes = match array.data_type() { + DataType::Boolean => 1, + DataType::Null => 0, + DataType::Utf8 => array.as_string::().value(row).len(), + DataType::LargeUtf8 => array.as_string::().value(row).len(), + DataType::Binary => array.as_binary::().value(row).len(), + DataType::LargeBinary => array.as_binary::().value(row).len(), + DataType::Struct(_) => { + array + .as_struct() + .columns() + .iter() + .try_fold(0usize, |sum, child| { + Ok::<_, error::Error>( + sum.saturating_add(estimate_value_size(child.as_ref(), row)?), + ) + })? + } + DataType::List(_) => { + let list = array.as_list::(); + let offsets = list.value_offsets(); + (offsets[row] as usize..offsets[row + 1] as usize).try_fold(0usize, |sum, index| { + Ok::<_, error::Error>( + sum.saturating_add(estimate_value_size(list.values().as_ref(), index)?), + ) + })? + } + DataType::Dictionary(_, _) => { + downcast_dictionary_array! { + array => { + match array.key(row) { + Some(index) => estimate_value_size(array.values().as_ref(), index)?, + None => 0, + } + }, + _ => return error::UnexpectedSnafu { violated: "invalid dictionary array" }.fail(), + } + } + other => other + .primitive_width() + .with_context(|| InvalidLogicalTableExportSnafu { + reason: format!("unsupported Metric Parquet type: {other}"), + })?, + }; + Ok(bytes.saturating_add(16)) +} + +fn rows_within_budget( + batch: &RecordBatch, + start: usize, + end: usize, + budget: usize, +) -> Result { + let mut bytes = 0usize; + let mut row = start; + while row < end { + let row_bytes = batch.columns().iter().try_fold(0usize, |sum, array| { + Ok::<_, error::Error>(sum.saturating_add(estimate_value_size(array.as_ref(), row)?)) + })?; + if row_bytes > budget.saturating_sub(bytes) { + break; + } + bytes += row_bytes; + row += 1; + } + ensure!( + row > start, + LogicalTableExportResourceSnafu { + reason: "one expanded logical row exceeds conversion byte budget" + } + ); + Ok(row - start) +} + +#[cfg(test)] +mod tests; diff --git a/src/operator/src/statement/export_logical_tables/tests.rs b/src/operator/src/statement/export_logical_tables/tests.rs new file mode 100644 index 00000000000..a85418fa2e6 --- /dev/null +++ b/src/operator/src/statement/export_logical_tables/tests.rs @@ -0,0 +1,578 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use arrow::array::{ + ArrayRef, DictionaryArray, Float64Array, Int32Array, ListArray, StringArray, StructArray, +}; +use arrow::buffer::OffsetBuffer; +use arrow::datatypes::{Field, Schema, UInt32Type}; +use bytes::Bytes; +use common_recordbatch::{RecordBatch as GreptimeRecordBatch, RecordBatches}; +use datafusion::parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use table::test_util::EmptyTable; +use table::test_util::table_info::test_table_info; + +use super::*; + +fn table(id: u32, name: &str, fields: Vec, physical: bool) -> TableRef { + let schema = Arc::new(datatypes::schema::Schema::try_from(Schema::new(fields)).unwrap()); + let mut info = test_table_info(id, name, "public", "greptime", schema); + info.meta.engine = METRIC_ENGINE_NAME.into(); + let (key, value) = if physical { + (PHYSICAL_TABLE_METADATA_KEY, "") + } else { + (LOGICAL_TABLE_METADATA_KEY, "phy") + }; + info.meta + .options + .extra_options + .insert(key.into(), value.into()); + EmptyTable::from_table_info(&info) +} + +fn export_limits() -> LogicalTableExportLimits { + let mut limits = LogicalTableExportLimits::default(); + limits.writer.row_group_rows = 1; + limits +} + +fn unit() -> LogicalTableExport { + let ts = Field::new( + "ts", + DataType::Timestamp(arrow::datatypes::TimeUnit::Millisecond, None), + false, + ); + let host = Field::new("host", DataType::Utf8, true); + let a = Field::new("a", DataType::Float64, true); + let b = Field::new("b", DataType::Float64, true); + let physical = table( + 1024, + "phy", + vec![ + Field::new(TABLE_ID, DataType::UInt32, false), + ts.clone(), + host.clone(), + a.clone(), + b.clone(), + ], + true, + ); + let tables = vec![ + table( + 1025, + "cpu.v1", + vec![host.clone(), a.clone(), ts.clone()], + false, + ), + table(1026, "empty", vec![ts.clone(), a], false), + table(1027, "requests", vec![ts, b, host], false), + ]; + LogicalTableExport::try_new(physical, &tables).unwrap() +} + +fn batch(ids: Vec>, hosts: Vec>) -> RecordBatch { + use arrow::array::TimestampMillisecondArray; + let rows = ids.len(); + let host: DictionaryArray = hosts.into_iter().collect(); + RecordBatch::try_from_iter_with_nullable([ + (TABLE_ID, Arc::new(UInt32Array::from(ids)) as ArrayRef, true), + ( + "ts", + Arc::new(TimestampMillisecondArray::from(vec![100; rows])), + false, + ), + ("host", Arc::new(host), true), + ( + "a", + Arc::new(Float64Array::from(vec![Some(1.5); rows])), + true, + ), + ("b", Arc::new(Float64Array::from(vec![None; rows])), true), + ]) + .unwrap() +} + +fn stream(batches: Vec) -> SendableRecordBatchStream { + let schema = Arc::new(datatypes::schema::Schema::try_from(batches[0].schema()).unwrap()); + let batches = batches + .into_iter() + .map(|batch| GreptimeRecordBatch::from_df_record_batch(schema.clone(), batch)) + .collect(); + RecordBatches::try_new(schema, batches).unwrap().as_stream() +} + +async fn read(store: &ObjectStore, path: &str) -> (SchemaRef, Vec) { + let bytes = store.read(path).await.unwrap().to_bytes(); + let builder = ParquetRecordBatchReaderBuilder::try_new(bytes).unwrap(); + let schema = builder.schema().clone(); + ( + schema, + builder + .build() + .unwrap() + .collect::>() + .unwrap(), + ) +} + +#[tokio::test] +async fn routes_across_batches_and_writes_empty_files() { + let unit = unit(); + let store = ObjectStore::new(object_store::services::Memory::default()).unwrap(); + let batches = vec![ + batch(vec![Some(1025), Some(1025)], vec![Some(""), None]), + batch( + vec![Some(1025), Some(1027), Some(1028)], + vec![Some("a"), Some("b"), Some("unselected")], + ), + ]; + let result = write_tables( + &unit, + stream(batches), + &store, + export_limits(), + &CancellationToken::new(), + &mut None, + ) + .await + .unwrap(); + assert_eq!( + result, + LogicalTableExportSummary { + rows: 4, + skipped_rows: 1, + files: 3 + } + ); + let (schema, cpu) = read(&store, "cpu.v1.parquet").await; + assert_eq!(schema.fields(), unit.logical_tables[&1025].schema.fields()); + let values: Vec<_> = cpu + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_string::() + .iter() + .map(|s| s.map(str::to_owned)) + }) + .collect(); + assert_eq!(values, vec![Some("".into()), None, Some("a".into())]); + let (schema, empty) = read(&store, "empty.parquet").await; + assert_eq!(schema.fields(), unit.logical_tables[&1026].schema.fields()); + assert!(empty.is_empty()); + let (schema, requests) = read(&store, "requests.parquet").await; + assert_eq!(schema.fields(), unit.logical_tables[&1027].schema.fields()); + assert_eq!(requests[0].column(1).null_count(), 1); +} + +#[tokio::test] +async fn rejects_invalid_order_ids_and_resource_exhaustion() { + let unit = unit(); + for (batches, limits, message) in [ + ( + vec![ + batch(vec![Some(1027)], vec![Some("a")]), + batch(vec![Some(1025)], vec![Some("b")]), + ], + LogicalTableExportLimits::default(), + "not ordered", + ), + ( + vec![batch(vec![None], vec![None])], + LogicalTableExportLimits::default(), + "null __table_id", + ), + ( + vec![batch(vec![Some(1025)], vec![Some("a")])], + LogicalTableExportLimits { + input_batch_bytes: 1, + ..Default::default() + }, + "input byte budget", + ), + ( + vec![batch(vec![Some(1025)], vec![Some("too big")])], + LogicalTableExportLimits { + conversion_bytes: 1, + ..Default::default() + }, + "one expanded logical row", + ), + ] { + let store = ObjectStore::new(object_store::services::Memory::default()).unwrap(); + let mut active = None; + let err = write_tables( + &unit, + stream(batches), + &store, + limits, + &CancellationToken::new(), + &mut active, + ) + .await + .unwrap_err(); + assert!(err.to_string().contains(message), "{err}"); + if let Some(writer) = active { + writer.writer.abort().await.unwrap(); + } + } +} + +#[tokio::test] +async fn existing_outputs_are_not_overwritten() { + let store = ObjectStore::new(object_store::services::Memory::default()).unwrap(); + store.write("cpu.v1.parquet", "keep").await.unwrap(); + let err = write_tables( + &unit(), + stream(vec![batch(vec![Some(1025)], vec![None])]), + &store, + LogicalTableExportLimits::default(), + &CancellationToken::new(), + &mut None, + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("already exists")); + assert_eq!( + store.read("cpu.v1.parquet").await.unwrap().to_bytes(), + Bytes::from_static(b"keep") + ); +} + +#[test] +fn dictionary_and_nested_histogram_values_are_bounded_before_expansion() { + let dictionary = DictionaryArray::::try_new( + UInt32Array::from(vec![0, 0, 0]), + Arc::new(StringArray::from(vec!["x".repeat(4096)])), + ) + .unwrap(); + let nested = ListArray::new( + Arc::new(Field::new("item", DataType::Int32, true)), + OffsetBuffer::new(vec![0i32, 2, 3, 5].into()), + Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])), + None, + ); + let histogram = StructArray::from(vec![( + Arc::new(Field::new("buckets", nested.data_type().clone(), true)), + Arc::new(nested) as ArrayRef, + )]); + let batch = RecordBatch::try_from_iter([ + ("tag", Arc::new(dictionary) as ArrayRef), + ("histogram", Arc::new(histogram)), + ]) + .unwrap(); + assert_eq!(rows_within_budget(&batch, 0, 3, 4300).unwrap(), 1); + // The dictionary and container overhead fit; the nested list elements do not. + assert!(rows_within_budget(&batch, 0, 3, 4180).is_err()); + assert_eq!(rows_within_budget(&batch, 0, 3, 15000).unwrap(), 3); +} + +#[test] +fn validates_selected_schemas_and_projects_only_selected_columns() { + let unit = unit(); + let selected = table( + 1025, + "only", + vec![Field::new("a", DataType::Float64, true)], + false, + ); + let one = + LogicalTableExport::try_new(unit.physical_table.clone(), std::slice::from_ref(&selected)) + .unwrap(); + assert_eq!(one.scan_projection, vec![0, 3]); + assert_eq!(one.logical_tables[&1025].projection, vec![1]); + assert!( + LogicalTableExport::try_new(unit.physical_table.clone(), &[selected.clone(), selected]) + .is_err() + ); + let unsafe_name = table(1030, "a/b", vec![], false); + assert!(LogicalTableExport::try_new(unit.physical_table.clone(), &[unsafe_name]).is_err()); + let wrong_type = table( + 1030, + "wrong", + vec![Field::new("a", DataType::Int32, true)], + false, + ); + assert!(LogicalTableExport::try_new(unit.physical_table, &[wrong_type]).is_err()); +} + +#[tokio::test] +async fn cancellation_drops_input_and_aborts_active_upload() { + let store = ObjectStore::new(object_store::services::Memory::default()).unwrap(); + let unit = unit(); + let batch = batch(vec![Some(1025)], vec![Some("a")]); + let schema = batch.schema(); + let token = CancellationToken::new(); + let trigger = token.clone(); + let batches = async_stream::stream! { + yield Ok(batch); + // Resuming the source proves the previous batch reached the writer. + trigger.cancel(); + std::future::pending::<()>().await; + }; + let df_stream = + datafusion::physical_plan::stream::RecordBatchStreamAdapter::new(schema, batches); + let stream = + common_recordbatch::adapter::RecordBatchStreamAdapter::try_new(Box::pin(df_stream)) + .unwrap(); + let result = export_stream(&unit, Box::pin(stream), &store, export_limits(), &token).await; + assert!(matches!( + result, + Err(error::Error::LogicalTableExportCancelled { .. }) + )); + assert!(!store.exists("cpu.v1.parquet").await.unwrap()); + assert!(!store.exists("empty.parquet").await.unwrap()); +} + +#[tokio::test] +async fn native_histogram_parquet_roundtrip() { + fn sample(data_type: &DataType) -> ArrayRef { + match data_type { + DataType::Struct(fields) => Arc::new(StructArray::new( + fields.clone(), + fields.iter().map(|f| sample(f.data_type())).collect(), + None, + )), + DataType::List(field) => Arc::new(ListArray::new( + field.clone(), + OffsetBuffer::new(vec![0i32, 1, 2].into()), + sample(field.data_type()), + None, + )), + DataType::Int32 => Arc::new(Int32Array::from(vec![Some(1), None])), + DataType::Int64 => Arc::new(arrow::array::Int64Array::from(vec![Some(2), None])), + DataType::Float64 => Arc::new(Float64Array::from(vec![Some(3.5), None])), + DataType::Timestamp(_, _) => { + Arc::new(arrow::array::TimestampMillisecondArray::from(vec![ + Some(4), + None, + ])) + } + other => panic!("unexpected histogram field {other}"), + } + } + let data_type = common_query::native_histogram::native_histogram_arrow_type(); + let field = Field::new("histogram", data_type.clone(), true); + let unit = LogicalTableExport::try_new( + table( + 1024, + "phy", + vec![Field::new(TABLE_ID, DataType::UInt32, false), field.clone()], + true, + ), + &[table(1025, "histogram", vec![field], false)], + ) + .unwrap(); + let batch = RecordBatch::try_from_iter_with_nullable([ + ( + TABLE_ID, + Arc::new(UInt32Array::from(vec![1025; 2])) as ArrayRef, + false, + ), + ("histogram", sample(&data_type), true), + ]) + .unwrap(); + let store = ObjectStore::new(object_store::services::Memory::default()).unwrap(); + let summary = export_stream( + &unit, + stream(vec![batch.clone()]), + &store, + export_limits(), + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(summary.rows, 2); + let (schema, batches) = read(&store, "histogram.parquet").await; + assert_eq!(schema.fields(), unit.logical_tables[&1025].schema.fields()); + assert_eq!( + arrow::compute::concat_batches(&schema, &batches).unwrap(), + batch.project(&[1]).unwrap() + ); +} + +struct PausedFileWriter { + inner: Option, + started: Arc, + release: Arc, +} + +impl object_store::layers::mock::oio::Write for PausedFileWriter { + async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> { + let mut inner = self.inner.take().unwrap(); + let started = self.started.clone(); + let release = self.release.clone(); + // Like SecureFs's blocking open, this task survives a dropped I/O future. + let (inner, result) = tokio::spawn(async move { + started.notify_one(); + release.notified().await; + let result = inner.write(bytes).await; + (inner, result) + }) + .await + .unwrap(); + self.inner = Some(inner); + result + } + + async fn close(&mut self) -> object_store::Result { + self.inner.as_mut().unwrap().close().await + } + + async fn abort(&mut self) -> object_store::Result<()> { + match self.inner.as_mut() { + Some(inner) => inner.abort().await, + None => Err(object_store::Error::new( + object_store::ErrorKind::Unsupported, + "open is pending", + )), + } + } +} + +#[tokio::test] +async fn cancellation_waits_for_file_creation_before_cleanup() { + use object_store::layers::mock::{MockLayerBuilder, MockWriterFactory}; + let directory = common_test_util::temp_dir::create_temp_dir("metric_export_pending_open"); + let access = + common_datasource::object_store::LocalFileAccess::sandboxed(directory.path()).unwrap(); + let store = build_backend_for_write( + &format!("{}/", directory.path().display()), + &HashMap::new(), + &access, + ) + .await + .unwrap(); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let factory: MockWriterFactory = Arc::new({ + let started = started.clone(); + let release = release.clone(); + move |_, _, inner| { + Box::new(PausedFileWriter { + inner: Some(inner), + started: started.clone(), + release: release.clone(), + }) + } + }); + let store = store.layer( + MockLayerBuilder::default() + .writer_factory(factory) + .build() + .unwrap(), + ); + let cancellation = CancellationToken::new(); + let unit = unit(); + let export = export_stream( + &unit, + stream(vec![batch(vec![Some(1025)], vec![Some("a")])]), + &store, + LogicalTableExportLimits::default(), + &cancellation, + ); + tokio::pin!(export); + tokio::select! { + result = &mut export => panic!("export completed before the file open: {result:?}"), + _ = started.notified() => {}, + } + cancellation.cancel(); + assert!(futures::poll!(&mut export).is_pending()); + release.notify_one(); + let result = export.await; + assert!(matches!( + result, + Err(error::Error::LogicalTableExportCancelled { .. }) + )); + assert!(!store.exists("cpu.v1.parquet").await.unwrap()); +} + +struct FailedAbortWriter(object_store::layers::mock::oio::Writer); + +impl object_store::layers::mock::oio::Write for FailedAbortWriter { + async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> { + self.0.write(bytes).await + } + async fn close(&mut self) -> object_store::Result { + self.0.close().await + } + async fn abort(&mut self) -> object_store::Result<()> { + Err(object_store::Error::new( + object_store::ErrorKind::PermissionDenied, + "injected abort failure", + )) + } +} + +#[tokio::test] +async fn cleanup_failure_preserves_resource_error() { + use object_store::layers::mock::{MockLayerBuilder, MockWriterFactory}; + let factory: MockWriterFactory = Arc::new(|_, _, inner| Box::new(FailedAbortWriter(inner))); + let store = ObjectStore::new(object_store::services::Memory::default()) + .unwrap() + .layer( + MockLayerBuilder::default() + .writer_factory(factory) + .build() + .unwrap(), + ); + let mut limits = export_limits(); + limits.writer.max_row_groups = 1; + let result = export_stream( + &unit(), + stream(vec![batch(vec![Some(1025); 2], vec![Some("a"); 2])]), + &store, + limits, + &CancellationToken::new(), + ) + .await; + assert!(matches!( + result, + Err(error::Error::LogicalTableExportResource { .. }) + )); +} + +#[tokio::test] +async fn validates_membership_by_table_route() { + use common_meta::kv_backend::TxnService; + use common_meta::kv_backend::memory::MemoryKvBackend; + + let unit = unit(); + for later_physical_id in [None, Some(2048), Some(1024)] { + let kv = Arc::new(MemoryKvBackend::default()); + let manager = TableRouteManager::new(kv.clone()); + for (&table_id, physical_id) in + unit.logical_tables + .keys() + .zip([Some(1024), later_physical_id, Some(1024)]) + { + if let Some(physical_id) = physical_id { + let (txn, _) = manager + .table_route_storage() + .build_create_txn(table_id, &TableRouteValue::logical(physical_id)) + .unwrap(); + assert!(kv.txn(txn).await.unwrap().succeeded); + } + } + let result = unit.validate_table_routes(&manager).await; + if later_physical_id == Some(1024) { + result.unwrap(); + } else { + assert!(matches!( + result, + Err(error::Error::InvalidLogicalTableExport { .. }) + )); + } + } +} diff --git a/src/partition/src/expr.rs b/src/partition/src/expr.rs index 1ef875dae08..cb15f5ae5d7 100644 --- a/src/partition/src/expr.rs +++ b/src/partition/src/expr.rs @@ -20,6 +20,7 @@ use api::v1::meta::Partition; use datafusion_common::{ScalarValue, ToDFSchema}; use datafusion_expr::Expr; use datafusion_expr::execution_props::ExecutionProps; +use datafusion_expr::physical_planning_context::PhysicalPlanningContext; use datafusion_physical_expr::{PhysicalExpr, create_physical_expr}; use datatypes::arrow; use datatypes::value::{ @@ -391,8 +392,13 @@ impl PartitionExpr { .context(error::ToDFSchemaSnafu)?; let execution_props = &ExecutionProps::default(); let expr = self.try_as_logical_expr()?; - create_physical_expr(&expr, &df_schema, execution_props) - .context(error::CreatePhysicalExprSnafu) + create_physical_expr( + &expr, + &df_schema, + execution_props, + &PhysicalPlanningContext::default(), + ) + .context(error::CreatePhysicalExprSnafu) } pub fn and(self, rhs: PartitionExpr) -> PartitionExpr { diff --git a/src/pipeline/tests/json_parse.rs b/src/pipeline/tests/json_parse.rs index c374c846cde..b0c32950518 100644 --- a/src/pipeline/tests/json_parse.rs +++ b/src/pipeline/tests/json_parse.rs @@ -23,6 +23,8 @@ use arrow_schema::extension::{ EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY, ExtensionType, }; use datatypes::extension::json::{Json2ExtensionType, JsonMetadata}; +use greptime_proto::v1::column_data_type_extension::TypeExt; +use greptime_proto::v1::{ColumnDataTypeExtension, JsonNativeTypeExtension}; const INPUT_VALUE_OBJ: &str = r#" [ @@ -138,7 +140,15 @@ transform: let output = common::parse_and_exec(INPUT_VALUE_OBJ, pipeline_yaml); assert_eq!(output.schema[0].datatype, ColumnDataType::Json as i32); - assert!(output.schema[0].datatype_extension.is_none()); + + let expected = Some(ColumnDataTypeExtension { + type_ext: Some(TypeExt::JsonNativeType(Box::new(JsonNativeTypeExtension { + datatype: ColumnDataType::Json as _, + datatype_extension: None, + }))), + }); + assert_eq!(output.schema[0].datatype_extension, expected); + assert_eq!( output.schema[0] .options diff --git a/src/promql/benches/bench_range_fn.rs b/src/promql/benches/bench_range_fn.rs index 268c3e67d23..459137d14dd 100644 --- a/src/promql/benches/bench_range_fn.rs +++ b/src/promql/benches/bench_range_fn.rs @@ -1129,6 +1129,7 @@ fn bench_range_manipulate_wall_time(c: &mut Criterion) { 0, (evaluations as i64 - 1) * RANGE_MANIPULATE_CADENCE_MS, RANGE_MANIPULATE_CADENCE_MS, + 0, window_points as i64 * RANGE_MANIPULATE_CADENCE_MS, "timestamp".to_string(), field_columns, diff --git a/src/promql/src/extension_plan.rs b/src/promql/src/extension_plan.rs index 29b31b7ca03..750ee41b5e8 100644 --- a/src/promql/src/extension_plan.rs +++ b/src/promql/src/extension_plan.rs @@ -29,9 +29,12 @@ pub use absent::{Absent, AbsentExec, AbsentStream}; use common_query::native_histogram::{SUM_FIELD, native_histogram_value_type}; use common_query::prometheus::is_prometheus_stale_nan; use datafusion::arrow::array::{Array, Float64Array, StructArray}; -use datafusion::arrow::datatypes::{ArrowPrimitiveType, TimestampMillisecondType}; -use datafusion::common::DFSchemaRef; +use datafusion::arrow::datatypes::{ + ArrowPrimitiveType, DataType, TimeUnit, TimestampMillisecondType, +}; +use datafusion::common::{Column, DFSchemaRef}; use datafusion::error::{DataFusionError, Result as DataFusionResult}; +use datafusion::logical_expr::{Expr, Extension, LogicalPlan}; use datatypes::data_type::DataType as _; pub use empty_metric::{EmptyMetric, EmptyMetricExec, EmptyMetricStream, build_special_time_expr}; pub use histogram_fold::{ @@ -47,6 +50,78 @@ pub use union_distinct_on::{UnionDistinctOn, UnionDistinctOnExec, UnionDistinctO pub type Millisecond = ::Native; +pub(crate) fn timestamp_unit(data_type: &DataType) -> datafusion::error::Result { + match data_type { + DataType::Timestamp(unit, _) => Ok(*unit), + _ => Err(datafusion::error::DataFusionError::Execution( + "Time index column is not a timestamp".into(), + )), + } +} + +pub(crate) fn nanoseconds_per_native_tick(unit: TimeUnit) -> i128 { + match unit { + TimeUnit::Second => 1_000_000_000, + TimeUnit::Millisecond => 1_000_000, + TimeUnit::Microsecond => 1_000, + TimeUnit::Nanosecond => 1, + } +} + +/// Recovers the offset serialized only by an immediately underlying normalize node. +/// +/// This is decode-only recovery for manipulators whose wire messages have no offset field. +/// Follow identity projections (as used by `timestamp()`), but stop at other nodes or changed +/// time columns to avoid applying an inner selector's offset again to an outer subquery. +pub(crate) fn local_offset(plan: &LogicalPlan, time_index: &str) -> Millisecond { + let Some(index) = plan.schema().index_of_column_by_name(None, time_index) else { + return 0; + }; + let (qualifier, field) = plan.schema().qualified_field(index); + let mut time_index = Column::new(qualifier.cloned(), field.name().clone()); + let mut plan = plan; + + loop { + match plan { + LogicalPlan::Extension(Extension { node }) => { + return node + .as_any() + .downcast_ref::() + .and_then(|normalize| normalize.offset_for_time_index(&time_index)) + .unwrap_or_default(); + } + LogicalPlan::Projection(projection) => { + let Some(output_index) = projection.schema.maybe_index_of_column(&time_index) + else { + return 0; + }; + let expr = &projection.expr[output_index]; + let source = match expr { + Expr::Column(column) => column, + Expr::Alias(alias) => { + let Expr::Column(column) = alias.expr.as_ref() else { + return 0; + }; + if alias.name != column.name { + return 0; + } + column + } + _ => return 0, + }; + let Some(input_index) = projection.input.schema().maybe_index_of_column(source) + else { + return 0; + }; + let (qualifier, field) = projection.input.schema().qualified_field(input_index); + time_index = Column::new(qualifier.cloned(), field.name().clone()); + plan = projection.input.as_ref(); + } + _ => return 0, + } + } +} + const METRIC_NUM_SERIES: &str = "num_series"; fn prometheus_stale_sample_column(column: &dyn Array) -> Option<(&dyn Array, &Float64Array)> { @@ -109,3 +184,87 @@ pub fn resolve_column_names( .map(|idx| resolve_column_name(*idx, schema, context, column_type)) .collect() } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; + use datafusion::common::ToDFSchema; + use datafusion::logical_expr::{EmptyRelation, Extension, LogicalPlan, Projection}; + use datafusion_expr::col; + + use super::*; + + fn input() -> LogicalPlan { + LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: Arc::new(Schema::new(vec![ + Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + Field::new( + "other_ts", + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + Field::new("value", DataType::Float64, true), + ])) + .to_dfschema_ref() + .unwrap(), + }) + } + + fn normalized() -> LogicalPlan { + LogicalPlan::Extension(Extension { + node: Arc::new(SeriesNormalize::new( + 1_000, + "timestamp", + false, + Vec::new(), + input(), + )), + }) + } + + #[test] + fn local_offset_tracks_identity_preserving_projections() { + let projection = + Projection::try_new(vec![col("timestamp"), col("value")], Arc::new(normalized())) + .unwrap(); + let projection = Projection::try_new( + vec![col("timestamp").alias("timestamp"), col("value")], + Arc::new(LogicalPlan::Projection(projection)), + ) + .unwrap(); + + assert_eq!( + 1_000, + local_offset(&LogicalPlan::Projection(projection), "timestamp") + ); + } + + #[test] + fn local_offset_rejects_a_different_timestamp_or_manipulator() { + let renamed = Projection::try_new( + vec![col("other_ts").alias("timestamp"), col("value")], + Arc::new(normalized()), + ) + .unwrap(); + assert_eq!( + 0, + local_offset(&LogicalPlan::Projection(renamed), "timestamp") + ); + + let divide = LogicalPlan::Extension(Extension { + node: Arc::new(SeriesDivide::new( + Vec::new(), + "timestamp".to_string(), + normalized(), + )), + }); + assert_eq!(0, local_offset(÷, "timestamp")); + } +} diff --git a/src/promql/src/extension_plan/absent.rs b/src/promql/src/extension_plan/absent.rs index 71af4130295..fa811b3df7d 100644 --- a/src/promql/src/extension_plan/absent.rs +++ b/src/promql/src/extension_plan/absent.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::cmp::Ordering; use std::collections::HashMap; use std::pin::Pin; @@ -20,6 +19,7 @@ use std::sync::Arc; use std::task::{Context, Poll}; use datafusion::arrow::array::Array; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{DFSchemaRef, Result as DataFusionResult}; use datafusion::execution::context::TaskContext; use datafusion::logical_expr::{Expr, LogicalPlan, UserDefinedLogicalNodeCore}; @@ -30,8 +30,8 @@ use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::expressions::Column as ColumnExpr; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, Partitioning, PlanProperties, - RecordBatchStream, SendableRecordBatchStream, + DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, InputDistributionRequirements, + Partitioning, PhysicalExpr, PlanProperties, RecordBatchStream, SendableRecordBatchStream, }; use datafusion_common::DFSchema; use datafusion_expr::{EmptyRelation, col}; @@ -325,8 +325,11 @@ pub struct AbsentExec { } impl ExecutionPlan for AbsentExec { - fn as_any(&self) -> &dyn Any { - self + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> datafusion_common::Result, + ) -> DataFusionResult { + Ok(TreeNodeRecursion::Continue) } fn schema(&self) -> SchemaRef { @@ -337,8 +340,8 @@ impl ExecutionPlan for AbsentExec { &self.properties } - fn required_input_distribution(&self) -> Vec { - vec![Distribution::SinglePartition] + fn input_distribution_requirements(&self) -> InputDistributionRequirements { + InputDistributionRequirements::new(vec![Distribution::SinglePartition]) } fn required_input_ordering(&self) -> Vec> { diff --git a/src/promql/src/extension_plan/empty_metric.rs b/src/promql/src/extension_plan/empty_metric.rs index 5a7678aab61..91c8a1666a2 100644 --- a/src/promql/src/extension_plan/empty_metric.rs +++ b/src/promql/src/extension_plan/empty_metric.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::collections::HashMap; use std::ops::Div; use std::pin::Pin; @@ -21,21 +20,24 @@ use std::task::{Context, Poll}; use datafusion::arrow::array::ArrayRef; use datafusion::arrow::datatypes::{DataType, TimeUnit}; +use datafusion::catalog::Session; use datafusion::common::arrow::datatypes::Field; use datafusion::common::stats::Precision; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{ DFSchema, DFSchemaRef, Result as DataFusionResult, Statistics, TableReference, }; use datafusion::datasource::{MemTable, provider_as_source}; use datafusion::error::DataFusionError; -use datafusion::execution::context::{SessionState, TaskContext}; +use datafusion::execution::context::TaskContext; +use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext; use datafusion::logical_expr::{ExprSchemable, LogicalPlan, UserDefinedLogicalNodeCore}; -use datafusion::physical_expr::{EquivalenceProperties, PhysicalExprRef}; +use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr, PhysicalExprRef}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, RecordBatchStream, - SendableRecordBatchStream, + SendableRecordBatchStream, StatisticsArgs, }; use datafusion::physical_planner::PhysicalPlanner; use datafusion::prelude::{Expr, col, lit}; @@ -113,14 +115,20 @@ impl EmptyMetric { pub fn to_execution_plan( &self, - session_state: &SessionState, + session: &dyn Session, physical_planner: &dyn PhysicalPlanner, + planning_ctx: &PhysicalPlanningContext, ) -> DataFusionResult> { let physical_expr = self .expr .as_ref() .map(|expr| { - physical_planner.create_physical_expr(expr, &self.time_index_schema, session_state) + physical_planner.create_physical_expr( + expr, + &self.time_index_schema, + session, + planning_ctx, + ) }) .transpose()?; let result_schema: SchemaRef = self.result_schema.inner().clone(); @@ -224,8 +232,11 @@ pub struct EmptyMetricExec { } impl ExecutionPlan for EmptyMetricExec { - fn as_any(&self) -> &dyn Any { - self + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> datafusion_common::Result, + ) -> DataFusionResult { + datafusion::physical_plan::apply_expression_roots(self.expr.iter(), f) } fn schema(&self) -> SchemaRef { @@ -273,9 +284,14 @@ impl ExecutionPlan for EmptyMetricExec { Some(self.metric.clone_inner()) } - fn partition_statistics(&self, partition: Option) -> DataFusionResult { + fn statistics_from_inputs( + &self, + _input_stats: &[Arc], + args: &StatisticsArgs, + ) -> DataFusionResult> { + let partition = args.partition(); if partition.is_some() { - return Ok(Statistics::new_unknown(self.schema().as_ref())); + return Ok(Arc::new(Statistics::new_unknown(self.schema().as_ref()))); } let estimated_row_num = if self.end > self.start { @@ -285,11 +301,11 @@ impl ExecutionPlan for EmptyMetricExec { }; let total_byte_size = estimated_row_num * std::mem::size_of::() as f64; - Ok(Statistics { + Ok(Arc::new(Statistics { num_rows: Precision::Inexact(estimated_row_num.floor() as _), total_byte_size: Precision::Inexact(total_byte_size.floor() as _), column_statistics: Statistics::unknown_column(&self.schema()), - }) + })) } fn name(&self) -> &str { @@ -429,7 +445,11 @@ mod test { ) .unwrap(); let empty_metric_exec = empty_metric - .to_execution_plan(&session_context.state(), &df_default_physical_planner) + .to_execution_plan( + &session_context.state(), + &df_default_physical_planner, + &PhysicalPlanningContext::default(), + ) .unwrap(); let result = @@ -544,7 +564,11 @@ mod test { let empty_metric = EmptyMetric::new(0, 200, 1000, "time".to_string(), "value".to_string(), None).unwrap(); let empty_metric_exec = empty_metric - .to_execution_plan(&session_context.state(), &df_default_physical_planner) + .to_execution_plan( + &session_context.state(), + &df_default_physical_planner, + &PhysicalPlanningContext::default(), + ) .unwrap(); let result = diff --git a/src/promql/src/extension_plan/histogram_fold.rs b/src/promql/src/extension_plan/histogram_fold.rs index 5a5f2f8d466..94769df28a4 100644 --- a/src/promql/src/extension_plan/histogram_fold.rs +++ b/src/promql/src/extension_plan/histogram_fold.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -25,6 +24,7 @@ use datafusion::arrow::compute::{SortOptions, concat_batches}; use datafusion::arrow::datatypes::{DataType, Float64Type, SchemaRef}; use datafusion::arrow::record_batch::RecordBatch; use datafusion::common::stats::Precision; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{DFSchema, DFSchemaRef, Statistics}; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::execution::TaskContext; @@ -37,7 +37,8 @@ use datafusion::physical_plan::expressions::{Column as PhyColumn, TryCastExpr as use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties, - Partitioning, PhysicalExpr, PlanProperties, RecordBatchStream, SendableRecordBatchStream, + InputDistributionRequirements, Partitioning, PhysicalExpr, PlanProperties, RecordBatchStream, + SendableRecordBatchStream, StatisticsArgs, }; use datafusion::prelude::{Column, Expr}; use datafusion_expr::{EmptyRelation, col}; @@ -526,8 +527,14 @@ pub struct HistogramFoldExec { } impl ExecutionPlan for HistogramFoldExec { - fn as_any(&self) -> &dyn Any { - self + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> datafusion_common::Result, + ) -> DataFusionResult { + datafusion::physical_plan::apply_expression_roots( + self.tag_columns.iter().chain(self.partition_exprs.iter()), + f, + ) } fn properties(&self) -> &Arc { @@ -574,8 +581,10 @@ impl ExecutionPlan for HistogramFoldExec { vec![Some(OrderingRequirements::Hard(vec![requirement]))] } - fn required_input_distribution(&self) -> Vec { - vec![Distribution::HashPartitioned(self.partition_exprs.clone())] + fn input_distribution_requirements(&self) -> InputDistributionRequirements { + InputDistributionRequirements::new(vec![Distribution::KeyPartitioned( + self.partition_exprs.clone(), + )]) } fn maintains_input_order(&self) -> Vec { @@ -664,12 +673,16 @@ impl ExecutionPlan for HistogramFoldExec { Some(self.metric.clone_inner()) } - fn partition_statistics(&self, _: Option) -> DataFusionResult { - Ok(Statistics { + fn statistics_from_inputs( + &self, + _input_stats: &[Arc], + _args: &StatisticsArgs, + ) -> DataFusionResult> { + Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, column_statistics: Statistics::unknown_column(&self.schema()), - }) + })) } fn name(&self) -> &str { diff --git a/src/promql/src/extension_plan/instant_manipulate.rs b/src/promql/src/extension_plan/instant_manipulate.rs index 25d62e6bee3..45857879d73 100644 --- a/src/promql/src/extension_plan/instant_manipulate.rs +++ b/src/promql/src/extension_plan/instant_manipulate.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; -use std::cmp::Ordering; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; @@ -23,21 +21,25 @@ use datafusion::arrow::array::{Array, TimestampMillisecondArray, UInt64Array}; use datafusion::arrow::datatypes::{DataType, SchemaRef}; use datafusion::arrow::record_batch::RecordBatch; use datafusion::common::stats::Precision; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{DFSchema, DFSchemaRef, ScalarValue}; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::execution::context::TaskContext; use datafusion::logical_expr::{ EmptyRelation, Expr, Extension, LogicalPlan, UserDefinedLogicalNodeCore, }; +use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::metrics::{ BaselineMetrics, Count, ExecutionPlanMetricsSet, MetricBuilder, MetricValue, MetricsSet, }; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, PlanProperties, RecordBatchStream, - SendableRecordBatchStream, Statistics, + ChildStats, DisplayAs, DisplayFormatType, ExecutionPlan, InputDistributionRequirements, + PhysicalExpr, PlanProperties, RecordBatchStream, SendableRecordBatchStream, Statistics, + StatisticsArgs, }; use datafusion_expr::col; use datatypes::arrow::compute; +use datatypes::timestamp::timestamp_array_to_primitive; use futures::{Stream, StreamExt, ready}; use greptime_proto::substrait_extension as pb; use prost::Message; @@ -46,8 +48,9 @@ use snafu::ResultExt; use crate::error::{DeserializeSnafu, Result}; use crate::extension_plan::series_divide::SeriesDivide; use crate::extension_plan::{ - METRIC_NUM_SERIES, Millisecond, is_prometheus_stale_sample, prometheus_stale_sample_column, - resolve_column_name, serialize_column_index, + METRIC_NUM_SERIES, Millisecond, is_prometheus_stale_sample, local_offset, + nanoseconds_per_native_tick, prometheus_stale_sample_column, resolve_column_name, + serialize_column_index, timestamp_unit, }; use crate::metrics::PROMQL_SERIES_COUNT; @@ -67,21 +70,52 @@ fn mixed_sample_fields(field: Option<&str>) -> [Option<&str>; 2] { /// This plan will try to align the input time series, for every timestamp between /// `start` and `end` with step `interval`. Find in the `lookback` range if data /// is missing at the given timestamp. -#[derive(Debug, PartialEq, Eq, Hash, PartialOrd)] +#[derive(Debug, PartialEq, Eq, Hash)] pub struct InstantManipulate { start: Millisecond, end: Millisecond, lookback_delta: Millisecond, interval: Millisecond, + offset: Millisecond, time_index_column: String, // Planner-provided tag-column hint for execution fast paths. tag_columns: Vec, /// Primary sample column used to derive the columns checked for staleness. field_column: Option, input: LogicalPlan, + output_schema: DFSchemaRef, unfix: Option, } +impl PartialOrd for InstantManipulate { + fn partial_cmp(&self, other: &Self) -> Option { + ( + self.start, + self.end, + self.lookback_delta, + self.interval, + self.offset, + &self.time_index_column, + &self.tag_columns, + &self.field_column, + &self.input, + &self.unfix, + ) + .partial_cmp(&( + other.start, + other.end, + other.lookback_delta, + other.interval, + other.offset, + &other.time_index_column, + &other.tag_columns, + &other.field_column, + &other.input, + &other.unfix, + )) + } +} + #[derive(Debug, PartialEq, Eq, Hash, PartialOrd)] struct UnfixIndices { pub time_index_idx: u64, @@ -98,7 +132,7 @@ impl UserDefinedLogicalNodeCore for InstantManipulate { } fn schema(&self) -> &DFSchemaRef { - self.input.schema() + &self.output_schema } fn expressions(&self) -> Vec { @@ -180,6 +214,8 @@ impl UserDefinedLogicalNodeCore for InstantManipulate { end: self.end, lookback_delta: self.lookback_delta, interval: self.interval, + offset: local_offset(&input, &time_index_column), + output_schema: Self::calculate_output_schema(&input, &time_index_column)?, time_index_column, tag_columns: Self::resolve_tag_columns(&input, &self.tag_columns), field_column, @@ -192,9 +228,11 @@ impl UserDefinedLogicalNodeCore for InstantManipulate { end: self.end, lookback_delta: self.lookback_delta, interval: self.interval, + offset: self.offset, time_index_column: self.time_index_column.clone(), tag_columns: Self::resolve_tag_columns(&input, &self.tag_columns), field_column: self.field_column.clone(), + output_schema: Self::calculate_output_schema(&input, &self.time_index_column)?, input, unfix: None, }) @@ -203,12 +241,45 @@ impl UserDefinedLogicalNodeCore for InstantManipulate { } impl InstantManipulate { + fn calculate_output_schema( + input: &LogicalPlan, + time_index_column: &str, + ) -> DataFusionResult { + let input_schema = input.schema(); + let time_index = input_schema + .index_of_column_by_name(None, time_index_column) + .ok_or_else(|| { + DataFusionError::Internal(format!( + "InstantManipulate time index {time_index_column} not found" + )) + })?; + let mut fields = (0..input_schema.fields().len()) + .map(|index| { + let (qualifier, field) = input_schema.qualified_field(index); + (qualifier.cloned(), field.clone()) + }) + .collect::>(); + let (qualifier, field) = input_schema.qualified_field(time_index); + fields[time_index] = ( + qualifier.cloned(), + Arc::new(field.as_ref().clone().with_data_type(DataType::Timestamp( + datafusion::arrow::datatypes::TimeUnit::Millisecond, + None, + ))), + ); + Ok(Arc::new(DFSchema::new_with_metadata( + fields, + input_schema.metadata().clone(), + )?)) + } + #[allow(clippy::too_many_arguments)] pub fn new( start: Millisecond, end: Millisecond, lookback_delta: Millisecond, interval: Millisecond, + offset: Millisecond, time_index_column: String, tag_columns: Vec, field_column: Option, @@ -219,6 +290,9 @@ impl InstantManipulate { end, lookback_delta, interval, + offset, + output_schema: Self::calculate_output_schema(&input, &time_index_column) + .unwrap_or_else(|_| input.schema().clone()), time_index_column, tag_columns, field_column, @@ -274,7 +348,27 @@ impl InstantManipulate { pub fn to_execution_plan(&self, exec_input: Arc) -> Arc { let reuse_tsid_column = matches!(self.tag_columns.as_slice(), [tag] if tag == "__tsid"); + let mut fields = exec_input.schema().fields().to_vec(); + let time_index = exec_input + .schema() + .index_of(&self.time_index_column) + .expect("time index column not found"); + fields[time_index] = Arc::new(fields[time_index].as_ref().clone().with_data_type( + DataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Millisecond, None), + )); + let output_schema = Arc::new(datafusion::arrow::datatypes::Schema::new_with_metadata( + fields, + exec_input.schema().metadata().clone(), + )); + let input_properties = exec_input.properties(); + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(output_schema.clone()), + input_properties.partitioning.clone(), + input_properties.emission_type, + input_properties.boundedness, + )); Arc::new(InstantManipulateExec { + offset: self.offset, start: self.start, end: self.end, lookback_delta: self.lookback_delta, @@ -283,6 +377,8 @@ impl InstantManipulate { field_column: self.field_column.clone(), reuse_tsid_column, input: exec_input, + output_schema, + properties, metric: ExecutionPlanMetricsSet::new(), }) } @@ -311,9 +407,10 @@ impl InstantManipulate { pub fn deserialize(bytes: &[u8]) -> Result { let pb_instant_manipulate = pb::InstantManipulate::decode(bytes).context(DeserializeSnafu)?; + let empty_schema = Arc::new(DFSchema::empty()); let placeholder_plan = LogicalPlan::EmptyRelation(EmptyRelation { produce_one_row: false, - schema: Arc::new(DFSchema::empty()), + schema: empty_schema.clone(), }); let unfix = UnfixIndices { @@ -326,9 +423,11 @@ impl InstantManipulate { end: pb_instant_manipulate.end, lookback_delta: pb_instant_manipulate.lookback_delta, interval: pb_instant_manipulate.interval, + offset: 0, time_index_column: String::new(), tag_columns: Vec::new(), field_column: None, + output_schema: empty_schema, input: placeholder_plan, unfix: Some(unfix), }) @@ -337,6 +436,7 @@ impl InstantManipulate { #[derive(Debug)] pub struct InstantManipulateExec { + offset: Millisecond, start: Millisecond, end: Millisecond, lookback_delta: Millisecond, @@ -346,24 +446,29 @@ pub struct InstantManipulateExec { reuse_tsid_column: bool, input: Arc, + output_schema: SchemaRef, + properties: Arc, metric: ExecutionPlanMetricsSet, } impl ExecutionPlan for InstantManipulateExec { - fn as_any(&self) -> &dyn Any { - self + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> datafusion_common::Result, + ) -> DataFusionResult { + Ok(TreeNodeRecursion::Continue) } fn schema(&self) -> SchemaRef { - self.input.schema() + self.output_schema.clone() } fn properties(&self) -> &Arc { - self.input.properties() + &self.properties } - fn required_input_distribution(&self) -> Vec { - self.input.required_input_distribution() + fn input_distribution_requirements(&self) -> InputDistributionRequirements { + self.input.input_distribution_requirements() } // Prevent reordering of input @@ -380,7 +485,18 @@ impl ExecutionPlan for InstantManipulateExec { children: Vec>, ) -> DataFusionResult> { assert!(!children.is_empty()); + let input = children[0].clone(); + // The child may expose native timestamps, but our output schema uses ms. + // Retain its execution properties, not its schema equivalences or ordering. + let input_properties = input.properties(); + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(self.output_schema.clone()), + input_properties.partitioning.clone(), + input_properties.emission_type, + input_properties.boundedness, + )); Ok(Arc::new(Self { + offset: self.offset, start: self.start, end: self.end, lookback_delta: self.lookback_delta, @@ -388,7 +504,9 @@ impl ExecutionPlan for InstantManipulateExec { time_index_column: self.time_index_column.clone(), field_column: self.field_column.clone(), reuse_tsid_column: self.reuse_tsid_column, - input: children[0].clone(), + input, + output_schema: self.output_schema.clone(), + properties, metric: self.metric.clone(), })) } @@ -413,6 +531,7 @@ impl ExecutionPlan for InstantManipulateExec { .column_with_name(&self.time_index_column) .expect("time index column not found") .0; + let time_unit = timestamp_unit(schema.field(time_index).data_type())?; let field_indices = mixed_sample_fields(self.field_column.as_deref()).map(|field| { field.and_then(|field| schema.column_with_name(field).map(|(index, _)| index)) }); @@ -421,15 +540,17 @@ impl ExecutionPlan for InstantManipulateExec { .filter(|(_, field)| field.data_type() == &DataType::UInt64) .map(|(index, _)| index); Ok(Box::pin(InstantManipulateStream { + offset: self.offset, start: self.start, end: self.end, lookback_delta: self.lookback_delta, interval: self.interval, time_index, + time_unit, field_indices, tsid_index, reuse_tsid_column: self.reuse_tsid_column && tsid_index.is_some(), - schema, + schema: self.output_schema.clone(), input, metric: baseline_metric, num_series, @@ -440,8 +561,16 @@ impl ExecutionPlan for InstantManipulateExec { Some(self.metric.clone_inner()) } - fn partition_statistics(&self, partition: Option) -> DataFusionResult { - let input_stats = self.input.partition_statistics(partition)?; + fn child_stats_requests(&self, partition: Option) -> Vec { + vec![ChildStats::At(partition)] + } + + fn statistics_from_inputs( + &self, + input_stats: &[Arc], + _args: &StatisticsArgs, + ) -> DataFusionResult> { + let input_stats = &input_stats[0]; let estimated_row_num = (self.end - self.start) as f64 / self.interval as f64; let estimated_total_bytes = input_stats @@ -453,12 +582,12 @@ impl ExecutionPlan for InstantManipulateExec { }) .unwrap_or(Precision::Absent); - Ok(Statistics { + Ok(Arc::new(Statistics { num_rows: Precision::Inexact(estimated_row_num.floor() as _), total_byte_size: estimated_total_bytes, // TODO(ruihang): support this column statistics column_statistics: Statistics::unknown_column(&self.schema()), - }) + })) } fn name(&self) -> &str { @@ -487,12 +616,14 @@ impl DisplayAs for InstantManipulateExec { } pub struct InstantManipulateStream { + offset: Millisecond, start: Millisecond, end: Millisecond, lookback_delta: Millisecond, interval: Millisecond, // Column index of TIME INDEX column's position in schema time_index: usize, + time_unit: datafusion::arrow::datatypes::TimeUnit, field_indices: [Option; 2], tsid_index: Option, reuse_tsid_column: bool, @@ -516,11 +647,10 @@ impl Stream for InstantManipulateStream { fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let poll = match ready!(self.input.poll_next_unpin(cx)) { Some(Ok(batch)) => { - if batch.num_rows() == 0 { - return Poll::Pending; - } let timer = std::time::Instant::now(); - self.num_series.add(1); + if batch.num_rows() != 0 { + self.num_series.add(1); + } let result = Ok(batch).and_then(|batch| self.manipulate(batch)); self.metric.elapsed_compute().add_elapsed(timer); Poll::Ready(Some(result)) @@ -543,22 +673,12 @@ impl InstantManipulateStream { /// lookback window `(eval_ts - lookback_delta, eval_ts]`; a sample at exactly /// `eval_ts - lookback_delta` is too old. pub fn manipulate(&self, input: RecordBatch) -> DataFusionResult { - let ts_column = input - .column(self.time_index) - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Execution( - "Time index Column downcast to TimestampMillisecondArray failed".into(), - ) - })?; - - // Early return for empty input + let ts_column = input.column(self.time_index); if ts_column.is_empty() { - return Ok(input); + // Returning Pending after consuming a ready batch without a wake can stall the stream. + return Ok(RecordBatch::new_empty(self.schema.clone())); } - - // Field columns for staleness checks, classified once per batch. + let scale = nanoseconds_per_native_tick(self.time_unit); let stale_sample_columns = self.field_indices.map(|index| { index.and_then(|index| prometheus_stale_sample_column(input.column(index).as_ref())) }); @@ -568,101 +688,82 @@ impl InstantManipulateStream { .flatten() .any(|column| is_prometheus_stale_sample(*column, row)) }; - - // Optimize iteration range based on actual data bounds - let first_ts = ts_column.value(0); - let last_ts = ts_column.value(ts_column.len() - 1); - // A sample at `t` is eligible for eval time `eval_ts` iff: - // t > eval_ts - lookback_delta <=> eval_ts < t + lookback_delta. - // Therefore the last eval timestamp for which the last sample is still eligible is: - // last_ts + lookback_delta - 1 (millisecond granularity). - let last_useful = if self.lookback_delta > 0 { - last_ts + self.lookback_delta - 1 + let (timestamps, _) = timestamp_array_to_primitive(ts_column).ok_or_else(|| { + DataFusionError::Execution("Time index column is not a timestamp".into()) + })?; + let timestamps = timestamps.values(); + let len = timestamps.len(); + // Shift the native-tick timeline in i128 before comparing samples. Doing + // this in the Arrow storage unit can overflow even when the shifted + // PromQL millisecond evaluation time is representable. + let to_nanoseconds = + |timestamp: i64| (timestamp as i128) * scale + (self.offset as i128) * 1_000_000; + let first_ns = to_nanoseconds(timestamps[0]); + let last_ns = to_nanoseconds(timestamps[len - 1]); + // An exact sample remains useful with zero lookback. Otherwise the lower + // boundary is exclusive, so subtract one nanosecond from its final window. + let last_useful = if self.lookback_delta == 0 { + last_ns } else { - last_ts + last_ns + (self.lookback_delta as i128) * 1_000_000 - 1 + }; + let first_ms = (first_ns + 999_999).div_euclid(1_000_000); + let last_ms = last_useful.div_euclid(1_000_000); + let query_start = self.start as i128; + let query_end = self.end as i128; + let interval = self.interval as i128; + let max_start = first_ms.max(query_start); + let min_end = last_ms.min(query_end); + let (aligned_start, aligned_end) = if max_start > min_end { + (1, 0) + } else { + ( + query_start + (max_start - query_start) / interval * interval, + query_end - (query_end - min_end) / interval * interval, + ) }; - - let max_start = first_ts.max(self.start); - let min_end = last_useful.min(self.end); - - let aligned_start = self.start + (max_start - self.start) / self.interval * self.interval; - let aligned_end = self.end - (self.end - min_end) / self.interval * self.interval; - let estimated_points = if aligned_end >= aligned_start { - ((aligned_end - aligned_start) / self.interval).saturating_add(1) as usize + (aligned_end - aligned_start) / interval + 1 } else { 0 }; - if estimated_points > MAX_INSTANT_MANIPULATE_OUTPUT_POINTS { + if estimated_points > MAX_INSTANT_MANIPULATE_OUTPUT_POINTS as i128 { return Err(DataFusionError::Execution(format!( "InstantManipulate output points exceed limit: {estimated_points} > {MAX_INSTANT_MANIPULATE_OUTPUT_POINTS}" ))); } + let estimated_points = estimated_points as usize; + let aligned_start = aligned_start as i64; + let aligned_end = aligned_end as i64; let mut take_indices = Vec::with_capacity(estimated_points); - - let mut cursor = 0; - - let aligned_ts_iter = (aligned_start..=aligned_end).step_by(self.interval as usize); let mut aligned_ts = Vec::with_capacity(estimated_points); - - // calculate the offsets to take - 'next: for expected_ts in aligned_ts_iter { - // first, search toward end to see if there is matched timestamp - while cursor < ts_column.len() { - let curr = ts_column.value(cursor); - match curr.cmp(&expected_ts) { - Ordering::Equal => { - if is_stale(cursor) { - // Ignore the stale marker. - } else { - take_indices.push(cursor as u64); - aligned_ts.push(expected_ts); - } - continue 'next; - } - Ordering::Greater => break, - Ordering::Less => {} + let mut cursor = 0; + for expected_ms in (aligned_start..=aligned_end).step_by(self.interval as usize) { + let expected = (expected_ms as i128) * 1_000_000; + let mut exact_candidate = None; + while cursor < len && to_nanoseconds(timestamps[cursor]) <= expected { + if to_nanoseconds(timestamps[cursor]) == expected && exact_candidate.is_none() { + exact_candidate = Some(cursor); } cursor += 1; } - if cursor == ts_column.len() { - cursor -= 1; - // short cut this loop - if ts_column.value(cursor) + self.lookback_delta <= expected_ts { - break; - } - } - - // then examine the value - let curr_ts = ts_column.value(cursor); - if curr_ts + self.lookback_delta <= expected_ts { + // Keep the first row among exact timestamp ties; otherwise use the + // latest preceding row. Zero lookback admits only exact samples. + // Test staleness after choosing: a selected stale marker suppresses + // this evaluation rather than falling back to an older finite value. + let Some(candidate) = exact_candidate.or_else(|| cursor.checked_sub(1)) else { continue; - } - if curr_ts > expected_ts { - // exceeds current expected timestamp, examine the previous value - if let Some(prev_cursor) = cursor.checked_sub(1) { - let prev_ts = ts_column.value(prev_cursor); - if prev_ts + self.lookback_delta > expected_ts { - // only use the point in the time range - if is_stale(prev_cursor) { - // Do not use a stale marker as the newest value. - continue; - } - // use this point - take_indices.push(prev_cursor as u64); - aligned_ts.push(expected_ts); - } - } - } else if is_stale(cursor) { - // Do not use a stale marker as the newest value. - } else { - // use this point - take_indices.push(cursor as u64); - aligned_ts.push(expected_ts); + }; + let candidate_ts = to_nanoseconds(timestamps[candidate]); + let lower = expected - (self.lookback_delta as i128) * 1_000_000; + if (candidate_ts == expected || candidate_ts > lower) + && candidate_ts <= expected + && !is_stale(candidate) + { + take_indices.push(candidate as u64); + aligned_ts.push(expected_ms); } } - - // take record batch and replace the time index column self.take_record_batch_optional(input, take_indices, aligned_ts) } @@ -696,7 +797,7 @@ impl InstantManipulateStream { arrays.push(compute::take(array, indices_array, None)?); } - let result = RecordBatch::try_new(record_batch.schema(), arrays) + let result = RecordBatch::try_new(self.schema.clone(), arrays) .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))?; Ok(result) } @@ -718,14 +819,20 @@ fn reuse_constant_column(array: &Arc, len: usize) -> DataFusionResult mod test { use common_query::native_histogram::build_histogram_array; use common_query::prometheus::PROMETHEUS_STALE_NAN_BITS; - use datafusion::arrow::array::Float64Array; + use datafusion::arrow::array::{ + Float64Array, TimestampMicrosecondArray, TimestampNanosecondArray, TimestampSecondArray, + }; use datafusion::arrow::buffer::NullBuffer; - use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; use datafusion::common::ToDFSchema; use datafusion::datasource::memory::MemorySourceConfig; use datafusion::datasource::source::DataSourceExec; - use datafusion::logical_expr::{EmptyRelation, LogicalPlan}; + use datafusion::logical_expr::{ + EmptyRelation, Extension, LogicalPlan, Projection, UserDefinedLogicalNodeCore, + }; + use datafusion::physical_plan::{ChildrenPropertiesMode, ReplaceChildrenOptions}; use datafusion::prelude::SessionContext; + use datafusion_expr::col; use super::*; use crate::extension_plan::test_util::{ @@ -746,6 +853,7 @@ mod test { Arc::new(prepare_test_data()) }; let normalize_exec = Arc::new(InstantManipulateExec { + offset: 0, start, end, lookback_delta, @@ -753,6 +861,8 @@ mod test { time_index_column: TIME_INDEX_COLUMN.to_string(), field_column: Some("value".to_string()), reuse_tsid_column: false, + output_schema: memory_exec.schema(), + properties: memory_exec.properties().clone(), input: memory_exec, metric: ExecutionPlanMetricsSet::new(), }); @@ -767,6 +877,322 @@ mod test { assert_eq!(result_literal, expected); } + #[tokio::test] + async fn native_timestamps_select_exact_samples_and_keep_ms_output() { + for (unit, ticks_per_ms) in [ + (TimeUnit::Microsecond, 1_000_i64), + (TimeUnit::Nanosecond, 1_000_000_i64), + ] { + let lower = 1_000 * ticks_per_ms; + let upper = 1_001 * ticks_per_ms; + let stale = f64::from_bits(PROMETHEUS_STALE_NAN_BITS); + for (name, timestamps, values, expected_timestamps, expected_values) in [ + ( + "exact upper sample", + vec![lower + 1, upper], + vec![1.0, 2.0], + vec![1_001], + vec![2.0], + ), + ( + "exclusive lower boundary and future sample", + vec![lower, upper + 1], + vec![1.0, 2.0], + vec![1_000], + vec![1.0], + ), + ( + "one native tick above lower boundary", + vec![lower + 1, upper + 1], + vec![1.0, 2.0], + vec![1_001], + vec![1.0], + ), + ( + "future stale marker does not suppress", + vec![lower + 1, upper + 1], + vec![1.0, stale], + vec![1_001], + vec![1.0], + ), + ( + "latest in-window stale marker suppresses", + vec![lower + 1, lower + 2, upper + 1], + vec![1.0, stale, 3.0], + vec![], + vec![], + ), + ] { + let schema = Arc::new(Schema::new(vec![ + Field::new(TIME_INDEX_COLUMN, DataType::Timestamp(unit, None), false), + Field::new("value", DataType::Float64, true), + ])); + let time: Arc = match unit { + TimeUnit::Microsecond => Arc::new(TimestampMicrosecondArray::from(timestamps)), + TimeUnit::Nanosecond => Arc::new(TimestampNanosecondArray::from(timestamps)), + _ => unreachable!(), + }; + let batch = RecordBatch::try_new( + schema.clone(), + vec![time, Arc::new(Float64Array::from(values))], + ) + .unwrap(); + let logical_input = LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: schema.clone().to_dfschema_ref().unwrap(), + }); + let plan = InstantManipulate::new( + 1_000, + 1_001, + 1, + 1, + 0, + TIME_INDEX_COLUMN.to_string(), + Vec::new(), + Some("value".to_string()), + logical_input.clone(), + ); + let output_schema = Arc::new(Schema::new(vec![ + Field::new( + TIME_INDEX_COLUMN, + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + Field::new("value", DataType::Float64, true), + ])); + assert_eq!(plan.schema().as_arrow(), output_schema.as_ref()); + + let rebuilt = InstantManipulate::deserialize(&plan.serialize()) + .unwrap() + .with_exprs_and_inputs(vec![], vec![logical_input]) + .unwrap(); + assert_eq!(rebuilt.schema(), plan.schema()); + assert_eq!(rebuilt.input.schema().as_arrow(), schema.as_ref()); + + let input = Arc::new(DataSourceExec::new(Arc::new( + MemorySourceConfig::try_new(&[vec![batch]], schema.clone(), None).unwrap(), + ))); + let exec = rebuilt.to_execution_plan(input); + assert_eq!(exec.schema(), output_schema); + assert_eq!(exec.children()[0].schema(), schema); + + let batches = + datafusion::physical_plan::collect(exec, SessionContext::default().task_ctx()) + .await + .unwrap(); + assert_eq!(batches.len(), 1, "{unit:?}: {name}"); + let output = &batches[0]; + assert_eq!(output.schema(), output_schema); + let timestamps = output + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = output + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + timestamps.values().as_ref(), + expected_timestamps.as_slice(), + "{unit:?}: {name}" + ); + assert_eq!( + values.values().as_ref(), + expected_values.as_slice(), + "{unit:?}: {name}" + ); + assert_eq!(values.null_count(), 0, "{unit:?}: {name}"); + } + } + } + + #[tokio::test] + async fn logical_normalize_offset_survives_rebuild_and_executes() { + for (name, time_unit, raw, offset, start, lookback_delta) in [ + ( + "millisecond offset", + TimeUnit::Millisecond, + 0, + 1_000, + 1_000, + 0, + ), + ( + "second timestamp with negative fractional offset", + TimeUnit::Second, + 1, + -500, + 500, + 0, + ), + ] { + let schema = Arc::new(Schema::new(vec![ + Field::new( + TIME_INDEX_COLUMN, + DataType::Timestamp(time_unit, None), + false, + ), + Field::new("value", DataType::Float64, true), + ])); + let input = LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: schema.clone().to_dfschema_ref().unwrap(), + }); + let normalize = crate::extension_plan::SeriesNormalize::new( + offset, + TIME_INDEX_COLUMN, + false, + Vec::new(), + input.clone(), + ); + let normalize = + crate::extension_plan::SeriesNormalize::deserialize(&normalize.serialize()) + .unwrap() + .with_exprs_and_inputs(vec![], vec![input.clone()]) + .unwrap(); + let normalized = LogicalPlan::Projection( + Projection::try_new( + vec![col(TIME_INDEX_COLUMN), col("value")], + Arc::new(LogicalPlan::Extension(Extension { + node: Arc::new(normalize), + })), + ) + .unwrap(), + ); + let fresh = InstantManipulate::new( + start, + start, + lookback_delta, + 1, + offset, + TIME_INDEX_COLUMN.to_string(), + Vec::new(), + Some("value".to_string()), + input.clone(), + ) + .with_exprs_and_inputs(vec![], vec![input.clone()]) + .unwrap(); + let serialized = InstantManipulate::new( + start, + start, + lookback_delta, + 1, + offset, + TIME_INDEX_COLUMN.to_string(), + Vec::new(), + Some("value".to_string()), + normalized.clone(), + ); + let decoded = InstantManipulate::deserialize(&serialized.serialize()) + .unwrap() + .with_exprs_and_inputs(vec![], vec![normalized]) + .unwrap(); + let timestamp: Arc = match time_unit { + TimeUnit::Millisecond => Arc::new(TimestampMillisecondArray::from(vec![raw])), + TimeUnit::Second => Arc::new(TimestampSecondArray::from(vec![raw])), + _ => unreachable!(), + }; + let batch = RecordBatch::try_new( + schema.clone(), + vec![timestamp, Arc::new(Float64Array::from(vec![7.0]))], + ) + .unwrap(); + for (mode, rebuilt) in [("fresh", fresh), ("decoded", decoded)] { + let rebuilt = rebuilt + .with_exprs_and_inputs(vec![], vec![input.clone()]) + .unwrap(); + assert_eq!(rebuilt.offset, offset, "{name}: {mode}"); + assert_eq!(rebuilt.input.schema(), input.schema(), "{name}: {mode}"); + + let empty_exec_input = Arc::new(DataSourceExec::new(Arc::new( + MemorySourceConfig::try_new(&[vec![]], schema.clone(), None).unwrap(), + ))); + let exec_input = Arc::new(DataSourceExec::new(Arc::new( + MemorySourceConfig::try_new(&[vec![batch.clone()]], schema.clone(), None) + .unwrap(), + ))); + let exec = rebuilt + .to_execution_plan(empty_exec_input) + .replace_children( + vec![exec_input], + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + .unwrap(); + let output = + datafusion::physical_plan::collect(exec, SessionContext::default().task_ctx()) + .await + .unwrap(); + let output = &output[0]; + assert_eq!(output.num_rows(), 1, "{name}: {mode}"); + assert_eq!( + output + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + start, + "{name}: {mode}" + ); + assert_eq!( + output + .column(1) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 7.0, + "{name}: {mode}" + ); + } + } + } + + #[test] + fn deserialized_ordering_preserves_column_indices() { + let mut wire = pb::InstantManipulate::default(); + let first = InstantManipulate::deserialize(&wire.encode_to_vec()).unwrap(); + wire.time_index_idx = 1; + let second = InstantManipulate::deserialize(&wire.encode_to_vec()).unwrap(); + assert_ne!(first, second); + assert_eq!(first.partial_cmp(&second), Some(std::cmp::Ordering::Less)); + wire.field_index_idx = 2; + let third = InstantManipulate::deserialize(&wire.encode_to_vec()).unwrap(); + assert_ne!(second, third); + assert_eq!(second.partial_cmp(&third), Some(std::cmp::Ordering::Less)); + + let input = LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: prepare_test_data().schema().to_dfschema_ref().unwrap(), + }); + let first = InstantManipulate::new( + 0, + 0, + 0, + 0, + 0, + TIME_INDEX_COLUMN.to_string(), + Vec::new(), + Some("value".to_string()), + input.clone(), + ); + let second = InstantManipulate::new( + 0, + 0, + 0, + 0, + 1, + TIME_INDEX_COLUMN.to_string(), + Vec::new(), + Some("value".to_string()), + input, + ); + assert_ne!(first, second); + assert_eq!(first.partial_cmp(&second), Some(std::cmp::Ordering::Less)); + } + #[test] fn pruning_should_keep_time_and_field_columns_for_exec() { let df_schema = prepare_test_data().schema().to_dfschema_ref().unwrap(); @@ -779,6 +1205,7 @@ mod test { 0, 0, 0, + 0, TIME_INDEX_COLUMN.to_string(), Vec::new(), Some("value".to_string()), @@ -811,6 +1238,7 @@ mod test { 0, 0, 0, + 0, TIME_INDEX_COLUMN.to_string(), vec!["__tsid".to_string()], Some("value".to_string()), @@ -853,6 +1281,7 @@ mod test { 0, 0, 0, + 0, TIME_INDEX_COLUMN.to_string(), vec!["__tsid".to_string()], Some("value".to_string()), @@ -886,6 +1315,7 @@ mod test { 0, 0, 0, + 0, TIME_INDEX_COLUMN.to_string(), vec!["__tsid".to_string()], Some("value".to_string()), @@ -927,6 +1357,7 @@ mod test { MemorySourceConfig::try_new(&[vec![batch]], schema, None).unwrap(), ))); let normalize_exec = Arc::new(InstantManipulateExec { + offset: 0, start: 0, end: 1_500, lookback_delta: 1_000, @@ -934,6 +1365,8 @@ mod test { time_index_column: TIME_INDEX_COLUMN.to_string(), field_column: Some("value".to_string()), reuse_tsid_column: true, + output_schema: input.schema(), + properties: input.properties().clone(), input, metric: ExecutionPlanMetricsSet::new(), }); @@ -988,6 +1421,7 @@ mod test { MemorySourceConfig::try_new(&[vec![batch]], schema, None).unwrap(), ))); let normalize_exec = Arc::new(InstantManipulateExec { + offset: 0, start: 0, end: 1_500, lookback_delta: 1_000, @@ -995,6 +1429,8 @@ mod test { time_index_column: TIME_INDEX_COLUMN.to_string(), field_column: Some("value".to_string()), reuse_tsid_column: true, + output_schema: input.schema(), + properties: input.properties().clone(), input, metric: ExecutionPlanMetricsSet::new(), }); @@ -1042,6 +1478,7 @@ mod test { ))); let too_many_points = MAX_INSTANT_MANIPULATE_OUTPUT_POINTS as Millisecond + 1; let normalize_exec = Arc::new(InstantManipulateExec { + offset: 0, start: 0, end: too_many_points, lookback_delta: too_many_points + 1, @@ -1049,6 +1486,8 @@ mod test { time_index_column: TIME_INDEX_COLUMN.to_string(), field_column: Some("value".to_string()), reuse_tsid_column: false, + output_schema: input.schema(), + properties: input.properties().clone(), input, metric: ExecutionPlanMetricsSet::new(), }); @@ -1336,6 +1775,290 @@ mod test { .await; } + #[test] + fn exact_ties_select_first_and_lookback_uses_latest() { + for (values, expected_timestamp) in [ + (vec![42.0, f64::from_bits(PROMETHEUS_STALE_NAN_BITS)], 1_000), + (vec![f64::from_bits(PROMETHEUS_STALE_NAN_BITS), 42.0], 1_050), + ] { + let schema = Arc::new(Schema::new(vec![ + Field::new( + TIME_INDEX_COLUMN, + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + Field::new("value", DataType::Float64, true), + ])); + let input = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(TimestampMillisecondArray::from(vec![1_000, 1_000])), + Arc::new(Float64Array::from(values)), + ], + ) + .unwrap(); + let stream = InstantManipulateStream { + offset: 0, + start: 1_000, + end: 1_050, + lookback_delta: 100, + interval: 50, + time_index: 0, + time_unit: TimeUnit::Millisecond, + field_indices: [Some(1), None], + tsid_index: None, + reuse_tsid_column: false, + schema: schema.clone(), + input: Box::pin( + datafusion::physical_plan::memory::MemoryStream::try_new(vec![], schema, None) + .unwrap(), + ), + metric: BaselineMetrics::new(&ExecutionPlanMetricsSet::new(), 0), + num_series: Count::new(), + }; + + let output = stream.manipulate(input).unwrap(); + let timestamps = output + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = output + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(timestamps.values(), &[expected_timestamp]); + assert_eq!(values.values(), &[42.0]); + } + } + + #[test] + fn empty_batches_preserve_stream_progression_and_output_schema() { + use datafusion::physical_plan::stream::RecordBatchStreamAdapter; + use futures::stream; + use futures::task::noop_waker_ref; + + let input_schema = Arc::new(Schema::new(vec![Field::new( + TIME_INDEX_COLUMN, + DataType::Timestamp(TimeUnit::Second, None), + false, + )])); + let output_schema = Arc::new(Schema::new(vec![Field::new( + TIME_INDEX_COLUMN, + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + )])); + let empty = RecordBatch::new_empty(input_schema.clone()); + let valid = RecordBatch::try_new( + input_schema.clone(), + vec![Arc::new(TimestampSecondArray::from(vec![1]))], + ) + .unwrap(); + let input = RecordBatchStreamAdapter::new( + input_schema, + stream::iter(vec![ + Ok(empty.clone()), + Ok(empty.clone()), + Ok(valid), + Ok(empty), + Err(DataFusionError::Execution("injected input error".into())), + ]), + ); + let mut stream = InstantManipulateStream { + offset: 0, + start: 1_000, + end: 1_000, + lookback_delta: 0, + interval: 1, + time_index: 0, + time_unit: TimeUnit::Second, + field_indices: [None, None], + tsid_index: None, + reuse_tsid_column: false, + schema: output_schema.clone(), + input: Box::pin(input), + metric: BaselineMetrics::new(&ExecutionPlanMetricsSet::new(), 0), + num_series: Count::new(), + }; + let waker = noop_waker_ref(); + let mut cx = Context::from_waker(waker); + + for _ in 0..2 { + let Poll::Ready(Some(Ok(batch))) = Pin::new(&mut stream).poll_next(&mut cx) else { + panic!("empty batch must be returned immediately"); + }; + assert_eq!(batch.num_rows(), 0); + assert_eq!(batch.schema(), output_schema); + } + + let Poll::Ready(Some(Ok(valid))) = Pin::new(&mut stream).poll_next(&mut cx) else { + panic!("valid batch must follow empty batches"); + }; + assert_eq!(valid.schema(), output_schema); + assert_eq!( + valid.column(0).data_type(), + &DataType::Timestamp(TimeUnit::Millisecond, None) + ); + assert_eq!( + valid + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values(), + &[1_000] + ); + + let Poll::Ready(Some(Ok(empty))) = Pin::new(&mut stream).poll_next(&mut cx) else { + panic!("empty batch after valid batch must be returned immediately"); + }; + assert_eq!(empty.num_rows(), 0); + assert_eq!(empty.schema(), output_schema); + + let Poll::Ready(Some(Err(error))) = Pin::new(&mut stream).poll_next(&mut cx) else { + panic!("input error must propagate"); + }; + assert!(error.to_string().contains("injected input error")); + assert!(matches!( + Pin::new(&mut stream).poll_next(&mut cx), + Poll::Ready(None) + )); + assert_eq!(stream.num_series.value(), 1); + } + + #[test] + fn extreme_alignment_retains_exact_sample() { + let schema = Arc::new(Schema::new(vec![ + Field::new( + TIME_INDEX_COLUMN, + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + Field::new("value", DataType::Float64, true), + ])); + let input = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(TimestampMillisecondArray::from(vec![i64::MAX])), + Arc::new(Float64Array::from(vec![7.0])), + ], + ) + .unwrap(); + let stream = InstantManipulateStream { + offset: 0, + start: i64::MIN + 1, + end: i64::MAX, + lookback_delta: 0, + interval: i64::MAX, + time_index: 0, + time_unit: TimeUnit::Millisecond, + field_indices: [Some(1), None], + tsid_index: None, + reuse_tsid_column: false, + schema: schema.clone(), + input: Box::pin( + datafusion::physical_plan::memory::MemoryStream::try_new(vec![], schema, None) + .unwrap(), + ), + metric: BaselineMetrics::new(&ExecutionPlanMetricsSet::new(), 0), + num_series: Count::new(), + }; + + let output = stream.manipulate(input).unwrap(); + let timestamps = output + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let values = output + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(timestamps.values(), &[i64::MAX]); + assert_eq!(values.values(), &[7.0]); + } + + #[test] + fn native_nanosecond_offset_uses_wide_shifted_timeline() { + for (raw, offset, eval) in [ + ( + 9_223_112_837_000_000_000_i64, + 259_200_000, + 9_223_372_037_000, + ), + ( + -9_223_112_837_000_000_000_i64, + -259_200_000, + -9_223_372_037_000, + ), + ] { + let schema = Arc::new(Schema::new(vec![ + Field::new( + TIME_INDEX_COLUMN, + DataType::Timestamp(TimeUnit::Nanosecond, None), + false, + ), + Field::new("value", DataType::Float64, true), + ])); + let input = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(TimestampNanosecondArray::from(vec![raw])), + Arc::new(Float64Array::from(vec![7.0])), + ], + ) + .unwrap(); + let stream = InstantManipulateStream { + offset, + start: eval, + end: eval, + lookback_delta: 300_000, + interval: 1, + time_index: 0, + time_unit: TimeUnit::Nanosecond, + field_indices: [Some(1), None], + tsid_index: None, + reuse_tsid_column: false, + schema: Arc::new(Schema::new(vec![ + Field::new( + TIME_INDEX_COLUMN, + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + Field::new("value", DataType::Float64, true), + ])), + input: Box::pin( + datafusion::physical_plan::memory::MemoryStream::try_new(vec![], schema, None) + .unwrap(), + ), + metric: BaselineMetrics::new(&ExecutionPlanMetricsSet::new(), 0), + num_series: Count::new(), + }; + let output = stream.manipulate(input).unwrap(); + assert_eq!(output.num_rows(), 1); + assert_eq!( + output + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + eval + ); + assert_eq!( + output + .column(1) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 7.0 + ); + } + } + #[tokio::test] async fn ordinary_nan_is_selected_for_exact_and_lookback() { let schema = Arc::new(Schema::new(vec![ @@ -1358,6 +2081,7 @@ mod test { MemorySourceConfig::try_new(&[vec![batch]], schema, None).unwrap(), ))); let exec = Arc::new(InstantManipulateExec { + offset: 0, start: 1_000, end: 1_500, lookback_delta: 1_000, @@ -1365,6 +2089,8 @@ mod test { time_index_column: TIME_INDEX_COLUMN.to_string(), field_column: Some("value".to_string()), reuse_tsid_column: false, + output_schema: input.schema(), + properties: input.properties().clone(), input, metric: ExecutionPlanMetricsSet::new(), }); @@ -1437,6 +2163,7 @@ mod test { MemorySourceConfig::try_new(&[vec![batch]], schema, None).unwrap(), ))); let exec = Arc::new(InstantManipulateExec { + offset: 0, start: 750, end: 1_500, lookback_delta: 1_001, @@ -1444,6 +2171,8 @@ mod test { time_index_column: TIME_INDEX_COLUMN.to_string(), field_column: Some("value".to_string()), reuse_tsid_column: false, + output_schema: input.schema(), + properties: input.properties().clone(), input, metric: ExecutionPlanMetricsSet::new(), }); @@ -1500,6 +2229,7 @@ mod test { MemorySourceConfig::try_new(&[vec![batch]], schema, None).unwrap(), ))); let exec = Arc::new(InstantManipulateExec { + offset: 0, start: 1_000, end: 1_500, lookback_delta: 1_001, @@ -1507,6 +2237,8 @@ mod test { time_index_column: TIME_INDEX_COLUMN.to_string(), field_column: Some("value".to_string()), reuse_tsid_column: false, + output_schema: input.schema(), + properties: input.properties().clone(), input, metric: ExecutionPlanMetricsSet::new(), }); @@ -1547,6 +2279,7 @@ mod test { MemorySourceConfig::try_new(&[vec![batch]], schema, None).unwrap(), ))); let exec = Arc::new(InstantManipulateExec { + offset: 0, start: 1_000, end: 1_500, lookback_delta: 1_000, @@ -1554,6 +2287,8 @@ mod test { time_index_column: TIME_INDEX_COLUMN.to_string(), field_column: Some("value".to_string()), reuse_tsid_column: false, + output_schema: input.schema(), + properties: input.properties().clone(), input, metric: ExecutionPlanMetricsSet::new(), }); diff --git a/src/promql/src/extension_plan/normalize.rs b/src/promql/src/extension_plan/normalize.rs index e3410be9267..1ca015b10f1 100644 --- a/src/promql/src/extension_plan/normalize.rs +++ b/src/promql/src/extension_plan/normalize.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; @@ -20,7 +19,8 @@ use std::task::{Context, Poll}; use common_query::native_histogram::{START_TIMESTAMP_FIELD, native_histogram_arrow_type}; use datafusion::arrow::array::{Array, BooleanArray, StructArray}; use datafusion::arrow::compute; -use datafusion::common::{DFSchema, DFSchemaRef, Result as DataFusionResult, Statistics}; +use datafusion::common::tree_node::TreeNodeRecursion; +use datafusion::common::{Column, DFSchema, DFSchemaRef, Result as DataFusionResult, Statistics}; use datafusion::error::DataFusionError; use datafusion::execution::context::TaskContext; use datafusion::logical_expr::{EmptyRelation, Expr, LogicalPlan, UserDefinedLogicalNodeCore}; @@ -29,8 +29,9 @@ use datafusion::physical_plan::metrics::{ BaselineMetrics, Count, ExecutionPlanMetricsSet, MetricBuilder, MetricValue, MetricsSet, }; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, PlanProperties, RecordBatchStream, - SendableRecordBatchStream, + ChildStats, DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, + InputDistributionRequirements, PhysicalExpr, PlanProperties, RecordBatchStream, + SendableRecordBatchStream, StatisticsArgs, }; use datafusion_expr::col; use datatypes::arrow::array::TimestampMillisecondArray; @@ -48,13 +49,13 @@ use crate::extension_plan::{ }; use crate::metrics::PROMQL_SERIES_COUNT; -/// Normalize the input record batch. Notice that for simplicity, this method assumes -/// the input batch only contains sample points from one time series. +/// Normalizes a single-series input batch and optionally removes Prometheus stale markers. /// -/// Roughly speaking, this method does these things: -/// - bias sample and native histogram start timestamps by offset -/// - sort the record batch based on timestamp column -/// - remove Prometheus stale markers (optional) +/// This node remains the serialized carrier of the selector offset. Native sample timestamps +/// stay raw: applying an offset can overflow native `i64` ticks even when evaluation is valid. +/// Manipulators instead apply it in `i128` during selection and produce millisecond outputs. +/// Histogram start timestamps are already millisecond payloads used for rate/reset, so their +/// offsets are applied here, preserving unknown zero values and nulls. #[derive(Debug, PartialEq, Eq, Hash, PartialOrd)] pub struct SeriesNormalize { offset: Millisecond, @@ -179,6 +180,14 @@ impl UserDefinedLogicalNodeCore for SeriesNormalize { } impl SeriesNormalize { + pub(crate) fn offset_for_time_index(&self, time_index: &Column) -> Option { + let index = self.input.schema().maybe_index_of_column(time_index)?; + let (qualifier, field) = self.input.schema().qualified_field(index); + (field.name() == &self.time_index_column_name + && time_index == &Column::new(qualifier.cloned(), field.name().clone())) + .then_some(self.offset) + } + pub fn new>( offset: Millisecond, time_index_column_name: N, @@ -271,27 +280,30 @@ pub struct SeriesNormalizeExec { } impl ExecutionPlan for SeriesNormalizeExec { - fn as_any(&self) -> &dyn Any { - self + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> datafusion_common::Result, + ) -> DataFusionResult { + Ok(TreeNodeRecursion::Continue) } fn schema(&self) -> SchemaRef { self.input.schema() } - fn required_input_distribution(&self) -> Vec { + fn input_distribution_requirements(&self) -> InputDistributionRequirements { if self.tag_columns.is_empty() { - return vec![Distribution::SinglePartition]; + return InputDistributionRequirements::new(vec![Distribution::SinglePartition]); } let schema = self.input.schema(); - vec![Distribution::HashPartitioned( + InputDistributionRequirements::new(vec![Distribution::KeyPartitioned( self.tag_columns .iter() // Safety: the tag column names is verified in the planning phase .map(|tag| Arc::new(ColumnExpr::new_with_schema(tag, &schema).unwrap()) as _) .collect(), - )] + )]) } fn properties(&self) -> &Arc { @@ -334,13 +346,8 @@ impl ExecutionPlan for SeriesNormalizeExec { let input = self.input.execute(partition, context)?; let schema = input.schema(); - let time_index = schema - .column_with_name(&self.time_index_column_name) - .expect("time index column not found") - .0; Ok(Box::pin(SeriesNormalizeStream { offset: self.offset, - time_index, filter_stale_markers: self.filter_stale_markers, schema, input, @@ -353,8 +360,16 @@ impl ExecutionPlan for SeriesNormalizeExec { Some(self.metric.clone_inner()) } - fn partition_statistics(&self, partition: Option) -> DataFusionResult { - self.input.partition_statistics(partition) + fn child_stats_requests(&self, partition: Option) -> Vec { + vec![ChildStats::At(partition)] + } + + fn statistics_from_inputs( + &self, + input_stats: &[Arc], + _args: &StatisticsArgs, + ) -> DataFusionResult> { + Ok(Arc::clone(&input_stats[0])) } fn name(&self) -> &str { @@ -380,8 +395,6 @@ impl DisplayAs for SeriesNormalizeExec { pub struct SeriesNormalizeStream { offset: Millisecond, - // Column index of TIME INDEX column's position in schema - time_index: usize, filter_stale_markers: bool, schema: SchemaRef, @@ -393,33 +406,12 @@ pub struct SeriesNormalizeStream { impl SeriesNormalizeStream { pub fn normalize(&self, input: RecordBatch) -> DataFusionResult { - let ts_column = input - .column(self.time_index) - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Execution( - "Time index Column downcast to TimestampMillisecondArray failed".into(), - ) - })?; - - let bias_timestamp = |timestamp: i64| { - timestamp.checked_add(self.offset).ok_or_else(|| { - DataFusionError::Execution("SeriesNormalize: timestamp offset overflow".into()) - }) - }; - - // bias the timestamp column by offset - let ts_column_biased = if self.offset == 0 { - Arc::new(ts_column.clone()) as _ - } else { - Arc::new(ts_column.try_unary::<_, TimestampMillisecondType, _>(&bias_timestamp)?) - }; + // Native sample timestamps remain raw. Manipulators apply the selector offset + // in wide nanosecond arithmetic, avoiding overflow in native Arrow storage. let mut columns = input.columns().to_vec(); - columns[self.time_index] = ts_column_biased; - // Offset selectors move samples into the evaluation timeline. Keep native histogram - // start timestamps on the same timeline for rate and reset calculations. + // Offset selectors move native histogram start timestamps onto the evaluation + // timeline for rate and reset calculations. These payloads are milliseconds. if self.offset != 0 { let native_histogram_type = native_histogram_arrow_type(); for column in &mut columns { @@ -444,11 +436,17 @@ impl SeriesNormalizeStream { if timestamp == 0 { Ok(0) } else { - bias_timestamp(timestamp) + timestamp.checked_add(self.offset).ok_or_else(|| { + DataFusionError::Execution( + "SeriesNormalize: histogram timestamp offset overflow".into(), + ) + }) } })?; - // Replace only the start timestamp child to preserve the histogram payload and - // null bitmap. + // Struct arrays are immutable, so rebuild the physical histogram payload with + // only its start-timestamp child replaced. Its logical schema stays unchanged: + // the selector offset affects histogram reset/rate metadata, not the native + // sample timestamp column consumed by later manipulators. let mut children = histograms.columns().to_vec(); children[start_timestamp_index] = Arc::new(start_timestamps); *column = Arc::new(StructArray::new( @@ -516,10 +514,12 @@ impl Stream for SeriesNormalizeStream { mod test { use common_query::native_histogram::{build_histogram_array, read_histogram}; use common_query::prometheus::PROMETHEUS_STALE_NAN_BITS; - use datafusion::arrow::array::Float64Array; + use datafusion::arrow::array::{ + DictionaryArray, Float64Array, TimestampMicrosecondArray, TimestampNanosecondArray, + }; use datafusion::arrow::buffer::NullBuffer; use datafusion::arrow::datatypes::{ - ArrowPrimitiveType, DataType, Field, Schema, TimestampMillisecondType, + ArrowPrimitiveType, DataType, Field, Int64Type, Schema, TimeUnit, TimestampMillisecondType, }; use datafusion::common::ToDFSchema; use datafusion::datasource::memory::MemorySourceConfig; @@ -530,7 +530,9 @@ mod test { use datatypes::arrow_array::StringArray; use super::*; + use crate::extension_plan::RangeManipulate; use crate::extension_plan::test_util::native_histogram; + use crate::range_array::RangeArray; const TIME_INDEX_COLUMN: &str = "timestamp"; @@ -630,11 +632,11 @@ mod test { "+---------------------+--------+------+\ \n| timestamp | value | path |\ \n+---------------------+--------+------+\ - \n| 1970-01-01T00:01:01 | 0.0 | foo |\ - \n| 1970-01-01T00:02:01 | 1.0 | foo |\ - \n| 1970-01-01T00:00:01 | 10.0 | foo |\ - \n| 1970-01-01T00:00:31 | 100.0 | foo |\ - \n| 1970-01-01T00:01:31 | 1000.0 | foo |\ + \n| 1970-01-01T00:01:00 | 0.0 | foo |\ + \n| 1970-01-01T00:02:00 | 1.0 | foo |\ + \n| 1970-01-01T00:00:00 | 10.0 | foo |\ + \n| 1970-01-01T00:00:30 | 100.0 | foo |\ + \n| 1970-01-01T00:01:30 | 1000.0 | foo |\ \n+---------------------+--------+------+", ); @@ -720,12 +722,104 @@ mod test { regular.start_timestamp = Some(500); let mut ordinary_nan = native_histogram(f64::NAN); ordinary_nan.start_timestamp = Some(0); + let mut unknown_start = native_histogram(7.0); + unknown_start.start_timestamp = None; let histograms = build_histogram_array(&[ Some(regular), Some(native_histogram(f64::from_bits(PROMETHEUS_STALE_NAN_BITS))), Some(ordinary_nan), + Some(unknown_start), None, ]); + for (unit, ticks_per_ms) in [ + (TimeUnit::Millisecond, 1_i64), + (TimeUnit::Microsecond, 1_000), + (TimeUnit::Nanosecond, 1_000_000), + ] { + let timestamp_array = |values: Vec| -> Arc { + match unit { + TimeUnit::Millisecond => Arc::new(TimestampMillisecondArray::from(values)), + TimeUnit::Microsecond => Arc::new(TimestampMicrosecondArray::from(values)), + TimeUnit::Nanosecond => Arc::new(TimestampNanosecondArray::from(values)), + TimeUnit::Second => unreachable!(), + } + }; + for offset in [-1_i64, 1] { + let timestamps = timestamp_array( + [1_000, 2_000, 3_000, 4_000, 5_000] + .into_iter() + .map(|timestamp| timestamp * ticks_per_ms) + .collect(), + ); + let schema = Arc::new(Schema::new(vec![ + Field::new(TIME_INDEX_COLUMN, timestamps.data_type().clone(), false), + Field::new("value", histograms.data_type().clone(), true), + ])); + let batch = + RecordBatch::try_new(schema.clone(), vec![timestamps, histograms.clone()]) + .unwrap(); + let input = Arc::new(DataSourceExec::new(Arc::new( + MemorySourceConfig::try_new(&[vec![batch]], schema, None).unwrap(), + ))); + let exec = Arc::new(SeriesNormalizeExec { + offset, + time_index_column_name: TIME_INDEX_COLUMN.to_string(), + filter_stale_markers: true, + tag_columns: Vec::new(), + input, + metric: ExecutionPlanMetricsSet::new(), + }); + let context = SessionContext::default(); + let batches = datafusion::physical_plan::collect(exec, context.task_ctx()) + .await + .unwrap(); + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + 4, + "unit={unit:?}, offset={offset}" + ); + let batch = batches.iter().find(|batch| batch.num_rows() == 4).unwrap(); + let expected_timestamps = timestamp_array( + [1_000, 3_000, 4_000, 5_000] + .into_iter() + .map(|timestamp| timestamp * ticks_per_ms) + .collect(), + ); + assert_eq!( + batch.column(0).to_data(), + expected_timestamps.to_data(), + "unit={unit:?}, offset={offset}" + ); + let values = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let regular = read_histogram(values, 0).unwrap().unwrap(); + assert_eq!( + (regular.sum, regular.start_timestamp), + (42.0, Some(500 + offset)), + "unit={unit:?}, offset={offset}" + ); + let ordinary_nan = read_histogram(values, 1).unwrap().unwrap(); + assert!(ordinary_nan.sum.is_nan()); + assert_eq!(ordinary_nan.start_timestamp, Some(0)); + let unknown_start = read_histogram(values, 2).unwrap().unwrap(); + assert_eq!( + (unknown_start.sum, unknown_start.start_timestamp), + (7.0, None) + ); + assert!(read_histogram(values, 3).unwrap().is_none()); + } + } + + let mut known_start = native_histogram(42.0); + known_start.start_timestamp = Some(500); + let mut sentinel_start = native_histogram(8.0); + sentinel_start.start_timestamp = Some(0); + let unknown_start = native_histogram(7.0); + let histograms = + build_histogram_array(&[Some(known_start), Some(sentinel_start), Some(unknown_start)]); let schema = Arc::new(Schema::new(vec![ Field::new( TIME_INDEX_COLUMN, @@ -737,47 +831,85 @@ mod test { let batch = RecordBatch::try_new( schema.clone(), vec![ - Arc::new(TimestampMillisecondArray::from(vec![ - 1_000, 2_000, 3_000, 4_000, - ])), + Arc::new(TimestampMillisecondArray::from(vec![1_000; 3])), histograms, ], ) .unwrap(); + let logical_input = LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: schema.clone().to_dfschema_ref().unwrap(), + }); + let normalized = + SeriesNormalize::new(1_000, TIME_INDEX_COLUMN, false, Vec::new(), logical_input); + let range = RangeManipulate::new( + 2_000, + 2_000, + 1, + 1_000, + 1, + TIME_INDEX_COLUMN.to_string(), + vec!["value".to_string()], + LogicalPlan::Extension(datafusion::logical_expr::Extension { + node: Arc::new(normalized), + }), + ) + .unwrap(); let input = Arc::new(DataSourceExec::new(Arc::new( MemorySourceConfig::try_new(&[vec![batch]], schema, None).unwrap(), ))); - let exec = Arc::new(SeriesNormalizeExec { + let normalized_input = Arc::new(SeriesNormalizeExec { offset: 1_000, time_index_column_name: TIME_INDEX_COLUMN.to_string(), - filter_stale_markers: true, + filter_stale_markers: false, tag_columns: Vec::new(), input, metric: ExecutionPlanMetricsSet::new(), }); - - let context = SessionContext::default(); - let batches = datafusion::physical_plan::collect(exec, context.task_ctx()) - .await - .unwrap(); - let batch = batches.iter().find(|batch| batch.num_rows() == 3).unwrap(); - let values = batch - .column(1) + let output = datafusion::physical_plan::collect( + range.to_execution_plan(normalized_input), + SessionContext::default().task_ctx(), + ) + .await + .unwrap(); + let values = RangeArray::try_new( + output[0] + .column(1) + .as_any() + .downcast_ref::>() + .unwrap() + .clone(), + ) + .unwrap(); + let values = values.get(0).unwrap(); + let values = values .as_any() .downcast_ref::() .unwrap(); - - let timestamps = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(timestamps.values(), &[2_000, 4_000, 5_000]); - let regular = read_histogram(values, 0).unwrap().unwrap(); - assert_eq!((regular.sum, regular.start_timestamp), (42.0, Some(1_500))); - let ordinary_nan = read_histogram(values, 1).unwrap().unwrap(); - assert!(ordinary_nan.sum.is_nan()); - assert_eq!(ordinary_nan.start_timestamp, Some(0)); - assert!(read_histogram(values, 2).unwrap().is_none()); + assert_eq!( + read_histogram(values, 0).unwrap().unwrap().start_timestamp, + Some(1_500) + ); + assert_eq!( + read_histogram(values, 1).unwrap().unwrap().start_timestamp, + Some(0) + ); + assert_eq!( + read_histogram(values, 2).unwrap().unwrap().start_timestamp, + None + ); + let timestamps = RangeArray::try_new( + output[0] + .column(2) + .as_any() + .downcast_ref::>() + .unwrap() + .clone(), + ) + .unwrap(); + assert_eq!( + timestamps.get(0).unwrap().to_data(), + TimestampMillisecondArray::from(vec![2_000; 3]).to_data() + ); } } diff --git a/src/promql/src/extension_plan/planner.rs b/src/promql/src/extension_plan/planner.rs index 1e6914a78d7..50cf4543d59 100644 --- a/src/promql/src/extension_plan/planner.rs +++ b/src/promql/src/extension_plan/planner.rs @@ -15,8 +15,9 @@ use std::sync::Arc; use async_trait::async_trait; +use datafusion::catalog::Session; use datafusion::error::Result as DfResult; -use datafusion::execution::context::SessionState; +use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext; use datafusion::logical_expr::{LogicalPlan, UserDefinedLogicalNode}; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_planner::{ExtensionPlanner, PhysicalPlanner}; @@ -36,7 +37,8 @@ impl ExtensionPlanner for PromExtensionPlanner { node: &dyn UserDefinedLogicalNode, _logical_inputs: &[&LogicalPlan], physical_inputs: &[Arc], - session_state: &SessionState, + session: &dyn Session, + planning_ctx: &PhysicalPlanningContext, ) -> DfResult>> { if let Some(node) = node.as_any().downcast_ref::() { Ok(Some(node.to_execution_plan(physical_inputs[0].clone()))) @@ -47,7 +49,11 @@ impl ExtensionPlanner for PromExtensionPlanner { } else if let Some(node) = node.as_any().downcast_ref::() { Ok(Some(node.to_execution_plan(physical_inputs[0].clone()))) } else if let Some(node) = node.as_any().downcast_ref::() { - Ok(Some(node.to_execution_plan(session_state, planner)?)) + Ok(Some(node.to_execution_plan( + session, + planner, + planning_ctx, + )?)) } else if let Some(node) = node.as_any().downcast_ref::() { Ok(Some(node.to_execution_plan(physical_inputs[0].clone())?)) } else if let Some(node) = node.as_any().downcast_ref::() { diff --git a/src/promql/src/extension_plan/range_manipulate.rs b/src/promql/src/extension_plan/range_manipulate.rs index 44c5f490947..96a0c97778d 100644 --- a/src/promql/src/extension_plan/range_manipulate.rs +++ b/src/promql/src/extension_plan/range_manipulate.rs @@ -12,8 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; @@ -21,11 +20,12 @@ use std::task::{Context, Poll}; use common_telemetry::{debug, warn}; use datafusion::arrow::array::{Array, ArrayRef, Int64Array, TimestampMillisecondArray}; use datafusion::arrow::compute; -use datafusion::arrow::datatypes::{Field, SchemaRef}; +use datafusion::arrow::datatypes::{DataType, Field, SchemaRef, TimeUnit}; use datafusion::arrow::error::ArrowError; use datafusion::arrow::record_batch::RecordBatch; use datafusion::common::stats::Precision; -use datafusion::common::{DFSchema, DFSchemaRef}; +use datafusion::common::tree_node::TreeNodeRecursion; +use datafusion::common::{DFSchema, DFSchemaRef, TableReference}; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::execution::context::TaskContext; use datafusion::logical_expr::{EmptyRelation, Expr, LogicalPlan, UserDefinedLogicalNodeCore}; @@ -34,11 +34,12 @@ use datafusion::physical_plan::metrics::{ BaselineMetrics, Count, ExecutionPlanMetricsSet, MetricBuilder, MetricValue, MetricsSet, }; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, PlanProperties, RecordBatchStream, - SendableRecordBatchStream, Statistics, + ChildStats, DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, + InputDistributionRequirements, PhysicalExpr, PlanProperties, RecordBatchStream, + SendableRecordBatchStream, Statistics, StatisticsArgs, }; -use datafusion::sql::TableReference; use datafusion_expr::col; +use datatypes::timestamp::timestamp_array_to_primitive; use futures::{Stream, StreamExt, ready}; use greptime_proto::substrait_extension as pb; use prost::Message; @@ -46,7 +47,8 @@ use snafu::ResultExt; use crate::error::{DeserializeSnafu, Result}; use crate::extension_plan::{ - METRIC_NUM_SERIES, Millisecond, resolve_column_name, serialize_column_index, + METRIC_NUM_SERIES, Millisecond, local_offset, nanoseconds_per_native_tick, resolve_column_name, + serialize_column_index, timestamp_unit, }; use crate::metrics::PROMQL_SERIES_COUNT; use crate::range_array::RangeArray; @@ -66,6 +68,7 @@ pub struct RangeManipulate { end: Millisecond, interval: Millisecond, range: Millisecond, + offset: Millisecond, time_index: String, field_columns: Vec, input: LogicalPlan, @@ -80,10 +83,12 @@ struct UnfixIndices { } impl RangeManipulate { + #[allow(clippy::too_many_arguments)] pub fn new( start: Millisecond, end: Millisecond, interval: Millisecond, + offset: Millisecond, range: Millisecond, time_index: String, field_columns: Vec, @@ -96,6 +101,7 @@ impl RangeManipulate { end, interval, range, + offset, time_index, field_columns, input, @@ -142,9 +148,21 @@ impl RangeManipulate { )); }; let ts_col_field = &columns[ts_col_index]; + let output_time_field = Arc::new( + ts_col_field + .as_ref() + .clone() + .with_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)), + ); + new_columns[ts_col_index] = ( + input_schema.qualified_field(ts_col_index).0.cloned(), + output_time_field.clone(), + ); let timestamp_range_field = Field::new( Self::build_timestamp_range_name(time_index), - RangeArray::convert_field(ts_col_field).data_type().clone(), + RangeArray::convert_field(output_time_field.as_ref()) + .data_type() + .clone(), ts_col_field.is_nullable(), ); new_columns.push((None, Arc::new(timestamp_range_field))); @@ -163,7 +181,7 @@ impl RangeManipulate { Ok(Arc::new(DFSchema::new_with_metadata( new_columns, - HashMap::new(), + input_schema.metadata().clone(), )?)) } @@ -177,6 +195,7 @@ impl RangeManipulate { properties.boundedness, )); Arc::new(RangeManipulateExec { + offset: self.offset, start: self.start, end: self.end, interval: self.interval, @@ -235,6 +254,7 @@ impl RangeManipulate { end: pb_range_manipulate.end, interval: pb_range_manipulate.interval, range: pb_range_manipulate.range, + offset: 0, time_index: String::new(), field_columns: Vec::new(), input: placeholder_plan, @@ -263,6 +283,10 @@ impl PartialOrd for RangeManipulate { Some(core::cmp::Ordering::Equal) => {} ord => return ord, } + match self.offset.partial_cmp(&other.offset) { + Some(core::cmp::Ordering::Equal) => {} + ord => return ord, + } match self.time_index.partial_cmp(&other.time_index) { Some(core::cmp::Ordering::Equal) => {} ord => return ord, @@ -383,6 +407,7 @@ impl UserDefinedLogicalNodeCore for RangeManipulate { end: self.end, interval: self.interval, range: self.range, + offset: local_offset(&input, &time_index), time_index, field_columns, input, @@ -398,6 +423,7 @@ impl UserDefinedLogicalNodeCore for RangeManipulate { end: self.end, interval: self.interval, range: self.range, + offset: self.offset, time_index: self.time_index.clone(), field_columns: self.field_columns.clone(), input, @@ -410,6 +436,7 @@ impl UserDefinedLogicalNodeCore for RangeManipulate { #[derive(Debug)] pub struct RangeManipulateExec { + offset: Millisecond, start: Millisecond, end: Millisecond, interval: Millisecond, @@ -425,8 +452,11 @@ pub struct RangeManipulateExec { } impl ExecutionPlan for RangeManipulateExec { - fn as_any(&self) -> &dyn Any { - self + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> datafusion_common::Result, + ) -> DataFusionResult { + Ok(TreeNodeRecursion::Continue) } fn schema(&self) -> SchemaRef { @@ -445,14 +475,17 @@ impl ExecutionPlan for RangeManipulateExec { vec![&self.input] } - fn required_input_distribution(&self) -> Vec { - let input_requirement = self.input.required_input_distribution(); + fn input_distribution_requirements(&self) -> InputDistributionRequirements { + let input_requirement = self + .input + .input_distribution_requirements() + .into_per_child(); if input_requirement.is_empty() { // if the input is EmptyMetric, its required_input_distribution() is empty so we can't // use its input distribution. - vec![Distribution::UnspecifiedDistribution] + InputDistributionRequirements::new(vec![Distribution::UnspecifiedDistribution]) } else { - input_requirement + InputDistributionRequirements::new(input_requirement) } } @@ -470,6 +503,7 @@ impl ExecutionPlan for RangeManipulateExec { properties.boundedness, )); Ok(Arc::new(Self { + offset: self.offset, start: self.start, end: self.end, interval: self.interval, @@ -515,14 +549,17 @@ impl ExecutionPlan for RangeManipulateExec { .0 }) .collect(); + let time_unit = timestamp_unit(schema.field(time_index).data_type())?; let aligned_ts_array = RangeManipulateStream::build_aligned_ts_array(self.start, self.end, self.interval); Ok(Box::pin(RangeManipulateStream { + offset: self.offset, start: self.start, end: self.end, interval: self.interval, range: self.range, time_index, + time_unit, field_columns, aligned_ts_array, output_schema: self.output_schema.clone(), @@ -536,8 +573,16 @@ impl ExecutionPlan for RangeManipulateExec { Some(self.metric.clone_inner()) } - fn partition_statistics(&self, partition: Option) -> DataFusionResult { - let input_stats = self.input.partition_statistics(partition)?; + fn child_stats_requests(&self, partition: Option) -> Vec { + vec![ChildStats::At(partition)] + } + + fn statistics_from_inputs( + &self, + input_stats: &[Arc], + _args: &StatisticsArgs, + ) -> DataFusionResult> { + let input_stats = &input_stats[0]; let estimated_row_num = (self.end - self.start) as f64 / self.interval as f64; let estimated_total_bytes = input_stats @@ -549,12 +594,12 @@ impl ExecutionPlan for RangeManipulateExec { }) .unwrap_or_default(); - Ok(Statistics { + Ok(Arc::new(Statistics { num_rows: Precision::Inexact(estimated_row_num as _), total_byte_size: estimated_total_bytes, // TODO(ruihang): support this column statistics column_statistics: Statistics::unknown_column(&self.schema()), - }) + })) } fn name(&self) -> &str { @@ -579,11 +624,13 @@ impl DisplayAs for RangeManipulateExec { } pub struct RangeManipulateStream { + offset: Millisecond, start: Millisecond, end: Millisecond, interval: Millisecond, range: Millisecond, time_index: usize, + time_unit: TimeUnit, field_columns: Vec, aligned_ts_array: ArrayRef, @@ -655,11 +702,35 @@ impl RangeManipulateStream { new_columns[*index] = new_column; } - // push timestamp range column - let ts_range_column = - RangeArray::from_ranges(input.column(self.time_index).clone(), ranges.clone()) - .map_err(|e| ArrowError::InvalidArgumentError(e.to_string()))? - .into_dict(); + // The timestamp range payload is always millisecond ABI. Shift in wide + // native precision before truncating toward zero, preserving null validity. + let scale = nanoseconds_per_native_tick(self.time_unit); + let (timestamps, _) = timestamp_array_to_primitive(input.column(self.time_index)) + .ok_or_else(|| { + DataFusionError::Execution("Time index column is not a timestamp".into()) + })?; + let timestamp_values = timestamps + .values() + .iter() + .enumerate() + .map(|(index, timestamp)| { + if !input.column(self.time_index).is_valid(index) { + return Ok(None); + } + let shifted_ns = (*timestamp as i128) * scale + (self.offset as i128) * 1_000_000; + i64::try_from(shifted_ns / 1_000_000) + .map(Some) + .map_err(|_| { + ArrowError::ComputeError( + "RangeManipulate timestamp payload overflow".into(), + ) + }) + }) + .collect::, _>>()?; + let timestamp_values = TimestampMillisecondArray::from(timestamp_values); + let ts_range_column = RangeArray::from_ranges(Arc::new(timestamp_values), ranges.clone()) + .map_err(|e| ArrowError::InvalidArgumentError(e.to_string()))? + .into_dict(); new_columns.push(Arc::new(ts_range_column)); // truncate other columns @@ -694,52 +765,58 @@ impl RangeManipulateStream { &self, input: &RecordBatch, ) -> DataFusionResult<(Vec<(u32, u32)>, (i64, i64))> { - let ts_column = input - .column(self.time_index) - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Execution( - "Time index Column downcast to TimestampMillisecondArray failed".into(), - ) - })?; - - let len = ts_column.len(); + let ts_column = input.column(self.time_index); + let scale = nanoseconds_per_native_tick(self.time_unit); + let (timestamps, _) = timestamp_array_to_primitive(ts_column).ok_or_else(|| { + DataFusionError::Execution("Time index column is not a timestamp".into()) + })?; + let timestamps = timestamps.values(); + let timestamp = + |index| (timestamps[index] as i128) * scale + (self.offset as i128) * 1_000_000; + let len = timestamps.len(); if len == 0 { return Ok((vec![], (self.start, self.end))); } - // shorten the range to calculate - let first_ts = ts_column.value(0); - // Preserve the query's alignment pattern when optimizing start time - let remainder = (first_ts - self.start).rem_euclid(self.interval); - let first_ts_aligned = if remainder == 0 { - first_ts - } else { - first_ts + (self.interval - remainder) - }; - let last_ts = ts_column.value(ts_column.len() - 1); - let last_ts_with_range = last_ts + self.range; - let remainder = (last_ts_with_range - self.start).rem_euclid(self.interval); + // Shorten the range using wide arithmetic so timestamps near the native + // type limits retain every query-aligned evaluation point. + let query_start = self.start as i128; + let query_end = self.end as i128; + let interval = self.interval as i128; + let first_ts = timestamp(0).div_euclid(1_000_000); + // Preserve the query's alignment pattern when optimizing start time. + let remainder = (first_ts - query_start).rem_euclid(interval); + let first_ts_aligned = first_ts + (interval - remainder).rem_euclid(interval); + let last_ts_with_range = + (timestamp(len - 1) + (self.range as i128) * 1_000_000).div_euclid(1_000_000); + let remainder = (last_ts_with_range - query_start).rem_euclid(interval); let last_ts_aligned = last_ts_with_range - remainder; - let start = self.start.max(first_ts_aligned); - let end = self.end.min(last_ts_aligned); + let start = query_start.max(first_ts_aligned); + let end = query_end.min(last_ts_aligned); if start > end { - return Ok((vec![], (start, end))); + return Ok((vec![], (self.start, self.end))); } - let mut ranges = Vec::with_capacity(((self.end - self.start) / self.interval + 1) as usize); + // The intersection is within the declared i64 query bounds. + let start = start as i64; + let end = end as i64; + let mut ranges = Vec::new(); - // calculate for every aligned timestamp (`curr_ts`), assume the ts column is ordered. + // Range membership is decided on shifted native ticks, before the + // timestamp-range payload is converted to its millisecond ABI. This + // keeps sub-millisecond samples distinct in a range; equal millisecond + // payload values are not a reason to deduplicate input samples. + // + // Calculate for every aligned timestamp (`curr_ts`), assuming ordered timestamps. let mut left = 0usize; let mut right = 0usize; for curr_ts in (start..=end).step_by(self.interval as _) { - let start_ts = curr_ts - self.range; + let start_ts = (curr_ts as i128) * 1_000_000 - (self.range as i128) * 1_000_000; - while left < len && ts_column.value(left) <= start_ts { + while left < len && timestamp(left) <= start_ts { left += 1; } right = right.max(left); - while right < len && ts_column.value(right) <= curr_ts { + while right < len && timestamp(right) <= (curr_ts as i128) * 1_000_000 { right += 1; } @@ -756,17 +833,24 @@ impl RangeManipulateStream { #[cfg(test)] mod test { - use datafusion::arrow::array::{ArrayRef, DictionaryArray, Float64Array, StringArray}; + use datafusion::arrow::array::{ + ArrayRef, DictionaryArray, Float64Array, StringArray, TimestampMicrosecondArray, + TimestampNanosecondArray, TimestampSecondArray, + }; + use datafusion::arrow::buffer::NullBuffer; use datafusion::arrow::datatypes::{ ArrowPrimitiveType, DataType, Field, Int64Type, Schema, TimestampMillisecondType, }; use datafusion::common::ToDFSchema; use datafusion::datasource::memory::MemorySourceConfig; use datafusion::datasource::source::DataSourceExec; - use datafusion::logical_expr::{EmptyRelation, LogicalPlan}; + use datafusion::logical_expr::{ + EmptyRelation, Extension, LogicalPlan, UserDefinedLogicalNodeCore, + }; use datafusion::physical_expr::Partitioning; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::memory::MemoryStream; + use datafusion::physical_plan::{ChildrenPropertiesMode, ReplaceChildrenOptions}; use datafusion::prelude::SessionContext; use datatypes::arrow::array::TimestampMillisecondArray; use futures::FutureExt; @@ -845,6 +929,7 @@ mod test { Boundedness::Bounded, )); let normalize_exec = Arc::new(RangeManipulateExec { + offset: 0, start, end, interval, @@ -888,6 +973,435 @@ mod test { assert_eq!(result_literal, expected); } + #[tokio::test] + async fn native_timestamps_preserve_range_membership_and_ms_payload() { + for (unit, ticks_per_ms) in [ + (TimeUnit::Microsecond, 1_000_i64), + (TimeUnit::Nanosecond, 1_000_000_i64), + ] { + let lower = 1_000 * ticks_per_ms; + let upper = 1_001 * ticks_per_ms; + // Exclude the lower boundary and future sample; retain both native + // samples in the same millisecond bucket and the exact upper sample. + let timestamps = vec![lower, lower + 1, lower + 2, upper, upper + 1]; + let time: ArrayRef = match unit { + TimeUnit::Microsecond => Arc::new(TimestampMicrosecondArray::from(timestamps)), + TimeUnit::Nanosecond => Arc::new(TimestampNanosecondArray::from(timestamps)), + _ => unreachable!(), + }; + let schema = Arc::new(Schema::new(vec![ + Field::new(TIME_INDEX_COLUMN, DataType::Timestamp(unit, None), false), + Field::new("value", DataType::Float64, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + time, + Arc::new(Float64Array::from(vec![10.0, 20.0, 30.0, 40.0, 50.0])), + ], + ) + .unwrap(); + let logical_input = LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: schema.clone().to_dfschema_ref().unwrap(), + }); + let plan = RangeManipulate::new( + 1_001, + 1_001, + 1, + 0, + 1, + TIME_INDEX_COLUMN.to_string(), + vec!["value".to_string()], + logical_input.clone(), + ) + .unwrap(); + let output_time = Field::new( + TIME_INDEX_COLUMN, + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ); + let output_schema = Arc::new(Schema::new(vec![ + output_time.clone(), + RangeArray::convert_field(&Field::new("value", DataType::Float64, true)), + Field::new( + RangeManipulate::build_timestamp_range_name(TIME_INDEX_COLUMN), + RangeArray::convert_field(&output_time).data_type().clone(), + false, + ), + ])); + assert_eq!(plan.schema().as_arrow(), output_schema.as_ref()); + + let rebuilt = RangeManipulate::deserialize(&plan.serialize()) + .unwrap() + .with_exprs_and_inputs(vec![], vec![logical_input]) + .unwrap(); + assert_eq!(rebuilt.schema(), plan.schema()); + assert_eq!(rebuilt.input.schema().as_arrow(), schema.as_ref()); + + let input = Arc::new(DataSourceExec::new(Arc::new( + MemorySourceConfig::try_new(&[vec![batch]], schema.clone(), None).unwrap(), + ))); + let exec = rebuilt.to_execution_plan(input); + assert_eq!(exec.schema(), output_schema); + assert_eq!(exec.children()[0].schema(), schema); + + let batches = + datafusion::physical_plan::collect(exec, SessionContext::default().task_ctx()) + .await + .unwrap(); + assert_eq!(batches.len(), 1, "{unit:?}"); + let output = &batches[0]; + assert_eq!(output.schema(), output_schema); + assert_eq!(output.num_rows(), 1); + assert_eq!( + output + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .as_ref(), + &[1_001] + ); + + // RangeArray packs offset/length into dictionary keys; Arrow dictionary + // equality treats those packed keys as indices and cannot compare them. + let values = RangeArray::try_new( + output + .column(1) + .as_any() + .downcast_ref::>() + .unwrap() + .clone(), + ) + .unwrap(); + assert_eq!(values.get_offset_length(0), Some((1, 3))); + assert_eq!( + values.get(0).unwrap().to_data(), + Float64Array::from(vec![20.0, 30.0, 40.0]).to_data() + ); + let timestamps = RangeArray::try_new( + output + .column(2) + .as_any() + .downcast_ref::>() + .unwrap() + .clone(), + ) + .unwrap(); + assert_eq!(timestamps.get_offset_length(0), Some((1, 3))); + assert_eq!( + timestamps.get(0).unwrap().to_data(), + TimestampMillisecondArray::from(vec![1_000, 1_000, 1_001]).to_data() + ); + } + } + + #[test] + fn logical_offset_participates_in_ordering() { + let input = LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: prepare_test_data().schema().to_dfschema_ref().unwrap(), + }); + let first = RangeManipulate::new( + 0, + 0, + 0, + 0, + 0, + TIME_INDEX_COLUMN.to_string(), + vec!["value_1".to_string()], + input.clone(), + ) + .unwrap(); + let second = RangeManipulate::new( + 0, + 0, + 0, + 1, + 0, + TIME_INDEX_COLUMN.to_string(), + vec!["value_1".to_string()], + input, + ) + .unwrap(); + assert_ne!(first, second); + assert_eq!(first.partial_cmp(&second), Some(std::cmp::Ordering::Less)); + } + + #[tokio::test] + async fn logical_normalize_offset_survives_rebuild_and_executes() { + for (name, time_unit, raw, offset, start, range, expected_payload) in [ + ( + "millisecond offset", + TimeUnit::Millisecond, + 0, + 1_000, + 1_000, + 1_000, + 1_000, + ), + ( + "negative native lower limit with positive window", + TimeUnit::Nanosecond, + -9_223_112_837_000_000_000, + -259_200_000, + -9_223_372_037_000, + 300_000, + -9_223_372_037_000, + ), + ( + "second timestamp with negative fractional offset", + TimeUnit::Second, + 1, + -500, + 1_000, + 1_000, + 500, + ), + ] { + let schema = Arc::new(Schema::new(vec![ + Field::new( + TIME_INDEX_COLUMN, + DataType::Timestamp(time_unit, None), + false, + ), + Field::new("value", DataType::Float64, true), + ])); + let input = LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: schema.clone().to_dfschema_ref().unwrap(), + }); + let normalize = crate::extension_plan::SeriesNormalize::new( + offset, + TIME_INDEX_COLUMN, + false, + Vec::new(), + input.clone(), + ); + let normalize = + crate::extension_plan::SeriesNormalize::deserialize(&normalize.serialize()) + .unwrap() + .with_exprs_and_inputs(vec![], vec![input.clone()]) + .unwrap(); + let normalized = LogicalPlan::Extension(Extension { + node: Arc::new(normalize), + }); + let fresh = RangeManipulate::new( + start, + start, + 1, + offset, + range, + TIME_INDEX_COLUMN.to_string(), + vec!["value".to_string()], + input.clone(), + ) + .unwrap() + .with_exprs_and_inputs(vec![], vec![input.clone()]) + .unwrap(); + let serialized = RangeManipulate::new( + start, + start, + 1, + offset, + range, + TIME_INDEX_COLUMN.to_string(), + vec!["value".to_string()], + normalized.clone(), + ) + .unwrap(); + let decoded = RangeManipulate::deserialize(&serialized.serialize()) + .unwrap() + .with_exprs_and_inputs(vec![], vec![normalized]) + .unwrap(); + let timestamp: ArrayRef = match time_unit { + TimeUnit::Millisecond => Arc::new(TimestampMillisecondArray::from(vec![raw])), + TimeUnit::Nanosecond => Arc::new(TimestampNanosecondArray::from(vec![raw])), + TimeUnit::Second => Arc::new(TimestampSecondArray::from(vec![raw])), + _ => unreachable!(), + }; + let batch = RecordBatch::try_new( + schema.clone(), + vec![timestamp, Arc::new(Float64Array::from(vec![7.0]))], + ) + .unwrap(); + for (mode, rebuilt) in [("fresh", fresh), ("decoded", decoded)] { + let rebuilt = rebuilt + .with_exprs_and_inputs(vec![], vec![input.clone()]) + .unwrap(); + assert_eq!(rebuilt.offset, offset, "{name}: {mode}"); + assert_eq!(rebuilt.input.schema(), input.schema(), "{name}: {mode}"); + + let empty_exec_input = Arc::new(DataSourceExec::new(Arc::new( + MemorySourceConfig::try_new(&[vec![]], schema.clone(), None).unwrap(), + ))); + let exec_input = Arc::new(DataSourceExec::new(Arc::new( + MemorySourceConfig::try_new(&[vec![batch.clone()]], schema.clone(), None) + .unwrap(), + ))); + let exec = rebuilt + .to_execution_plan(empty_exec_input) + .replace_children( + vec![exec_input], + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + .unwrap(); + let output = + datafusion::physical_plan::collect(exec, SessionContext::default().task_ctx()) + .await + .unwrap(); + assert_eq!(output.len(), 1, "{name}: {mode}"); + let output = &output[0]; + assert_eq!(output.num_rows(), 1, "{name}: {mode}"); + assert_eq!( + output + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + start, + "{name}: {mode}" + ); + let values = RangeArray::try_new( + output + .column(1) + .as_any() + .downcast_ref::>() + .unwrap() + .clone(), + ) + .unwrap(); + assert_eq!(values.get_offset_length(0), Some((0, 1)), "{name}: {mode}"); + assert_eq!( + values.get(0).unwrap().to_data(), + Float64Array::from(vec![7.0]).to_data(), + "{name}: {mode}" + ); + let timestamps = RangeArray::try_new( + output + .column(2) + .as_any() + .downcast_ref::>() + .unwrap() + .clone(), + ) + .unwrap(); + assert_eq!( + timestamps.get_offset_length(0), + Some((0, 1)), + "{name}: {mode}" + ); + assert_eq!( + timestamps.get(0).unwrap().to_data(), + TimestampMillisecondArray::from(vec![expected_payload]).to_data(), + "{name}: {mode}" + ); + } + } + } + + #[tokio::test] + async fn range_payload_preserves_null_timestamp_and_rejects_offset_overflow() { + let schema = Arc::new(Schema::new(vec![ + Field::new(TIME_INDEX_COLUMN, TimestampMillisecondType::DATA_TYPE, true), + Field::new("value", DataType::Float64, true), + ])); + let null_timestamp = + TimestampMillisecondArray::new(vec![1_000].into(), Some(NullBuffer::from(vec![false]))); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(null_timestamp), + Arc::new(Float64Array::from(vec![7.0])), + ], + ) + .unwrap(); + let input = Arc::new(DataSourceExec::new(Arc::new( + MemorySourceConfig::try_new(&[vec![batch]], schema.clone(), None).unwrap(), + ))); + let plan = RangeManipulate::new( + 1_000, + 1_000, + 1, + 0, + 1, + TIME_INDEX_COLUMN.to_string(), + vec!["value".to_string()], + LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: schema.clone().to_dfschema_ref().unwrap(), + }), + ) + .unwrap(); + let output = datafusion::physical_plan::collect( + plan.to_execution_plan(input), + SessionContext::default().task_ctx(), + ) + .await + .unwrap(); + let timestamps = RangeArray::try_new( + output[0] + .column(2) + .as_any() + .downcast_ref::>() + .unwrap() + .clone(), + ) + .unwrap(); + let payload = timestamps.get(0).unwrap(); + let payload = payload + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(payload.len(), 1); + assert!(!payload.is_valid(0)); + + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(TimestampMillisecondArray::from(vec![0, i64::MAX])), + Arc::new(Float64Array::from(vec![7.0, 8.0])), + ], + ) + .unwrap(); + let input = Arc::new(DataSourceExec::new(Arc::new( + MemorySourceConfig::try_new(&[vec![batch]], schema.clone(), None).unwrap(), + ))); + let normalized = crate::extension_plan::SeriesNormalize::new( + 1, + TIME_INDEX_COLUMN, + false, + Vec::new(), + LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: schema.to_dfschema_ref().unwrap(), + }), + ); + let plan = RangeManipulate::new( + 1, + 1, + 1, + 1, + 1, + TIME_INDEX_COLUMN.to_string(), + vec!["value".to_string()], + LogicalPlan::Extension(Extension { + node: Arc::new(normalized), + }), + ) + .unwrap(); + let error = datafusion::physical_plan::collect( + plan.to_execution_plan(input), + SessionContext::default().task_ctx(), + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("timestamp payload overflow")); + } + #[tokio::test] async fn pruning_should_keep_time_and_value_columns_for_exec() { let schema = Arc::new(Schema::new(vec![ @@ -905,6 +1419,7 @@ mod test { 0, 310_000, 30_000, + 0, 90_000, TIME_INDEX_COLUMN.to_string(), vec!["value_1".to_string(), "value_2".to_string()], @@ -1042,11 +1557,13 @@ mod test { let empty_stream = MemoryStream::try_new(vec![], schema.clone(), None).unwrap(); let stream = RangeManipulateStream { + offset: 0, start: 1758093274000, // ends in 4000 end: 1758093334000, // ends in 4000 interval: 30000, // 30s step range: 60000, // 60s lookback time_index: 0, + time_unit: TimeUnit::Millisecond, field_columns: vec![], aligned_ts_array: Arc::new(TimestampMillisecondArray::from(vec![0i64; 0])), output_schema: schema.clone(), @@ -1092,6 +1609,69 @@ mod test { } } + #[tokio::test] + async fn no_intersection_batch_is_skipped_and_stream_continues() { + let schema = Arc::new(Schema::new(vec![ + Field::new( + TIME_INDEX_COLUMN, + TimestampMillisecondType::DATA_TYPE, + false, + ), + Field::new("value", DataType::Float64, false), + ])); + let input = LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: schema.clone().to_dfschema_ref().unwrap(), + }); + let plan = RangeManipulate::new( + 0, + 50, + 10, + 0, + 1, + TIME_INDEX_COLUMN.to_string(), + vec!["value".to_string()], + input, + ) + .unwrap(); + let no_intersection = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(TimestampMillisecondArray::from(vec![100])), + Arc::new(Float64Array::from(vec![1.0])), + ], + ) + .unwrap(); + let intersection = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(TimestampMillisecondArray::from(vec![20])), + Arc::new(Float64Array::from(vec![2.0])), + ], + ) + .unwrap(); + let input = Arc::new(DataSourceExec::new(Arc::new( + MemorySourceConfig::try_new(&[vec![no_intersection, intersection]], schema, None) + .unwrap(), + ))); + + let batches = datafusion::physical_plan::collect( + plan.to_execution_plan(input), + SessionContext::default().task_ctx(), + ) + .await + .unwrap(); + + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 1); + let timestamps = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(timestamps.value(0), 20); + } + fn calculate_range_for_test( query_start: i64, query_end: i64, @@ -1106,11 +1686,13 @@ mod test { )])); let empty_stream = MemoryStream::try_new(vec![], schema.clone(), None).unwrap(); let stream = RangeManipulateStream { + offset: 0, start: query_start, end: query_end, interval, range, time_index: 0, + time_unit: TimeUnit::Millisecond, field_columns: vec![], aligned_ts_array: Arc::new(TimestampMillisecondArray::from(vec![0i64; 0])), output_schema: schema.clone(), @@ -1216,7 +1798,7 @@ mod test { 10, 0, vec![100], - (100, 50), + (0, 50), ), ]; @@ -1227,6 +1809,15 @@ mod test { } } + #[test] + fn calculate_range_keeps_extreme_range_tail() { + let (ranges, bounds) = + calculate_range_for_test(i64::MAX - 1, i64::MAX, 1, i64::MAX, &[i64::MAX]); + + assert_eq!(bounds, (i64::MAX, i64::MAX)); + assert_eq!(ranges, vec![(0, 1)]); + } + #[test] fn calculate_range_matches_bruteforce_oracle_for_deterministic_cases() { let cases = vec![ @@ -1345,6 +1936,17 @@ mod test { let (actual, (start, end)) = calculate_range_for_test(query_start, query_end, interval, range, ×tamps); let expected = calculate_range_oracle(×tamps, start, end, interval, range); + let expected = if actual.is_empty() && !expected.is_empty() { + assert!( + expected.iter().all(|(_, len)| *len == 0), + "case={case}, timestamps={timestamps:?}, query=({query_start}, {query_end}), \ + interval={interval}, range={range}, bounds=({start}, {end}): \ + no-intersection output must have no selected samples" + ); + vec![] + } else { + expected + }; assert_eq!( actual, expected, "case={case}, timestamps={timestamps:?}, query=({query_start}, {query_end}), \ diff --git a/src/promql/src/extension_plan/scalar_calculate.rs b/src/promql/src/extension_plan/scalar_calculate.rs index 567fd65b5eb..e71333837f3 100644 --- a/src/promql/src/extension_plan/scalar_calculate.rs +++ b/src/promql/src/extension_plan/scalar_calculate.rs @@ -12,25 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::collections::HashMap; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; use datafusion::common::stats::Precision; -use datafusion::common::{DFSchema, DFSchemaRef, Result as DataFusionResult, Statistics}; +use datafusion::common::tree_node::TreeNodeRecursion; +use datafusion::common::{ + DFSchema, DFSchemaRef, Result as DataFusionResult, Statistics, TableReference, +}; use datafusion::error::DataFusionError; use datafusion::execution::context::TaskContext; use datafusion::logical_expr::{EmptyRelation, LogicalPlan, UserDefinedLogicalNodeCore}; use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, Partitioning, PlanProperties, - RecordBatchStream, SendableRecordBatchStream, + ChildStats, DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, + InputDistributionRequirements, Partitioning, PhysicalExpr, PlanProperties, RecordBatchStream, + SendableRecordBatchStream, StatisticsArgs, }; use datafusion::prelude::Expr; -use datafusion::sql::TableReference; use datafusion_expr::col; use datatypes::arrow::array::{Array, ArrayRef, Float64Array, TimestampMillisecondArray}; use datatypes::arrow::compute::{CastOptions, cast_with_options, concat_batches}; @@ -129,7 +131,10 @@ impl ScalarCalculate { .output_schema .fields() .iter() - .map(|field| Field::new(field.name(), field.data_type().clone(), field.is_nullable())) + .map(|field| { + Field::new(field.name(), field.data_type().clone(), field.is_nullable()) + .with_metadata(field.metadata().clone()) + }) .collect(); let input_schema = exec_input.schema(); let ts_index = input_schema @@ -138,7 +143,10 @@ impl ScalarCalculate { let val_index = input_schema .index_of(&self.field_column) .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))?; - let schema = Arc::new(Schema::new(fields)); + let schema = Arc::new(Schema::new_with_metadata( + fields, + input_schema.metadata().clone(), + )); let properties = exec_input.properties(); let properties = Arc::new(PlanProperties::new( EquivalenceProperties::new(schema.clone()), @@ -389,8 +397,11 @@ struct ScalarCalculateExec { } impl ExecutionPlan for ScalarCalculateExec { - fn as_any(&self) -> &dyn Any { - self + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> datafusion_common::Result, + ) -> DataFusionResult { + Ok(TreeNodeRecursion::Continue) } fn schema(&self) -> SchemaRef { @@ -405,8 +416,8 @@ impl ExecutionPlan for ScalarCalculateExec { vec![true; self.children().len()] } - fn required_input_distribution(&self) -> Vec { - vec![Distribution::SinglePartition] + fn input_distribution_requirements(&self) -> InputDistributionRequirements { + InputDistributionRequirements::new(vec![Distribution::SinglePartition]) } fn children(&self) -> Vec<&Arc> { @@ -469,8 +480,16 @@ impl ExecutionPlan for ScalarCalculateExec { Some(self.metric.clone_inner()) } - fn partition_statistics(&self, partition: Option) -> DataFusionResult { - let input_stats = self.input.partition_statistics(partition)?; + fn child_stats_requests(&self, partition: Option) -> Vec { + vec![ChildStats::At(partition)] + } + + fn statistics_from_inputs( + &self, + input_stats: &[Arc], + _args: &StatisticsArgs, + ) -> DataFusionResult> { + let input_stats = &input_stats[0]; let estimated_row_num = (self.end - self.start) as f64 / self.interval as f64; let estimated_total_bytes = input_stats @@ -482,12 +501,12 @@ impl ExecutionPlan for ScalarCalculateExec { }) .unwrap_or_default(); - Ok(Statistics { + Ok(Arc::new(Statistics { num_rows: Precision::Inexact(estimated_row_num as _), total_byte_size: estimated_total_bytes, // TODO(ruihang): support this column statistics column_statistics: Statistics::unknown_column(&self.schema()), - }) + })) } fn name(&self) -> &str { diff --git a/src/promql/src/extension_plan/series_divide.rs b/src/promql/src/extension_plan/series_divide.rs index 9e6a46d991a..b850d9b2ddc 100644 --- a/src/promql/src/extension_plan/series_divide.rs +++ b/src/promql/src/extension_plan/series_divide.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; @@ -20,6 +19,7 @@ use std::task::{Context, Poll}; use datafusion::arrow::array::{Array, ArrayRef, UInt64Array}; use datafusion::arrow::datatypes::{DataType, SchemaRef}; use datafusion::arrow::record_batch::RecordBatch; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{DFSchema, DFSchemaRef}; use datafusion::error::Result as DataFusionResult; use datafusion::execution::context::TaskContext; @@ -30,8 +30,8 @@ use datafusion::physical_plan::metrics::{ BaselineMetrics, Count, ExecutionPlanMetricsSet, MetricBuilder, MetricValue, MetricsSet, }; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, PlanProperties, RecordBatchStream, - SendableRecordBatchStream, + DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, InputDistributionRequirements, + PhysicalExpr, PlanProperties, RecordBatchStream, SendableRecordBatchStream, }; use datafusion_expr::col; use datatypes::arrow::compute; @@ -334,8 +334,11 @@ pub struct SeriesDivideExec { } impl ExecutionPlan for SeriesDivideExec { - fn as_any(&self) -> &dyn Any { - self + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> datafusion_common::Result, + ) -> DataFusionResult { + Ok(TreeNodeRecursion::Continue) } fn schema(&self) -> SchemaRef { @@ -346,18 +349,18 @@ impl ExecutionPlan for SeriesDivideExec { self.input.properties() } - fn required_input_distribution(&self) -> Vec { + fn input_distribution_requirements(&self) -> InputDistributionRequirements { if self.tag_columns.is_empty() { - return vec![Distribution::SinglePartition]; + return InputDistributionRequirements::new(vec![Distribution::SinglePartition]); } let schema = self.input.schema(); - vec![Distribution::HashPartitioned( + InputDistributionRequirements::new(vec![Distribution::KeyPartitioned( self.tag_columns .iter() // Safety: the tag column names is verified in the planning phase .map(|tag| Arc::new(ColumnExpr::new_with_schema(tag, &schema).unwrap()) as _) .collect(), - )] + )]) } fn required_input_ordering(&self) -> Vec> { @@ -554,7 +557,9 @@ impl Stream for SeriesDivideStream { } error => return Poll::Ready(error), }; - self.buffer.push(batch); + if batch.num_rows() != 0 { + self.buffer.push(batch); + } continue; } } @@ -624,7 +629,8 @@ impl SeriesDivideStream { #[cfg(test)] mod test { use datafusion::arrow::array::{ - DictionaryArray, Int32Array, LargeStringArray, StringArray, StringViewArray, UInt32Array, + DictionaryArray, Int32Array, Int64Array, LargeStringArray, StringArray, StringViewArray, + UInt32Array, }; use datafusion::arrow::datatypes::{DataType, Field, Int32Type, Schema, UInt32Type}; use datafusion::common::ToDFSchema; @@ -1245,4 +1251,216 @@ mod test { assert!(tsid_array.iter().all(|v| v == Some(expected_tsid))); } } + + /// The input stream may legally start with a batch that has zero rows. + /// `SeriesDivideStream` must skip it and keep working on the real rows that + /// follow (here: two batches of the very same tagged series). + #[tokio::test] + async fn test_stream_leading_empty_batch_then_same_tags() { + let schema = Arc::new(Schema::new(vec![ + Field::new("host", DataType::Utf8, false), + Field::new("path", DataType::Utf8, false), + Field::new( + "time_index", + DataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Millisecond, None), + false, + ), + ])); + + let make_batch = |hosts: Vec<&str>, paths: Vec<&str>, times: Vec| { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(hosts)), + Arc::new(StringArray::from(paths)), + Arc::new(datafusion::arrow::array::TimestampMillisecondArray::from( + times, + )), + ], + ) + .unwrap() + }; + + let memory_exec: Arc = Arc::new(DataSourceExec::new(Arc::new( + MemorySourceConfig::try_new( + &[vec![ + RecordBatch::new_empty(schema.clone()), + make_batch( + vec!["server1", "server1"], + vec!["/var/log", "/var/log"], + vec![1000, 2000], + ), + make_batch(vec!["server1"], vec!["/var/log"], vec![3000]), + ]], + schema.clone(), + None, + ) + .unwrap(), + ))); + + let divide_exec = Arc::new(SeriesDivideExec { + tag_columns: vec!["host".to_string(), "path".to_string()], + time_index_column: "time_index".to_string(), + input: memory_exec, + metric: ExecutionPlanMetricsSet::new(), + }); + + let mut stream = divide_exec + .execute(0, SessionContext::default().task_ctx()) + .unwrap(); + + let batch = stream.next().await.unwrap().unwrap(); + assert_eq!(batch.num_rows(), 3); + let expected = [ + ("server1", "/var/log", 1000i64), + ("server1", "/var/log", 2000), + ("server1", "/var/log", 3000), + ]; + for (row, (host, path, time)) in expected.into_iter().enumerate() { + assert_eq!( + string_array_value_at_index(batch.column(0), row), + Some(host) + ); + assert_eq!( + string_array_value_at_index(batch.column(1), row), + Some(path) + ); + assert_eq!( + batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap() + .value(row), + time + ); + } + assert!(stream.next().await.is_none()); + } + + /// Several leading empty batches (and one empty batch in the middle of a + /// series) must not disturb series division: the two series are still + /// emitted separately, including the last one flushed at end of input. + #[tokio::test] + async fn test_stream_repeated_empty_batches_and_two_series() { + let schema = Arc::new(Schema::new(vec![ + Field::new("host", DataType::Utf8, false), + Field::new("value", DataType::Int64, false), + Field::new( + "time_index", + DataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Millisecond, None), + false, + ), + ])); + + let make_batch = |hosts: Vec<&str>, values: Vec| { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(hosts)), + Arc::new(Int64Array::from(values.clone())), + Arc::new(datafusion::arrow::array::TimestampMillisecondArray::from( + values.iter().map(|v| v * 1000).collect::>(), + )), + ], + ) + .unwrap() + }; + + let memory_exec: Arc = Arc::new(DataSourceExec::new(Arc::new( + MemorySourceConfig::try_new( + &[vec![ + RecordBatch::new_empty(schema.clone()), + RecordBatch::new_empty(schema.clone()), + make_batch(vec!["server1", "server1"], vec![1, 2]), + RecordBatch::new_empty(schema.clone()), + make_batch(vec!["server1"], vec![3]), + make_batch(vec!["server2", "server2"], vec![4, 5]), + ]], + schema.clone(), + None, + ) + .unwrap(), + ))); + + let divide_exec = Arc::new(SeriesDivideExec { + tag_columns: vec!["host".to_string()], + time_index_column: "time_index".to_string(), + input: memory_exec, + metric: ExecutionPlanMetricsSet::new(), + }); + + let mut stream = divide_exec + .execute(0, SessionContext::default().task_ctx()) + .unwrap(); + + for (expected_host, expected_values) in + [("server1", vec![1i64, 2, 3]), ("server2", vec![4, 5])] + { + let batch = stream.next().await.unwrap().unwrap(); + assert_eq!(batch.num_rows(), expected_values.len()); + for (row, value) in expected_values.into_iter().enumerate() { + assert_eq!( + string_array_value_at_index(batch.column(0), row), + Some(expected_host) + ); + assert_eq!( + batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap() + .value(row), + value + ); + } + } + assert!(stream.next().await.is_none()); + } + + /// A stream that only ever yields zero-row batches must terminate with no + /// batches instead of failing. + #[tokio::test] + async fn test_stream_all_empty_batches_yields_nothing() { + let schema = Arc::new(Schema::new(vec![ + Field::new("host", DataType::Utf8, false), + Field::new("value", DataType::Int64, false), + Field::new( + "time_index", + DataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Millisecond, None), + false, + ), + ])); + + let memory_exec: Arc = Arc::new(DataSourceExec::new(Arc::new( + MemorySourceConfig::try_new( + &[vec![ + RecordBatch::new_empty(schema.clone()), + RecordBatch::new_empty(schema.clone()), + RecordBatch::new_empty(schema.clone()), + ]], + schema.clone(), + None, + ) + .unwrap(), + ))); + + let divide_exec = Arc::new(SeriesDivideExec { + tag_columns: vec!["host".to_string()], + time_index_column: "time_index".to_string(), + input: memory_exec, + metric: ExecutionPlanMetricsSet::new(), + }); + + let mut stream = divide_exec + .execute(0, SessionContext::default().task_ctx()) + .unwrap(); + assert!(stream.next().await.is_none()); + + let collected = + datafusion::physical_plan::collect(divide_exec, SessionContext::default().task_ctx()) + .await + .unwrap(); + assert!(collected.is_empty()); + } } diff --git a/src/promql/src/extension_plan/union_distinct_on.rs b/src/promql/src/extension_plan/union_distinct_on.rs index a7d8fd48594..774904a27aa 100644 --- a/src/promql/src/extension_plan/union_distinct_on.rs +++ b/src/promql/src/extension_plan/union_distinct_on.rs @@ -12,15 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; -use ahash::{HashSet, RandomState}; +use ahash::HashSet; use datafusion::arrow::array::UInt64Array; use datafusion::arrow::datatypes::SchemaRef; use datafusion::arrow::record_batch::RecordBatch; +use datafusion::common::hash_utils::RandomState as FixedState; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{DFSchema, DFSchemaRef}; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::execution::context::TaskContext; @@ -29,8 +30,9 @@ use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, Partitioning, PlanProperties, - RecordBatchStream, SendableRecordBatchStream, hash_utils, + DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, InputDistributionRequirements, + Partitioning, PhysicalExpr, PlanProperties, RecordBatchStream, SendableRecordBatchStream, + hash_utils, }; use datafusion_expr::col; use datatypes::arrow::compute; @@ -179,7 +181,7 @@ impl UnionDistinctOn { output_schema, metric: ExecutionPlanMetricsSet::new(), properties, - random_state: RandomState::new(), + random_state: FixedState::with_seed(0), }) } @@ -347,21 +349,27 @@ pub struct UnionDistinctOnExec { metric: ExecutionPlanMetricsSet, properties: Arc, - /// Shared the `RandomState` for the hashing algorithm - random_state: RandomState, + /// Shared deterministic hash state for the hashing algorithm. + random_state: FixedState, } impl ExecutionPlan for UnionDistinctOnExec { - fn as_any(&self) -> &dyn Any { - self + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> datafusion_common::Result, + ) -> DataFusionResult { + Ok(TreeNodeRecursion::Continue) } fn schema(&self) -> SchemaRef { self.output_schema.clone() } - fn required_input_distribution(&self) -> Vec { - vec![Distribution::SinglePartition, Distribution::SinglePartition] + fn input_distribution_requirements(&self) -> InputDistributionRequirements { + InputDistributionRequirements::new(vec![ + Distribution::SinglePartition, + Distribution::SinglePartition, + ]) } fn properties(&self) -> &Arc { @@ -452,7 +460,7 @@ pub struct UnionDistinctOnStream { /// Include time index compare_keys: Vec, output_schema: SchemaRef, - random_state: RandomState, + random_state: FixedState, lhs_signatures: HashSet, hashes: Vec, phase: StreamPhase, @@ -911,12 +919,17 @@ mod test { } impl ExecutionPlan for TestExec { - fn name(&self) -> &str { - "TestExec" + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> datafusion_common::Result, + ) -> DataFusionResult { + Ok(TreeNodeRecursion::Continue) } - fn as_any(&self) -> &dyn Any { - self + fn name(&self) -> &str { + "TestExec" } fn properties(&self) -> &Arc { @@ -965,7 +978,7 @@ mod test { right: None, compare_keys: vec![1, 0], output_schema, - random_state: RandomState::new(), + random_state: FixedState::with_seed(0), lhs_signatures: HashSet::default(), hashes: Vec::new(), phase: StreamPhase::Left, diff --git a/src/promql/src/functions/aggr_over_time.rs b/src/promql/src/functions/aggr_over_time.rs index 493c2d78518..f76e50c52b9 100644 --- a/src/promql/src/functions/aggr_over_time.rs +++ b/src/promql/src/functions/aggr_over_time.rs @@ -157,7 +157,10 @@ fn evaluate_presence( display_name = prom_avg_over_time )] pub fn avg_over_time(_: &TimestampMillisecondArray, values: &Float64Array) -> Option { - compute::sum(values).map(|result| result / values.len() as f64) + // `sum` already skips null slots and yields `None` for an all-null window, so only the + // divisor needs to count samples instead of slots. + let sample_count = values.len() - values.null_count(); + compute::sum(values).map(|result| result / sample_count as f64) } /// The minimum value of all points in the specified interval. @@ -524,26 +527,21 @@ pub fn present_over_time(_: &TimestampMillisecondArray, values: &Float64Array) - display_name = prom_stdvar_over_time )] pub fn stdvar_over_time(_: &TimestampMillisecondArray, values: &Float64Array) -> Option { - if values.is_empty() { - None - } else { - let mut count = 0; - let mut mean: f64 = 0.0; - let mut result: f64 = 0.0; - for value in values { - let value = value.unwrap(); - let new_count = count + 1; - let delta1 = value - mean; - let new_mean = delta1 / new_count as f64 + mean; - let delta2 = value - new_mean; - let new_result = result + delta1 * delta2; + let mut count = 0; + let mut mean: f64 = 0.0; + let mut result: f64 = 0.0; + for value in values.iter().flatten() { + let new_count = count + 1; + let delta1 = value - mean; + let new_mean = delta1 / new_count as f64 + mean; + let delta2 = value - new_mean; + let new_result = result + delta1 * delta2; - count += 1; - mean = new_mean; - result = new_result; - } - Some(result / count as f64) + count = new_count; + mean = new_mean; + result = new_result; } + (count > 0).then(|| result / count as f64) } /// the population standard deviation of the values in the specified interval. @@ -554,35 +552,32 @@ pub fn stdvar_over_time(_: &TimestampMillisecondArray, values: &Float64Array) -> display_name = prom_stddev_over_time )] pub fn stddev_over_time(_: &TimestampMillisecondArray, values: &Float64Array) -> Option { - if values.is_empty() { - None - } else { - let mut count = 0.0; - let mut mean = 0.0; - let mut comp_mean = 0.0; - let mut deviations_sum_sq = 0.0; - let mut comp_deviations_sum_sq = 0.0; - for v in values { - count += 1.0; - let current_value = v.unwrap(); - let delta = current_value - (mean + comp_mean); - let (new_mean, new_comp_mean) = compensated_sum_inc(delta / count, mean, comp_mean); - mean = new_mean; - comp_mean = new_comp_mean; - let (new_deviations_sum_sq, new_comp_deviations_sum_sq) = compensated_sum_inc( - delta * (current_value - (mean + comp_mean)), - deviations_sum_sq, - comp_deviations_sum_sq, - ); - deviations_sum_sq = new_deviations_sum_sq; - comp_deviations_sum_sq = new_comp_deviations_sum_sq; - } - Some(((deviations_sum_sq + comp_deviations_sum_sq) / count).sqrt()) + let mut count = 0.0; + let mut mean = 0.0; + let mut comp_mean = 0.0; + let mut deviations_sum_sq = 0.0; + let mut comp_deviations_sum_sq = 0.0; + for current_value in values.iter().flatten() { + count += 1.0; + let delta = current_value - (mean + comp_mean); + let (new_mean, new_comp_mean) = compensated_sum_inc(delta / count, mean, comp_mean); + mean = new_mean; + comp_mean = new_comp_mean; + let (new_deviations_sum_sq, new_comp_deviations_sum_sq) = compensated_sum_inc( + delta * (current_value - (mean + comp_mean)), + deviations_sum_sq, + comp_deviations_sum_sq, + ); + deviations_sum_sq = new_deviations_sum_sq; + comp_deviations_sum_sq = new_comp_deviations_sum_sq; } + (count > 0.0).then(|| ((deviations_sum_sq + comp_deviations_sum_sq) / count).sqrt()) } #[cfg(test)] mod test { + use datafusion::arrow::buffer::NullBuffer; + use super::*; use crate::functions::test_util::{invoke_range_udf, simple_range_udf_runner}; @@ -1719,4 +1714,57 @@ mod test { vec![Some(0.0), Some(3.249615361854384)], ); } + + /// Timestamps and value ranges shared by the null-sample assertions below. + fn null_sample_range_arrays() -> (RangeArray, RangeArray) { + let ts_array = Arc::new(TimestampMillisecondArray::from_iter_values([ + 0i64, 1000, 2000, 3000, + ])); + // Samples are 2.0@0 and 8.0@3000; the null slots keep a payload that would skew every + // aggregate if it were read. + let values_array = Arc::new(Float64Array::new( + vec![2.0, 1000.0, -1000.0, 8.0].into(), + Some(NullBuffer::from_iter([true, false, false, true])), + )); + // The second window holds no sample at all. + let ranges = [(0, 4), (1, 2)]; + + ( + RangeArray::from_ranges(ts_array, ranges).unwrap(), + RangeArray::from_ranges(values_array, ranges).unwrap(), + ) + } + + #[test] + fn avg_over_time_divides_by_sample_count() { + let (ts_array, value_array) = null_sample_range_arrays(); + simple_range_udf_runner( + AvgOverTime::scalar_udf(), + ts_array, + value_array, + vec![], + vec![Some(5.0), None], + ); + } + + #[test] + fn stdvar_and_stddev_over_time_skip_null_samples() { + let (ts_array, value_array) = null_sample_range_arrays(); + simple_range_udf_runner( + StdvarOverTime::scalar_udf(), + ts_array, + value_array, + vec![], + vec![Some(9.0), None], + ); + + let (ts_array, value_array) = null_sample_range_arrays(); + simple_range_udf_runner( + StddevOverTime::scalar_udf(), + ts_array, + value_array, + vec![], + vec![Some(3.0), None], + ); + } } diff --git a/src/promql/src/functions/changes.rs b/src/promql/src/functions/changes.rs index c143d29bef7..8630c914a8f 100644 --- a/src/promql/src/functions/changes.rs +++ b/src/promql/src/functions/changes.rs @@ -43,8 +43,8 @@ mod test { use super::*; use crate::functions::test_util::{ - self, STALE_NAN, TinyPrng, assert_execution_error, build_test_range_arrays, - invoke_range_udf, simple_range_udf_runner, + self, STALE_NAN, assert_execution_error, build_test_range_arrays, invoke_range_udf, + simple_range_udf_runner, }; use crate::range_array::RangeArray; @@ -151,43 +151,17 @@ mod test { changes_oracle, Changes::scalar_udf(), ); - assert_eq!(expected, vec![Some(2.0), Some(0.0)]); + assert_eq!(expected, vec![Some(0.0), None]); } #[test] fn changes_range_array_seeded_differential() { - let mut prng = TinyPrng(0x2f6e_2b1d_834a_90c5); - let raw_values = (0..48) - .map(|_| match prng.next_index(12) { - 0 => -0.0, - 1 => 0.0, - 2 => -2.0, - 3 => -1.0, - 4 => 1.0, - 5 => 2.0, - 6 => f64::INFINITY, - 7 => f64::NEG_INFINITY, - 8 | 9 => f64::NAN, - _ => STALE_NAN, - }) - .collect::>(); - let values = raw_values.iter().copied().map(Some).collect(); - let mut timestamp_ranges = Vec::new(); - let mut value_ranges = Vec::new(); - for _ in 0..32 { - let length = prng.next_index(13) as u32; - timestamp_ranges.push((prng.next_index(65 - length as usize) as u32, length)); - value_ranges.push((prng.next_index(49 - length as usize) as u32, length)); - } + test_util::run_seeded_differential(changes_oracle, Changes::scalar_udf(), false); + } - test_util::run_oracle_ranges( - values, - raw_values, - timestamp_ranges, - value_ranges, - changes_oracle, - Changes::scalar_udf(), - ); + #[test] + fn changes_range_array_seeded_differential_with_nulls() { + test_util::run_seeded_differential(changes_oracle, Changes::scalar_udf(), true); } #[test] @@ -263,6 +237,8 @@ mod test { vec![Some(0.0), Some(0.0), Some(0.0)], ); + // The raw payload under the null slot would look like two changes; the sample sequence + // is 10 -> 10, which is none. The last window holds no sample at all. let values = Arc::new(Float64Array::new( vec![10.0, 7.0, 10.0].into(), Some(NullBuffer::from(vec![true, false, true])), @@ -276,10 +252,10 @@ mod test { ])); simple_range_udf_runner( Changes::scalar_udf(), - RangeArray::from_ranges(timestamps, [(0, 3)]).unwrap(), - RangeArray::from_ranges(values, [(0, 3)]).unwrap(), + RangeArray::from_ranges(timestamps, [(0, 3), (1, 1)]).unwrap(), + RangeArray::from_ranges(values, [(0, 3), (1, 1)]).unwrap(), vec![], - vec![Some(2.0)], + vec![Some(0.0), None], ); } } diff --git a/src/promql/src/functions/edge_count.rs b/src/promql/src/functions/edge_count.rs index 9109c742073..7cd5907a5ee 100644 --- a/src/promql/src/functions/edge_count.rs +++ b/src/promql/src/functions/edge_count.rs @@ -81,7 +81,11 @@ fn calc( .unwrap(); let requested_edges = validate_windows(×tamp_ranges, &value_ranges, name)?; let raw_values = values.values(); - let direct = should_scan_direct(requested_edges, raw_values.len()); + // A NULL field value means the series has no sample at that timestamp. The prefix sums + // encode edges between physically adjacent slots, which no longer holds once nulls are + // skipped, so a null-bearing input falls back to scanning each window. + let has_nulls = values.null_count() > 0; + let direct = has_nulls || should_scan_direct(requested_edges, raw_values.len()); let prefix = (!direct).then(|| build_prefix(raw_values.as_ref(), kind)); let mut result = Vec::with_capacity(value_ranges.len()); @@ -90,6 +94,7 @@ fn calc( let end = checked_end(offset, len, index, name)?; let count = match len { 0 => None, + _ if has_nulls => count_edges_skipping_nulls(values, offset, end, kind), 1 => Some(0), _ if direct => Some(count_edges(raw_values.as_ref(), offset, end, kind)), _ => { @@ -172,6 +177,29 @@ fn count_edges(values: &[f64], offset: usize, end: usize, kind: EdgeKind) -> u64 count } +/// Counts edges between consecutive samples in `[offset, end)`, treating null slots as +/// absent. Returns `None` when the window holds no sample. +fn count_edges_skipping_nulls( + values: &Float64Array, + offset: usize, + end: usize, + kind: EdgeKind, +) -> Option { + let raw_values = values.values(); + let mut count = 0; + let mut previous = None; + for index in offset..end { + if values.is_null(index) { + continue; + } + let current = raw_values[index]; + if let Some(previous) = previous.replace(current) { + count += u64::from(is_edge(previous, current, kind)); + } + } + previous.is_some().then_some(count) +} + fn is_edge(previous: f64, current: f64, kind: EdgeKind) -> bool { match kind { EdgeKind::Changes => previous != current && !(previous.is_nan() && current.is_nan()), diff --git a/src/promql/src/functions/extrapolate_rate.rs b/src/promql/src/functions/extrapolate_rate.rs index f072400d3ed..263ae6337ba 100644 --- a/src/promql/src/functions/extrapolate_rate.rs +++ b/src/promql/src/functions/extrapolate_rate.rs @@ -181,12 +181,16 @@ impl ExtrapolatedRate() .expect("validated by extract_range_dict") .values(); - let all_values = value_dict + let value_array = value_dict .values() .as_any() .downcast_ref::() - .expect("validated by extract_range_dict") - .values(); + .expect("validated by extract_range_dict"); + // A NULL field value means the series has no sample at that timestamp, so the padding + // under a null slot must not be read. Skip the per-window null scan when the whole + // backing array is null-free, which is the common case. + let has_nulls = value_array.null_count() > 0; + let all_values = value_array.values(); let eval_ts = eval_ts_array.values(); let mut result_builder = Float64Builder::with_capacity(num_windows); @@ -198,7 +202,7 @@ impl ExtrapolatedRate ExtrapolatedRate bounds, + None => { + result_builder.append_null(); + continue; + } + } + } else { + (offset, end.saturating_sub(1), length) + }; + + if sample_count < 2 { result_builder.append_null(); continue; } - let end = offset + length; - let first_value = all_values[offset]; - let last_value = all_values[end - 1]; + let first_value = all_values[first_index]; + let last_value = all_values[last_index]; let mut result_value = last_value - first_value; if IS_COUNTER { - result_value = match &mut reset_index { - Some(reset_index) => reset_index.add_resets(result_value, offset, end), - None => add_counter_resets(result_value, &all_values[offset..end]), + result_value = if has_nulls { + add_counter_resets_between_samples( + result_value, + value_array, + first_index, + last_index, + ) + } else { + match &mut reset_index { + Some(reset_index) => reset_index.add_resets(result_value, offset, end), + None => add_counter_resets(result_value, &all_values[offset..end]), + } }; } - let first_ts = all_timestamps[offset]; - let last_ts = all_timestamps[end - 1]; + let first_ts = all_timestamps[first_index]; + let last_ts = all_timestamps[last_index]; let range_end = eval_ts[index]; let range_start = range_end - range_length; let sampled_interval_ms = (last_ts - first_ts) as f64; - let average_interval_ms = sampled_interval_ms / (length - 1) as f64; + let average_interval_ms = sampled_interval_ms / (sample_count - 1) as f64; let mut duration_to_start_ms = (first_ts - range_start) as f64; - let duration_to_end_ms = (range_end - last_ts) as f64; + let mut duration_to_end_ms = (range_end - last_ts) as f64; + let extrapolation_threshold = average_interval_ms * 1.1; - // Counters cannot be negative, so Prometheus allows the extrapolation window to snap - // back to the inferred zero point instead of extending into negative values. + // Mirror Prometheus extrapolation: extend to the real range boundary when a sample is + // close enough, otherwise only half an average sampling interval, which is the guess + // for where the series actually starts or ends. + if duration_to_start_ms >= extrapolation_threshold { + duration_to_start_ms = average_interval_ms / 2.0; + } + // Counters cannot be negative, so the extrapolation can snap back to the inferred + // zero point instead of extending into negative values. Prometheus applies this + // after the threshold clamp, so it can only shorten the leading extrapolation. if IS_COUNTER && result_value > 0.0 && first_value >= 0.0 { let duration_to_zero = sampled_interval_ms * (first_value / result_value); if duration_to_zero < duration_to_start_ms { duration_to_start_ms = duration_to_zero; } } - - let extrapolation_threshold = average_interval_ms * 1.1; - let mut extrapolated_interval_ms = sampled_interval_ms; - - // Mirror Prometheus extrapolation: extend to the real range boundary when a sample is - // close enough, otherwise add half an average sampling interval on that side. - if duration_to_start_ms < extrapolation_threshold { - extrapolated_interval_ms += duration_to_start_ms; - } else { - extrapolated_interval_ms += average_interval_ms / 2.0; - } - if duration_to_end_ms < extrapolation_threshold { - extrapolated_interval_ms += duration_to_end_ms; - } else { - extrapolated_interval_ms += average_interval_ms / 2.0; + if duration_to_end_ms >= extrapolation_threshold { + duration_to_end_ms = average_interval_ms / 2.0; } - let mut factor = extrapolated_interval_ms / sampled_interval_ms; + // Samples sharing one timestamp leave nothing to extrapolate over. + let mut factor = if sampled_interval_ms == 0.0 { + 1.0 + } else { + (sampled_interval_ms + duration_to_start_ms + duration_to_end_ms) + / sampled_interval_ms + }; if IS_RATE { factor /= range_length_secs; @@ -381,6 +407,52 @@ impl<'a> CounterResetIndex<'a> { } } +/// Same additions [`add_counter_resets`] performs, over the samples in `[first, last]` instead +/// of over every slot. +fn add_counter_resets_between_samples( + result: f64, + values: &Float64Array, + first: usize, + last: usize, +) -> f64 { + let raw_values = values.values(); + let mut result = result; + let mut previous = raw_values[first]; + for index in first + 1..=last { + if values.is_null(index) { + continue; + } + let current = raw_values[index]; + if current < previous { + result += previous; + } + previous = current; + } + result +} + +/// Locates the samples inside `[offset, offset + length)`, returning the first and last +/// non-null index together with the number of non-null slots. Returns `None` when the +/// window holds no sample. +fn valid_window_bounds( + values: &Float64Array, + offset: usize, + length: usize, +) -> Option<(usize, usize, usize)> { + let mut first = None; + let mut last = 0; + let mut count = 0; + for index in offset..offset + length { + if values.is_null(index) { + continue; + } + first.get_or_insert(index); + last = index; + count += 1; + } + first.map(|first| (first, last, count)) +} + fn extract_eval_timestamps( columnar_value: &ColumnarValue, func_name: &str, @@ -453,9 +525,11 @@ impl Display for ExtrapolatedRate { mod test { use datafusion::arrow::array::ArrayRef; + use datafusion::arrow::buffer::NullBuffer; use datafusion_common::ScalarValue; use super::*; + use crate::functions::test_util::TinyPrng; /// Range length is fixed to 5 fn extrapolated_rate_runner( @@ -607,6 +681,263 @@ mod test { assert_counter_windows_match_single(&values, &ranges); } + /// Builds a value array whose null slots keep a distinguishable raw payload, so a + /// function that reads the padding instead of the samples produces a different result. + fn values_with_nulls(values: Vec>, padding: f64) -> Arc { + let raw = values + .iter() + .map(|value| value.unwrap_or(padding)) + .collect::>(); + Arc::new(Float64Array::new( + raw.into(), + Some(NullBuffer::from_iter( + values.iter().map(|value| value.is_some()), + )), + )) + } + + fn nullable_rate_runner( + timestamps: Vec, + values: Arc, + ranges: Vec<(u32, u32)>, + eval_timestamps: Vec, + range_length: i64, + ) -> Vec> { + let ts_array = Arc::new(TimestampMillisecondArray::from_iter_values(timestamps)); + let ts_range = RangeArray::from_ranges(ts_array, ranges.clone()).unwrap(); + let value_range = RangeArray::from_ranges(values, ranges).unwrap(); + let input = vec![ + ColumnarValue::Array(Arc::new(ts_range.into_dict())), + ColumnarValue::Array(Arc::new(value_range.into_dict())), + ColumnarValue::Array(Arc::new(TimestampMillisecondArray::from_iter_values( + eval_timestamps, + ))), + ColumnarValue::Array(Arc::new(Int64Array::from(vec![range_length]))), + ]; + let output = extract_array( + &ExtrapolatedRate::::new(range_length) + .calc(&input) + .unwrap(), + ) + .unwrap(); + let output = output.as_any().downcast_ref::().unwrap(); + output.iter().collect() + } + + #[test] + fn rate_uses_samples_not_null_padding() { + // Samples are 1.0@0 and 4.0@3000; the padding under the null slots would add two more. + let output = nullable_rate_runner::( + vec![0, 1000, 2000, 3000], + values_with_nulls(vec![Some(1.0), None, None, Some(4.0)], 99.0), + vec![(0, 4)], + vec![3000], + 4000, + ); + + assert_eq!(output, vec![Some(1.0)]); + } + + #[test] + fn rate_returns_null_for_windows_without_enough_samples() { + let output = nullable_rate_runner::( + vec![0, 1000, 2000], + values_with_nulls(vec![None, Some(2.0), None], 7.0), + vec![(0, 3), (0, 2), (2, 1)], + vec![2000, 2000, 2000], + 4000, + ); + + assert_eq!(output, vec![None, None, None]); + } + + #[test] + fn increase_corrects_counter_reset_between_samples() { + // The sample sequence is 5.0 -> 3.0, one reset. Reading the padding would see + // 5.0 -> 100.0 -> 3.0 and charge the correction against the wrong value. + let output = nullable_rate_runner::( + vec![0, 1000, 2000], + values_with_nulls(vec![Some(5.0), None, Some(3.0)], 100.0), + vec![(0, 3)], + vec![2000], + 2000, + ); + + assert_eq!(output, vec![Some(3.0)]); + } + + #[test] + fn delta_extrapolates_from_sample_timestamps() { + // The window spans (-1000, 3000] but its samples only cover 1000..2000, so the + // extrapolation adds half an average interval on the leading side. + let output = nullable_rate_runner::( + vec![0, 1000, 2000, 3000], + values_with_nulls(vec![None, Some(2.0), Some(5.0), None], 42.0), + vec![(0, 4)], + vec![3000], + 4000, + ); + + assert_eq!(output, vec![Some(7.5)]); + } + + /// Line-by-line port of Prometheus `extrapolatedRate` (promql/functions.go), float path + /// without start timestamps. Kept as a second implementation so that the order of the + /// threshold clamp and the counter zero-snap stays pinned to the upstream one. + fn prometheus_extrapolated_rate( + timestamps: &[i64], + values: &[f64], + eval_ts: i64, + range_ms: i64, + is_counter: bool, + is_rate: bool, + ) -> Option { + if values.len() < 2 { + return None; + } + let num_samples_minus_one = values.len() - 1; + let first_t = timestamps[0]; + let last_t = timestamps[num_samples_minus_one]; + let mut result = values[num_samples_minus_one] - values[0]; + if is_counter { + for index in 1..values.len() { + if values[index] < values[index - 1] { + result += values[index - 1]; + } + } + } + + let range_start = eval_ts - range_ms; + let mut duration_to_start = (first_t - range_start) as f64 / 1000.0; + let mut duration_to_end = (eval_ts - last_t) as f64 / 1000.0; + let sampled_interval = (last_t - first_t) as f64 / 1000.0; + let average_duration_between_samples = sampled_interval / num_samples_minus_one as f64; + let extrapolation_threshold = average_duration_between_samples * 1.1; + + if duration_to_start >= extrapolation_threshold { + duration_to_start = average_duration_between_samples / 2.0; + } + if is_counter { + let mut duration_to_zero = duration_to_start; + if result > 0.0 && values[0] >= 0.0 { + duration_to_zero = sampled_interval * (values[0] / result); + } + if duration_to_zero < duration_to_start { + duration_to_start = duration_to_zero; + } + } + if duration_to_end >= extrapolation_threshold { + duration_to_end = average_duration_between_samples / 2.0; + } + + let mut factor = 1.0; + if sampled_interval != 0.0 { + factor = (sampled_interval + duration_to_start + duration_to_end) / sampled_interval; + } + if is_rate { + factor /= range_ms as f64 / 1000.0; + } + Some(result * factor) + } + + fn assert_matches_prometheus( + timestamps: &[i64], + values: &[f64], + ranges: &[(u32, u32)], + eval_timestamps: &[i64], + range_ms: i64, + ) { + let actual = nullable_rate_runner::( + timestamps.to_vec(), + Arc::new(Float64Array::from(values.to_vec())), + ranges.to_vec(), + eval_timestamps.to_vec(), + range_ms, + ); + + for (index, ((offset, length), eval_ts)) in ranges.iter().zip(eval_timestamps).enumerate() { + let window = *offset as usize..(*offset + *length) as usize; + let expected = prometheus_extrapolated_rate( + ×tamps[window.clone()], + &values[window], + *eval_ts, + range_ms, + IS_COUNTER, + IS_RATE, + ); + match (actual[index], expected) { + (None, None) => {} + (Some(actual), Some(expected)) => assert!( + (actual - expected).abs() <= expected.abs() * 1e-9, + "window {index} {:?}: got {actual}, Prometheus gives {expected}", + ranges[index] + ), + (actual, expected) => { + panic!("window {index}: got {actual:?}, Prometheus gives {expected:?}") + } + } + } + } + + #[test] + fn extrapolation_matches_prometheus_on_seeded_windows() { + let mut prng = TinyPrng(0x51ed_270b_8f26_1a37); + // Uneven spacing so the average interval, and with it the extrapolation threshold, + // differs from window to window. + let timestamps = (0..48) + .scan(0i64, |clock, _| { + *clock += 1_000 + prng.next_index(4) as i64 * 500; + Some(*clock) + }) + .collect::>(); + // A counter that resets a few times, so the zero-snap branch is reached with both + // small and large leading values. + let values = (0..48) + .scan(0.0f64, |counter, _| { + *counter = match prng.next_index(8) { + 0 => 0.0, + 1 => *counter / 2.0, + _ => *counter + prng.next_index(50) as f64, + }; + Some(*counter) + }) + .collect::>(); + + let mut ranges = Vec::new(); + let mut eval_timestamps = Vec::new(); + for _ in 0..32 { + let length = 2 + prng.next_index(10) as u32; + let offset = prng.next_index(48 - length as usize) as u32; + ranges.push((offset, length)); + // Land the range boundary at varying distances from the samples, so the clamp + // fires on neither, one, or both sides. + let last = timestamps[(offset + length - 1) as usize]; + eval_timestamps.push(last + prng.next_index(5) as i64 * 500); + } + + assert_matches_prometheus::( + ×tamps, + &values, + &ranges, + &eval_timestamps, + 20_000, + ); + assert_matches_prometheus::( + ×tamps, + &values, + &ranges, + &eval_timestamps, + 20_000, + ); + assert_matches_prometheus::( + ×tamps, + &values, + &ranges, + &eval_timestamps, + 20_000, + ); + } + #[test] fn rate_rejects_wrong_input_arity() { let err = ExtrapolatedRate::::new(5) @@ -684,7 +1015,7 @@ mod test { ts_range, value_range, timestamps, - vec![2.0, 5.0, 0.0, 2.5, 0.0, 0.0], + vec![1.5, 5.0, 0.0, 2.5, 0.0, 0.0], ); } @@ -715,8 +1046,7 @@ mod test { ts_range, value_range, timestamps, - // `2.0` is because that `duration_to_zero` less than `extrapolation_threshold` - vec![2.0, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5], + vec![1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5], ); } @@ -789,7 +1119,7 @@ mod test { // that two `2.0` is because `duration_to_start` are shrunk to // `duration_to_zero`, and causes `duration_to_zero` less than // `extrapolation_threshold`. - vec![2.0, 1.5, 1.5, 1.5, 2.0, 1.5, 1.5, 1.5], + vec![1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5], ); } @@ -809,7 +1139,7 @@ mod test { ts_range, value_range, timestamps, - vec![4.0, 3.5, 3.5, 4.0], + vec![3.5, 3.5, 3.5, 3.5], ); } @@ -984,7 +1314,7 @@ mod test { ts_range, value_range, timestamps, - vec![400.0, 300.0, 300.0, 300.0, 400.0, 300.0, 300.0, 300.0], + vec![300.0, 300.0, 300.0, 300.0, 300.0, 300.0, 300.0, 300.0], ); } @@ -1015,7 +1345,7 @@ mod test { ts_range, value_range, timestamps, - vec![400.0, 300.0, 300.0, 300.0, 300.0, 300.0, 300.0, 300.0], + vec![300.0, 300.0, 300.0, 300.0, 300.0, 300.0, 300.0, 300.0], ); } diff --git a/src/promql/src/functions/idelta.rs b/src/promql/src/functions/idelta.rs index 0772b0bf1ee..35ce44ba270 100644 --- a/src/promql/src/functions/idelta.rs +++ b/src/promql/src/functions/idelta.rs @@ -101,12 +101,12 @@ impl IDelta { .unwrap() .values(); - let value_values = value_range.values(); - let value_values = value_values - .as_any() - .downcast_ref::() - .unwrap() - .values(); + let value_array = value_range.values(); + let value_array = value_array.as_any().downcast_ref::().unwrap(); + // A NULL field value means the series has no sample at that timestamp, so the last two + // samples are not necessarily the last two slots. + let has_nulls = value_array.null_count() > 0; + let value_values = value_array.values(); let mut result_builder = Float64Builder::with_capacity(ts_range.len()); @@ -122,20 +122,29 @@ impl IDelta { value_len )), )?; - if len < 2 { - result_builder.append_null(); - continue; - } + let (last_position, prev_position) = if has_nulls { + match last_two_samples(value_array, value_offset, len) { + Some(positions) => positions, + None => { + result_builder.append_null(); + continue; + } + } + } else { + if len < 2 { + result_builder.append_null(); + continue; + } + (len - 1, len - 2) + }; - let last_offset = ts_offset + len - 1; - let prev_offset = last_offset - 1; + let last_offset = ts_offset + last_position; + let prev_offset = ts_offset + prev_position; let sampled_interval = (ts_values[last_offset] - ts_values[prev_offset]) as f64 / 1000.0; - let last_value_offset = value_offset + len - 1; - let prev_value_offset = last_value_offset - 1; - let last_value = value_values[last_value_offset]; - let prev_value = value_values[prev_value_offset]; + let last_value = value_values[value_offset + last_position]; + let prev_value = value_values[value_offset + prev_position]; if !IS_RATE { result_builder.append_value(last_value - prev_value); @@ -157,6 +166,22 @@ impl IDelta { } } +/// Locates the last two samples inside `[offset, offset + len)`, returning their positions +/// relative to `offset`. Returns `None` when the window holds fewer than two samples. +fn last_two_samples(values: &Float64Array, offset: usize, len: usize) -> Option<(usize, usize)> { + let mut last = None; + for position in (0..len).rev() { + if values.is_null(offset + position) { + continue; + } + match last { + None => last = Some(position), + Some(last) => return Some((last, position)), + } + } + None +} + impl Display for IDelta { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "PromQL Idelta Function (is_rate: {IS_RATE})",) @@ -166,6 +191,8 @@ impl Display for IDelta { #[cfg(test)] mod test { + use datafusion::arrow::buffer::NullBuffer; + use super::*; use crate::functions::test_util::simple_range_udf_runner; @@ -207,4 +234,33 @@ mod test { vec![Some(0.5), Some(0.0), None, Some(3.0), None, None], ); } + + #[test] + fn idelta_uses_last_two_samples_not_last_two_slots() { + let ts_array = Arc::new(TimestampMillisecondArray::from_iter_values([ + 0i64, 1000, 2000, 3000, + ])); + // Samples are 1.0@0 and 4.0@1000; the trailing slots only carry padding. + let values_array = Arc::new(Float64Array::new( + vec![1.0, 4.0, 100.0, 200.0].into(), + Some(NullBuffer::from_iter([true, true, false, false])), + )); + let ranges = [(0, 4), (2, 2), (3, 1)]; + + simple_range_udf_runner( + IDelta::::scalar_udf(), + RangeArray::from_ranges(ts_array.clone(), ranges).unwrap(), + RangeArray::from_ranges(values_array.clone(), ranges).unwrap(), + vec![], + vec![Some(3.0), None, None], + ); + + simple_range_udf_runner( + IDelta::::scalar_udf(), + RangeArray::from_ranges(ts_array, ranges).unwrap(), + RangeArray::from_ranges(values_array, ranges).unwrap(), + vec![], + vec![Some(3.0), None, None], + ); + } } diff --git a/src/promql/src/functions/native_histogram.rs b/src/promql/src/functions/native_histogram.rs index 2d0fa16b087..1fcb9c553ad 100644 --- a/src/promql/src/functions/native_histogram.rs +++ b/src/promql/src/functions/native_histogram.rs @@ -14,7 +14,6 @@ //! Native histogram PromQL helpers. -use std::any::Any; use std::hash::{Hash, Hasher}; use std::mem::size_of; use std::sync::Arc; @@ -170,10 +169,6 @@ impl Hash for NativeHistogramAnnotationUdf { } impl ScalarUDFImpl for NativeHistogramAnnotationUdf { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.name } @@ -1517,10 +1512,6 @@ impl Hash for MixedRangeUdf { } impl ScalarUDFImpl for MixedRangeUdf { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.output.name() } diff --git a/src/promql/src/functions/quantile.rs b/src/promql/src/functions/quantile.rs index 35f0c1aa81b..3794057da5e 100644 --- a/src/promql/src/functions/quantile.rs +++ b/src/promql/src/functions/quantile.rs @@ -93,14 +93,14 @@ impl QuantileOverTime { )), )?; - let all_values = value_range - .values() - .as_any() - .downcast_ref::() - .unwrap() - .values(); + let value_array = value_range.values(); + let value_array = value_array.as_any().downcast_ref::().unwrap(); + // A NULL field value means the series has no sample at that timestamp, so a window's + // samples are not simply its slots. + let has_nulls = value_array.null_count() > 0; let mut result_builder = Float64Builder::with_capacity(ts_range.len()); let mut scratch = Vec::new(); + let mut samples = Vec::new(); match quantile_col { ColumnarValue::Scalar(quantile_scalar) => { @@ -125,11 +125,14 @@ impl QuantileOverTime { )), )?; - match quantile_with_scratch( - &all_values[value_offset..value_offset + value_len], - quantile, - &mut scratch, - ) { + let window = window_samples( + value_array, + has_nulls, + value_offset, + value_len, + &mut samples, + ); + match window_quantile(window, quantile, &mut scratch) { Some(value) => result_builder.append_value(value), None => result_builder.append_null(), } @@ -173,11 +176,14 @@ impl QuantileOverTime { } else { quantile_array.value(index) }; - match quantile_with_scratch( - &all_values[value_offset..value_offset + value_len], - quantile, - &mut scratch, - ) { + let window = window_samples( + value_array, + has_nulls, + value_offset, + value_len, + &mut samples, + ); + match window_quantile(window, quantile, &mut scratch) { Some(value) => result_builder.append_value(value), None => result_builder.append_null(), } @@ -190,6 +196,40 @@ impl QuantileOverTime { } } +/// Returns the samples of the window `[offset, offset + len)`, collecting the non-null ones +/// into `samples` when the backing array has nulls and borrowing the slice otherwise. +fn window_samples<'a>( + values: &'a Float64Array, + has_nulls: bool, + offset: usize, + len: usize, + samples: &'a mut Vec, +) -> &'a [f64] { + let raw_values = values.values(); + if !has_nulls { + return &raw_values[offset..offset + len]; + } + samples.clear(); + samples.extend( + (offset..offset + len) + .filter(|index| values.is_valid(*index)) + .map(|index| raw_values[index]), + ); + samples +} + +/// Quantile of one range window, or `None` when the window holds no sample. +/// +/// Prometheus returns an empty vector for a range without float samples rather than the NaN +/// that [`quantile_impl`] yields for an empty slice, so the emptiness check belongs here and +/// not in the shared kernel. +fn window_quantile(values: &[f64], quantile: f64, scratch: &mut Vec) -> Option { + if values.is_empty() { + return None; + } + quantile_with_scratch(values, quantile, scratch) +} + /// Refer to pub(crate) fn quantile_impl(values: &[f64], quantile: f64) -> Option { let mut scratch = Vec::new(); @@ -226,6 +266,9 @@ fn quantile_with_scratch(values: &[f64], quantile: f64, scratch: &mut Vec) #[cfg(test)] mod tests { + use datafusion::arrow::array::TimestampMillisecondArray; + use datafusion::arrow::buffer::NullBuffer; + use super::*; #[test] @@ -276,4 +319,64 @@ mod tests { let q = 0.25; assert_eq!(quantile_impl(values, q).unwrap(), 2.0); } + + #[test] + fn quantile_over_time_ranks_samples_only() { + let ts_array = Arc::new(TimestampMillisecondArray::from_iter_values([ + 0i64, 1000, 2000, + ])); + // Samples are 1.0 and 4.0; ranking the padding too would pull the median down. + let values_array = Arc::new(Float64Array::new( + vec![1.0, -100.0, 4.0].into(), + Some(NullBuffer::from_iter([true, false, true])), + )); + // The second window holds no sample, the third holds no slot at all. + let ranges = [(0, 3), (1, 1), (3, 0)]; + + let input = vec![ + ColumnarValue::Array(Arc::new( + RangeArray::from_ranges(ts_array, ranges) + .unwrap() + .into_dict(), + )), + ColumnarValue::Array(Arc::new( + RangeArray::from_ranges(values_array, ranges) + .unwrap() + .into_dict(), + )), + ColumnarValue::Scalar(ScalarValue::Float64(Some(0.5))), + ]; + let output = extract_array(&QuantileOverTime::quantile_over_time(&input).unwrap()).unwrap(); + let output = output.as_any().downcast_ref::().unwrap(); + + assert_eq!( + output.iter().collect::>(), + vec![Some(2.5), None, None] + ); + } + + #[test] + fn quantile_over_time_keeps_nan_for_an_invalid_quantile() { + let ts_array = Arc::new(TimestampMillisecondArray::from_iter_values([0i64, 1000])); + let values_array = Arc::new(Float64Array::from_iter_values([1.0, 4.0])); + let ranges = [(0, 2)]; + + let input = vec![ + ColumnarValue::Array(Arc::new( + RangeArray::from_ranges(ts_array, ranges) + .unwrap() + .into_dict(), + )), + ColumnarValue::Array(Arc::new( + RangeArray::from_ranges(values_array, ranges) + .unwrap() + .into_dict(), + )), + ColumnarValue::Scalar(ScalarValue::Float64(None)), + ]; + let output = extract_array(&QuantileOverTime::quantile_over_time(&input).unwrap()).unwrap(); + let output = output.as_any().downcast_ref::().unwrap(); + + assert!(output.value(0).is_nan()); + } } diff --git a/src/promql/src/functions/quantile_aggr.rs b/src/promql/src/functions/quantile_aggr.rs index 6d755cadf7b..c818d6220a3 100644 --- a/src/promql/src/functions/quantile_aggr.rs +++ b/src/promql/src/functions/quantile_aggr.rs @@ -82,7 +82,6 @@ impl QuantileAccumulator { } let q = match &args.exprs[0] - .as_any() .downcast_ref::() .map(|lit| lit.value()) { diff --git a/src/promql/src/functions/resets.rs b/src/promql/src/functions/resets.rs index 66e83cde311..7a5c0db71c1 100644 --- a/src/promql/src/functions/resets.rs +++ b/src/promql/src/functions/resets.rs @@ -43,8 +43,8 @@ mod test { use super::*; use crate::functions::test_util::{ - self, STALE_NAN, TinyPrng, assert_execution_error, build_test_range_arrays, - invoke_range_udf, simple_range_udf_runner, + self, STALE_NAN, assert_execution_error, build_test_range_arrays, invoke_range_udf, + simple_range_udf_runner, }; use crate::range_array::RangeArray; @@ -151,43 +151,17 @@ mod test { resets_oracle, Resets::scalar_udf(), ); - assert_eq!(expected, vec![Some(1.0), Some(0.0)]); + assert_eq!(expected, vec![Some(0.0), None]); } #[test] fn resets_range_array_seeded_differential() { - let mut prng = TinyPrng(0x2f6e_2b1d_834a_90c5); - let raw_values = (0..48) - .map(|_| match prng.next_index(12) { - 0 => -0.0, - 1 => 0.0, - 2 => -2.0, - 3 => -1.0, - 4 => 1.0, - 5 => 2.0, - 6 => f64::INFINITY, - 7 => f64::NEG_INFINITY, - 8 | 9 => f64::NAN, - _ => STALE_NAN, - }) - .collect::>(); - let values = raw_values.iter().copied().map(Some).collect(); - let mut timestamp_ranges = Vec::new(); - let mut value_ranges = Vec::new(); - for _ in 0..32 { - let length = prng.next_index(13) as u32; - timestamp_ranges.push((prng.next_index(65 - length as usize) as u32, length)); - value_ranges.push((prng.next_index(49 - length as usize) as u32, length)); - } + test_util::run_seeded_differential(resets_oracle, Resets::scalar_udf(), false); + } - test_util::run_oracle_ranges( - values, - raw_values, - timestamp_ranges, - value_ranges, - resets_oracle, - Resets::scalar_udf(), - ); + #[test] + fn resets_range_array_seeded_differential_with_nulls() { + test_util::run_seeded_differential(resets_oracle, Resets::scalar_udf(), true); } #[test] @@ -263,6 +237,8 @@ mod test { vec![Some(0.0), Some(0.0), Some(0.0)], ); + // The raw payload under the null slot would look like a reset; the sample sequence is + // 10 -> 10, which is none. The last window holds no sample at all. let values = Arc::new(Float64Array::new( vec![10.0, 7.0, 10.0].into(), Some(NullBuffer::from(vec![true, false, true])), @@ -276,10 +252,10 @@ mod test { ])); simple_range_udf_runner( Resets::scalar_udf(), - RangeArray::from_ranges(timestamps, [(0, 3)]).unwrap(), - RangeArray::from_ranges(values, [(0, 3)]).unwrap(), + RangeArray::from_ranges(timestamps, [(0, 3), (1, 1)]).unwrap(), + RangeArray::from_ranges(values, [(0, 3), (1, 1)]).unwrap(), vec![], - vec![Some(1.0)], + vec![Some(0.0), None], ); } } diff --git a/src/promql/src/functions/test_util.rs b/src/promql/src/functions/test_util.rs index e52f52e3bbb..6148e06c391 100644 --- a/src/promql/src/functions/test_util.rs +++ b/src/promql/src/functions/test_util.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use datafusion::arrow::array::{Float64Array, TimestampMillisecondArray}; +use datafusion::arrow::buffer::NullBuffer; use datafusion::common::DataFusionError; use datafusion::logical_expr::ScalarUDF; use datafusion::physical_plan::ColumnarValue; @@ -144,12 +145,54 @@ impl TinyPrng { } } -/// Run the oracle-based differential test: build range arrays, verify raw bits and -/// null validity, invoke the UDF via [`simple_range_udf_runner`], and return the -/// expected values for further assertions. +/// Run [`run_oracle_ranges`] over a seeded mix of signed zeros, infinities, NaNs and stale +/// markers spread across random windows. `nullable` marks roughly a quarter of the slots as +/// null while keeping their raw payload, which moves the UDF off its null-free fast path. +pub fn run_seeded_differential(oracle: fn(&[f64]) -> Option, udf: ScalarUDF, nullable: bool) { + let mut prng = TinyPrng(0x2f6e_2b1d_834a_90c5); + let raw_values = (0..48) + .map(|_| match prng.next_index(12) { + 0 => -0.0, + 1 => 0.0, + 2 => -2.0, + 3 => -1.0, + 4 => 1.0, + 5 => 2.0, + 6 => f64::INFINITY, + 7 => f64::NEG_INFINITY, + 8 | 9 => f64::NAN, + _ => STALE_NAN, + }) + .collect::>(); + let values = raw_values + .iter() + .map(|value| (!nullable || prng.next_index(4) != 0).then_some(*value)) + .collect::>(); + let mut timestamp_ranges = Vec::new(); + let mut value_ranges = Vec::new(); + for _ in 0..32 { + let length = prng.next_index(13) as u32; + timestamp_ranges.push((prng.next_index(65 - length as usize) as u32, length)); + value_ranges.push((prng.next_index(49 - length as usize) as u32, length)); + } + + run_oracle_ranges( + values, + raw_values, + timestamp_ranges, + value_ranges, + oracle, + udf, + ); +} + +/// Run the oracle-based differential test: build range arrays, invoke the UDF via +/// [`simple_range_udf_runner`], and return the expected values for further assertions. /// /// `oracle` is the behavior-specific function (e.g. `changes_oracle` or `resets_oracle`) -/// that computes the expected count for a slice of raw f64 values. +/// that computes the expected count for a window's samples. Null slots are dropped before +/// the oracle runs: a NULL field value means the series has no sample at that timestamp, +/// so the raw payload underneath it is not a sample. pub fn run_oracle_ranges( values: Vec>, raw_values: Vec, @@ -166,29 +209,37 @@ pub fn run_oracle_ranges( .all(|((_, timestamp_length), (_, value_length))| timestamp_length == value_length) ); assert_eq!(values.len(), raw_values.len()); + // `values` marks the null slots, `raw_values` carries the payload of every slot including + // the null ones. The two must agree wherever a sample exists. + for (index, (value, raw)) in values.iter().zip(&raw_values).enumerate() { + assert!( + value.is_none_or(|value| value.to_bits() == raw.to_bits()), + "values[{index}] and raw_values[{index}] disagree on the sample" + ); + } let nulls = values.iter().map(Option::is_none).collect::>(); let expected = value_ranges .iter() - .map(|(offset, length)| oracle(&raw_values[*offset as usize..(*offset + *length) as usize])) + .map(|(offset, length)| { + let samples = (*offset as usize..(*offset + *length) as usize) + .filter(|index| !nulls[*index]) + .map(|index| raw_values[index]) + .collect::>(); + oracle(&samples) + }) .collect::>(); let timestamp_values = (0..64) .map(|value| Some(i64::from(value) * 1_000)) .collect::>(); let timestamp_array = Arc::new(TimestampMillisecondArray::from_iter(timestamp_values)); - let value_array = Arc::new(Float64Array::from_iter(values)); - for (index, ((actual, expected), is_null)) in value_array - .values() - .iter() - .zip(&raw_values) - .zip(nulls) - .enumerate() - { - assert_eq!(actual.to_bits(), expected.to_bits()); - assert_eq!(value_array.is_null(index), is_null); - } - + // Build from the raw payload plus a null bitmap instead of `from_iter`, which would zero + // the null slots and hide a function that reads them. + let value_array = Arc::new(Float64Array::new( + raw_values.clone().into(), + Some(NullBuffer::from_iter(nulls.iter().map(|null| !null))), + )); let timestamp_ranges = RangeArray::from_ranges(timestamp_array, timestamp_ranges).unwrap(); let value_ranges = RangeArray::from_ranges(value_array, value_ranges).unwrap(); simple_range_udf_runner( diff --git a/src/query/Cargo.toml b/src/query/Cargo.toml index 9e43bb53e43..2a32292614f 100644 --- a/src/query/Cargo.toml +++ b/src/query/Cargo.toml @@ -43,6 +43,7 @@ datafusion-expr.workspace = true datafusion-expr-common.workspace = true datafusion-functions.workspace = true datafusion-optimizer.workspace = true +datafusion-pg-catalog.workspace = true datafusion-physical-expr.workspace = true datafusion-proto.workspace = true datafusion-sql.workspace = true diff --git a/src/query/src/analyze.rs b/src/query/src/analyze.rs index dec73e4093c..34f1c69ee30 100644 --- a/src/query/src/analyze.rs +++ b/src/query/src/analyze.rs @@ -16,7 +16,6 @@ //! //! The code skeleton is taken from `datafusion/physical-plan/src/analyze.rs` -use std::any::Any; use std::fmt::Display; use std::sync::Arc; @@ -30,11 +29,12 @@ use datafusion::execution::TaskContext; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, accept, + ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, + InputDistributionRequirements, PlanProperties, ReplaceChildrenOptions, accept, }; use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion}; use datafusion_common::{DataFusionError, assert_eq_or_internal_err, internal_err}; -use datafusion_physical_expr::{Distribution, EquivalenceProperties, Partitioning}; +use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr}; use futures::StreamExt; use serde::Serialize; use serde_json::{Value, json}; @@ -110,7 +110,6 @@ pub fn analyze_plan_metrics_to_json_value( verbose: bool, ) -> serde_json::Result { let input = plan - .as_any() .downcast_ref::() .map(|exec| exec.input().clone()) .unwrap_or_else(|| plan.clone()); @@ -125,7 +124,7 @@ pub fn analyze_plan_metrics_to_json_value( })); let _ = input.apply(|plan| { - if let Some(merge_scan) = plan.as_any().downcast_ref::() { + if let Some(merge_scan) = plan.downcast_ref::() { for (node, metric) in merge_scan.sub_stage_metrics().into_iter().enumerate() { stages.push(json!({ "stage": 1, @@ -157,11 +156,6 @@ impl ExecutionPlan for DistAnalyzeExec { "DistAnalyzeExec" } - /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.properties } @@ -170,14 +164,25 @@ impl ExecutionPlan for DistAnalyzeExec { vec![&self.input] } - /// AnalyzeExec is handled specially so this value is ignored - fn required_input_distribution(&self) -> Vec { - vec![] + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DfResult, + ) -> DfResult { + Ok(TreeNodeRecursion::Continue) } - fn with_new_children( + /// AnalyzeExec is handled specially so this value is ignored + fn input_distribution_requirements(&self) -> InputDistributionRequirements { + // AnalyzeExec is handled specially so this value is ignored. + InputDistributionRequirements::new(vec![ + datafusion_physical_expr::Distribution::UnspecifiedDistribution, + ]) + } + + fn replace_children( self: Arc, mut children: Vec>, + _options: ReplaceChildrenOptions, ) -> DfResult> { assert_eq_or_internal_err!( children.len(), @@ -191,6 +196,17 @@ impl ExecutionPlan for DistAnalyzeExec { ))) } + #[allow(deprecated)] + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DfResult> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + } + fn execute( &self, partition: usize, @@ -293,7 +309,7 @@ fn create_output_batch( // Find merge scan and append its sub_stage_metrics input.apply(|plan| { - if let Some(merge_scan) = plan.as_any().downcast_ref::() { + if let Some(merge_scan) = plan.downcast_ref::() { let sub_stage_metrics = merge_scan.sub_stage_metrics(); for (node, metric) in sub_stage_metrics.into_iter().enumerate() { builder.append_metric(1, node as _, metrics_to_string(metric, format)?); @@ -421,7 +437,14 @@ mod tests { AnalyzeFormat::TEXT, )); - assert!(ExecutionPlan::with_new_children(analyze, vec![]).is_err()); + assert!( + ExecutionPlan::replace_children( + analyze, + vec![], + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + .is_err() + ); } #[test] @@ -432,14 +455,14 @@ mod tests { AnalyzeFormat::TEXT, )); - let result = ExecutionPlan::with_new_children( + let result = ExecutionPlan::replace_children( analyze, vec![empty_plan("first"), empty_plan("second")], + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), ); if let Ok(plan) = result { let retained = plan - .as_any() .downcast_ref::() .unwrap() .input() @@ -460,8 +483,13 @@ mod tests { )); let replacement = empty_plan("replacement"); - let rebuilt = ExecutionPlan::with_new_children(analyze, vec![replacement]).unwrap(); - let rebuilt = rebuilt.as_any().downcast_ref::().unwrap(); + let rebuilt = ExecutionPlan::replace_children( + analyze, + vec![replacement], + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) + .unwrap(); + let rebuilt = rebuilt.downcast_ref::().unwrap(); assert_eq!(rebuilt.input().schema().field(0).name(), "replacement"); } diff --git a/src/query/src/datafusion.rs b/src/query/src/datafusion.rs index 28644b8ccc9..7df99f059ca 100644 --- a/src/query/src/datafusion.rs +++ b/src/query/src/datafusion.rs @@ -16,6 +16,7 @@ mod error; mod json_expr_planner; +mod pg_oid_alias_expr_planner; mod planner; use std::any::Any; @@ -110,7 +111,7 @@ fn query_load_region_id(plan: &Arc) -> Option { while let Some(plan) = stack.pop() { if plan.name() == REGION_SCAN_EXEC_NAME - && let Some(scan) = plan.as_any().downcast_ref::() + && let Some(scan) = plan.downcast_ref::() && let Some(scan_region_id) = scan.query_load_region_id() { match region_id { @@ -139,7 +140,7 @@ fn query_stat_counters(plan: &Arc) -> Option() + && let Some(scan) = plan.downcast_ref::() && let Some(scan_counters) = scan.query_stat_counters() { match &counters { @@ -519,8 +520,7 @@ impl DatafusionQueryEngine { // let config = state.config_options(); // skip optimize AnalyzeExec plan - let optimized_plan = if let Some(analyze_plan) = plan.as_any().downcast_ref::() - { + let optimized_plan = if let Some(analyze_plan) = plan.downcast_ref::() { let format = if let Some(format) = ctx.query_ctx().explain_format() && format.to_lowercase() == "json" { @@ -814,17 +814,20 @@ mod tests { use api::v1::SemanticType; use arrow::array::{ArrayRef, UInt64Array}; use arrow_schema::SortOptions; + use async_trait::async_trait; use catalog::RegisterTableRequest; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, NUMBERS_TABLE_ID}; use common_error::ext::BoxedError; - use common_recordbatch::{EmptyRecordBatchStream, SendableRecordBatchStream, util}; + use common_recordbatch::{ + EmptyRecordBatchStream, RecordBatch, SendableRecordBatchStream, util, + }; use datafusion::physical_plan::display::{DisplayAs, DisplayFormatType}; use datafusion::physical_plan::expressions::PhysicalSortExpr; use datafusion::physical_plan::joins::{HashJoinExec, JoinOn, PartitionMode}; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion::physical_plan::{ExecutionPlan, PhysicalExpr}; use datafusion::prelude::{col, lit}; - use datafusion_common::{JoinType, NullEquality}; + use datafusion_common::{JoinType, NullEquality, ScalarValue}; use datafusion_physical_expr::expressions::Column; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{ColumnSchema, SchemaRef}; @@ -835,12 +838,13 @@ mod tests { PartitionRange, PrepareRequest, QueryScanContext, RegionScanner, ScannerProperties, }; use store_api::storage::{RegionId, ScanRequest}; + use table::metadata::{TableInfoBuilder, TableMetaBuilder}; use table::table::numbers::{NUMBERS_TABLE_NAME, NumbersTable}; use table::table::scan::RegionScanExec; use super::*; use crate::options::QueryOptions; - use crate::parser::QueryLanguageParser; + use crate::parser::{QueryLanguageParser, QueryStatement}; use crate::part_sort::PartSortExec; use crate::query_engine::{QueryEngineFactory, QueryEngineRef}; @@ -1422,4 +1426,341 @@ mod tests { assert!(right_update_calls.load(Ordering::Relaxed) > 0); assert!(right_last_filter_len.load(Ordering::Relaxed) > 0); } + #[derive(Default)] + struct RecordingMutationHandler { + inserts: std::sync::Mutex>, + } + + #[async_trait] + impl common_function::handlers::TableMutationHandler for RecordingMutationHandler { + async fn insert( + &self, + request: table::requests::InsertRequest, + _ctx: session::context::QueryContextRef, + ) -> common_query::error::Result { + self.inserts.lock().unwrap().push(request); + Ok(common_query::Output::new_with_affected_rows(1)) + } + + async fn delete( + &self, + _request: table::requests::DeleteRequest, + _ctx: session::context::QueryContextRef, + ) -> common_query::error::Result { + unimplemented!("unexpected delete") + } + + async fn flush( + &self, + _request: table::requests::FlushTableRequest, + _ctx: session::context::QueryContextRef, + ) -> common_query::error::Result { + unimplemented!("unexpected flush") + } + + async fn compact( + &self, + _request: table::requests::CompactTableRequest, + _ctx: session::context::QueryContextRef, + ) -> common_query::error::Result { + unimplemented!("unexpected compact") + } + + async fn build_index( + &self, + _request: table::requests::BuildIndexTableRequest, + _ctx: session::context::QueryContextRef, + ) -> common_query::error::Result { + unimplemented!("unexpected build_index") + } + + async fn flush_region( + &self, + _region_id: store_api::storage::RegionId, + _ctx: session::context::QueryContextRef, + ) -> common_query::error::Result { + unimplemented!("unexpected flush_region") + } + + async fn compact_region( + &self, + _region_id: store_api::storage::RegionId, + _ctx: session::context::QueryContextRef, + ) -> common_query::error::Result { + unimplemented!("unexpected compact_region") + } + + async fn discard_unflushed_data( + &self, + _region_id: store_api::storage::RegionId, + _ctx: session::context::QueryContextRef, + ) -> common_query::error::Result { + unimplemented!("unexpected discard_unflushed_data") + } + + async fn discard_unflushed_data_by_table( + &self, + _table_name: table::table_name::TableName, + _ctx: session::context::QueryContextRef, + ) -> common_query::error::Result { + unimplemented!("unexpected discard_unflushed_data_by_table") + } + } + + fn native_schema() -> Arc { + Arc::new(Schema::new(vec![ + ColumnSchema::new("dim", ConcreteDataType::date_datatype(), true), + ColumnSchema::new("amount", ConcreteDataType::decimal128_datatype(30, 2), true), + ColumnSchema::new( + "elapsed", + ConcreteDataType::duration_millisecond_datatype(), + true, + ), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new( + "updated_at", + ConcreteDataType::timestamp_millisecond_datatype(), + true, + ), + ColumnSchema::new("marker", ConcreteDataType::uint8_datatype(), true), + ColumnSchema::new("payload", ConcreteDataType::binary_datatype(), true), + ColumnSchema::new("epoch", ConcreteDataType::uint64_datatype(), true), + ])) + } + + fn register_native_tables(catalog: &catalog::memory::MemoryCatalogManager) { + let schema = native_schema(); + let meta = TableMetaBuilder::empty() + .schema(schema.clone()) + .primary_key_indices(vec![]) + .value_indices((0..schema.num_columns()).collect()) + .next_column_id(8) + .build() + .unwrap(); + let info = TableInfoBuilder::default() + .name("native_regression") + .table_id(9001) + .table_version(0) + .meta(meta) + .build() + .unwrap(); + catalog + .register_table_sync(RegisterTableRequest { + catalog: DEFAULT_CATALOG_NAME.to_string(), + schema: DEFAULT_SCHEMA_NAME.to_string(), + table_name: "native_regression".to_string(), + table_id: 9001, + table: table::test_util::EmptyTable::from_table_info(&info), + }) + .unwrap(); + + let schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("dim", ConcreteDataType::date_datatype(), true), + ColumnSchema::new("amount", ConcreteDataType::decimal128_datatype(20, 2), true), + ColumnSchema::new( + "elapsed", + ConcreteDataType::duration_millisecond_datatype(), + true, + ), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ])); + let rows = RecordBatch::new( + schema, + vec![ + Arc::new(datatypes::vectors::DateVector::from_slice([0, 2])) as VectorRef, + Arc::new( + datatypes::vectors::Decimal128Vector::from_slice([10000, 20000]) + .with_precision_and_scale(20, 2) + .unwrap(), + ) as VectorRef, + Arc::new(datatypes::vectors::DurationMillisecondVector::from_values( + [10, 20], + )) as VectorRef, + Arc::new(datatypes::vectors::TimestampMillisecondVector::from_slice( + [1, 2], + )) as VectorRef, + ], + ) + .unwrap(); + catalog + .register_table_sync(RegisterTableRequest { + catalog: DEFAULT_CATALOG_NAME.to_string(), + schema: DEFAULT_SCHEMA_NAME.to_string(), + table_name: "native_aggregate".to_string(), + table_id: 9002, + table: table::test_util::MemTable::table("native_aggregate", rows), + }) + .unwrap(); + } + + async fn run_sql(engine: &QueryEngineRef, sql: &str) -> Vec { + let stmt = QueryLanguageParser::parse_sql(sql, &QueryContext::arc()).unwrap(); + let plan = engine + .planner() + .plan(&stmt, QueryContext::arc()) + .await + .unwrap(); + match engine + .execute(plan, QueryContext::arc()) + .await + .unwrap() + .data + { + OutputData::Stream(stream) => util::collect(stream).await.unwrap(), + _ => unreachable!(), + } + } + + #[tokio::test] + async fn test_native_executor_null_insert_and_aggregates() { + let catalog = catalog::memory::new_memory_catalog_manager().unwrap(); + register_native_tables(&catalog); + let handler = Arc::new(RecordingMutationHandler::default()); + let engine = QueryEngineFactory::new( + catalog, + None, + Some(handler.clone()), + None, + None, + false, + QueryOptions::default(), + ) + .query_engine(); + + // The CASTs force Insert::can_extract_values false. This records the + // QueryEngine DML path selected by operator/src/statement/dml.rs. + let insert_sql = "INSERT INTO native_regression (dim, amount, elapsed, ts, updated_at, marker, payload, epoch) VALUES (NULL, NULL, NULL, CAST(-62135596799999 AS TIMESTAMP(3)), CAST(NULL AS TIMESTAMP(3)), CAST(1 AS UInt8), X'0102', CAST(1 AS UInt64))"; + let stmt = QueryLanguageParser::parse_sql(insert_sql, &QueryContext::arc()).unwrap(); + let QueryStatement::Sql(sql::statements::statement::Statement::Insert(insert)) = &stmt + else { + unreachable!() + }; + assert!(!insert.can_extract_values()); + let plan = engine + .planner() + .plan(&stmt, QueryContext::arc()) + .await + .unwrap(); + assert!(matches!( + engine + .execute(plan, QueryContext::arc()) + .await + .unwrap() + .data, + OutputData::AffectedRows(1) + )); + + let request = handler.inserts.lock().unwrap().pop().unwrap(); + for (name, data_type) in [ + ("dim", ConcreteDataType::date_datatype()), + ("amount", ConcreteDataType::decimal128_datatype(30, 2)), + ("elapsed", ConcreteDataType::duration_millisecond_datatype()), + ( + "updated_at", + ConcreteDataType::timestamp_millisecond_datatype(), + ), + ] { + let vector = request.columns_values.get(name).unwrap(); + assert_eq!(vector.len(), 1, "{name}"); + assert_eq!(vector.data_type(), data_type, "{name}"); + assert!(vector.is_null(0), "{name}"); + } + assert!(!request.columns_values["ts"].is_null(0)); + for (name, data_type) in [ + ("marker", ConcreteDataType::uint8_datatype()), + ("payload", ConcreteDataType::binary_datatype()), + ("epoch", ConcreteDataType::uint64_datatype()), + ] { + assert_eq!( + request.columns_values[name].data_type(), + data_type, + "{name}" + ); + } + + let batches = run_sql( + &engine, + "SELECT MIN(dim), MAX(dim), SUM(amount), SUM(elapsed) FROM native_aggregate", + ) + .await; + let batch = &batches[0]; + assert_eq!(batch.num_rows(), 1); + for (index, (data_type, value)) in [ + ( + ConcreteDataType::date_datatype(), + ScalarValue::Date32(Some(0)), + ), + ( + ConcreteDataType::date_datatype(), + ScalarValue::Date32(Some(2)), + ), + ( + ConcreteDataType::decimal128_datatype(30, 2), + ScalarValue::Decimal128(Some(30000), 30, 2), + ), + ( + ConcreteDataType::duration_millisecond_datatype(), + ScalarValue::DurationMillisecond(Some(30)), + ), + ] + .into_iter() + .enumerate() + { + assert_eq!(batch.schema.column_schemas()[index].data_type, data_type); + assert_eq!( + ScalarValue::try_from_array(batch.column(index).as_ref(), 0).unwrap(), + value + ); + } + + let batches = run_sql( + &engine, + "SELECT aggregate.amount_sum + aggregate.amount_sum AS amount_add, \ + aggregate.elapsed_sum + aggregate.elapsed_sum AS elapsed_add, \ + CASE WHEN aggregate.min_dim < CAST('1970-01-02' AS DATE) THEN true ELSE false END AS min_before, \ + CASE WHEN aggregate.max_dim > CAST('1970-01-01' AS DATE) THEN true ELSE false END AS max_after \ + FROM (SELECT MIN(dim) AS min_dim, MAX(dim) AS max_dim, SUM(amount) AS amount_sum, SUM(elapsed) AS elapsed_sum \ + FROM native_aggregate) AS aggregate", + ) + .await; + let batch = &batches[0]; + assert_eq!(batch.num_rows(), 1); + for (index, (data_type, value)) in [ + ( + ConcreteDataType::decimal128_datatype(31, 2), + ScalarValue::Decimal128(Some(60000), 31, 2), + ), + ( + ConcreteDataType::duration_millisecond_datatype(), + ScalarValue::DurationMillisecond(Some(60)), + ), + ( + ConcreteDataType::boolean_datatype(), + ScalarValue::Boolean(Some(true)), + ), + ( + ConcreteDataType::boolean_datatype(), + ScalarValue::Boolean(Some(true)), + ), + ] + .into_iter() + .enumerate() + { + assert_eq!(batch.schema.column_schemas()[index].data_type, data_type); + assert_eq!( + ScalarValue::try_from_array(batch.column(index).as_ref(), 0).unwrap(), + value + ); + } + } } diff --git a/src/query/src/datafusion/pg_oid_alias_expr_planner.rs b/src/query/src/datafusion/pg_oid_alias_expr_planner.rs new file mode 100644 index 00000000000..6eb704b58df --- /dev/null +++ b/src/query/src/datafusion/pg_oid_alias_expr_planner.rs @@ -0,0 +1,340 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use arrow_schema::DataType; +use datafusion_common::{DFSchema, ExprSchema, Result, ScalarValue}; +use datafusion_expr::expr::BinaryExpr; +use datafusion_expr::planner::{ExprPlanner, PlannerResult, RawBinaryExpr}; +use datafusion_expr::{Expr, Operator}; +use datafusion_pg_catalog::pg_catalog::oid_field::{OID_ALIAS_KEY, kind}; +use sqlparser::ast::BinaryOperator; + +/// Rewrites PostgreSQL's regproc zero sentinel before DataFusion type coercion. +#[derive(Debug)] +pub(crate) struct PgOidAliasExprPlanner; + +impl ExprPlanner for PgOidAliasExprPlanner { + fn plan_binary_op( + &self, + expr: RawBinaryExpr, + schema: &DFSchema, + ) -> Result> { + let RawBinaryExpr { + op, + mut left, + mut right, + } = expr; + + let operator = match op { + BinaryOperator::Eq => Operator::Eq, + BinaryOperator::NotEq => Operator::NotEq, + _ => return Ok(PlannerResult::Original(RawBinaryExpr { op, left, right })), + }; + + let (column, zero_on_left) = match (&left, &right) { + (Expr::Literal(value, _), Expr::Column(column)) if is_integral_zero(value) => { + (column, true) + } + (Expr::Column(column), Expr::Literal(value, _)) if is_integral_zero(value) => { + (column, false) + } + _ => return Ok(PlannerResult::Original(RawBinaryExpr { op, left, right })), + }; + + // A raw SQL column is resolved against the schema before the default + // coercion planner runs. Do not infer alias semantics from casts or any + // other expression shape. + let Ok(field) = schema.field_from_column(column) else { + return Ok(PlannerResult::Original(RawBinaryExpr { op, left, right })); + }; + if field.metadata().get(OID_ALIAS_KEY).map(String::as_str) != Some(kind::REGPROC) { + return Ok(PlannerResult::Original(RawBinaryExpr { op, left, right })); + } + + let Some(sentinel) = regproc_zero_sentinel(field.data_type()) else { + return Ok(PlannerResult::Original(RawBinaryExpr { op, left, right })); + }; + + let sentinel = Expr::Literal(sentinel, None); + if zero_on_left { + left = sentinel; + } else { + right = sentinel; + } + + Ok(PlannerResult::Planned(Expr::BinaryExpr(BinaryExpr::new( + Box::new(left), + operator, + Box::new(right), + )))) + } +} + +fn is_integral_zero(value: &ScalarValue) -> bool { + matches!( + value, + ScalarValue::Int8(Some(0)) + | ScalarValue::Int16(Some(0)) + | ScalarValue::Int32(Some(0)) + | ScalarValue::Int64(Some(0)) + | ScalarValue::UInt8(Some(0)) + | ScalarValue::UInt16(Some(0)) + | ScalarValue::UInt32(Some(0)) + | ScalarValue::UInt64(Some(0)) + ) +} + +fn regproc_zero_sentinel(data_type: &DataType) -> Option { + match data_type { + DataType::Utf8 => Some(ScalarValue::Utf8(Some("-".to_string()))), + DataType::LargeUtf8 => Some(ScalarValue::LargeUtf8(Some("-".to_string()))), + DataType::Utf8View => Some(ScalarValue::Utf8View(Some("-".to_string()))), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use arrow_schema::{Field, Fields}; + use datafusion_common::Column; + use datafusion_expr::ExprSchemable; + use datafusion_expr::expr::Cast; + use datafusion_expr::simplify::SimplifyContext; + use datafusion_optimizer::simplify_expressions::ExprSimplifier; + + use super::*; + + fn schema(data_type: DataType, alias: Option<&str>) -> DFSchema { + let mut field = Field::new("typreceive", data_type, true); + if let Some(alias) = alias { + field = field.with_metadata(HashMap::from([( + OID_ALIAS_KEY.to_string(), + alias.to_string(), + )])); + } + DFSchema::from_unqualified_fields(Fields::from(vec![field]), HashMap::new()).unwrap() + } + + fn column() -> Expr { + Expr::Column(Column::new_unqualified("typreceive")) + } + + fn plan(expr: RawBinaryExpr, schema: &DFSchema) -> PlannerResult { + PgOidAliasExprPlanner.plan_binary_op(expr, schema).unwrap() + } + + fn assert_planned_sentinel( + planned: PlannerResult, + operator: Operator, + zero_on_left: bool, + sentinel: ScalarValue, + ) -> Expr { + let PlannerResult::Planned(Expr::BinaryExpr(expr)) = planned else { + panic!("expected a planned binary expression"); + }; + assert_eq!(expr.op, operator); + let literal = Expr::Literal(sentinel, None); + if zero_on_left { + assert_eq!(expr.left.as_ref(), &literal); + assert_eq!(expr.right.as_ref(), &column()); + } else { + assert_eq!(expr.left.as_ref(), &column()); + assert_eq!(expr.right.as_ref(), &literal); + } + Expr::BinaryExpr(expr) + } + + #[test] + fn rewrites_zero_regproc_comparisons_in_both_operand_orders() { + let schema = schema(DataType::Utf8, Some(kind::REGPROC)); + + for (sql_operator, operator) in [ + (BinaryOperator::Eq, Operator::Eq), + (BinaryOperator::NotEq, Operator::NotEq), + ] { + for zero_on_left in [true, false] { + let zero = Expr::Literal(ScalarValue::Int64(Some(0)), None); + let (left, right) = if zero_on_left { + (zero, column()) + } else { + (column(), zero) + }; + let planned = assert_planned_sentinel( + plan( + RawBinaryExpr { + op: sql_operator.clone(), + left, + right, + }, + &schema, + ), + operator, + zero_on_left, + ScalarValue::Utf8(Some("-".to_string())), + ); + assert!(planned.nullable(&schema).unwrap()); + } + } + } + + #[test] + fn rewrites_every_integral_zero_with_the_column_string_storage_type() { + let zero_literals = [ + ScalarValue::Int8(Some(0)), + ScalarValue::Int16(Some(0)), + ScalarValue::Int32(Some(0)), + ScalarValue::Int64(Some(0)), + ScalarValue::UInt8(Some(0)), + ScalarValue::UInt16(Some(0)), + ScalarValue::UInt32(Some(0)), + ScalarValue::UInt64(Some(0)), + ]; + let string_types = [ + (DataType::Utf8, ScalarValue::Utf8(Some("-".to_string()))), + ( + DataType::LargeUtf8, + ScalarValue::LargeUtf8(Some("-".to_string())), + ), + ( + DataType::Utf8View, + ScalarValue::Utf8View(Some("-".to_string())), + ), + ]; + + for (data_type, sentinel) in string_types { + let schema = schema(data_type, Some(kind::REGPROC)); + for zero in &zero_literals { + assert_planned_sentinel( + plan( + RawBinaryExpr { + op: BinaryOperator::Eq, + left: column(), + right: Expr::Literal(zero.clone(), None), + }, + &schema, + ), + Operator::Eq, + false, + sentinel.clone(), + ); + } + } + } + + #[test] + fn leaves_non_matching_comparisons_untouched() { + let regproc = schema(DataType::Utf8, Some(kind::REGPROC)); + let int32_regproc = schema(DataType::Int32, Some(kind::REGPROC)); + let untagged = schema(DataType::Utf8, None); + let regtype = schema(DataType::Utf8, Some(kind::REGTYPE)); + + let cases = [ + ( + RawBinaryExpr { + op: BinaryOperator::Eq, + left: column(), + right: Expr::Literal(ScalarValue::Int64(Some(1)), None), + }, + ®proc, + ), + ( + RawBinaryExpr { + op: BinaryOperator::Eq, + left: column(), + right: Expr::Literal(ScalarValue::Int64(None), None), + }, + ®proc, + ), + ( + RawBinaryExpr { + op: BinaryOperator::Lt, + left: column(), + right: Expr::Literal(ScalarValue::Int64(Some(0)), None), + }, + ®proc, + ), + ( + RawBinaryExpr { + op: BinaryOperator::Eq, + left: column(), + right: Expr::Literal(ScalarValue::Int64(Some(0)), None), + }, + &int32_regproc, + ), + ( + RawBinaryExpr { + op: BinaryOperator::Eq, + left: column(), + right: Expr::Literal(ScalarValue::Int64(Some(0)), None), + }, + &untagged, + ), + ( + RawBinaryExpr { + op: BinaryOperator::Eq, + left: column(), + right: Expr::Literal(ScalarValue::Int64(Some(0)), None), + }, + ®type, + ), + ]; + + for (expr, schema) in cases { + assert!(matches!(plan(expr, schema), PlannerResult::Original(_))); + } + } + + #[test] + fn leaves_casts_and_the_adbc_array_receiver_predicate_untouched() { + let schema = schema(DataType::Utf8, Some(kind::REGPROC)); + let cast = Expr::Cast(Cast::new(Box::new(column()), DataType::Utf8)); + let expr = RawBinaryExpr { + op: BinaryOperator::NotEq, + left: cast, + right: Expr::Literal(ScalarValue::Utf8(Some("array_recv".to_string())), None), + }; + + assert!(matches!(plan(expr, &schema), PlannerResult::Original(_))); + } + + #[test] + fn type_coercion_keeps_regproc_as_a_string_after_the_rewrite() { + let schema = Arc::new(schema(DataType::Utf8, Some(kind::REGPROC))); + let planned = assert_planned_sentinel( + plan( + RawBinaryExpr { + op: BinaryOperator::NotEq, + left: column(), + right: Expr::Literal(ScalarValue::Int64(Some(0)), None), + }, + &schema, + ), + Operator::NotEq, + false, + ScalarValue::Utf8(Some("-".to_string())), + ); + let simplifier = ExprSimplifier::new( + SimplifyContext::builder() + .with_schema(schema.clone()) + .build(), + ); + let coerced = simplifier.coerce(planned, &schema).unwrap(); + + assert!(!format!("{coerced}").contains("CAST")); + assert!(!format!("{coerced:?}").contains("Int64")); + } +} diff --git a/src/query/src/datafusion/planner.rs b/src/query/src/datafusion/planner.rs index 856f2189d24..342209a6386 100644 --- a/src/query/src/datafusion/planner.rs +++ b/src/query/src/datafusion/planner.rs @@ -19,7 +19,8 @@ use std::sync::Arc; use arrow_schema::DataType; use catalog::table_source::DfTableSourceProvider; use common_function::function::FunctionContext; -use datafusion::common::TableReference; +use datafusion::catalog::TableFunctionArgs; +use datafusion::common::{DFSchema, TableReference}; use datafusion::datasource::cte_worktable::CteWorkTable; use datafusion::datasource::file_format::{FileFormatFactory, format_as_file_type}; use datafusion::datasource::provider_as_source; @@ -33,12 +34,13 @@ use datafusion_common::config::ConfigOptions; use datafusion_common::file_options::file_type::FileType; use datafusion_expr::planner::{ExprPlanner, TypePlanner}; use datafusion_expr::var_provider::is_system_variables; -use datafusion_expr::{AggregateUDF, ScalarUDF, TableSource, WindowUDF}; +use datafusion_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, TableSource, WindowUDF}; use datafusion_sql::parser::Statement as DfStatement; use session::context::QueryContextRef; use snafu::{Location, ResultExt}; use crate::datafusion::json_expr_planner::JsonExprPlanner; +use crate::datafusion::pg_oid_alias_expr_planner::PgOidAliasExprPlanner; use crate::error::{CatalogSnafu, Result}; use crate::query_engine::{DefaultPlanDecoder, QueryEngineState}; @@ -90,6 +92,7 @@ impl DfContextProviderAdapter { let mut expr_planners = SessionStateDefaults::default_expr_planners(); expr_planners.insert(0, Arc::new(JsonExprPlanner)); + expr_planners.insert(0, Arc::new(PgOidAliasExprPlanner)); Ok(Self { engine_state, @@ -161,6 +164,13 @@ impl ContextProvider for DfContextProviderAdapter { ) } + fn get_higher_order_meta(&self, name: &str) -> Option> { + self.session_state + .higher_order_functions() + .get(name) + .cloned() + } + fn get_aggregate_meta(&self, name: &str) -> Option> { self.engine_state.aggr_function(name).map_or_else( || self.session_state.aggregate_functions().get(name).cloned(), @@ -200,6 +210,14 @@ impl ContextProvider for DfContextProviderAdapter { names } + fn higher_order_function_names(&self) -> Vec { + self.session_state + .higher_order_functions() + .keys() + .cloned() + .collect() + } + fn udaf_names(&self) -> Vec { let mut names = self.engine_state.aggr_names(); names.extend(self.session_state.aggregate_functions().keys().cloned()); @@ -228,22 +246,47 @@ impl ContextProvider for DfContextProviderAdapter { name: &str, args: Vec, ) -> DfResult> { - if let Some(tbl_func) = self.engine_state.table_function(name) { - let provider = tbl_func.create_table_provider(&args)?; - Ok(provider_as_source(provider)) + // Constant-fold the args before resolving the table function. DataFusion's + // SQL planner does not fold table-function arguments (constant folding + // happens later, in the analyzer), but table functions such as + // `generate_series`/`range` are resolved during planning and require + // literal bounds. Folding here lets immutable-UDF bounds like + // `array_upper(ARRAY[...], 1)` reach them as concrete literals. + // Non-constant args are returned unchanged by the simplifier. + let simplify_info = datafusion_expr::simplify::SimplifyContext::builder() + .with_config_options(Arc::clone(self.session_state.config_options())) + .with_query_execution_start_time( + self.session_state + .execution_props() + .query_execution_start_time, + ) + .build(); + let simplifier = + datafusion_optimizer::simplify_expressions::ExprSimplifier::new(simplify_info); + let schema = DFSchema::empty(); + let args = args + .into_iter() + .map(|arg| { + simplifier + .coerce(arg, &schema) + .and_then(|arg| simplifier.simplify(arg)) + }) + .collect::>>()?; + let table_args = TableFunctionArgs::new(&args, &self.session_state); + let tbl_func = if let Some(tbl_func) = self.engine_state.table_function(name) { + tbl_func } else { - let tbl_func = self - .session_state + self.session_state .table_functions() .get(name) .cloned() .ok_or_else(|| { DataFusionError::Plan(format!("table function '{name}' not found")) - })?; - let provider = tbl_func.create_table_provider(&args)?; + })? + }; + let provider = tbl_func.create_table_provider_with_args(table_args)?; - Ok(provider_as_source(provider)) - } + Ok(provider_as_source(provider)) } fn create_cte_work_table( @@ -260,6 +303,154 @@ impl ContextProvider for DfContextProviderAdapter { } fn get_type_planner(&self) -> Option> { - None + // Provide the SQL planner with Postgres oid-alias type names + // (`regclass`, `regproc`, `regtype`, `regnamespace`, `oid`, ...) and + // `pg_catalog.`-qualified builtins. DataFusion rejects these as + // "Unsupported SQL type" otherwise. The planner maps each to its Arrow + // type so reverse / column-operand casts like `prorettype::regtype::text` + // parse. Forward name->oid casts (`'x'::regclass`) are resolved earlier, + // at SQL-parse time, by the `PostgresCompatibilityParser`'s built-in + // `RewriteRegCastToSubquery` rule. + // Stateless, so a fresh instance per query is cheap. + Some(Arc::new( + datafusion_pg_catalog::pg_catalog::oid_type_planner::PgOidTypePlanner, + )) + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + + use common_base::Plugins; + use datafusion::catalog::{TableFunction, TableFunctionArgs, TableFunctionImpl, TableProvider}; + use datafusion::datasource::MemTable; + use datafusion::execution::SessionStateBuilder; + use datafusion::execution::context::{SessionConfig, SessionContext, SessionState}; + use datafusion_common::ScalarValue; + use datafusion_expr::expr::BinaryExpr; + use datafusion_expr::{Expr, Operator, lit}; + use session::context::QueryContext; + + use super::*; + use crate::options::QueryOptions; + + #[derive(Debug, Default)] + struct RecordingTableFunction { + called: AtomicBool, + args: Mutex>, + target_partitions: Mutex>, + } + + impl TableFunctionImpl for RecordingTableFunction { + fn call_with_args(&self, args: TableFunctionArgs) -> DfResult> { + let session_state = args + .session() + .as_any() + .downcast_ref::() + .expect("table function must receive the SessionState"); + *self.args.lock().unwrap() = args.exprs().to_vec(); + *self.target_partitions.lock().unwrap() = + Some(session_state.config().target_partitions()); + self.called.store(true, Ordering::SeqCst); + + Ok(Arc::new(MemTable::try_new( + Arc::new(arrow_schema::Schema::empty()), + vec![vec![]], + )?)) + } + } + + fn query_engine_state() -> Arc { + Arc::new(QueryEngineState::new( + catalog::memory::new_memory_catalog_manager().unwrap(), + None, + None, + None, + None, + None, + false, + Plugins::default(), + QueryOptions::default(), + )) + } + + async fn context_provider( + engine_state: Arc, + session_state: SessionState, + ) -> DfContextProviderAdapter { + DfContextProviderAdapter::try_new(engine_state, session_state, None, QueryContext::arc()) + .await + .unwrap() + } + + fn plus(left: Expr, right: Expr) -> Expr { + Expr::BinaryExpr(BinaryExpr { + left: Box::new(left), + op: Operator::Plus, + right: Box::new(right), + }) + } + + #[tokio::test] + async fn table_function_arguments_are_folded_before_engine_function_creation() { + let engine_state = query_engine_state(); + let function = Arc::new(RecordingTableFunction::default()); + engine_state.register_table_function(Arc::new(TableFunction::new( + "capture_engine_args".to_string(), + function.clone(), + ))); + let provider = context_provider(engine_state.clone(), engine_state.session_state()).await; + + provider + .get_table_function_source("capture_engine_args", vec![plus(lit(1_i64), lit(2_i64))]) + .unwrap(); + + assert!(function.called.load(Ordering::SeqCst)); + assert_eq!( + *function.args.lock().unwrap(), + vec![Expr::Literal(ScalarValue::Int64(Some(3)), None)] + ); + } + + #[tokio::test] + async fn table_function_argument_simplification_errors_are_propagated() { + let engine_state = query_engine_state(); + let function = Arc::new(RecordingTableFunction::default()); + engine_state.register_table_function(Arc::new(TableFunction::new( + "reject_invalid_args".to_string(), + function.clone(), + ))); + let provider = context_provider(engine_state.clone(), engine_state.session_state()).await; + + let error = match provider + .get_table_function_source("reject_invalid_args", vec![plus(lit(true), lit(1_i64))]) + { + Ok(_) => panic!("invalid table-function argument must fail planning"), + Err(error) => error, + }; + + assert!(!error.to_string().is_empty()); + assert!(!function.called.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn session_table_function_receives_table_function_args_session() { + let engine_state = query_engine_state(); + let session_state = SessionStateBuilder::new_from_existing(engine_state.session_state()) + .with_config(SessionConfig::new().with_target_partitions(7)) + .build(); + let session_context = SessionContext::new_with_state(session_state); + let function = Arc::new(RecordingTableFunction::default()); + session_context.register_udtf("capture_session_args", function.clone()); + let provider = context_provider(engine_state, session_context.state()).await; + + provider + .get_table_function_source("capture_session_args", vec![lit(1_i64)]) + .unwrap(); + + assert!(function.called.load(Ordering::SeqCst)); + assert_eq!(*function.target_partitions.lock().unwrap(), Some(7)); } } diff --git a/src/query/src/dist_plan/analyzer.rs b/src/query/src/dist_plan/analyzer.rs index 433b66519b9..d928671fc5d 100644 --- a/src/query/src/dist_plan/analyzer.rs +++ b/src/query/src/dist_plan/analyzer.rs @@ -248,7 +248,7 @@ impl DistPlannerAnalyzer { /// Try push down as many nodes as possible fn try_push_down(&self, plan: LogicalPlan) -> DfResult { let plan = plan.transform(&Self::inspect_plan_with_subquery)?; - let mut rewriter = PlanRewriter::default(); + let mut rewriter = PlanRewriter::new(&plan.data); let result = plan.data.rewrite(&mut rewriter)?.data; Self::assign_merge_scan_remote_dyn_filter_producer_ids(result) } @@ -302,7 +302,7 @@ impl DistPlannerAnalyzer { } fn handle_subquery(subquery: Subquery) -> DfResult { - let mut rewriter = PlanRewriter::default(); + let mut rewriter = PlanRewriter::new(&subquery.subquery); let mut rewrote_subquery = subquery .subquery .as_ref() @@ -391,6 +391,9 @@ enum RewriterStatus { #[derive(Debug, Default)] struct PlanRewriter { + /// Whether the whole plan this rewriter walks can be encoded to Substrait. + /// Used by [`PlanRewriter::should_expand`] to skip the per-node encoding check. + whole_plan_encodable: bool, /// Current level in the tree level: usize, /// Simulated stack for the `rewrite` recursion @@ -447,6 +450,17 @@ struct PlanRewriter { } impl PlanRewriter { + /// `plan` must be the root of the tree that is about to be rewritten, otherwise + /// [`PlanRewriter::should_expand`] may skip a check it has to perform. + fn new(plan: &LogicalPlan) -> Self { + Self { + whole_plan_encodable: DFLogicalSubstraitConvertor + .encode(plan, DefaultSerializer) + .is_ok(), + ..Default::default() + } + } + fn get_parent(&self) -> Option<&LogicalPlan> { // level starts from 1, it's safe to minus by 1 self.stack @@ -467,7 +481,16 @@ impl PlanRewriter { .collect::>() .join("\n"), ); - if let Err(e) = DFLogicalSubstraitConvertor.encode(plan, DefaultSerializer) { + // Substrait encoding recurses from the root to the leaves, so a root that encodes + // proves that every node below it encodes as well. `plan` here always comes from + // `self.stack`, which holds untouched sub-trees of that root, so one check on the + // root covers the whole descent. Each check builds a fresh `SessionState`, which + // is the dominant cost on deep PromQL plans. + // When the root does not encode, the plan holds at least one node that has to stay + // on the frontend and only the per-node check locates it. + if !self.whole_plan_encodable + && let Err(e) = DFLogicalSubstraitConvertor.encode(plan, DefaultSerializer) + { debug!( "PlanRewriter: plan cannot be converted to substrait with error={e:?}, expanding now: {plan}" ); @@ -645,13 +668,9 @@ impl PlanRewriter { } if let LogicalPlan::TableScan(table_scan) = plan - && let Some(source) = table_scan - .source - .as_any() - .downcast_ref::() + && let Some(source) = table_scan.source.downcast_ref::() && let Some(provider) = source .table_provider - .as_any() .downcast_ref::() { let table = provider.table(); diff --git a/src/query/src/dist_plan/analyzer/fallback.rs b/src/query/src/dist_plan/analyzer/fallback.rs index 79dfaf904be..86c7f50628d 100644 --- a/src/query/src/dist_plan/analyzer/fallback.rs +++ b/src/query/src/dist_plan/analyzer/fallback.rs @@ -45,14 +45,11 @@ impl TreeNodeRewriter for FallbackPlanRewriter { plan: Self::Node, ) -> DfResult> { if let LogicalPlan::TableScan(table_scan) = &plan { - let partition_cols = if let Some(source) = table_scan - .source - .as_any() - .downcast_ref::() + let partition_cols = if let Some(source) = + table_scan.source.downcast_ref::() { if let Some(provider) = source .table_provider - .as_any() .downcast_ref::() { if provider.table().table_type() == TableType::Base { diff --git a/src/query/src/dist_plan/analyzer/test.rs b/src/query/src/dist_plan/analyzer/test.rs index c42eb0e285a..36ea83fadae 100644 --- a/src/query/src/dist_plan/analyzer/test.rs +++ b/src/query/src/dist_plan/analyzer/test.rs @@ -28,17 +28,18 @@ use datafusion::datasource::DefaultTableSource; use datafusion::execution::SessionState; use datafusion::functions_aggregate::expr_fn::avg; use datafusion::functions_aggregate::min_max::{max, min}; +use datafusion::functions_nested::expr_fn::make_array; use datafusion::prelude::SessionContext; use datafusion_common::tree_node::TreeNodeRecursion; -use datafusion_common::{ExprSchema, JoinType, ScalarValue}; +use datafusion_common::{ExprSchema, JoinType, ScalarValue, TableReference}; use datafusion_expr::expr::{Exists, ScalarFunction}; +use datafusion_expr::utils::split_conjunction; use datafusion_expr::{ AggregateUDF, Expr, ExprSchemable as _, Extension, LogicalPlanBuilder, Operator, Subquery, binary_expr, col, lit, }; use datafusion_functions::datetime::date_bin; use datafusion_functions::datetime::expr_fn::now; -use datafusion_sql::TableReference; use datatypes::data_type::ConcreteDataType; use datatypes::schema::{ColumnSchema, SchemaBuilder, SchemaRef}; use futures::Stream; @@ -301,6 +302,53 @@ fn find_merge_scan(plan: &LogicalPlan) -> Option<&MergeScanLogicalPlan> { plan.inputs().into_iter().find_map(find_merge_scan) } +fn find_table_scan<'a>( + plan: &'a LogicalPlan, + table_name: &str, +) -> Option<&'a datafusion_expr::logical_plan::TableScan> { + if let LogicalPlan::TableScan(table_scan) = plan + && table_scan.table_name.to_string() == table_name + { + return Some(table_scan); + } + + plan.inputs() + .into_iter() + .find_map(|input| find_table_scan(input, table_name)) +} + +fn find_merge_scan_for_table<'a>( + plan: &'a LogicalPlan, + table_name: &str, +) -> Option<&'a MergeScanLogicalPlan> { + if let LogicalPlan::Extension(extension) = plan + && let Some(merge_scan) = extension + .node + .as_any() + .downcast_ref::() + && find_table_scan(merge_scan.input(), table_name).is_some() + { + return Some(merge_scan); + } + + plan.inputs() + .into_iter() + .find_map(|input| find_merge_scan_for_table(input, table_name)) +} + +fn has_filter_above_table_scan(plan: &LogicalPlan, table_name: &str, predicate: &Expr) -> bool { + if let LogicalPlan::Filter(filter) = plan + && split_conjunction(&filter.predicate).contains(&predicate) + && find_table_scan(filter.input.as_ref(), table_name).is_some() + { + return true; + } + + plan.inputs() + .into_iter() + .any(|input| has_filter_above_table_scan(input, table_name, predicate)) +} + #[test] fn frontend_only_histogram_folds_stay_above_merge_scan() { let table = TestTable::table_with_name(0, "t".to_string()); @@ -367,6 +415,47 @@ fn frontend_only_histogram_folds_stay_above_merge_scan() { } } +/// `Categorizer` calls `Unnest` commutative, so the Substrait check in `should_expand` is +/// the only thing keeping it on the frontend. It also sits above an encodable projection, +/// which pins the rewriter to the whole-plan check rather than to the root node alone. +#[test] +fn unencodable_unnest_stays_above_merge_scan() { + let table = TestTable::table_with_name(0, "t".to_string()); + let table_source = Arc::new(DefaultTableSource::new(Arc::new( + DfTableProviderAdapter::new(table), + ))); + let plan = LogicalPlanBuilder::scan_with_filters("t", table_source, None, vec![]) + .unwrap() + .project(vec![ + col("pk1"), + make_array(vec![col("number")]).alias("numbers"), + ]) + .unwrap() + .unnest_column("numbers") + .unwrap() + .build() + .unwrap(); + assert!( + DFLogicalSubstraitConvertor + .encode(&plan, DefaultSerializer) + .is_err() + ); + + let result = DistPlannerAnalyzer {} + .analyze(plan, &ConfigOptions::default()) + .unwrap(); + let result_text = result.to_string(); + assert!(result_text.contains("Unnest:"), "{result_text}"); + + let remote_input = find_merge_scan(&result).unwrap().input(); + let remote_text = remote_input.to_string(); + assert!(!remote_text.contains("Unnest:"), "{remote_text}"); + assert!(remote_text.contains("make_array"), "{remote_text}"); + DFLogicalSubstraitConvertor + .encode(remote_input, DefaultSerializer) + .unwrap(); +} + #[test] fn dictionary_literals_are_unwrapped_before_pushdown() { let table = Arc::new(Table::new( @@ -2544,34 +2633,37 @@ fn test_join_side_local_filter_pushdown_into_merge_scan() { let result = DistPlannerAnalyzer {}.analyze(plan, &config).unwrap(); assert_remote_table_scan_filters_are_safe(&result); - let plan_str = result.to_string(); - // After PushDownFilter runs, the predicate `t1.pk1 = Utf8("v")` should appear - // inside the left MergeScan's remote_input. The pre-MergeScan optimizer may - // combine it with join-derived IS NOT NULL pushdowns, so it may not appear as - // a standalone Filter: line. It must still be in TableScan partial_filters - // and below the Inner Join. + let predicate = col("t1.pk1").eq(lit("v")); + let t1_remote_input = find_merge_scan_for_table(&result, "t1") + .expect("expected MergeScan for t1") + .input(); + let t1_scan = find_table_scan(t1_remote_input, "t1").expect("expected t1 TableScan"); assert!( - plan_str.contains("t1.pk1 = Utf8(\"v\")"), - "Expected predicate t1.pk1 = Utf8(\"v\") in plan, got:\n{plan_str}" - ); - assert!( - plan_str.contains( - "TableScan: t1, partial_filters=[t1.pk1 = Utf8(\"v\"), t1.number IS NOT NULL]" - ), - "Expected t1 TableScan partial_filters to contain pushed predicate, got:\n{plan_str}" + t1_scan + .filters + .iter() + .flat_map(|filter| split_conjunction(filter)) + .any(|filter| filter == &predicate), + "expected t1 TableScan to contain the pushed predicate: {t1_remote_input}" ); - // Find the position of the filter and verify it appears after a MergeScan - // opening (i.e., inside remote_input) rather than before the Join. - let filter_pos = plan_str - .find("TableScan: t1, partial_filters=[t1.pk1 = Utf8(\"v\"), t1.number IS NOT NULL]") - .unwrap(); - let join_pos = plan_str.find("Inner Join").unwrap(); - // The filter should be after the Join (meaning it was pushed down below the Join, - // into a MergeScan's remote_input) + // Inexact provider pushdown must retain the predicate in an ancestor Filter. assert!( - filter_pos > join_pos, - "Filter should be pushed below Join (into MergeScan remote_input), but found before Join" + has_filter_above_table_scan(t1_remote_input, "t1", &predicate), + "expected an ancestor Filter for t1 to retain the pushed predicate: {t1_remote_input}" + ); + + let t2_remote_input = find_merge_scan_for_table(&result, "t2") + .expect("expected MergeScan for t2") + .input(); + assert!( + !find_table_scan(t2_remote_input, "t2") + .expect("expected t2 TableScan") + .filters + .iter() + .flat_map(|filter| split_conjunction(filter)) + .any(|filter| filter == &predicate), + "t2 TableScan must not contain t1's predicate: {t2_remote_input}" ); } diff --git a/src/query/src/dist_plan/commutativity.rs b/src/query/src/dist_plan/commutativity.rs index 1a70fd8e1d9..bc41516ba3c 100644 --- a/src/query/src/dist_plan/commutativity.rs +++ b/src/query/src/dist_plan/commutativity.rs @@ -328,6 +328,9 @@ impl Categorizer { | Expr::WindowFunction(_) | Expr::InSubquery(_) | Expr::ScalarSubquery(_) + | Expr::HigherOrderFunction(_) + | Expr::Lambda(_) + | Expr::LambdaVariable(_) | Expr::Wildcard { .. } => Commutativity::Unimplemented, Expr::Alias(alias) => Self::check_expr(&alias.expr), diff --git a/src/query/src/dist_plan/dyn_filter_bridge.rs b/src/query/src/dist_plan/dyn_filter_bridge.rs index fcde3dc0083..4761c7aea93 100644 --- a/src/query/src/dist_plan/dyn_filter_bridge.rs +++ b/src/query/src/dist_plan/dyn_filter_bridge.rs @@ -346,10 +346,6 @@ mod tests { impl Eq for UnserializableExpr {} impl datafusion_physical_expr::PhysicalExpr for UnserializableExpr { - fn as_any(&self) -> &dyn Any { - self - } - fn data_type( &self, _input_schema: &arrow_schema::Schema, @@ -766,7 +762,7 @@ mod tests { captured_dyn_filters[0].filter_id.to_string() ); assert_eq!(decoded_children.len(), 1); - assert!(decoded_children[0].as_any().is::()); + assert!(decoded_children[0].is::()); } #[test] diff --git a/src/query/src/dist_plan/merge_scan.rs b/src/query/src/dist_plan/merge_scan.rs index 26474b7ca22..2efdd7b1fa2 100644 --- a/src/query/src/dist_plan/merge_scan.rs +++ b/src/query/src/dist_plan/merge_scan.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; #[cfg(test)] use std::cell::Cell; use std::sync::{Arc, Mutex}; @@ -25,6 +24,7 @@ use arrow_schema::{ }; use async_stream::stream; use common_catalog::parse_catalog_and_schema_from_db_string; +use common_error::ext::BoxedError; use common_plugins::GREPTIME_EXEC_READ_COST; use common_query::request::QueryRequest; use common_recordbatch::adapter::{RecordBatchMetrics, region_scan_output_bytes}; @@ -40,9 +40,10 @@ use datafusion::physical_plan::metrics::{ use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, - SendableRecordBatchStream, + SendableRecordBatchStream, apply_expression_roots, }; use datafusion_common::stats::Precision; +use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_common::{Column as ColumnExpr, DFSchemaRef, DataFusionError, Result, Statistics}; use datafusion_expr::{Expr, Extension, FetchType, LogicalPlan, UserDefinedLogicalNodeCore}; use datafusion_physical_expr::expressions::Column; @@ -746,7 +747,7 @@ impl MergeScanExec { } let mut stream = do_get_result.map_err(|e| { MERGE_SCAN_ERRORS_TOTAL.inc(); - DataFusionError::External(Box::new(e)) + DataFusionError::External(Box::new(BoxedError::new(e))) })?; if let Some(subscriber_rollback) = subscriber_rollback.as_mut() { @@ -815,7 +816,8 @@ impl MergeScanExec { let poll_elapsed = poll_timer.elapsed(); poll_duration += poll_elapsed; - let batch = batch.map_err(|e| DataFusionError::External(Box::new(e)))?; + let batch = batch + .map_err(|e| DataFusionError::External(Box::new(BoxedError::new(e))))?; let df_batch = batch.into_df_record_batch(); if !Arc::ptr_eq(&advertised_schema, df_batch.schema_ref()) { validate_remote_schema( @@ -911,7 +913,7 @@ impl MergeScanExec { } pub fn try_with_new_distribution(&self, distribution: Distribution) -> Option { - let Distribution::HashPartitioned(hash_exprs) = distribution else { + let Distribution::KeyPartitioned(hash_exprs) = distribution else { // not applicable return None; }; @@ -926,8 +928,7 @@ impl MergeScanExec { let hash_expr_col_names: HashSet<_> = hash_exprs .iter() .filter_map(|expr| { - expr.as_any() - .downcast_ref::() + expr.downcast_ref::() .map(|col_expr| col_expr.name()) }) .collect(); @@ -949,8 +950,7 @@ impl MergeScanExec { let overlaps: Vec<_> = hash_exprs .iter() .filter(|expr| { - expr.as_any() - .downcast_ref::() + expr.downcast_ref::() .is_some_and(|col_expr| all_partition_col_aliases.contains(col_expr.name())) }) .cloned() @@ -1153,10 +1153,6 @@ impl Drop for PartitionMetrics { } impl ExecutionPlan for MergeScanExec { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> ArrowSchemaRef { self.arrow_schema.clone() } @@ -1169,6 +1165,24 @@ impl ExecutionPlan for MergeScanExec { vec![] } + fn apply_expressions( + &self, + f: &mut dyn FnMut( + &Arc, + ) -> Result, + ) -> Result { + let captured_remote_dyn_filters = self.captured_remote_dyn_filters(); + apply_expression_roots( + captured_remote_dyn_filters + .into_iter() + .map(|captured_dyn_filter| { + captured_dyn_filter.alive_dyn_filter + as Arc + }), + f, + ) + } + // DataFusion will swap children unconditionally. // But since this node is leaf node, it's safe to just return self. fn with_new_children( @@ -1250,14 +1264,14 @@ impl ExecutionPlan for MergeScanExec { Some(self.metric.clone_inner()) } - fn partition_statistics(&self, partition: Option) -> Result { + fn partition_statistics(&self, partition: Option) -> Result> { if partition.is_some() { - return Ok(Statistics::new_unknown(&self.arrow_schema)); + return Ok(Arc::new(Statistics::new_unknown(&self.arrow_schema))); } let mut statistics = Statistics::new_unknown(&self.arrow_schema); statistics.num_rows = self.estimated_num_rows(); - Ok(statistics) + Ok(Arc::new(statistics)) } fn name(&self) -> &str { @@ -1411,6 +1425,8 @@ mod tests { use arrow_schema::{DataType as TestArrowDataType, Field, TimeUnit}; use async_trait::async_trait; use common_base::Plugins; + use common_error::ext::{ErrorExt, PlainError}; + use common_error::status_code::StatusCode; use common_meta::peer::Peer; use common_query::request::{ INITIAL_REMOTE_DYN_FILTER_REGISTRATIONS_EXTENSION_KEY, InitialDynFilterRegs, @@ -1422,12 +1438,14 @@ mod tests { use datafusion::config::ConfigOptions; use datafusion::execution::SessionStateBuilder; use datafusion::physical_plan::filter_pushdown::ChildFilterPushdownResult; + use datafusion::physical_plan::repartition::RepartitionExec; + use datafusion::physical_plan::{StatisticsArgs, StatisticsContext}; use datafusion_common::TableReference; use datafusion_expr::{LogicalPlanBuilder, col, lit}; - use datafusion_physical_expr::Distribution; use datafusion_physical_expr::expressions::{ Column, DynamicFilterPhysicalExpr, lit as physical_lit, }; + use datafusion_physical_expr::{Distribution, PhysicalExpr}; use datatypes::prelude::{ConcreteDataType, VectorRef}; use datatypes::schema::{ColumnSchema, Schema}; use datatypes::vectors::{Int64Vector, StringVector, TimestampMillisecondVector}; @@ -1435,6 +1453,7 @@ mod tests { use session::ReadPreference; use session::context::QueryContext; use session::query_id::QueryId; + use snafu::IntoError; use table::table::scan::REGION_SCAN_EXEC_NAME; use table::table_name::TableName; use tokio::sync::{Notify, oneshot}; @@ -1501,6 +1520,12 @@ mod tests { .unwrap() } + fn merge_scan_statistics(exec: &MergeScanExec) -> Arc { + StatisticsContext::new() + .compute(exec, &StatisticsArgs::new()) + .unwrap() + } + fn task_context_with_engine_state( state: Arc, query_ctx: QueryContextRef, @@ -1649,9 +1674,7 @@ mod tests { .build() .unwrap(); assert_eq!( - merge_scan_exec_with_plan(regions.clone(), limited, 10) - .partition_statistics(None) - .unwrap() + merge_scan_statistics(&merge_scan_exec_with_plan(regions.clone(), limited, 10)) .num_rows, Precision::Inexact(100) ); @@ -1665,10 +1688,12 @@ mod tests { .build() .unwrap(); assert_eq!( - merge_scan_exec_with_plan(vec![RegionId::new(1024, 1)], large_limit, 10) - .partition_statistics(None) - .unwrap() - .num_rows, + merge_scan_statistics(&merge_scan_exec_with_plan( + vec![RegionId::new(1024, 1)], + large_limit, + 10, + )) + .num_rows, Precision::Inexact(large_bound) ); @@ -1678,17 +1703,16 @@ mod tests { .build() .unwrap(); assert_eq!( - merge_scan_exec_with_plan(regions.clone(), uncapped.clone(), 10) - .partition_statistics(None) - .unwrap() - .num_rows, + merge_scan_statistics(&merge_scan_exec_with_plan( + regions.clone(), + uncapped.clone(), + 10, + )) + .num_rows, Precision::Absent ); assert_eq!( - merge_scan_exec_with_plan(Vec::new(), uncapped, 10) - .partition_statistics(None) - .unwrap() - .num_rows, + merge_scan_statistics(&merge_scan_exec_with_plan(Vec::new(), uncapped, 10)).num_rows, Precision::Inexact(0) ); @@ -1702,10 +1726,12 @@ mod tests { .build() .unwrap(); assert_eq!( - merge_scan_exec_with_plan(regions.clone(), global_aggregate, 10) - .partition_statistics(None) - .unwrap() - .num_rows, + merge_scan_statistics(&merge_scan_exec_with_plan( + regions.clone(), + global_aggregate, + 10, + )) + .num_rows, Precision::Inexact(2) ); @@ -1725,10 +1751,7 @@ mod tests { .build() .unwrap(); assert_eq!( - merge_scan_exec_with_plan(regions, grouping_sets, 10) - .partition_statistics(None) - .unwrap() - .num_rows, + merge_scan_statistics(&merge_scan_exec_with_plan(regions, grouping_sets, 10)).num_rows, Precision::Absent ); } @@ -1813,7 +1836,7 @@ mod tests { } #[tokio::test] - async fn failed_do_get_rolls_back_new_subscriber_without_starting_fanout() { + async fn failed_do_get_preserves_status_code_and_rolls_back_subscriber() { let handler = Arc::new(FailingRegionQueryHandler::default()); let query_ctx = QueryContext::arc(); let state = Arc::new(QueryEngineState::new( @@ -1869,8 +1892,12 @@ mod tests { ) .unwrap(); - let mut stream = exec.to_stream(task_ctx, 0).unwrap(); - assert!(stream.next().await.unwrap().is_err()); + let mut stream = common_recordbatch::adapter::RecordBatchStreamAdapter::try_new( + exec.to_stream(task_ctx, 0).unwrap(), + ) + .unwrap(); + let error = stream.next().await.unwrap().unwrap_err(); + assert_eq!(error.status_code(), StatusCode::RequestOutdated); assert_eq!(handler.do_get_calls.load(Ordering::SeqCst), 1); assert!(handler.saw_subscriber.load(Ordering::SeqCst)); @@ -1880,6 +1907,46 @@ mod tests { assert!(!entries[0].fanout_started_for_test()); } + #[tokio::test] + async fn repartitioned_merge_scan_later_stream_error_preserves_status_code() { + let region_id = RegionId::new(1024, 1); + let handler = Arc::new(TestRegionQueryHandler::with_responses(vec![( + region_id, + int64_schema(&["a", "b"]), + vec![Err(common_recordbatch::error::ExternalSnafu.into_error( + BoxedError::new(PlainError::new( + "neutral stream error".to_string(), + StatusCode::RequestOutdated, + )), + ))], + )])); + let merge_scan = Arc::new(merge_scan_exec_with_handler( + vec![region_id], + expected_int64_schema(), + handler, + 1, + )); + let repartition = + RepartitionExec::try_new(merge_scan, Partitioning::RoundRobinBatch(2)).unwrap(); + assert_eq!( + repartition + .properties() + .output_partitioning() + .partition_count(), + 2 + ); + + let mut stream = common_recordbatch::adapter::RecordBatchStreamAdapter::try_new( + repartition + .execute(0, Arc::new(TaskContext::default())) + .unwrap(), + ) + .unwrap(); + + let error = stream.next().await.unwrap().unwrap_err(); + assert_eq!(error.status_code(), StatusCode::RequestOutdated); + } + #[tokio::test] async fn aborting_pending_do_get_poll_rolls_back_subscriber_without_starting_fanout() { let handler = Arc::new(PendingDoGetHandler::default()); @@ -2080,10 +2147,9 @@ mod tests { assert_eq!(registry_manager.registry_count(), 0); } - #[derive(Clone)] struct TestRegionResponse { advertised_schema: Arc, - batches: Vec, + batches: Vec>, } #[derive(Default)] @@ -2100,7 +2166,7 @@ mod tests { region_id, TestRegionResponse { advertised_schema: batch.schema.clone(), - batches: vec![batch], + batches: vec![Ok(batch)], }, ) }) @@ -2109,7 +2175,13 @@ mod tests { } fn with_responses( - responses: impl IntoIterator, Vec)>, + responses: impl IntoIterator< + Item = ( + RegionId, + Arc, + Vec>, + ), + >, ) -> Self { let responses = responses .into_iter() @@ -2129,19 +2201,17 @@ mod tests { struct TestRecordBatchStream { schema: Arc, - batches: Vec, - index: usize, + batches: Vec>, } impl Stream for TestRecordBatchStream { type Item = common_recordbatch::error::Result; fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - if let Some(batch) = self.batches.get(self.index).cloned() { - self.index += 1; - Poll::Ready(Some(Ok(batch))) - } else { + if self.batches.is_empty() { Poll::Ready(None) + } else { + Poll::Ready(Some(self.batches.remove(0))) } } } @@ -2202,10 +2272,13 @@ mod tests { }), Ordering::SeqCst, ); - crate::error::UnimplementedSnafu { - operation: "test do_get failure", - } - .fail() + Err(crate::error::Error::QueryExecution { + source: BoxedError::new(PlainError::new( + "neutral do_get error".to_string(), + StatusCode::RequestOutdated, + )), + location: snafu::Location::default(), + }) } async fn handle_remote_dyn_filter_update( @@ -2523,8 +2596,19 @@ mod tests { .expect("test handler needs a response for every requested region"); Ok(Box::pin(TestRecordBatchStream { schema: response.advertised_schema.clone(), - batches: response.batches.clone(), - index: 0, + batches: response + .batches + .iter() + .map(|batch| match batch { + Ok(batch) => Ok(batch.clone()), + Err(error) => Err(common_recordbatch::error::ExternalSnafu.into_error( + BoxedError::new(PlainError::new( + error.to_string(), + error.status_code(), + )), + )), + }) + .collect(), })) } @@ -3085,7 +3169,7 @@ mod tests { Arc::new(TestRegionQueryHandler::with_responses(vec![( RegionId::new(1024, 1), remote_schema, - vec![batch], + vec![Ok(batch)], )])), 1, )) @@ -3122,7 +3206,7 @@ mod tests { Arc::new(TestRegionQueryHandler::with_responses(vec![( region_id, advertised_schema, - vec![batch], + vec![Ok(batch)], )])), 1, ); @@ -3155,7 +3239,7 @@ mod tests { Arc::new(TestRegionQueryHandler::with_responses(vec![( region_id, advertised_schema, - vec![batch], + vec![Ok(batch)], )])), 1, ); @@ -3190,7 +3274,7 @@ mod tests { Arc::new(TestRegionQueryHandler::with_responses(vec![( region_id, advertised_schema, - vec![batch], + vec![Ok(batch)], )])), 1, )) @@ -3251,7 +3335,7 @@ mod tests { // A distribution that differs from the current partitioning but shares a // column name present in partition_cols, so try_with_new_distribution // produces a clone instead of returning None. - let new_dist = Distribution::HashPartitioned(vec![ + let new_dist = Distribution::KeyPartitioned(vec![ Arc::new(Column::new("col1", 0)), Arc::new(Column::new("col2", 1)), ]); @@ -3267,6 +3351,24 @@ mod tests { ); } + #[test] + fn merge_scan_apply_expressions_exposes_remote_dyn_filter_id() { + let query_ctx = QueryContext::arc(); + let exec = + remote_dyn_filter_test_exec(Arc::new(TestRegionQueryHandler::default()), query_ctx); + let dyn_filter = install_remote_dyn_filter(&exec); + let expected_expression_id = dyn_filter.expression_id(); + let mut expression_ids = Vec::new(); + + exec.apply_expressions(&mut |expr| { + expression_ids.push(expr.expression_id()); + Ok(TreeNodeRecursion::Continue) + }) + .unwrap(); + + assert_eq!(expression_ids, vec![expected_expression_id]); + } + #[test] fn remote_dyn_filter_preflight_removes_parent_filter_after_dn_runtime_is_ready() { let remote_dyn_filter_producer_id = RemoteDynFilterProducerId::new(42); diff --git a/src/query/src/dist_plan/merge_sort.rs b/src/query/src/dist_plan/merge_sort.rs index 2c8f4d9fd8a..e95849d5b5f 100644 --- a/src/query/src/dist_plan/merge_sort.rs +++ b/src/query/src/dist_plan/merge_sort.rs @@ -16,7 +16,6 @@ //! `SortPreservingMergeExec` operator in datafusion //! -use std::any::Any; use std::fmt; use std::sync::Arc; @@ -27,12 +26,14 @@ use datafusion::physical_plan::projection::{ProjectionExec, make_with_child, upd use datafusion::physical_plan::sorts::sort::SortExec; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SendableRecordBatchStream, - Statistics, + ChildStats, ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, + InputDistributionRequirements, PlanProperties, ReplaceChildrenOptions, + SendableRecordBatchStream, Statistics, StatisticsArgs, apply_expression_roots, }; +use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_common::{DataFusionError, Result}; use datafusion_expr::{Extension, LogicalPlan, SortExpr, UserDefinedLogicalNodeCore}; -use datafusion_physical_expr::{Distribution, LexOrdering, OrderingRequirements}; +use datafusion_physical_expr::{LexOrdering, OrderingRequirements, PhysicalExpr}; /// MergeSort Logical Plan, have same field as `Sort`, but indicate it is a merge sort, /// which assume each input partition is a sorted stream, and will use `SortPreserveingMergeExec` @@ -94,7 +95,7 @@ impl MergeSortExec { fn input_with_fetch(&self, fetch: Option) -> Arc { let input = Arc::clone(self.inner.input()); - if let Some(sort) = input.as_any().downcast_ref::() + if let Some(sort) = input.downcast_ref::() && sort.preserve_partitioning() && sort.expr() == self.inner.expr() { @@ -149,7 +150,7 @@ impl ExecutionPlan for MergeSortExec { /// `MergeSortExec` delegates most behavior to DataFusion's /// `SortPreservingMergeExec`, but it must not expose itself as that type. /// DataFusion's `EnforceSorting` optimizer recognizes a bare - /// `SortPreservingMergeExec` via `as_any().downcast_ref::<...>()` and may + /// `SortPreservingMergeExec` via `downcast_ref::<...>()` and may /// replace it with an unordered `CoalescePartitionsExec(fetch)` when the /// parent does not require sorted output. /// @@ -165,10 +166,6 @@ impl ExecutionPlan for MergeSortExec { /// below `MergeSortExec` when `MergeScanExec` cannot preserve per-partition /// ordering. This opacity is specifically about protecting the merge stage /// itself from the `EnforceSorting` rewrite above. - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { self.inner.properties() } @@ -193,8 +190,8 @@ impl ExecutionPlan for MergeSortExec { }) } - fn required_input_distribution(&self) -> Vec { - self.inner.required_input_distribution() + fn input_distribution_requirements(&self) -> InputDistributionRequirements { + self.inner.input_distribution_requirements() } fn benefits_from_input_partitioning(&self) -> Vec { @@ -205,7 +202,7 @@ impl ExecutionPlan for MergeSortExec { /// ordered. This is the contract that makes `EnforceSorting` insert a /// `SortExec` below `MergeSortExec` when the input cannot preserve ordering. /// - /// The opacity of `MergeSortExec::as_any`, not this requirement, is what + /// The opacity of `MergeSortExec`'s downcast identity, not this requirement, is what /// prevents DataFusion from rewriting the merge stage itself as a bare /// `SortPreservingMergeExec`. fn required_input_ordering(&self) -> Vec> { @@ -220,9 +217,17 @@ impl ExecutionPlan for MergeSortExec { self.inner.children() } - fn with_new_children( + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + apply_expression_roots(self.inner.expr().iter().map(|sort_expr| &sort_expr.expr), f) + } + + fn replace_children( self: Arc, mut children: Vec>, + options: ReplaceChildrenOptions, ) -> Result> { if children.len() != 1 { return Err(DataFusionError::Internal(format!( @@ -231,11 +236,31 @@ impl ExecutionPlan for MergeSortExec { ))); } - Ok(Arc::new(Self::new( - self.inner.expr().clone(), - children.swap_remove(0), - self.inner.fetch(), - ))) + match options.children_properties { + ChildrenPropertiesMode::Keep => Ok(Arc::new(Self { + inner: SortPreservingMergeExec::new( + self.inner.expr().clone(), + children.swap_remove(0), + ) + .with_fetch(self.inner.fetch()), + })), + ChildrenPropertiesMode::Recompute => Ok(Arc::new(Self::new( + self.inner.expr().clone(), + children.swap_remove(0), + self.inner.fetch(), + ))), + } + } + + #[allow(deprecated)] + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + self.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) } fn execute( @@ -250,8 +275,16 @@ impl ExecutionPlan for MergeSortExec { self.inner.metrics() } - fn partition_statistics(&self, partition: Option) -> Result { - self.inner.partition_statistics(partition) + fn child_stats_requests(&self, partition: Option) -> Vec { + self.inner.child_stats_requests(partition) + } + + fn statistics_from_inputs( + &self, + input_stats: &[Arc], + args: &StatisticsArgs, + ) -> Result> { + self.inner.statistics_from_inputs(input_stats, args) } fn cardinality_effect(&self) -> CardinalityEffect { @@ -421,10 +454,6 @@ mod tests { "PreserveOrderProbeExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { self.inner.properties() } @@ -433,6 +462,13 @@ mod tests { vec![&self.inner] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, mut children: Vec>, @@ -489,7 +525,6 @@ mod tests { assert_eq!(merge_sort.name(), "MergeSortExec"); assert!( merge_sort - .as_any() .downcast_ref::() .is_none(), "MergeSortExec must stay opaque to EnforceSorting's bare SortPreservingMerge rewrite" @@ -503,7 +538,7 @@ mod tests { assert!(!tree.contains("SortPreservingMergeExec")); let fetched = merge_sort.with_fetch(Some(2)).unwrap(); - assert!(fetched.as_any().downcast_ref::().is_some()); + assert!(fetched.downcast_ref::().is_some()); assert_eq!(fetched.fetch(), Some(2)); } @@ -545,12 +580,9 @@ mod tests { let fetched = merge_sort.with_fetch(Some(2)).unwrap(); - assert!(fetched.as_any().downcast_ref::().is_some()); + assert!(fetched.downcast_ref::().is_some()); assert_eq!(fetched.fetch(), Some(2)); - let child_sort = fetched.children()[0] - .as_any() - .downcast_ref::() - .unwrap(); + let child_sort = fetched.children()[0].downcast_ref::().unwrap(); assert_eq!(child_sort.fetch(), Some(2)); assert!(child_sort.preserve_partitioning()); } @@ -568,14 +600,12 @@ mod tests { let preserved_spm = bare_spm.with_preserve_order(true).unwrap(); assert!( preserved_spm - .as_any() .downcast_ref::() .is_some(), "bare SPM should rebuild as bare SPM" ); assert!( preserved_spm.children()[0] - .as_any() .downcast_ref::() .unwrap() .preserve_order @@ -585,14 +615,12 @@ mod tests { let preserved_merge_sort = merge_sort.with_preserve_order(true).unwrap(); assert!( preserved_merge_sort - .as_any() .downcast_ref::() .is_some(), "MergeSortExec must rewrap the preserve-order child as MergeSortExec" ); assert!( preserved_merge_sort - .as_any() .downcast_ref::() .is_none(), "MergeSortExec must not expose a bare SPM after with_preserve_order" @@ -605,7 +633,6 @@ mod tests { ); assert!( preserved_merge_sort.children()[0] - .as_any() .downcast_ref::() .unwrap() .preserve_order @@ -637,7 +664,6 @@ mod tests { .expect("SPM should accept a narrowing projection that preserves the sort key"); assert!( swapped_spm - .as_any() .downcast_ref::() .is_some(), "bare SPM should rebuild as bare SPM" @@ -657,15 +683,11 @@ mod tests { .expect("MergeSortExec should accept the same projection swap as SPM"); assert!( - swapped_merge_sort - .as_any() - .downcast_ref::() - .is_some(), + swapped_merge_sort.downcast_ref::().is_some(), "MergeSortExec must rewrap projection swaps as MergeSortExec" ); assert!( swapped_merge_sort - .as_any() .downcast_ref::() .is_none(), "MergeSortExec must not expose a bare SPM after projection swap" @@ -673,7 +695,6 @@ mod tests { assert_eq!(swapped_merge_sort.fetch(), Some(1)); assert!( swapped_merge_sort.children()[0] - .as_any() .downcast_ref::() .is_some(), "the projection should move below MergeSortExec" @@ -754,7 +775,6 @@ mod tests { .plan; assert!( optimized_spm - .as_any() .downcast_ref::() .is_some(), "this regression test must exercise EnforceSorting's bare SPM -> CoalescePartitionsExec rewrite" @@ -776,14 +796,12 @@ mod tests { .plan; assert!( optimized_merge_sort - .as_any() .downcast_ref::() .is_some(), "MergeSortExec must stay opaque to the bare SPM rewrite" ); assert!( optimized_merge_sort - .as_any() .downcast_ref::() .is_none(), "MergeSortExec(fetch) is the required distributed TopK merge stage, not an unordered coalesce" diff --git a/src/query/src/dist_plan/planner.rs b/src/query/src/dist_plan/planner.rs index f2aaa2bbf84..130a123fc66 100644 --- a/src/query/src/dist_plan/planner.rs +++ b/src/query/src/dist_plan/planner.rs @@ -22,9 +22,11 @@ use async_trait::async_trait; use catalog::CatalogManagerRef; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_telemetry::debug; +use datafusion::catalog::Session; use datafusion::common::Result; use datafusion::datasource::DefaultTableSource; use datafusion::execution::context::SessionState; +use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_planner::{ExtensionPlanner, PhysicalPlanner}; use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion, TreeNodeVisitor}; @@ -60,15 +62,21 @@ pub struct MergeSortExtensionPlanner {} impl MergeSortExtensionPlanner { fn ordering( - session_state: &SessionState, + planner: &dyn PhysicalPlanner, + session: &dyn Session, + planning_ctx: &PhysicalPlanningContext, merge_sort: &MergeSortLogicalPlan, ) -> Result { let ordering = merge_sort .expr .iter() .map(|sort_expr| { - let physical_expr = session_state - .create_physical_expr(sort_expr.expr.clone(), merge_sort.input.schema())?; + let physical_expr = planner.create_physical_expr( + &sort_expr.expr, + merge_sort.input.schema(), + session, + planning_ctx, + )?; Ok(PhysicalSortExpr::new( physical_expr, SortOptions { @@ -91,11 +99,12 @@ impl MergeSortExtensionPlanner { impl ExtensionPlanner for MergeSortExtensionPlanner { async fn plan_extension( &self, - _planner: &dyn PhysicalPlanner, + planner: &dyn PhysicalPlanner, node: &dyn UserDefinedLogicalNode, _logical_inputs: &[&LogicalPlan], physical_inputs: &[Arc], - session_state: &SessionState, + session: &dyn Session, + planning_ctx: &PhysicalPlanningContext, ) -> Result>> { if let Some(merge_sort) = node.as_any().downcast_ref::() { if let LogicalPlan::Extension(ext) = &merge_sort.input.as_ref() @@ -110,14 +119,14 @@ impl ExtensionPlanner for MergeSortExtensionPlanner { "Expect MergeSort to have one physical input".to_string(), ) })?; - if input.as_any().downcast_ref::().is_none() { + if input.downcast_ref::().is_none() { return Err(DataFusionError::Internal(format!( "Expect MergeSort's input is a MergeScanExec, found {:?}", physical_inputs ))); } - let ordering = Self::ordering(session_state, merge_sort)?; + let ordering = Self::ordering(planner, session, planning_ctx, merge_sort)?; Ok(Some(Arc::new(MergeSortExec::new( ordering, input, @@ -163,7 +172,8 @@ impl ExtensionPlanner for DistExtensionPlanner { node: &dyn UserDefinedLogicalNode, _logical_inputs: &[&LogicalPlan], _physical_inputs: &[Arc], - session_state: &SessionState, + session: &dyn Session, + _planning_ctx: &PhysicalPlanningContext, ) -> Result>> { let Some(merge_scan) = node.as_any().downcast_ref::() else { return Ok(None); @@ -171,9 +181,9 @@ impl ExtensionPlanner for DistExtensionPlanner { let input_plan = merge_scan.input(); let fallback = |logical_plan| async move { - let optimized_plan = self.optimize_input_logical_plan(session_state, logical_plan)?; + let optimized_plan = self.optimize_input_logical_plan(session, logical_plan)?; planner - .create_physical_plan(&optimized_plan, session_state) + .create_physical_plan(&optimized_plan, session) .await .map(Some) }; @@ -196,6 +206,14 @@ impl ExtensionPlanner for DistExtensionPlanner { // TODO(ruihang): generate different execution plans for different variant merge operation let schema = merge_scan.schema().as_arrow(); + let session_state = session + .as_any() + .downcast_ref::() + .ok_or_else(|| { + DataFusionError::Internal( + "MergeScan requires a SessionState for physical planning".to_string(), + ) + })?; let query_ctx = session_state .config() .get_extension() @@ -208,7 +226,7 @@ impl ExtensionPlanner for DistExtensionPlanner { schema, self.region_query_handler.clone(), query_ctx, - session_state.config().target_partitions(), + session.config().target_partitions(), merge_scan.partition_cols().clone(), merge_scan.remote_dyn_filter_producer_id(), self.enable_per_region_metrics, @@ -420,11 +438,21 @@ impl DistExtensionPlanner { /// Input logical plan is analyzed. Thus only call logical optimizer to optimize it. fn optimize_input_logical_plan( &self, - session_state: &SessionState, + session: &dyn Session, plan: &LogicalPlan, ) -> Result { - let state = session_state.clone(); - state.optimizer().optimize(plan.clone(), &state, |_, _| {}) + let session_state = session + .as_any() + .downcast_ref::() + .ok_or_else(|| { + DataFusionError::Internal( + "MergeScan requires a SessionState for logical optimization".to_string(), + ) + })?; + + session_state + .optimizer() + .optimize(plan.clone(), session_state, |_, _| {}) } } @@ -448,10 +476,9 @@ impl TreeNodeVisitor<'_> for TableNameExtractor { fn f_down(&mut self, node: &Self::Node) -> Result { match node { LogicalPlan::TableScan(scan) => { - if let Some(source) = scan.source.as_any().downcast_ref::() + if let Some(source) = scan.source.downcast_ref::() && let Some(provider) = source .table_provider - .as_any() .downcast_ref::() { if provider.table().table_type() == TableType::Base { diff --git a/src/query/src/dist_plan/predicate_extractor.rs b/src/query/src/dist_plan/predicate_extractor.rs index febf6c31203..778f5b9fc94 100644 --- a/src/query/src/dist_plan/predicate_extractor.rs +++ b/src/query/src/dist_plan/predicate_extractor.rs @@ -433,16 +433,16 @@ impl DataFusionExprConverter { Expr::Cast(cast_expr) => { // For safe casts, unwrap to the inner expression // For unsafe casts, skip with debug logging - if Self::is_safe_cast_for_partition_pruning(&cast_expr.data_type) { + if Self::is_safe_cast_for_partition_pruning(cast_expr.field.data_type()) { Self::convert_to_operand(&cast_expr.expr) } else { debug!( "Skipping unsafe cast for partition pruning: {:?}", - cast_expr.data_type + cast_expr.field.data_type() ); Err(datafusion_common::DataFusionError::Plan(format!( "Cast to {:?} not supported for partition pruning", - cast_expr.data_type + cast_expr.field.data_type() ))) } } @@ -638,10 +638,10 @@ mod tests { fn test_dictionary_cast_preserves_partition_constraint() { let dictionary_type = DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)); - let filter = col("tag").eq(Expr::Cast(datafusion_expr::expr::Cast { - expr: Box::new(lit("b")), - data_type: dictionary_type, - })); + let filter = col("tag").eq(Expr::Cast(datafusion_expr::Cast::new( + Box::new(lit("b")), + dictionary_type, + ))); let partition_expr = DataFusionExprConverter::convert(&filter).unwrap(); assert_eq!( @@ -1011,10 +1011,10 @@ mod tests { let cases = vec![ FilterTestCase::new( "safe_cast", - Expr::Cast(datafusion_expr::Cast { - expr: Box::new(col("user_id")), - data_type: DataType::Int64, - }) + Expr::Cast(datafusion_expr::Cast::new( + Box::new(col("user_id")), + DataType::Int64, + )) .eq(lit(100i64)), vec![PartitionExpr::new( Operand::Column("user_id".to_string()), @@ -1025,10 +1025,10 @@ mod tests { ), FilterTestCase::new( "cast_with_alias", - Expr::Cast(datafusion_expr::Cast { - expr: Box::new(col("user_id").alias("uid")), - data_type: DataType::Int64, - }) + Expr::Cast(datafusion_expr::Cast::new( + Box::new(col("user_id").alias("uid")), + DataType::Int64, + )) .eq(lit(100i64)), vec![PartitionExpr::new( Operand::Column("user_id".to_string()), @@ -1039,12 +1039,12 @@ mod tests { ), FilterTestCase::new( "unsafe_cast", - Expr::Cast(datafusion_expr::Cast { - expr: Box::new(col("user_id")), - data_type: DataType::List(std::sync::Arc::new( + Expr::Cast(datafusion_expr::Cast::new( + Box::new(col("user_id")), + DataType::List(std::sync::Arc::new( datafusion::arrow::datatypes::Field::new("item", DataType::Int32, true), )), - }) + )) .eq(lit(100i64)), vec![], vec!["user_id"], @@ -1122,10 +1122,10 @@ mod tests { let in_expr = col("user_id") .alias("uid") .in_list(vec![lit(100i64), lit(200i64)], false); - let cast_expr = Expr::Cast(datafusion_expr::Cast { - expr: Box::new(col("user_id")), - data_type: DataType::Int64, - }); + let cast_expr = Expr::Cast(datafusion_expr::Cast::new( + Box::new(col("user_id")), + DataType::Int64, + )); let between_expr = cast_expr.between(lit(300i64), lit(400i64)); in_expr.or(between_expr) }, diff --git a/src/query/src/dist_plan/remote_dyn_filter_receiver.rs b/src/query/src/dist_plan/remote_dyn_filter_receiver.rs index b36e725bed0..562cfd3c77c 100644 --- a/src/query/src/dist_plan/remote_dyn_filter_receiver.rs +++ b/src/query/src/dist_plan/remote_dyn_filter_receiver.rs @@ -18,8 +18,9 @@ use std::hash::{Hash, Hasher}; use std::sync::Arc; use async_trait::async_trait; +use datafusion::catalog::Session; use datafusion::common::Result; -use datafusion::execution::context::SessionState; +use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext; use datafusion::physical_expr::utils::conjunction; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::expressions::Column; @@ -176,7 +177,7 @@ fn remap_physical_expr_columns( expr: Arc, input_schema: &datafusion::arrow::datatypes::Schema, ) -> Result> { - if let Some(column) = expr.as_any().downcast_ref::() { + if let Some(column) = expr.downcast_ref::() { return Ok(Arc::new(Column::new_with_schema( column.name(), input_schema, @@ -205,7 +206,8 @@ impl ExtensionPlanner for RemoteDynFilterReceiverExtensionPlanner { node: &dyn UserDefinedLogicalNode, _logical_inputs: &[&LogicalPlan], physical_inputs: &[Arc], - _session_state: &SessionState, + _session: &dyn Session, + _planning_ctx: &PhysicalPlanningContext, ) -> Result>> { let Some(receiver) = node .as_any() diff --git a/src/query/src/dummy_catalog.rs b/src/query/src/dummy_catalog.rs index 3d1dfe5f375..2131423e3fe 100644 --- a/src/query/src/dummy_catalog.rs +++ b/src/query/src/dummy_catalog.rs @@ -70,10 +70,6 @@ impl DummyCatalogList { } impl CatalogProviderList for DummyCatalogList { - fn as_any(&self) -> &dyn Any { - self - } - fn register_catalog( &self, _name: String, @@ -98,10 +94,6 @@ struct DummyCatalogProvider { } impl CatalogProvider for DummyCatalogProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn schema_names(&self) -> Vec { vec![] } @@ -119,10 +111,6 @@ struct DummySchemaProvider { #[async_trait] impl SchemaProvider for DummySchemaProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn table_names(&self) -> Vec { vec![] } @@ -162,10 +150,6 @@ impl fmt::Debug for DummyTableProvider { #[async_trait] impl TableProvider for DummyTableProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { let schema = self.metadata.schema.arrow_schema(); if !supports_pk_dictionary_encoding(self.engine.name()) { @@ -382,6 +366,8 @@ struct FlowScanDecision { memtable_max_sequence: Option, /// Whether to skip SST files for memtable-only incremental source scans. skip_sst_files: bool, + /// Whether this source scan must enforce the exact sequence range. + exact_sequence_range: bool, } impl FlowScanDecision { @@ -392,6 +378,7 @@ impl FlowScanDecision { memtable_min_sequence: None, memtable_max_sequence: None, skip_sst_files: false, + exact_sequence_range: false, } } } @@ -406,6 +393,7 @@ fn decide_flow_scan(query_ctx: &QueryContext, region_id: RegionId) -> Result Result RegionId { @@ -754,6 +745,64 @@ mod tests { assert_eq!(request.memtable_max_sequence, None); } + #[test] + fn test_scan_request_from_sequence_range_context_uses_exact_source_scan() { + let region_id = test_region_id(); + let query_ctx = QueryContextBuilder::default() + .extensions(HashMap::from([ + ( + FLOW_INCREMENTAL_MODE.to_string(), + FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE.to_string(), + ), + ( + FLOW_INCREMENTAL_AFTER_SEQS.to_string(), + format!(r#"{{"{}":10}}"#, region_id.as_u64()), + ), + ])) + .snapshot_seqs(Arc::new(RwLock::new(HashMap::from([( + region_id.as_u64(), + 42_u64, + )])))) + .sst_min_sequences(Arc::new(RwLock::new(HashMap::from([( + region_id.as_u64(), + 7_u64, + )])))) + .build(); + + let request = scan_request_from_query_context(region_id, &query_ctx).unwrap(); + + assert!(request.exact_sequence_range); + assert!(!request.skip_sst_files); + assert_eq!(request.memtable_min_sequence, Some(10)); + assert_eq!(request.memtable_max_sequence, Some(42)); + assert_eq!(request.sst_min_sequence, None); + } + + #[test] + fn test_scan_request_from_sequence_range_context_binds_snapshot_on_scan() { + let region_id = test_region_id(); + let query_ctx = QueryContextBuilder::default() + .extensions(HashMap::from([ + ( + FLOW_INCREMENTAL_MODE.to_string(), + FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE.to_string(), + ), + ( + FLOW_INCREMENTAL_AFTER_SEQS.to_string(), + format!(r#"{{"{}":10}}"#, region_id.as_u64()), + ), + ])) + .build(); + + let request = scan_request_from_query_context(region_id, &query_ctx).unwrap(); + + assert!(request.exact_sequence_range); + assert!(request.snapshot_on_scan); + assert_eq!(request.memtable_min_sequence, Some(10)); + assert_eq!(request.memtable_max_sequence, None); + assert!(!request.skip_sst_files); + } + #[test] fn test_scan_request_from_query_context_keeps_snapshot_fields() { let region_id = test_region_id(); @@ -824,6 +873,32 @@ mod tests { assert!(!request.snapshot_on_scan); } + #[test] + fn test_apply_cached_snapshot_to_request_preserves_exact_sequence_range() { + let region_id = test_region_id(); + let query_ctx = QueryContextBuilder::default() + .snapshot_seqs(Arc::new(RwLock::new(HashMap::from([( + region_id.as_u64(), + 10_u64, + )])))) + .build(); + let mut request = ScanRequest { + memtable_min_sequence: Some(10), + snapshot_on_scan: true, + exact_sequence_range: true, + ..Default::default() + }; + + apply_cached_snapshot_to_request(&query_ctx, region_id, false, &mut request); + + assert_eq!(request.memtable_min_sequence, Some(10)); + assert_eq!(request.memtable_max_sequence, Some(10)); + assert!(request.exact_sequence_range); + assert!(!request.skip_sst_files); + assert_eq!(request.sst_min_sequence, None); + assert!(!request.snapshot_on_scan); + } + #[test] fn test_apply_cached_snapshot_to_request_skips_sink_scan() { let region_id = test_region_id(); @@ -891,6 +966,44 @@ mod tests { assert_eq!(request.memtable_min_sequence, Some(55)); assert_eq!(request.sst_min_sequence, None); assert!(request.skip_sst_files); + assert!(!request.exact_sequence_range); + } + + #[test] + fn test_scan_request_from_sequence_range_context_excludes_sink_scan() { + let region_id = test_region_id(); + let query_ctx = QueryContextBuilder::default() + .extensions(HashMap::from([ + ( + FLOW_INCREMENTAL_MODE.to_string(), + FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE.to_string(), + ), + ( + FLOW_INCREMENTAL_AFTER_SEQS.to_string(), + format!(r#"{{"{}":55}}"#, region_id.as_u64()), + ), + ( + FLOW_SINK_TABLE_ID.to_string(), + region_id.table_id().to_string(), + ), + ])) + .snapshot_seqs(Arc::new(RwLock::new(HashMap::from([( + region_id.as_u64(), + 88_u64, + )])))) + .sst_min_sequences(Arc::new(RwLock::new(HashMap::from([( + region_id.as_u64(), + 77_u64, + )])))) + .build(); + + let request = scan_request_from_query_context(region_id, &query_ctx).unwrap(); + + assert!(!request.exact_sequence_range); + assert!(!request.skip_sst_files); + assert_eq!(request.memtable_min_sequence, None); + assert_eq!(request.memtable_max_sequence, None); + assert_eq!(request.sst_min_sequence, None); } #[test] @@ -949,6 +1062,27 @@ mod tests { assert!(matches!(err, Error::InvalidQueryContextExtension { .. })); } + #[test] + fn test_scan_request_from_sequence_range_rejects_missing_source_bound() { + let region_id = test_region_id(); + let query_ctx = QueryContextBuilder::default() + .extensions(HashMap::from([ + ( + FLOW_INCREMENTAL_MODE.to_string(), + FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE.to_string(), + ), + ( + FLOW_INCREMENTAL_AFTER_SEQS.to_string(), + r#"{"9":55}"#.to_string(), + ), + ])) + .build(); + + let err = scan_request_from_query_context(region_id, &query_ctx).unwrap_err(); + assert!(matches!(err, Error::InvalidQueryContextExtension { .. })); + assert_eq!(err.status_code(), StatusCode::InvalidArguments); + } + #[test] fn test_scan_request_from_query_context_rejects_invalid_incremental_json() { let region_id = test_region_id(); diff --git a/src/query/src/error.rs b/src/query/src/error.rs index fdfeaafde97..a2946415570 100644 --- a/src/query/src/error.rs +++ b/src/query/src/error.rs @@ -498,8 +498,23 @@ impl From for DataFusionError { #[cfg(test)] mod tests { + use common_error::ext::PlainError; + use super::*; + #[test] + fn test_datafusion_external_boxed_error_status_code() { + let error = Error::DataFusion { + error: DataFusionError::External(Box::new(BoxedError::new(PlainError::new( + "neutral error".to_string(), + StatusCode::RequestOutdated, + )))), + location: Location::default(), + }; + + assert_eq!(error.status_code(), StatusCode::RequestOutdated); + } + #[test] fn test_build_backend_delegates_error_metadata() { let source = common_datasource::error::LocalFileAccessDisabledSnafu { diff --git a/src/query/src/log_query/planner.rs b/src/query/src/log_query/planner.rs index 05aeb0795d2..5bc8d034a12 100644 --- a/src/query/src/log_query/planner.rs +++ b/src/query/src/log_query/planner.rs @@ -17,12 +17,11 @@ use catalog::table_source::DfTableSourceProvider; use common_function::utils::escape_like_pattern; use datafusion::datasource::DefaultTableSource; use datafusion::execution::SessionState; -use datafusion_common::{DFSchema, ScalarValue}; +use datafusion_common::{DFSchema, ScalarValue, TableReference}; use datafusion_expr::utils::{conjunction, disjunction}; use datafusion_expr::{ BinaryExpr, Expr, ExprSchemable, LogicalPlan, LogicalPlanBuilder, Operator, col, lit, not, }; -use datafusion_sql::TableReference; use datatypes::schema::Schema; use log_query::{AggFunc, BinaryOperator, EqualValue, LogExpr, LogQuery, TimeFilter}; use snafu::{OptionExt, ResultExt}; @@ -56,11 +55,9 @@ impl LogQueryPlanner { .await .context(CatalogSnafu)?; let schema = table_source - .as_any() .downcast_ref::() .context(UnknownTableSnafu)? .table_provider - .as_any() .downcast_ref::() .context(UnknownTableSnafu)? .table() diff --git a/src/query/src/metrics.rs b/src/query/src/metrics.rs index 5aa7bf82822..d6d62c99c4f 100644 --- a/src/query/src/metrics.rs +++ b/src/query/src/metrics.rs @@ -280,7 +280,7 @@ fn collect_region_watermarks(plan: Arc) -> Vec() + if let Some(merge_scan) = plan.downcast_ref::() && !merge_scan.is_flow_sink_scan() { merge_merge_scan_region_watermarks( diff --git a/src/query/src/optimizer.rs b/src/query/src/optimizer.rs index 480c5046c2a..fec001e7506 100644 --- a/src/query/src/optimizer.rs +++ b/src/query/src/optimizer.rs @@ -16,6 +16,7 @@ pub mod const_normalization; pub mod constant_term; pub mod count_nest_aggr; pub mod count_wildcard; +pub mod enforce_sorting; pub mod global_limit; pub(crate) mod insert_assignment; pub(crate) mod json_schema_concretize; diff --git a/src/query/src/optimizer/const_normalization.rs b/src/query/src/optimizer/const_normalization.rs index 620797208e3..d29f0aa4f39 100644 --- a/src/query/src/optimizer/const_normalization.rs +++ b/src/query/src/optimizer/const_normalization.rs @@ -561,11 +561,11 @@ enum CastInputKind { /// Returns the input expression and target type for `CAST` and `TRY_CAST` expressions. fn extract_cast_input(expr: &Expr) -> Option<(CastInputKind, &Expr, &DataType)> { match expr { - Expr::Cast(Cast { expr, data_type }) => { - Some((CastInputKind::Cast, expr.as_ref(), data_type)) + Expr::Cast(Cast { expr, field }) => { + Some((CastInputKind::Cast, expr.as_ref(), field.data_type())) } - Expr::TryCast(TryCast { expr, data_type }) => { - Some((CastInputKind::TryCast, expr.as_ref(), data_type)) + Expr::TryCast(TryCast { expr, field }) => { + Some((CastInputKind::TryCast, expr.as_ref(), field.data_type())) } _ => None, } @@ -921,7 +921,6 @@ mod tests { .await .unwrap(); let filter = physical_plan - .as_any() .downcast_ref::() .expect("regex residual must remain a FilterExec"); assert!(matches!( @@ -1332,6 +1331,24 @@ mod tests { expected_greptime: "Filter: CAST(t.ts_ms AS Timestamp(ns)) = TimestampNanosecond(5000000000, None)\n TableScan: t", expected_datafusion: "Filter: t.ts_ms = TimestampMillisecond(5000, None)\n TableScan: t", }, + Case { + name: "timestamp widening try_cast exact", + fields: vec![Field::new( + "ts_ms", + DataType::Timestamp(ArrowTimeUnit::Millisecond, None), + false, + )], + predicate: try_cast( + col("ts_ms"), + DataType::Timestamp(ArrowTimeUnit::Nanosecond, None), + ) + .eq(lit(ScalarValue::TimestampNanosecond( + Some(5_000_000_000), + None, + ))), + expected_greptime: "Filter: TRY_CAST(t.ts_ms AS Timestamp(ns)) = TimestampNanosecond(5000000000, None)\n TableScan: t", + expected_datafusion: "Filter: t.ts_ms = TimestampMillisecond(5000, None)\n TableScan: t", + }, ]; for case in cases { @@ -1506,10 +1523,6 @@ mod tests { #[async_trait] impl TableProvider for ExactPushdownProvider { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> arrow_schema::SchemaRef { self.schema.clone() } diff --git a/src/query/src/optimizer/constant_term.rs b/src/query/src/optimizer/constant_term.rs index 47f3faf59d7..70ddcc2f647 100644 --- a/src/query/src/optimizer/constant_term.rs +++ b/src/query/src/optimizer/constant_term.rs @@ -76,10 +76,6 @@ impl PartialEq for PreCompiledMatchesTermExpr { impl Eq for PreCompiledMatchesTermExpr {} impl PhysicalExpr for PreCompiledMatchesTermExpr { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn data_type( &self, _input_schema: &arrow_schema::Schema, @@ -166,10 +162,10 @@ impl PhysicalOptimizerRule for MatchesConstantTermOptimizer { ) -> DfResult> { let res = plan .transform_down(&|plan: Arc| { - if let Some(filter) = plan.as_any().downcast_ref::() { + if let Some(filter) = plan.downcast_ref::() { let pred = filter.predicate().clone(); let new_pred = pred.transform_down(&|expr: Arc| { - if let Some(func) = expr.as_any().downcast_ref::() { + if let Some(func) = expr.downcast_ref::() { if !func.name().eq_ignore_ascii_case("matches_term") { return Ok(Transformed::no(expr)); } @@ -178,7 +174,7 @@ impl PhysicalOptimizerRule for MatchesConstantTermOptimizer { return Ok(Transformed::no(expr)); } - if let Some(lit) = args[1].as_any().downcast_ref::() + if let Some(lit) = args[1].downcast_ref::() && let ScalarValue::Utf8(Some(term)) = lit.value() { let finder = MatchesTermFinder::new(term); @@ -248,6 +244,7 @@ mod tests { use datafusion::physical_plan::get_plan_string; use datafusion_common::{Column, DFSchema}; use datafusion_expr::expr::ScalarFunction; + use datafusion_expr::physical_planning_context::PhysicalPlanningContext; use datafusion_expr::{Expr, Literal, ScalarUDF}; use datafusion_physical_expr::{ScalarFunctionExpr, create_physical_expr}; use datatypes::prelude::ConcreteDataType; @@ -343,6 +340,7 @@ mod tests { )), &DFSchema::try_from(batch.schema().clone()).unwrap(), &Default::default(), + &PhysicalPlanningContext::default(), ) .unwrap(); @@ -357,16 +355,11 @@ mod tests { .optimize(Arc::new(filter), &Default::default()) .unwrap(); - let optimized_filter = optimized_plan - .as_any() - .downcast_ref::() - .unwrap(); + let optimized_filter = optimized_plan.downcast_ref::().unwrap(); let predicate = optimized_filter.predicate(); // The predicate should be a PreCompiledMatchesTermExpr - assert!( - std::any::TypeId::of::() == predicate.as_any().type_id() - ); + assert!(predicate.is::()); } #[test] @@ -413,6 +406,7 @@ mod tests { )), &DFSchema::try_from(batch.schema().clone()).unwrap(), &Default::default(), + &PhysicalPlanningContext::default(), ) .unwrap(); @@ -426,14 +420,11 @@ mod tests { .optimize(Arc::new(filter), &Default::default()) .unwrap(); - let optimized_filter = optimized_plan - .as_any() - .downcast_ref::() - .unwrap(); + let optimized_filter = optimized_plan.downcast_ref::().unwrap(); let predicate = optimized_filter.predicate(); // The predicate should still be a ScalarFunctionExpr - assert!(std::any::TypeId::of::() == predicate.as_any().type_id()); + assert!(predicate.is::()); } #[tokio::test] diff --git a/src/query/src/optimizer/count_wildcard.rs b/src/query/src/optimizer/count_wildcard.rs index 1c6a5b814b3..3c829fbef0c 100644 --- a/src/query/src/optimizer/count_wildcard.rs +++ b/src/query/src/optimizer/count_wildcard.rs @@ -16,13 +16,12 @@ use datafusion::datasource::DefaultTableSource; use datafusion_common::tree_node::{ Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeVisitor, }; -use datafusion_common::{Column, Result as DataFusionResult, ScalarValue}; +use datafusion_common::{Column, Result as DataFusionResult, ScalarValue, TableReference}; use datafusion_expr::expr::{AggregateFunction, WindowFunction}; use datafusion_expr::utils::COUNT_STAR_EXPANSION; use datafusion_expr::{Expr, LogicalPlan, WindowFunctionDefinition, col, lit}; use datafusion_optimizer::AnalyzerRule; use datafusion_optimizer::utils::NamePreserver; -use datafusion_sql::TableReference; use table::table::adapter::DfTableProviderAdapter; /// A replacement to DataFusion's [`CountWildcardRule`]. This rule @@ -155,13 +154,9 @@ impl TreeNodeVisitor<'_> for TimeIndexFinder { } if let LogicalPlan::TableScan(table_scan) = &node - && let Some(source) = table_scan - .source - .as_any() - .downcast_ref::() + && let Some(source) = table_scan.source.downcast_ref::() && let Some(adapter) = source .table_provider - .as_any() .downcast_ref::() { let table_info = adapter.table().table_info(); @@ -206,9 +201,8 @@ mod test { use common_recordbatch::{RecordBatch, SendableRecordBatchStream}; use datafusion::functions_aggregate::count::count_all; use datafusion::functions_aggregate::min_max::max; - use datafusion_common::Column; + use datafusion_common::{Column, TableReference}; use datafusion_expr::LogicalPlanBuilder; - use datafusion_sql::TableReference; use datatypes::data_type::ConcreteDataType; use datatypes::schema::{ColumnSchema, Schema, SchemaBuilder}; use datatypes::vectors::{Int64Vector, TimestampMillisecondVector, VectorRef}; diff --git a/src/query/src/optimizer/enforce_sorting.rs b/src/query/src/optimizer/enforce_sorting.rs new file mode 100644 index 00000000000..b50127fa060 --- /dev/null +++ b/src/query/src/optimizer/enforce_sorting.rs @@ -0,0 +1,91 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Sorting enforcement that runs after GreptimeDB's custom physical rules. +//! +//! DataFusion 55 moved the standalone `EnforceSorting` phases into +//! `EnsureRequirements`. GreptimeDB still needs to rerun those phases after +//! custom rules modify scan partitioning and distribution. + +use std::sync::Arc; + +use datafusion::physical_optimizer::PhysicalOptimizerRule; +use datafusion::physical_optimizer::enforce_sorting::replace_with_order_preserving_variants::{ + OrderPreservationContext, replace_with_order_preserving_variants, +}; +use datafusion::physical_optimizer::enforce_sorting::sort_pushdown::{ + SortPushDown, assign_initial_requirements, pushdown_sorts, +}; +use datafusion::physical_optimizer::enforce_sorting::{ + PlanWithCorrespondingCoalescePartitions, PlanWithCorrespondingSort, ensure_sorting, + parallelize_sorts, replace_with_partial_sort, +}; +use datafusion::physical_plan::ExecutionPlan; +use datafusion_common::Result; +use datafusion_common::config::ConfigOptions; +use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode}; + +/// Runs the standalone sorting-enforcement pipeline removed in DataFusion 55. +#[derive(Debug)] +pub struct EnforceSorting; + +impl PhysicalOptimizerRule for EnforceSorting { + fn optimize( + &self, + plan: Arc, + config: &ConfigOptions, + ) -> Result> { + // Phase 1: ensure sorting requirements and remove redundant sorts. + let sorting = PlanWithCorrespondingSort::new_default(plan); + let sorting = sorting.transform_up(ensure_sorting)?.data; + + // Phase 2: optionally turn CoalescePartitions + Sort into parallel + // sorts followed by a SortPreservingMerge. + let plan = if config.optimizer.repartition_sorts { + let parallel = PlanWithCorrespondingCoalescePartitions::new_default(sorting.plan) + .transform_up(parallelize_sorts) + .data()?; + parallel.plan + } else { + sorting.plan + }; + + // Phase 3: use order-preserving executor variants where appropriate. + let variants = OrderPreservationContext::new_default(plan); + let variants = variants + .transform_up(|context| { + replace_with_order_preserving_variants(context, false, true, config) + }) + .data()?; + + // Phase 4: push sorts down through order-preserving operators. + let mut pushdown = SortPushDown::new_default(variants.plan); + assign_initial_requirements(&mut pushdown); + let pushed = pushdown_sorts(pushdown)?; + + // Phase 5: exploit an already-satisfied prefix on unbounded inputs. + pushed + .plan + .transform_up(|plan| Ok(Transformed::yes(replace_with_partial_sort(plan)?))) + .data() + } + + fn name(&self) -> &str { + "EnforceSorting" + } + + fn schema_check(&self) -> bool { + true + } +} diff --git a/src/query/src/optimizer/global_limit.rs b/src/query/src/optimizer/global_limit.rs index 4b2ff721f64..5abb9953b7b 100644 --- a/src/query/src/optimizer/global_limit.rs +++ b/src/query/src/optimizer/global_limit.rs @@ -21,7 +21,9 @@ use datafusion::physical_plan::filter::FilterExec; use datafusion::physical_plan::limit::GlobalLimitExec; use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; -use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; +use datafusion::physical_plan::{ + ChildrenPropertiesMode, ExecutionPlan, ExecutionPlanProperties, ReplaceChildrenOptions, +}; use datafusion_common::Result as DfResult; use datafusion_physical_expr::{Distribution, OrderingRequirements, Partitioning}; @@ -57,7 +59,7 @@ impl EnsureGlobalLimitForFetch { let plan = if children.is_empty() { plan } else { - let required_input_distribution = plan.required_input_distribution(); + let required_input_distribution = plan.input_distribution_requirements(); let required_input_ordering = plan.required_input_ordering(); let maintains_input_order = plan.maintains_input_order(); let child_parent = ParentContext { @@ -72,7 +74,7 @@ impl EnsureGlobalLimitForFetch { .enumerate() .map(|(idx, child)| { let required_distribution = required_input_distribution - .get(idx) + .child_distribution(idx) .cloned() .unwrap_or(Distribution::UnspecifiedDistribution); let partitioning_to_restore = @@ -101,7 +103,10 @@ impl EnsureGlobalLimitForFetch { Self::optimize_plan(Arc::clone(child), parent) }) .collect::>>()?; - plan.with_new_children(children)? + plan.replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + )? }; let Some(fetch) = plan.fetch() else { @@ -111,7 +116,7 @@ impl EnsureGlobalLimitForFetch { if parent .global_fetch .is_some_and(|parent_fetch| parent_fetch <= fetch) - || !plan.as_any().is::() + || !plan.is::() || plan.output_partitioning().partition_count() <= 1 { return Ok(plan); @@ -149,10 +154,10 @@ impl Default for ParentContext { fn provided_global_fetch(plan: &Arc) -> Option { let fetch = plan.fetch()?; - (plan.as_any().is::() - || plan.as_any().is::() - || plan.as_any().is::() - || plan.as_any().is::()) + (plan.is::() + || plan.is::() + || plan.is::() + || plan.is::()) .then_some(fetch) } @@ -196,7 +201,7 @@ fn partitioning_to_restore_for( child: &Arc, required_distribution: &Distribution, ) -> Option { - if !matches!(required_distribution, Distribution::HashPartitioned(_)) + if !matches!(required_distribution, Distribution::KeyPartitioned(_)) || child.output_partitioning().partition_count() <= 1 { return None; @@ -233,7 +238,7 @@ fn inherited_partitioning_to_restore( let satisfies_parent_distribution = matches!( parent.required_distribution, - Distribution::HashPartitioned(_) + Distribution::KeyPartitioned(_) ) && plan .output_partitioning() .satisfaction( @@ -272,7 +277,7 @@ mod tests { let optimized = EnsureGlobalLimitForFetch::optimize_plan(filter, ParentContext::default()).unwrap(); - assert!(optimized.as_any().is::()); + assert!(optimized.is::()); assert_eq!(optimized.fetch(), Some(1)); assert_eq!(optimized.output_partitioning().partition_count(), 1); } @@ -295,7 +300,7 @@ mod tests { let projection = optimized.children()[0]; let coalesce = projection.children()[0]; - assert!(coalesce.as_any().is::()); + assert!(coalesce.is::()); assert_eq!(coalesce.fetch(), Some(5)); } @@ -310,8 +315,8 @@ mod tests { EnsureGlobalLimitForFetch::optimize_plan(merge, ParentContext::default()).unwrap(); let child = optimized.children()[0]; - assert!(optimized.as_any().is::()); - assert!(child.as_any().is::()); + assert!(optimized.is::()); + assert!(child.is::()); } #[test] @@ -325,8 +330,8 @@ mod tests { EnsureGlobalLimitForFetch::optimize_plan(merge, ParentContext::default()).unwrap(); let child = optimized.children()[0]; - assert!(optimized.as_any().is::()); - assert!(child.as_any().is::()); + assert!(optimized.is::()); + assert!(child.is::()); assert_eq!(child.fetch(), Some(5)); } @@ -340,8 +345,8 @@ mod tests { EnsureGlobalLimitForFetch::optimize_plan(merge, ParentContext::default()).unwrap(); let child = optimized.children()[0]; - assert!(optimized.as_any().is::()); - assert!(child.as_any().is::()); + assert!(optimized.is::()); + assert!(child.is::()); } #[test] @@ -354,10 +359,10 @@ mod tests { EnsureGlobalLimitForFetch::optimize_plan(merge, ParentContext::default()).unwrap(); let child = optimized.children()[0]; - assert!(optimized.as_any().is::()); - assert!(child.as_any().is::()); + assert!(optimized.is::()); + assert!(child.is::()); assert_eq!(child.fetch(), Some(5)); - assert!(child.children()[0].as_any().is::()); + assert!(child.children()[0].is::()); } #[test] @@ -371,8 +376,8 @@ mod tests { EnsureGlobalLimitForFetch::optimize_plan(merge, ParentContext::default()).unwrap(); let child = optimized.children()[0]; - assert!(optimized.as_any().is::()); - assert!(child.as_any().is::()); + assert!(optimized.is::()); + assert!(child.is::()); assert_eq!(child.fetch(), Some(1)); } @@ -396,10 +401,7 @@ mod tests { None, ) .unwrap(); - let merge = optimized - .as_any() - .downcast_ref::() - .unwrap(); + let merge = optimized.downcast_ref::().unwrap(); assert_eq!(merge.expr(), &actual_ordering); } @@ -423,9 +425,9 @@ mod tests { let projection = optimized.children()[0]; let child = projection.children()[0]; - assert!(optimized.as_any().is::()); - assert!(projection.as_any().is::()); - assert!(child.as_any().is::()); + assert!(optimized.is::()); + assert!(projection.is::()); + assert!(child.is::()); assert_eq!(child.fetch(), Some(1)); } @@ -455,13 +457,13 @@ mod tests { let optimized = EnsureGlobalLimitForFetch::optimize_plan(join, ParentContext::default()).unwrap(); let left = optimized.children()[0]; - let repartition = left.as_any().downcast_ref::().unwrap(); + let repartition = left.downcast_ref::().unwrap(); assert!(matches!( repartition.partitioning(), Partitioning::Hash(_, 3) )); - assert!(repartition.input().as_any().is::()); + assert!(repartition.input().is::()); assert_eq!(repartition.input().fetch(), Some(1)); } @@ -499,16 +501,15 @@ mod tests { EnsureGlobalLimitForFetch::optimize_plan(join, ParentContext::default()).unwrap(); let projection = optimized.children()[0]; let repartition = projection.children()[0] - .as_any() .downcast_ref::() .unwrap(); - assert!(projection.as_any().is::()); + assert!(projection.is::()); assert!(matches!( repartition.partitioning(), Partitioning::Hash(_, 3) )); - assert!(repartition.input().as_any().is::()); + assert!(repartition.input().is::()); assert_eq!(repartition.input().fetch(), Some(1)); } @@ -542,17 +543,16 @@ mod tests { let outer_projection = optimized.children()[0]; let inner_projection = outer_projection.children()[0]; let repartition = inner_projection.children()[0] - .as_any() .downcast_ref::() .unwrap(); - assert!(outer_projection.as_any().is::()); - assert!(inner_projection.as_any().is::()); + assert!(outer_projection.is::()); + assert!(inner_projection.is::()); assert!(matches!( repartition.partitioning(), Partitioning::Hash(_, 3) )); - assert!(repartition.input().as_any().is::()); + assert!(repartition.input().is::()); assert_eq!(repartition.input().fetch(), Some(1)); } diff --git a/src/query/src/optimizer/insert_assignment.rs b/src/query/src/optimizer/insert_assignment.rs index ad77736ce0a..8645668b590 100644 --- a/src/query/src/optimizer/insert_assignment.rs +++ b/src/query/src/optimizer/insert_assignment.rs @@ -163,11 +163,14 @@ fn retarget_assignment_cast( let expr = unalias_mut(expr); let Expr::Cast(Cast { expr: source, - data_type: DataType::Timestamp(unit, None), + field, }) = expr else { return Ok(false); }; + let DataType::Timestamp(unit, None) = field.data_type() else { + return Ok(false); + }; let unit = *unit; if !matches!( diff --git a/src/query/src/optimizer/json_type_concretize.rs b/src/query/src/optimizer/json_type_concretize.rs index 5ec45cd7f63..db94daa9e03 100644 --- a/src/query/src/optimizer/json_type_concretize.rs +++ b/src/query/src/optimizer/json_type_concretize.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::Any; use std::collections::HashMap; use arrow_schema::DataType; @@ -50,11 +51,7 @@ impl OptimizerRule for JsonTypeConcretizeRule { plan.transform_down(|plan| match &plan { LogicalPlan::TableScan(table_scan) => { - let Some(source) = table_scan - .source - .as_any() - .downcast_ref::() - else { + let Some(source) = table_scan.source.downcast_ref::() else { return Ok(Transformed::no(plan)); }; @@ -95,12 +92,12 @@ fn apply_json_type_hint( return false; } - if let Some(adapter) = provider.as_any().downcast_ref::() { + if let Some(adapter) = (provider as &dyn Any).downcast_ref::() { adapter.with_json_type_hint(json_types); return true; } - if let Some(adapter) = provider.as_any().downcast_ref::() { + if let Some(adapter) = (provider as &dyn Any).downcast_ref::() { adapter.with_json_type_hint(json_types); return true; } diff --git a/src/query/src/optimizer/parallelize_scan.rs b/src/query/src/optimizer/parallelize_scan.rs index dd2ba062901..8e0113e69df 100644 --- a/src/query/src/optimizer/parallelize_scan.rs +++ b/src/query/src/optimizer/parallelize_scan.rs @@ -55,12 +55,10 @@ impl ParallelizeScan { let result = plan .transform_down(|plan| { - if let Some(sort_exec) = plan.as_any().downcast_ref::() { + if let Some(sort_exec) = plan.downcast_ref::() { // save the first order expr first_order_expr = Some(sort_exec.expr().first()).cloned(); - } else if let Some(region_scan_exec) = - plan.as_any().downcast_ref::() - { + } else if let Some(region_scan_exec) = plan.downcast_ref::() { let expected_partition_num = config.execution.target_partitions; if region_scan_exec.is_partition_set() || region_scan_exec.scanner_type().as_str() == "SinglePartition" diff --git a/src/query/src/optimizer/pass_distribution.rs b/src/query/src/optimizer/pass_distribution.rs index c45d4695d93..732168e962f 100644 --- a/src/query/src/optimizer/pass_distribution.rs +++ b/src/query/src/optimizer/pass_distribution.rs @@ -18,7 +18,9 @@ use datafusion::config::ConfigOptions; use datafusion::physical_optimizer::PhysicalOptimizerRule; use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::repartition::RepartitionExec; -use datafusion::physical_plan::{ExecutionPlan, Partitioning}; +use datafusion::physical_plan::{ + ChildrenPropertiesMode, ExecutionPlan, Partitioning, ReplaceChildrenOptions, +}; use datafusion_common::Result as DfResult; use datafusion_physical_expr::Distribution; use datafusion_physical_expr::utils::map_columns_before_projection; @@ -68,8 +70,8 @@ impl PassDistribution { current_req: Option, ) -> DfResult> { // If this is a MergeScanExec, try to apply the current requirement. - if let Some(merge_scan) = plan.as_any().downcast_ref::() - && let Some(Distribution::HashPartitioned(hash_exprs)) = current_req.as_ref() + if let Some(merge_scan) = plan.downcast_ref::() + && let Some(Distribution::KeyPartitioned(hash_exprs)) = current_req.as_ref() { if let Partitioning::Hash(current_hash_exprs, _) = &merge_scan.properties().partitioning && *current_hash_exprs == *hash_exprs @@ -78,7 +80,7 @@ impl PassDistribution { } if let Some(new_plan) = merge_scan - .try_with_new_distribution(Distribution::HashPartitioned(hash_exprs.clone())) + .try_with_new_distribution(Distribution::KeyPartitioned(hash_exprs.clone())) { // Leaf node; no children to process return Ok(Arc::new(new_plan) as _); @@ -97,10 +99,10 @@ impl PassDistribution { return Ok(plan); } - let required = plan.required_input_distribution(); + let required = plan.input_distribution_requirements(); let mut new_children = Vec::with_capacity(children.len()); for (idx, child) in children.into_iter().enumerate() { - let child_req = match required.get(idx) { + let child_req = match required.child_distribution(idx) { Some(Distribution::UnspecifiedDistribution) if idx == 0 => { Self::map_hash_requirement_through_projection(plan.as_ref(), ¤t_req) } @@ -121,7 +123,10 @@ impl PassDistribution { if unchanged { Ok(plan) } else { - plan.with_new_children(new_children) + plan.replace_children( + new_children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) } } @@ -129,11 +134,11 @@ impl PassDistribution { plan: &dyn ExecutionPlan, current_req: &Option, ) -> Option { - let Some(Distribution::HashPartitioned(required_exprs)) = current_req else { + let Some(Distribution::KeyPartitioned(required_exprs)) = current_req else { return None; }; - let projection = plan.as_any().downcast_ref::()?; + let projection = plan.downcast_ref::()?; let proj_exprs = projection .expr() .iter() @@ -141,7 +146,7 @@ impl PassDistribution { .collect::>(); let mapped = map_columns_before_projection(required_exprs, &proj_exprs); - (mapped.len() == required_exprs.len()).then_some(Distribution::HashPartitioned(mapped)) + (mapped.len() == required_exprs.len()).then_some(Distribution::KeyPartitioned(mapped)) } } @@ -261,12 +266,8 @@ mod tests { let optimized = PassDistribution .optimize(join, &ConfigOptions::default()) .unwrap(); - let hash_join = optimized.as_any().downcast_ref::().unwrap(); - let left_projection = hash_join - .left() - .as_any() - .downcast_ref::() - .unwrap(); + let hash_join = optimized.downcast_ref::().unwrap(); + let left_projection = hash_join.left().downcast_ref::().unwrap(); let left_partitioning = left_projection.input().output_partitioning(); let right_partitioning = hash_join.right().output_partitioning(); @@ -293,7 +294,7 @@ mod tests { fn merge_scan_rejects_hash_requirement_on_partition_key_subset() { let merge_scan = test_merge_scan_exec(test_schema()); - let new_plan = merge_scan.try_with_new_distribution(Distribution::HashPartitioned(vec![ + let new_plan = merge_scan.try_with_new_distribution(Distribution::KeyPartitioned(vec![ partition_column(DATA_SCHEMA_TSID_COLUMN_NAME, 1), ])); @@ -355,12 +356,7 @@ mod tests { fn column_names(exprs: &[Arc]) -> Vec<&str> { exprs .iter() - .map(|expr| { - expr.as_any() - .downcast_ref::() - .unwrap() - .name() - }) + .map(|expr| expr.downcast_ref::().unwrap().name()) .collect() } } diff --git a/src/query/src/optimizer/promql_tsid_narrow_join.rs b/src/query/src/optimizer/promql_tsid_narrow_join.rs index 6dd0b5e4bb2..c26247179ed 100644 --- a/src/query/src/optimizer/promql_tsid_narrow_join.rs +++ b/src/query/src/optimizer/promql_tsid_narrow_join.rs @@ -55,7 +55,7 @@ impl PhysicalOptimizerRule for PromqlTsidNarrowJoin { impl PromqlTsidNarrowJoin { fn rewrite_join(plan: Arc) -> DfResult>> { - let Some(hash_join) = plan.as_any().downcast_ref::() else { + let Some(hash_join) = plan.downcast_ref::() else { return Ok(Transformed::no(plan)); }; @@ -103,8 +103,8 @@ impl PromqlTsidNarrowJoin { for (left, right) in hash_join.on() { let (Some(left_col), Some(right_col)) = ( - left.as_any().downcast_ref::(), - right.as_any().downcast_ref::(), + left.downcast_ref::(), + right.downcast_ref::(), ) else { return false; }; @@ -199,7 +199,7 @@ mod tests { let optimized = PromqlTsidNarrowJoin .optimize(join, &ConfigOptions::default()) .unwrap(); - let optimized_join = optimized.as_any().downcast_ref::().unwrap(); + let optimized_join = optimized.downcast_ref::().unwrap(); assert_eq!(optimized_join.partition_mode(), &PartitionMode::CollectLeft); assert_eq!(optimized.schema(), original_schema); @@ -262,7 +262,7 @@ mod tests { let optimized = PromqlTsidNarrowJoin .optimize(join, &ConfigOptions::default()) .unwrap(); - let optimized_join = optimized.as_any().downcast_ref::().unwrap(); + let optimized_join = optimized.downcast_ref::().unwrap(); assert_eq!(optimized_join.partition_mode(), &PartitionMode::CollectLeft); } @@ -317,7 +317,7 @@ mod tests { let optimized = PromqlTsidNarrowJoin .optimize(join, &ConfigOptions::default()) .unwrap(); - let optimized_join = optimized.as_any().downcast_ref::().unwrap(); + let optimized_join = optimized.downcast_ref::().unwrap(); assert_eq!(optimized_join.partition_mode(), &PartitionMode::Partitioned); } diff --git a/src/query/src/optimizer/remove_duplicate.rs b/src/query/src/optimizer/remove_duplicate.rs index 65d191e7b53..155fb1ff95b 100644 --- a/src/query/src/optimizer/remove_duplicate.rs +++ b/src/query/src/optimizer/remove_duplicate.rs @@ -16,8 +16,8 @@ use std::sync::Arc; use datafusion::config::ConfigOptions; use datafusion::physical_optimizer::PhysicalOptimizerRule; -use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::physical_plan::{ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions}; use datafusion_common::Result as DfResult; use datafusion_common::tree_node::{Transformed, TreeNode}; @@ -35,29 +35,18 @@ impl PhysicalOptimizerRule for RemoveDuplicate { plan: Arc, _config: &ConfigOptions, ) -> DfResult> { - Self::do_optimize(plan) - } - - fn name(&self) -> &str { - "RemoveDuplicateRule" - } - - fn schema_check(&self) -> bool { - false - } -} - -impl RemoveDuplicate { - fn do_optimize(plan: Arc) -> DfResult> { let result = plan .transform_down(|plan| { - if plan.as_any().is::() { + if plan.is::() { // check child let child = plan.children()[0].clone(); - if child.as_any().type_id() == plan.as_any().type_id() { + if child.is::() { // remove child let grand_child = child.children()[0].clone(); - let new_plan = plan.with_new_children(vec![grand_child])?; + let new_plan = plan.replace_children( + vec![grand_child], + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + )?; return Ok(Transformed::yes(new_plan)); } } @@ -65,7 +54,14 @@ impl RemoveDuplicate { Ok(Transformed::no(plan)) })? .data; - Ok(result) } + + fn name(&self) -> &str { + "RemoveDuplicateRule" + } + + fn schema_check(&self) -> bool { + true + } } diff --git a/src/query/src/optimizer/scan_hint.rs b/src/query/src/optimizer/scan_hint.rs index ce1009f8811..edb73559139 100644 --- a/src/query/src/optimizer/scan_hint.rs +++ b/src/query/src/optimizer/scan_hint.rs @@ -19,7 +19,6 @@ use arrow_schema::SortOptions; use common_function::aggrs::aggr_wrapper::aggr_state_func_name; use common_recordbatch::OrderOption; use common_recordbatch::filter::SimpleFilterEvaluator; -use common_time::timestamp::TimeUnit; use datafusion::datasource::DefaultTableSource; use datafusion_common::tree_node::{Transformed, TreeNodeRewriter}; use datafusion_common::{Column, Result}; @@ -77,19 +76,11 @@ impl ScanHintRule { let LogicalPlan::TableScan(mut table_scan) = plan else { return Ok(Transformed::no(plan)); }; - let Some(source) = table_scan - .source - .as_any() - .downcast_ref::() - else { + let Some(source) = table_scan.source.downcast_ref::() else { return Ok(Transformed::no(LogicalPlan::TableScan(table_scan))); }; // The provider in the region server is [DummyTableProvider]. - let Some(original) = source - .table_provider - .as_any() - .downcast_ref::() - else { + let Some(original) = source.table_provider.downcast_ref::() else { return Ok(Transformed::no(LogicalPlan::TableScan(table_scan))); }; @@ -139,8 +130,7 @@ impl ScanHintRule { /// predicate later rejects that row. Only recognized tag/time predicates are /// allowed: tags select whole series, and supported time predicates constrain /// the scan window before row selection. Field or unrecognized predicates are - /// conservatively rejected. Finer-than-millisecond timestamps are also excluded - /// because instant evaluation can conflate distinct samples at that precision. + /// conservatively rejected. /// /// This checks only attached predicates; the path allowlist separately rejects /// residual Filter nodes between InstantManipulate and the scan. @@ -149,14 +139,6 @@ impl ScanHintRule { provider: &DummyTableProvider, ) -> bool { let metadata = provider.region_metadata(); - // Instant evaluation is millisecond-based, so finer time units can - // conflate timestamps and must not use the LastRow hint. - if !matches!( - metadata.time_index_type().unit(), - TimeUnit::Second | TimeUnit::Millisecond - ) { - return false; - } for filter in &table_scan.filters { let Some(filter) = SimpleFilterEvaluator::try_new(filter) else { return false; @@ -447,6 +429,10 @@ fn single_evaluation_node_allowed(node: &LogicalPlan) -> bool { /// This whitelist assumes the planner preserves time-index and series identity; /// it is not a proof that an arbitrary plan does so. +/// +/// Identity projections preserve the selected samples. Only the planner's named +/// seconds/milliseconds-to-milliseconds casts are accepted; a microsecond or +/// nanosecond cast could collapse a future sample onto the evaluation boundary. fn single_evaluation_projection_expr_allowed( expr: &Expr, projection: &datafusion_expr::logical_plan::Projection, @@ -461,7 +447,7 @@ fn single_evaluation_projection_expr_allowed( }; alias.name == column.name && matches!( - cast.data_type, + cast.field.data_type(), DataType::Timestamp(ArrowTimeUnit::Millisecond, None) ) && matches!( @@ -602,11 +588,8 @@ mod test { let mut requests = Vec::new(); plan.apply_with_subqueries(|node| { if let LogicalPlan::TableScan(scan) = node - && let Some(source) = scan.source.as_any().downcast_ref::() - && let Some(provider) = source - .table_provider - .as_any() - .downcast_ref::() + && let Some(source) = scan.source.downcast_ref::() + && let Some(provider) = source.table_provider.downcast_ref::() { requests.push((scan.table_name.to_string(), provider.scan_request())); } @@ -635,6 +618,7 @@ mod test { 1000, 1000, 1000, + 0, "ts".to_string(), vec![], Some("v0".to_string()), @@ -682,26 +666,49 @@ mod test { } fn last_value_aggregate(input: LogicalPlan) -> LogicalPlan { - LogicalPlanBuilder::from(input) + let aggregate = LogicalPlanBuilder::from(input) .aggregate( vec![col("k0")], - vec![Expr::AggregateFunction(AggregateFunction { - func: last_value_udaf(), - params: AggregateFunctionParams { - args: vec![col("v0")], - distinct: false, - filter: None, - order_by: vec![Sort { - expr: col("ts"), - asc: true, - nulls_first: true, - }], - null_treatment: None, - }, - })], + vec![ + Expr::AggregateFunction(AggregateFunction { + func: last_value_udaf(), + params: AggregateFunctionParams { + args: vec![col("v0")], + distinct: false, + filter: None, + order_by: vec![Sort { + expr: col("ts"), + asc: true, + nulls_first: true, + }], + null_treatment: None, + }, + }), + Expr::AggregateFunction(AggregateFunction { + func: last_value_udaf(), + params: AggregateFunctionParams { + args: vec![col("ts")], + distinct: false, + filter: None, + order_by: vec![Sort { + expr: col("ts"), + asc: true, + nulls_first: true, + }], + null_treatment: None, + }, + }), + ], ) .unwrap() .build() + .unwrap(); + let timestamp = aggregate.schema().field(2).name().clone(); + + LogicalPlanBuilder::from(aggregate) + .project(vec![col("k0"), col(timestamp).alias("ts")]) + .unwrap() + .build() .unwrap() } @@ -712,6 +719,7 @@ mod test { 1000, 1000, 1000, + 0, "ts".to_string(), vec![], Some("v0".to_string()), @@ -741,6 +749,7 @@ mod test { outer_end, 1000, 1000, + 0, "ts".to_string(), vec![], Some("v0".to_string()), @@ -768,6 +777,7 @@ mod test { end, 1000, 1000, + 0, "ts".to_string(), vec![], Some("v0".to_string()), @@ -928,21 +938,39 @@ mod test { None, ) .unwrap() + .project(vec![ + Expr::Column(Column::new(Some("left"), "ts")), + Expr::Column(Column::new(Some("left"), "v0")), + ]) + .unwrap() .build() .unwrap(); let nonlast_aggregate = LogicalPlanBuilder::from(scan_plan(provider(), "aggregate")) .aggregate( vec![col("k0")], - vec![Expr::AggregateFunction(AggregateFunction { - func: max_udaf(), - params: AggregateFunctionParams { - args: vec![col("v0")], - distinct: false, - filter: None, - order_by: vec![], - null_treatment: None, - }, - })], + vec![ + Expr::AggregateFunction(AggregateFunction { + func: max_udaf(), + params: AggregateFunctionParams { + args: vec![col("v0")], + distinct: false, + filter: None, + order_by: vec![], + null_treatment: None, + }, + }), + Expr::AggregateFunction(AggregateFunction { + func: max_udaf(), + params: AggregateFunctionParams { + args: vec![col("ts")], + distinct: false, + filter: None, + order_by: vec![], + null_treatment: None, + }, + }) + .alias("ts"), + ], ) .unwrap() .build() @@ -953,6 +981,7 @@ mod test { 1000, 1000, 1000, + 0, 1000, "ts".to_string(), vec!["v0".to_string()], @@ -1023,7 +1052,49 @@ mod test { } #[test] - fn single_evaluation_rejects_microsecond_and_nanosecond_time_index_casts() { + fn single_evaluation_uses_last_row_for_microsecond_and_nanosecond_time_indexes() { + for timestamp_type in [ + ConcreteDataType::timestamp_microsecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype(), + ] { + let direct_provider = Arc::new(mock_table_provider_with_timestamp( + RegionId::new(1, 1), + timestamp_type.clone(), + )); + let direct = ScanHintRule + .rewrite( + single_evaluation(scan_plan(direct_provider, "direct")), + &OptimizerContext::default(), + ) + .unwrap() + .data; + assert_eq!( + scan_requests(&direct)[0].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); + + let projection_provider = Arc::new(mock_table_provider_with_timestamp( + RegionId::new(1, 1), + timestamp_type, + )); + let projection = LogicalPlanBuilder::from(scan_plan(projection_provider, "projection")) + .project(vec![col("ts")]) + .unwrap() + .build() + .unwrap(); + let projected = ScanHintRule + .rewrite(single_evaluation(projection), &OptimizerContext::default()) + .unwrap() + .data; + assert_eq!( + scan_requests(&projected)[0].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); + } + } + + #[test] + fn single_evaluation_rejects_lossy_microsecond_and_nanosecond_time_index_casts() { for timestamp_type in [ ConcreteDataType::timestamp_microsecond_datatype(), ConcreteDataType::timestamp_nanosecond_datatype(), @@ -1081,7 +1152,7 @@ mod test { #[test] fn single_evaluation_rejects_projection_expressions_that_change_rows() { let invalid_projections = [ - vec![col("ts").alias("renamed")], + vec![col("ts").alias("renamed"), col("ts")], vec![ Expr::BinaryExpr(datafusion_expr::expr::BinaryExpr::new( Box::new(col("v0")), @@ -1089,6 +1160,7 @@ mod test { Box::new(lit(1.0_f64)), )) .alias("v0"), + col("ts"), ], vec![ Expr::Cast(Cast::new( @@ -1097,7 +1169,10 @@ mod test { )) .alias("ts"), ], - vec![Expr::Cast(Cast::new(Box::new(col("v0")), DataType::Int64)).alias("v0")], + vec![ + Expr::Cast(Cast::new(Box::new(col("v0")), DataType::Int64)).alias("v0"), + col("ts"), + ], vec![ Expr::Cast(Cast::new( Box::new(col("ts")), @@ -1417,6 +1492,7 @@ mod test { 1000, 1000, 1000, + 0, "ts".to_string(), vec![], Some("v0".to_string()), @@ -1441,6 +1517,7 @@ mod test { 2000, 1000, 1000, + 0, "ts".to_string(), vec![], Some("v0".to_string()), diff --git a/src/query/src/optimizer/scan_hint/vector_search.rs b/src/query/src/optimizer/scan_hint/vector_search.rs index 1a4d274ccaa..90f45378ecd 100644 --- a/src/query/src/optimizer/scan_hint/vector_search.rs +++ b/src/query/src/optimizer/scan_hint/vector_search.rs @@ -18,11 +18,10 @@ use common_function::scalars::vector::distance::{ VEC_COS_DISTANCE, VEC_DOT_PRODUCT, VEC_L2SQ_DISTANCE, }; use common_telemetry::debug; -use datafusion_common::ScalarValue; +use datafusion_common::{ScalarValue, TableReference}; use datafusion_expr::logical_plan::FetchType; use datafusion_expr::utils::split_conjunction; use datafusion_expr::{Expr, SortExpr}; -use datafusion_sql::TableReference; use datatypes::types::parse_string_to_vector_type_value; use store_api::storage::{VectorDistanceMetric, VectorSearchRequest}; @@ -488,11 +487,8 @@ mod tests { let mut request = None; plan.apply_with_subqueries(|node| { if let LogicalPlan::TableScan(scan) = node - && let Some(source) = scan.source.as_any().downcast_ref::() - && let Some(provider) = source - .table_provider - .as_any() - .downcast_ref::() + && let Some(source) = scan.source.downcast_ref::() + && let Some(provider) = source.table_provider.downcast_ref::() { request = Some(provider.scan_request()); } @@ -620,6 +616,7 @@ mod tests { 1000, 1000, 1000, + 0, "ts".to_string(), vec![], Some("v".to_string()), @@ -670,6 +667,7 @@ mod tests { 1000, 1000, 1000, + 0, "ts".to_string(), vec![], Some("v".to_string()), diff --git a/src/query/src/optimizer/string_normalization.rs b/src/query/src/optimizer/string_normalization.rs index 46993ee45e3..2fc2d3957f9 100644 --- a/src/query/src/optimizer/string_normalization.rs +++ b/src/query/src/optimizer/string_normalization.rs @@ -87,8 +87,8 @@ impl TreeNodeRewriter for StringNormalizationConverter { /// Otherwise - no modifications applied fn f_up(&mut self, expr: Expr) -> Result> { let new_expr = match expr { - Expr::Cast(Cast { expr, data_type }) => { - let expr = match data_type { + Expr::Cast(Cast { expr, field }) => { + let expr = match field.data_type() { DataType::Timestamp(_, _) => match *expr { Expr::Literal(value, _) => match value { ScalarValue::Utf8(Some(s)) => trim_utf_expr(s), @@ -98,10 +98,7 @@ impl TreeNodeRewriter for StringNormalizationConverter { }, _ => *expr, }; - Expr::Cast(Cast { - expr: Box::new(expr), - data_type, - }) + Expr::Cast(Cast::new_from_field(Box::new(expr), field)) } expr => expr, }; diff --git a/src/query/src/optimizer/type_conversion.rs b/src/query/src/optimizer/type_conversion.rs index 40bb43e791e..536628bce4c 100644 --- a/src/query/src/optimizer/type_conversion.rs +++ b/src/query/src/optimizer/type_conversion.rs @@ -63,6 +63,7 @@ impl ExtensionAnalyzerRule for TypeConversionRule { projected_schema, filters, fetch, + statistics_requests, }) => { let mut converter = TypeConverter::new(projected_schema.clone(), ctx.query_ctx()); let rewrite_filters = filters @@ -76,6 +77,7 @@ impl ExtensionAnalyzerRule for TypeConversionRule { projected_schema, filters: rewrite_filters, fetch, + statistics_requests, }))) } LogicalPlan::Projection { .. } => { @@ -359,10 +361,9 @@ mod tests { use std::sync::Arc; use datafusion_common::arrow::datatypes::Field; - use datafusion_common::{Column, DFSchema, NullEquality}; + use datafusion_common::{Column, DFSchema, NullEquality, TableReference}; use datafusion_expr::expr::{Cast, Exists}; use datafusion_expr::{Join, JoinConstraint, JoinType, Literal, LogicalPlanBuilder, Subquery}; - use datafusion_sql::TableReference; use session::context::QueryContext; use super::*; @@ -487,10 +488,10 @@ mod tests { use datafusion_common::arrow::datatypes::TimeUnit as ArrowTimeUnit; let mut converter = TypeConverter::new(Arc::new(DFSchema::empty()), QueryContext::arc()); - let expr = Expr::Cast(Cast { - expr: Box::new("2009-02-13 23:31:30".lit()), - data_type: DataType::Timestamp(ArrowTimeUnit::Millisecond, None), - }); + let expr = Expr::Cast(Cast::new( + Box::new("2009-02-13 23:31:30".lit()), + DataType::Timestamp(ArrowTimeUnit::Millisecond, None), + )); assert_eq!(converter.f_up(expr.clone()).unwrap().data, expr); } diff --git a/src/query/src/optimizer/windowed_sort.rs b/src/query/src/optimizer/windowed_sort.rs index 55a48c18691..3e4ccaf91c4 100644 --- a/src/query/src/optimizer/windowed_sort.rs +++ b/src/query/src/optimizer/windowed_sort.rs @@ -16,7 +16,6 @@ use std::sync::Arc; use arrow_schema::DataType; use datafusion::physical_optimizer::PhysicalOptimizerRule; -use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::coop::CooperativeExec; use datafusion::physical_plan::filter::FilterExec; @@ -24,6 +23,7 @@ use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::sorts::sort::SortExec; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; +use datafusion::physical_plan::{ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions}; use datafusion_common::Result as DataFusionResult; use datafusion_common::tree_node::{Transformed, TreeNode}; use datafusion_physical_expr::expressions::{CastExpr, Column as PhysicalColumn}; @@ -71,7 +71,7 @@ impl WindowedSortPhysicalRule { ) -> DataFusionResult> { let result = plan .transform_down(|plan| { - if let Some(sort_exec) = plan.as_any().downcast_ref::() { + if let Some(sort_exec) = plan.downcast_ref::() { // TODO: support multiple expr in windowed sort if sort_exec.expr().len() != 1 { return Ok(Transformed::no(plan)); @@ -88,10 +88,7 @@ impl WindowedSortPhysicalRule { let input_schema = sort_input.schema(); let first_sort_expr = sort_exec.expr().first(); - if let Some(column_expr) = first_sort_expr - .expr - .as_any() - .downcast_ref::() + if let Some(column_expr) = first_sort_expr.expr.downcast_ref::() && matches!( input_schema.field(column_expr.index()).data_type(), DataType::Timestamp(_, _) @@ -165,26 +162,26 @@ fn fetch_partition_range(input: Arc) -> DataFusionResult() { + if plan.is::() { return Ok(Transformed::no(plan)); } // Unappliable case, reset the state. - if plan.as_any().is::() - || plan.as_any().is::() - || plan.as_any().is::() - || plan.as_any().is::() + if plan.is::() + || plan.is::() + || plan.is::() + || plan.is::() { partition_ranges = None; } // only a very limited set of plans can exist between region scan and sort exec // other plans might make this optimize wrong, so be safe here by limiting it - if !(plan.as_any().is::() || plan.as_any().is::()) { + if !(plan.is::() || plan.is::()) { partition_ranges = None; } - if let Some(region_scan_exec) = plan.as_any().downcast_ref::() { + if let Some(region_scan_exec) = plan.downcast_ref::() { // `PerSeries` distribution is not supported in windowed sort. if region_scan_exec.distribution() == Some(store_api::storage::TimeSeriesDistribution::PerSeries) @@ -216,11 +213,11 @@ fn is_time_index_expr( plan: &Arc, expr: &Arc, ) -> DataFusionResult { - if let Some(column_expr) = expr.as_any().downcast_ref::() { + if let Some(column_expr) = expr.downcast_ref::() { return is_time_index_column(plan, column_expr); } - if let Some(cast_expr) = expr.as_any().downcast_ref::() { + if let Some(cast_expr) = expr.downcast_ref::() { return if matches!(cast_expr.cast_type(), DataType::Timestamp(_, _)) { is_time_index_expr(plan, cast_expr.expr()) } else { @@ -228,7 +225,7 @@ fn is_time_index_expr( }; } - if let Some(scalar_function_expr) = expr.as_any().downcast_ref::() { + if let Some(scalar_function_expr) = expr.downcast_ref::() { return if is_supported_time_index_wrapper(scalar_function_expr) && scalar_function_expr.args().len() == 1 { @@ -245,14 +242,14 @@ fn is_time_index_column( plan: &Arc, column_expr: &PhysicalColumn, ) -> DataFusionResult { - if let Some(projection) = plan.as_any().downcast_ref::() { + if let Some(projection) = plan.downcast_ref::() { let Some(projection_expr) = projection.expr().get(column_expr.index()) else { return Ok(false); }; return is_time_index_expr(projection.input(), &projection_expr.expr); } - if let Some(filter) = plan.as_any().downcast_ref::() { + if let Some(filter) = plan.downcast_ref::() { let child_column_expr = filter .projection() .as_ref() @@ -268,7 +265,7 @@ fn is_time_index_column( return is_time_index_expr(filter.input(), &child_expr); } - if let Some(region_scan_exec) = plan.as_any().downcast_ref::() { + if let Some(region_scan_exec) = plan.downcast_ref::() { let schema = plan.schema(); let column_field = schema.field(column_expr.index()); return Ok( @@ -285,9 +282,9 @@ fn is_time_index_column( } fn passthrough_child(plan: &dyn ExecutionPlan) -> Option> { - if plan.as_any().is::() - || plan.as_any().is::() - || plan.as_any().is::() + if plan.is::() + || plan.is::() + || plan.is::() { return schema_preserving_child(plan); } @@ -314,13 +311,16 @@ fn remove_repartition( plan: Arc, ) -> DataFusionResult>> { plan.transform_down(|plan| { - if plan.as_any().is::() { + if plan.is::() { // Checks child. let maybe_repartition = plan.children()[0]; - if maybe_repartition.as_any().is::() { + if maybe_repartition.is::() { let maybe_scan = maybe_repartition.children()[0]; - if maybe_scan.as_any().is::() { - let new_filter = plan.clone().with_new_children(vec![maybe_scan.clone()])?; + if maybe_scan.is::() { + let new_filter = plan.clone().replace_children( + vec![maybe_scan.clone()], + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + )?; return Ok(Transformed::yes(new_filter)); } } diff --git a/src/query/src/options.rs b/src/query/src/options.rs index 24cd3853f07..a49021f2f15 100644 --- a/src/query/src/options.rs +++ b/src/query/src/options.rs @@ -39,6 +39,7 @@ pub const QUERY_ENABLE_REMOTE_DYNAMIC_FILTER_PUSHDOWN: &str = "query.enable_remote_dynamic_filter_pushdown"; pub const FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY: &str = "memtable_only"; +pub const FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE: &str = "sequence_range"; /// Query spill mode controlling disk manager behavior. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] @@ -131,6 +132,7 @@ impl Default for QueryOptions { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FlowIncrementalMode { MemtableOnly, + SequenceRange, } #[derive(Debug, Clone, PartialEq, Eq, Default)] @@ -167,6 +169,9 @@ impl FlowQueryExtensions { v if v.eq_ignore_ascii_case(FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY) => { Ok(FlowIncrementalMode::MemtableOnly) } + v if v.eq_ignore_ascii_case(FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE) => { + Ok(FlowIncrementalMode::SequenceRange) + } _ => Err(invalid_query_context_extension(format!( "Invalid value for {}: {}", FLOW_INCREMENTAL_MODE, value @@ -197,21 +202,25 @@ impl FlowQueryExtensions { }) .transpose()?; - if matches!(incremental_mode, Some(FlowIncrementalMode::MemtableOnly)) { + if matches!( + incremental_mode, + Some(FlowIncrementalMode::MemtableOnly | FlowIncrementalMode::SequenceRange) + ) { + let mode = if incremental_mode == Some(FlowIncrementalMode::MemtableOnly) { + FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY + } else { + FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE + }; let after_seqs = incremental_after_seqs.as_ref().ok_or_else(|| { invalid_query_context_extension(format!( "{} is required when {}={}.", - FLOW_INCREMENTAL_AFTER_SEQS, - FLOW_INCREMENTAL_MODE, - FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY + FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE, mode )) })?; if after_seqs.is_empty() { return Err(invalid_query_context_extension(format!( "{} must not be empty when {}={}.", - FLOW_INCREMENTAL_AFTER_SEQS, - FLOW_INCREMENTAL_MODE, - FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY + FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE, mode ))); } } @@ -231,19 +240,24 @@ impl FlowQueryExtensions { if matches!( self.incremental_mode, - Some(FlowIncrementalMode::MemtableOnly) + Some(FlowIncrementalMode::MemtableOnly | FlowIncrementalMode::SequenceRange) ) { + let mode = if self.incremental_mode == Some(FlowIncrementalMode::MemtableOnly) { + FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY + } else { + FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE + }; let after_seqs = self.incremental_after_seqs.as_ref().ok_or_else(|| { invalid_query_context_extension(format!( - "{} is required when {}=memtable_only.", - FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE + "{} is required when {}={}.", + FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE, mode )) })?; if !after_seqs.contains_key(&source_region_id.as_u64()) { return Err(invalid_query_context_extension(format!( - "Missing region {} in {} when {}=memtable_only.", - source_region_id, FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE + "Missing region {} in {} when {}={}.", + source_region_id, FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE, mode ))); } } @@ -510,6 +524,57 @@ mod flow_extension_tests { assert_eq!(parsed.sink_table_id, Some(1024)); } + #[test] + fn test_parse_flow_extensions_sequence_range_success() { + let exts = HashMap::from([ + ( + FLOW_INCREMENTAL_MODE.to_string(), + FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE.to_string(), + ), + ( + FLOW_INCREMENTAL_AFTER_SEQS.to_string(), + r#"{"1":10}"#.to_string(), + ), + ]); + + let parsed = FlowQueryExtensions::parse_flow_extensions(&exts) + .unwrap() + .unwrap(); + assert_eq!( + parsed.incremental_mode, + Some(FlowIncrementalMode::SequenceRange) + ); + assert_eq!( + parsed.incremental_after_seqs, + Some(HashMap::from([(1, 10)])) + ); + } + + #[test] + fn test_parse_flow_extensions_sequence_range_rejects_empty_after_seqs() { + let exts = HashMap::from([ + ( + FLOW_INCREMENTAL_MODE.to_string(), + FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE.to_string(), + ), + (FLOW_INCREMENTAL_AFTER_SEQS.to_string(), "{}".to_string()), + ]); + + let err = FlowQueryExtensions::parse_flow_extensions(&exts).unwrap_err(); + assert!(format!("{err}").contains(FLOW_INCREMENTAL_AFTER_SEQS)); + } + + #[test] + fn test_parse_flow_extensions_sequence_range_requires_after_seqs() { + let exts = HashMap::from([( + FLOW_INCREMENTAL_MODE.to_string(), + FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE.to_string(), + )]); + + let err = FlowQueryExtensions::parse_flow_extensions(&exts).unwrap_err(); + assert!(format!("{err}").contains(FLOW_INCREMENTAL_AFTER_SEQS)); + } + #[test] fn test_parse_flow_extensions_mode_requires_after_seqs() { let exts = HashMap::from([( diff --git a/src/query/src/part_sort.rs b/src/query/src/part_sort.rs index f1574ec3346..7f9a63df8df 100644 --- a/src/query/src/part_sort.rs +++ b/src/query/src/part_sort.rs @@ -18,7 +18,6 @@ //! partition ([`PartitionRange`]) independently based on the provided physical //! sort expressions. -use std::any::Any; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; @@ -38,7 +37,9 @@ use datafusion::physical_plan::filter_pushdown::{ use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, + apply_expression_roots, }; +use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_common::{DataFusionError, ScalarValue, internal_err}; use datafusion_expr::Operator; use datafusion_physical_expr::expressions::{ @@ -208,10 +209,6 @@ impl ExecutionPlan for PartSortExec { "PartSortExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.input.schema() } @@ -224,6 +221,27 @@ impl ExecutionPlan for PartSortExec { vec![&self.input] } + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> datafusion_common::Result, + ) -> datafusion_common::Result { + let dynamic_filter = self + .dynamic_filter + .as_ref() + .map(|filter| filter.clone() as Arc); + apply_expression_roots( + std::iter::once(&self.expression.expr).chain(dynamic_filter.as_ref()), + f, + ) + } + + fn dynamic_expressions_produced(&self) -> Vec> { + self.dynamic_filter + .iter() + .map(|filter| filter.clone() as Arc) + .collect() + } + fn with_new_children( self: Arc, children: Vec>, @@ -1847,6 +1865,63 @@ mod test { .await; } + #[test] + fn dynamic_expressions_produced_returns_topk_filter_arc() { + let unit = TimeUnit::Millisecond; + let schema = Arc::new(Schema::new(vec![Field::new( + "ts", + DataType::Timestamp(unit, None), + false, + )])); + let partition_range = PartitionRange { + start: Timestamp::new(0, unit.into()), + end: Timestamp::new(10, unit.into()), + num_rows: 0, + identifier: 0, + }; + let sort_expr = PhysicalSortExpr { + expr: Arc::new(Column::new("ts", 0)), + options: SortOptions::default(), + }; + + let limited = PartSortExec::try_new( + sort_expr.clone(), + Some(1), + vec![vec![partition_range]], + Arc::new(MockInputExec::new(vec![vec![]], schema.clone())), + ) + .unwrap(); + let expected = limited.dynamic_filter.as_ref().unwrap().clone() as Arc; + let produced = limited.dynamic_expressions_produced(); + assert_eq!(produced.len(), 1); + assert!(Arc::ptr_eq(&produced[0], &expected)); + + let mut applied_dynamic_filter = None; + limited + .apply_expressions(&mut |expr| { + if expr.expression_id().is_some() { + applied_dynamic_filter = Some(expr.clone()); + } + Ok(TreeNodeRecursion::Continue) + }) + .unwrap(); + let applied_dynamic_filter = applied_dynamic_filter.unwrap(); + assert!(Arc::ptr_eq(&produced[0], &applied_dynamic_filter)); + assert_eq!( + produced[0].expression_id(), + applied_dynamic_filter.expression_id() + ); + + let unlimited = PartSortExec::try_new( + sort_expr, + None, + vec![vec![partition_range]], + Arc::new(MockInputExec::new(vec![vec![]], schema)), + ) + .unwrap(); + assert!(unlimited.dynamic_expressions_produced().is_empty()); + } + #[test] fn test_topk_buffer_is_bounded_and_updates_dynamic_filter() { let unit = TimeUnit::Millisecond; diff --git a/src/query/src/plan.rs b/src/query/src/plan.rs index b4e5e664b69..26635a9d990 100644 --- a/src/query/src/plan.rs +++ b/src/query/src/plan.rs @@ -40,10 +40,9 @@ impl TreeNodeRewriter for TableNamesExtractAndRewriter { ) -> datafusion::error::Result> { match node { LogicalPlan::TableScan(mut scan) => { - if let Some(source) = scan.source.as_any().downcast_ref::() + if let Some(source) = scan.source.downcast_ref::() && let Some(provider) = source .table_provider - .as_any() .downcast_ref::() && provider.table().table_type() == TableType::Base { diff --git a/src/query/src/planner.rs b/src/query/src/planner.rs index 8a65a94adf0..9a577c26f9c 100644 --- a/src/query/src/planner.rs +++ b/src/query/src/planner.rs @@ -143,13 +143,15 @@ impl DfLogicalPlanner { // notice format is already set in query context, so can be ignore here Ok(LogicalPlan::Analyze(Analyze { verbose, + format: ExplainFormat::Indent, input: plan, schema, + analyze_level: None, + analyze_categories: None, })) } else { let stringified_plans = vec![plan.to_stringified(PlanType::InitialLogicalPlan)]; - // default to configuration value let options = self.session_state.config().options(); let format = format .map(|x| ExplainFormat::from_str(&x)) @@ -163,6 +165,7 @@ impl DfLogicalPlanner { stringified_plans, schema, logical_optimization_succeeded: false, + show_statistics: None, })) } } @@ -500,7 +503,8 @@ impl DfLogicalPlanner { if let DfExpr::Cast(cast) = e && let DfExpr::Placeholder(ph) = &*cast.expr { - placeholder_types.insert(ph.id.clone(), Some(cast.data_type.clone())); + placeholder_types + .insert(ph.id.clone(), Some(cast.field.data_type().clone())); casted_placeholders.insert(ph.id.clone()); } diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs index 7f1d5d35fbe..c51cdb05831 100644 --- a/src/query/src/promql/planner.rs +++ b/src/query/src/promql/planner.rs @@ -48,15 +48,14 @@ use datafusion::optimizer::simplify_expressions::ExprSimplifier; use datafusion::prelude as df_prelude; use datafusion::prelude::{Column, Expr as DfExpr, JoinType}; use datafusion::scalar::ScalarValue; -use datafusion::sql::TableReference; use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRewriter}; -use datafusion_common::{DFSchema, NullEquality}; +use datafusion_common::{DFSchema, NullEquality, TableReference}; use datafusion_expr::expr::WindowFunctionParams; use datafusion_expr::expr_fn::when; use datafusion_expr::simplify::SimplifyContext; use datafusion_expr::utils::{conjunction, disjunction}; use datafusion_expr::{ - ExprSchemable, Literal, Projection, SortExpr, TableScan, TableSource, col, lit, + ExprSchemable, Literal, Projection, SortExpr, TableScanBuilder, TableSource, col, lit, }; use datafusion_functions::core::coalesce; use datatypes::arrow::datatypes::{DataType as ArrowDataType, TimeUnit as ArrowTimeUnit}; @@ -587,6 +586,7 @@ impl PromPlanner { self.ctx.start, self.ctx.end, self.ctx.interval, + 0, range_ms, time_index_column, self.ctx.field_columns.clone(), @@ -1336,10 +1336,8 @@ impl PromPlanner { let mut field_expr = field_expr_builder(lhs, rhs)?; if is_comparison_op && should_return_bool { - field_expr = DfExpr::Cast(Cast { - expr: Box::new(field_expr), - data_type: ArrowDataType::Float64, - }); + field_expr = + DfExpr::Cast(Cast::new(Box::new(field_expr), ArrowDataType::Float64)); } Ok(LogicalPlan::Extension(Extension { @@ -1405,10 +1403,8 @@ impl PromPlanner { }; if is_comparison_op && should_return_bool { - binary_expr = DfExpr::Cast(Cast { - expr: Box::new(binary_expr), - data_type: ArrowDataType::Float64, - }); + binary_expr = + DfExpr::Cast(Cast::new(Box::new(binary_expr), ArrowDataType::Float64)); } Ok(binary_expr) }; @@ -1474,10 +1470,8 @@ impl PromPlanner { }; if is_comparison_op && should_return_bool { - binary_expr = DfExpr::Cast(Cast { - expr: Box::new(binary_expr), - data_type: ArrowDataType::Float64, - }); + binary_expr = + DfExpr::Cast(Cast::new(Box::new(binary_expr), ArrowDataType::Float64)); } Ok(binary_expr) }; @@ -1696,10 +1690,10 @@ impl PromPlanner { None => binary_expr_builder(lhs, rhs)?, }; if is_comparison_op && should_return_bool { - binary_expr = DfExpr::Cast(Cast { - expr: Box::new(binary_expr), - data_type: ArrowDataType::Float64, - }); + binary_expr = DfExpr::Cast(Cast::new( + Box::new(binary_expr), + ArrowDataType::Float64, + )); } Ok(binary_expr) }) @@ -1943,6 +1937,11 @@ impl PromPlanner { if let Some(empty_plan) = self.setup_context().await? { return Ok(empty_plan); } + let offset_ms = match offset { + Some(Offset::Pos(duration)) => duration.as_millis() as Millisecond, + Some(Offset::Neg(duration)) => -(duration.as_millis() as Millisecond), + None => 0, + }; let normalize = self .selector_to_series_normalize_plan(offset, matchers, false) .await?; @@ -1974,8 +1973,48 @@ impl PromPlanner { DfExpr::Column(Column::new(qualifier.cloned(), field.name().clone())) }) .collect::>(); - project_exprs - .push(build_special_time_expr(&time_index_column).alias(×tamp_value_column)); + // `timestamp()` preserves the shifted selector timeline even though + // SeriesNormalize now retains raw native timestamp storage. Decimal + // arithmetic shifts before truncating to milliseconds. + let unit_factor = match col(&time_index_column) + .get_type(normalize.schema()) + .context(DataFusionPlanningSnafu)? + { + ArrowDataType::Timestamp(ArrowTimeUnit::Second, _) => (1_000_i128, 4, 0), + ArrowDataType::Timestamp(ArrowTimeUnit::Millisecond, _) => (1, 1, 0), + ArrowDataType::Timestamp(ArrowTimeUnit::Microsecond, _) => (1, 4, 3), + ArrowDataType::Timestamp(ArrowTimeUnit::Nanosecond, _) => (1, 7, 6), + _ => unreachable!("time index is a timestamp"), + }; + let sample_time = col(&time_index_column) + .cast_to(&ArrowDataType::Int64, normalize.schema()) + .context(DataFusionPlanningSnafu)? + .cast_to(&ArrowDataType::Decimal128(19, 0), normalize.schema()) + .context(DataFusionPlanningSnafu)?; + let sample_time = DfExpr::BinaryExpr(BinaryExpr { + left: Box::new(sample_time), + op: Operator::Multiply, + right: Box::new(lit(ScalarValue::Decimal128( + Some(unit_factor.0), + unit_factor.1, + unit_factor.2, + ))), + }); + let sample_time = DfExpr::BinaryExpr(BinaryExpr { + left: Box::new(sample_time), + op: Operator::Plus, + right: Box::new(lit(ScalarValue::Decimal128(Some(offset_ms as i128), 19, 0))), + }) + .cast_to(&ArrowDataType::Int64, normalize.schema()) + .context(DataFusionPlanningSnafu)? + .cast_to(&ArrowDataType::Float64, normalize.schema()) + .context(DataFusionPlanningSnafu)?; + let sample_time = DfExpr::BinaryExpr(BinaryExpr { + left: Box::new(sample_time), + op: Operator::Divide, + right: Box::new(lit(1000.0)), + }); + project_exprs.push(sample_time.alias(×tamp_value_column)); let normalize = LogicalPlanBuilder::from(normalize) .project(project_exprs) .context(DataFusionPlanningSnafu)? @@ -1992,6 +2031,7 @@ impl PromPlanner { self.ctx.end, self.ctx.lookback_delta, self.ctx.interval, + offset_ms, time_index_column, if self.ctx.use_tsid { vec![DATA_SCHEMA_TSID_COLUMN_NAME.to_string()] @@ -2067,6 +2107,11 @@ impl PromPlanner { ensure!(!range.is_zero(), ZeroRangeSelectorSnafu); let range_ms = range.as_millis() as _; self.ctx.range = Some(range_ms); + let offset_ms = match offset { + Some(Offset::Pos(duration)) => duration.as_millis() as Millisecond, + Some(Offset::Neg(duration)) => -(duration.as_millis() as Millisecond), + None => 0, + }; // Some functions like rate may require special fields in the RangeManipulate plan // so we can't skip RangeManipulate. @@ -2081,6 +2126,7 @@ impl PromPlanner { self.ctx.start, self.ctx.end, self.ctx.interval, + offset_ms, // TODO(ruihang): convert via Timestamp datatypes to support different time units range_ms, self.ctx @@ -2142,8 +2188,6 @@ impl PromPlanner { ), }) }; - let preserve_any_value = - Self::field_columns_are_alternative_samples(input.schema(), &self.ctx.field_columns); let (mut func_exprs, new_tags) = self.create_function_expr( func, args.literals.clone(), @@ -2158,10 +2202,15 @@ impl PromPlanner { func_exprs.push(tsid_col); } + // A row survives as long as one field column produced a sample, and the fields without + // one stay NULL, which is the shape a selector already emits. Requiring every field to + // be non-NULL would drop one field's samples because another field has none in the same + // window — the reason alternative float/histogram columns already needed this form. A + // single field column reduces to the same predicate either way. let builder = LogicalPlanBuilder::from(input) .project(func_exprs) .context(DataFusionPlanningSnafu)? - .filter(self.create_empty_values_filter_expr(preserve_any_value)?) + .filter(self.create_empty_values_filter_expr(true)?) .context(DataFusionPlanningSnafu)?; let builder = match func.name { @@ -2339,14 +2388,18 @@ impl PromPlanner { None => 0, }; let mut scan_filters = Self::matchers_to_expr(label_matchers.clone(), table_schema)?; - if let Some(time_index_filter) = self.build_time_index_filter(offset_duration)? { + if let Some(time_index_filter) = + self.build_time_index_filter(offset_duration, table_schema)? + { scan_filters.push(time_index_filter); } - table_scan = LogicalPlanBuilder::from(table_scan) - .filter(conjunction(scan_filters).unwrap()) // Safety: `scan_filters` is not empty. - .context(DataFusionPlanningSnafu)? - .build() - .context(DataFusionPlanningSnafu)?; + if let Some(filter) = conjunction(scan_filters) { + table_scan = LogicalPlanBuilder::from(table_scan) + .filter(filter) + .context(DataFusionPlanningSnafu)? + .build() + .context(DataFusionPlanningSnafu)?; + } // make a projection plan if there is any `__field__` matcher if let Some(field_matchers) = &self.ctx.field_column_matcher { @@ -2691,11 +2744,9 @@ impl PromPlanner { fn table_from_source(&self, source: &Arc) -> Result { Ok(source - .as_any() .downcast_ref::() .context(UnknownTableSnafu)? .table_provider - .as_any() .downcast_ref::() .context(UnknownTableSnafu)? .table()) @@ -2718,72 +2769,99 @@ impl PromPlanner { Ok(table_ref) } - fn build_time_index_filter(&self, offset_duration: i64) -> Result> { + fn build_time_index_filter( + &self, + offset_duration: i64, + schema: &DFSchemaRef, + ) -> Result> { let start = self.ctx.start; let end = self.ctx.end; if end < start { return InvalidTimeRangeSnafu { start, end }.fail(); } - let lookback_delta = self.ctx.lookback_delta; - let range = self.ctx.range.unwrap_or_default(); - let interval = self.ctx.interval; let time_index_expr = self.create_time_index_column_expr()?; - let num_points = (end - start) / interval; - - // Prometheus semantics: - // - Instant selector lookback: (eval_ts - lookback_delta, eval_ts] - // - Range selector: (eval_ts - range, eval_ts] - // - // So samples positioned exactly at the lower boundary must be excluded. We align the scan - // lower bound with Prometheus by shifting it forward by 1ms (millisecond granularity), - // while still using a `>=` filter. - let selector_window = if range == 0 { lookback_delta } else { range }; - let lower_exclusive_adjustment = if selector_window > 0 { 1 } else { 0 }; - - // Scan a continuous time range - if (end - start) / interval > MAX_SCATTER_POINTS || interval <= INTERVAL_1H { - let single_time_range = time_index_expr - .clone() - .gt_eq(DfExpr::Literal( - ScalarValue::TimestampMillisecond( - Some( - self.ctx.start - offset_duration - selector_window - + lower_exclusive_adjustment, - ), - None, - ), - None, - )) - .and(time_index_expr.lt_eq(DfExpr::Literal( - ScalarValue::TimestampMillisecond(Some(self.ctx.end - offset_duration), None), - None, - ))); - return Ok(Some(single_time_range)); - } - - // Otherwise scan scatter ranges separately - let mut filters = Vec::with_capacity(num_points as usize + 1); - for timestamp in (start..=end).step_by(interval as usize) { - filters.push( + let time_index_name = self.ctx.time_index_column.as_ref().unwrap(); + let unit = schema + .index_of_column_by_name(None, time_index_name) + .and_then(|index| match schema.field(index).data_type() { + ArrowDataType::Timestamp(unit, _) => Some(*unit), + _ => None, + }) + .unwrap_or(ArrowTimeUnit::Millisecond); + let native_value = |milliseconds: i128| match unit { + ArrowTimeUnit::Second => milliseconds.div_euclid(1_000), + ArrowTimeUnit::Millisecond => milliseconds, + ArrowTimeUnit::Microsecond => milliseconds * 1_000, + ArrowTimeUnit::Nanosecond => milliseconds * 1_000_000, + }; + let scalar = |milliseconds: i128| -> Option { + let value = i64::try_from(native_value(milliseconds)).ok()?; + Some(match unit { + ArrowTimeUnit::Second => ScalarValue::TimestampSecond(Some(value), None), + ArrowTimeUnit::Millisecond => ScalarValue::TimestampMillisecond(Some(value), None), + ArrowTimeUnit::Microsecond => ScalarValue::TimestampMicrosecond(Some(value), None), + ArrowTimeUnit::Nanosecond => ScalarValue::TimestampNanosecond(Some(value), None), + }) + }; + let window = self.ctx.range.unwrap_or(self.ctx.lookback_delta); + let filter = |lower_ms: i128, upper_ms: i128| { + let lower_value = native_value(lower_ms); + let upper_value = native_value(upper_ms); + if lower_value > i128::from(i64::MAX) || upper_value < i128::from(i64::MIN) { + return Some(lit(false)); + } + let lower_filter = (lower_value >= i128::from(i64::MIN)).then(|| { + let lower = DfExpr::Literal(scalar(lower_ms).unwrap(), None); + if window == 0 { + time_index_expr.clone().gt_eq(lower) + } else if unit == ArrowTimeUnit::Millisecond + && let Some(inclusive_lower) = lower_ms + .checked_add(1) + .and_then(|lower| i64::try_from(lower).ok()) + .and_then(|lower| scalar(i128::from(lower))) + { + time_index_expr + .clone() + .gt_eq(DfExpr::Literal(inclusive_lower, None)) + } else { + time_index_expr.clone().gt(lower) + } + }); + let upper_filter = (upper_value <= i128::from(i64::MAX)).then(|| { time_index_expr .clone() - .gt_eq(DfExpr::Literal( - ScalarValue::TimestampMillisecond( - Some( - timestamp - offset_duration - selector_window - + lower_exclusive_adjustment, - ), - None, - ), - None, - )) - .and(time_index_expr.clone().lt_eq(DfExpr::Literal( - ScalarValue::TimestampMillisecond(Some(timestamp - offset_duration), None), - None, - ))), - ) - } + .lt_eq(DfExpr::Literal(scalar(upper_ms).unwrap(), None)) + }); + // An underflowing lower bound must not discard a representable upper + // bound: without it, LastRow could retain a future row and discard the + // older eligible sample before the manipulator can check its time. + match (lower_filter, upper_filter) { + (Some(lower), Some(upper)) => Some(lower.and(upper)), + (Some(filter), None) | (None, Some(filter)) => Some(filter), + (None, None) => None, + } + }; + let bounds = |timestamp: i64| { + let upper = i128::from(timestamp) - i128::from(offset_duration); + (upper - i128::from(window), upper) + }; + let num_points = (end as i128 - start as i128) / self.ctx.interval as i128; + if num_points > MAX_SCATTER_POINTS as i128 || self.ctx.interval <= INTERVAL_1H { + let (lower, _) = bounds(start); + let (_, upper) = bounds(end); + return Ok(filter(lower, upper)); + } + let mut filters = Vec::new(); + for timestamp in (start..=end).step_by(self.ctx.interval as usize) { + let (lower, upper) = bounds(timestamp); + let Some(filter) = filter(lower, upper) else { + // A point whose native bounds cannot be represented may cover the whole native + // time domain, so its disjunct cannot be omitted. + return Ok(None); + }; + filters.push(filter); + } Ok(filters.into_iter().reduce(DfExpr::or)) } @@ -2884,14 +2962,16 @@ impl PromPlanner { self.ctx.tag_columns.clone() }; - let is_time_index_ms = scan_table + let time_index_data_type = scan_table .schema() .timestamp_column() .with_context(|| TimeIndexNotFoundSnafu { table: maybe_phy_table_ref.to_quoted_string(), })? .data_type - == ConcreteDataType::timestamp_millisecond_datatype(); + .clone(); + let is_time_index_second = + time_index_data_type == ConcreteDataType::timestamp_second_datatype(); let scan_projection = if table_id_filter.is_some() { let mut required_columns = HashSet::new(); @@ -2944,8 +3024,11 @@ impl PromPlanner { .context(DataFusionPlanningSnafu)?; } - if !is_time_index_ms { - // cast to ms if time_index not in Millisecond precision + if is_time_index_second { + // Promote seconds so millisecond offsets remain exact; retain finer precision. + // Later manipulators compare native sample ticks, while PromQL evaluation and + // emitted timestamps remain millisecond-based, so this projection must not + // silently truncate a finer-grained time index. let expr: Vec<_> = self .create_field_column_exprs()? .into_iter() @@ -2959,10 +3042,10 @@ impl PromPlanner { DATA_SCHEMA_TSID_COLUMN_NAME.to_string(), )))) .chain(Some(DfExpr::Alias(Alias { - expr: Box::new(DfExpr::Cast(Cast { - expr: Box::new(self.create_time_index_column_expr()?), - data_type: ArrowDataType::Timestamp(ArrowTimeUnit::Millisecond, None), - })), + expr: Box::new(DfExpr::Cast(Cast::new( + Box::new(self.create_time_index_column_expr()?), + ArrowDataType::Timestamp(ArrowTimeUnit::Millisecond, None), + ))), relation: Some(table_ref.clone()), name: self .ctx @@ -2980,8 +3063,15 @@ impl PromPlanner { .context(DataFusionPlanningSnafu)? .build() .context(DataFusionPlanningSnafu)?; - } else if table_id_filter.is_some() { - // Drop the internal `__table_id` column after filtering. + } else if table_id_filter.is_some() + || time_index_data_type == ConcreteDataType::timestamp_microsecond_datatype() + || time_index_data_type == ConcreteDataType::timestamp_nanosecond_datatype() + { + // Drop the internal `__table_id` column after filtering and preserve PromQL's + // field/tag/timestamp column order for native microsecond/nanosecond timestamps. + // Keeping the original time column also lets the existing ordering hints + // use PerSeries scans without a cast, repartition, and sort. This benefits + // multi-evaluation selectors too; only a single evaluation can use LastRow. let project_exprs = self .create_field_column_exprs()? .into_iter() @@ -3099,13 +3189,12 @@ impl PromPlanner { projection.sort_unstable(); projection.dedup(); - let new_scan = TableScan::try_new( - scan.table_name.clone(), - scan.source.clone(), - Some(projection), - scan.filters, - scan.fetch, - )?; + let new_scan = + TableScanBuilder::new(scan.table_name.clone(), scan.source.clone()) + .with_projection(Some(projection)) + .with_filters(scan.filters) + .with_fetch(scan.fetch) + .build()?; Ok(Transformed::yes(LogicalPlan::TableScan(new_scan))) } LogicalPlan::Projection(proj) => { @@ -3328,10 +3417,10 @@ impl PromPlanner { } if func.name == "predict_linear" { - other_input_exprs[0] = DfExpr::Cast(Cast { - expr: Box::new(other_input_exprs[0].clone()), - data_type: ArrowDataType::Int64, - }); + other_input_exprs[0] = DfExpr::Cast(Cast::new( + Box::new(other_input_exprs[0].clone()), + ArrowDataType::Int64, + )); } let timestamp_range = DfExpr::Column(Column::from_name( @@ -3617,10 +3706,10 @@ impl PromPlanner { if all_field_columns_are_native_histogram_ranges { ScalarFunc::Udf(native_histogram_drop_udf(func.name)) } else { - other_input_exprs[0] = DfExpr::Cast(Cast { - expr: Box::new(other_input_exprs[0].clone()), - data_type: ArrowDataType::Int64, - }); + other_input_exprs[0] = DfExpr::Cast(Cast::new( + Box::new(other_input_exprs[0].clone()), + ArrowDataType::Int64, + )); ScalarFunc::Udf(Arc::new(PredictLinear::scalar_udf())) } } @@ -5279,10 +5368,10 @@ impl PromPlanner { false }; if is_comparison_op && should_return_bool { - Some(DfExpr::Cast(Cast { - expr: Box::new(expr), - data_type: ArrowDataType::Float64, - })) + Some(DfExpr::Cast(Cast::new( + Box::new(expr), + ArrowDataType::Float64, + ))) } else { Some(expr) } @@ -5377,18 +5466,12 @@ impl PromPlanner { let cast_float = |expr| { if matches!( &expr, - DfExpr::Cast(Cast { - data_type: ArrowDataType::Float64, - .. - }) + DfExpr::Cast(Cast { field, .. }) if field.data_type() == &ArrowDataType::Float64 ) || matches!(&expr, DfExpr::Literal(ScalarValue::Float64(_), _)) { expr } else { - DfExpr::Cast(Cast { - expr: Box::new(expr), - data_type: ArrowDataType::Float64, - }) + DfExpr::Cast(Cast::new(Box::new(expr), ArrowDataType::Float64)) } }; match token.id() { @@ -5983,21 +6066,29 @@ impl PromPlanner { ) -> Result<(LogicalPlan, LogicalPlan, bool)> { let marker = OTLP_AGGREGATION_TEMPORALITY_LABEL; let left_has_marker = left_context.tag_columns.iter().any(|tag| tag == marker); - let (present, add_to_left) = if left_has_marker { - (&left, false) - } else { - (&right, true) + let (data_type, value_type, add_to_left) = { + let (present, add_to_left) = if left_has_marker { + (&left, false) + } else { + (&right, true) + }; + let data_type = present + .schema() + .fields() + .iter() + .find(|field| field.name() == marker) + .map(|field| field.data_type().clone()) + .with_context(|| ColumnNotFoundSnafu { + col: marker.to_string(), + })?; + let value_type = Self::string_value_data_type(&data_type) + .cloned() + .with_context(|| UnexpectedPlanExprSnafu { + desc: format!("temporality match label {marker} must be a string"), + })?; + (data_type, value_type, add_to_left) }; - let data_type = present - .schema() - .fields() - .iter() - .find(|field| field.name() == marker) - .map(|field| field.data_type().clone()) - .with_context(|| ColumnNotFoundSnafu { - col: marker.to_string(), - })?; - let null = Self::string_scalar_value(&data_type, None).with_context(|| { + let null = Self::string_scalar_value(&value_type, None).with_context(|| { UnexpectedPlanExprSnafu { desc: format!("temporality match label {marker} must be a string"), } @@ -6020,7 +6111,28 @@ impl PromPlanner { .build() .context(DataFusionPlanningSnafu) }; - + if data_type != value_type { + let present = if add_to_left { &mut right } else { &mut left }; + let visible = present + .schema() + .iter() + .map(|(qualifier, field)| { + let column = + DfExpr::Column(Column::new(qualifier.cloned(), field.name().clone())); + if field.name() == marker { + DfExpr::Cast(Cast::new(Box::new(column), value_type.clone())) + .alias_qualified(qualifier.cloned(), field.name().clone()) + } else { + column + } + }) + .collect::>(); + *present = LogicalPlanBuilder::from(present.clone()) + .project(visible) + .context(DataFusionPlanningSnafu)? + .build() + .context(DataFusionPlanningSnafu)?; + } if add_to_left { left = add_marker(left)?; left_context.tag_columns.push(marker.to_string()); @@ -6044,10 +6156,7 @@ impl PromPlanner { let column = if &data_type == value_type { column } else { - DfExpr::Cast(Cast { - expr: Box::new(column), - data_type: value_type.clone(), - }) + DfExpr::Cast(Cast::new(Box::new(column), value_type.clone())) }; DfExpr::ScalarFunction(ScalarFunction { func: coalesce(), @@ -6256,7 +6365,8 @@ impl PromPlanner { result }; - // AND/UNLESS preserve the complete left operand schema and metadata. + // AND/UNLESS preserve the complete left operand's visible columns and values; encoded + // markers are decoded. self.ctx = output_context; Ok(result) } @@ -6617,11 +6727,7 @@ impl PromPlanner { if source_type == target_type { expr } else { - DfExpr::Cast(Cast { - expr: Box::new(expr), - data_type: target_type.clone(), - }) - .alias(col.clone()) + DfExpr::Cast(Cast::new(Box::new(expr), target_type.clone())).alias(col.clone()) } } else { DfExpr::Literal( @@ -6645,11 +6751,8 @@ impl PromPlanner { if data_type == &ArrowDataType::Float64 { expr.alias(output_col) } else { - DfExpr::Cast(Cast { - expr: Box::new(expr), - data_type: ArrowDataType::Float64, - }) - .alias(output_col) + DfExpr::Cast(Cast::new(Box::new(expr), ArrowDataType::Float64)) + .alias(output_col) } } else { DfExpr::Literal(ScalarValue::Float64(None), None).alias(output_col) @@ -6675,13 +6778,13 @@ impl PromPlanner { && col == left_field_col && left_field.2 != target_field_type { - DfExpr::Cast(Cast { - expr: Box::new(DfExpr::Column(Column::new( + DfExpr::Cast(Cast::new( + Box::new(DfExpr::Column(Column::new( left_field.1.clone(), left_field_col, ))), - data_type: target_field_type.clone(), - }) + target_field_type.clone(), + )) .alias(left_field_col.clone()) } else if target_tag_types.contains_key(col) { aligned_label_expr(col, &left_tag_types) @@ -6706,11 +6809,8 @@ impl PromPlanner { } else if !mixed_sample_types && col == left_field_col { let expr = DfExpr::Column(Column::new(right_field.1.clone(), right_field_col)); if right_field.2 != target_field_type { - DfExpr::Cast(Cast { - expr: Box::new(expr), - data_type: target_field_type.clone(), - }) - .alias(left_field_col.clone()) + DfExpr::Cast(Cast::new(Box::new(expr), target_field_type.clone())) + .alias(left_field_col.clone()) } else if left_field_col != right_field_col { expr.alias(left_field_col.clone()) } else { @@ -8832,7 +8932,7 @@ mod test { \n Projection: some_metric.timestamp, value AS value, some_metric.tag_0 [timestamp:Timestamp(ms), value:Float64, tag_0:Utf8]\ \n Projection: some_metric.timestamp, __promql_timestamp_value_ AS value, some_metric.tag_0 [timestamp:Timestamp(ms), value:Float64, tag_0:Utf8]\ \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_timestamp_value_:Float64]\ - \n Projection: some_metric.tag_0, some_metric.timestamp, some_metric.field_0, CAST(CAST(some_metric.timestamp AS Int64) AS Float64) / Float64(1000) AS __promql_timestamp_value_ [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_timestamp_value_:Float64]\ + \n Projection: some_metric.tag_0, some_metric.timestamp, some_metric.field_0, CAST(CAST(CAST(CAST(some_metric.timestamp AS Int64) AS Decimal128(19, 0)) * Decimal128(1,1,0) + Decimal128(0,19,0) AS Int64) AS Float64) / Float64(1000) AS __promql_timestamp_value_ [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_timestamp_value_:Float64]\ \n PromSeriesDivide: tags=[\"tag_0\"] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ \n Sort: some_metric.tag_0 ASC NULLS FIRST, some_metric.timestamp ASC NULLS FIRST [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ \n Filter: some_metric.tag_0 != Utf8(\"bar\") AND some_metric.timestamp >= TimestampMillisecond(-999, None) AND some_metric.timestamp <= TimestampMillisecond(100000000, None) [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ @@ -9051,7 +9151,17 @@ mod test { let manipulate = find_instant_manipulate(&plan).unwrap(); let exec = manipulate.to_execution_plan(Arc::new(DataSourceExec::new(Arc::new( - MemorySourceConfig::try_new(&[], Arc::new(ArrowSchema::empty()), None).unwrap(), + MemorySourceConfig::try_new( + &[], + Arc::new( + datafusion_expr::UserDefinedLogicalNodeCore::inputs(manipulate)[0] + .schema() + .as_arrow() + .clone(), + ), + None, + ) + .unwrap(), )))); assert!(format!("{exec:?}").contains("reuse_tsid_column: true")); } @@ -10814,6 +10924,7 @@ mod test { 1_000, 5_000, 1_000, + 0, "timestamp".to_string(), Vec::new(), Some(greptime_native_histogram().to_string()), @@ -11443,6 +11554,7 @@ mod test { 3000, 3000, 1000, + 0, 3000, "timestamp".to_string(), planner.ctx.field_columns.clone(), @@ -12135,6 +12247,101 @@ mod test { } } + #[tokio::test] + async fn native_scan_bounds_preserve_zero_lookback_and_overflow() { + let table_provider = build_test_table_provider( + &[(DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string())], + 1, + 1, + ) + .await; + let mut planner = PromPlanner { + table_provider, + ctx: PromPlannerContext::from_eval_stmt(&build_eval_stmt("some_metric")), + promql_annotations: None, + }; + planner.ctx.time_index_column = Some("timestamp".to_string()); + planner.ctx.start = 1_000; + planner.ctx.lookback_delta = 0; + let schema = Arc::new( + DFSchema::try_from(ArrowSchema::new(vec![Field::new( + "timestamp", + ArrowDataType::Timestamp(ArrowTimeUnit::Nanosecond, None), + false, + )])) + .unwrap(), + ); + for (end, interval, windows) in [ + (1_000, 1_000, 1), + (2_000, 1_000, 1), + (7_201_000, 7_200_000, 2), + ] { + planner.ctx.end = end; + planner.ctx.interval = interval; + let filter = planner + .build_time_index_filter(0, &schema) + .unwrap() + .unwrap() + .to_string(); + assert_eq!(filter.matches(">=").count(), windows, "{filter}"); + assert!( + filter.contains("TimestampNanosecond(1000000000, None)"), + "{filter}" + ); + } + planner.ctx.end = i64::MAX; + let filter = planner + .build_time_index_filter(0, &schema) + .unwrap() + .unwrap() + .to_string(); + assert!( + filter.contains("timestamp >= TimestampNanosecond(1000000000, None)"), + "{filter}" + ); + + // A lookback subtraction can underflow milliseconds while the upper bound remains + // representable. Keep that upper bound so LastRow cannot select a future sample. + let ms_schema = Arc::new( + DFSchema::try_from(ArrowSchema::new(vec![Field::new( + "timestamp", + ArrowDataType::Timestamp(ArrowTimeUnit::Millisecond, None), + false, + )])) + .unwrap(), + ); + planner.ctx.start = i64::MIN + 100; + planner.ctx.end = planner.ctx.start; + planner.ctx.lookback_delta = 200; + let filter = planner + .build_time_index_filter(0, &ms_schema) + .unwrap() + .unwrap() + .to_string(); + assert_eq!( + filter, + format!( + "timestamp <= TimestampMillisecond({}, None)", + i64::MIN + 100 + ) + ); + + // The lower bound can also overflow while converting milliseconds to native nanoseconds. + // Its representable upper bound still has to reach the scan. + planner.ctx.start = 0; + planner.ctx.end = 0; + planner.ctx.lookback_delta = 300_000; + let filter = planner + .build_time_index_filter(9_223_372_036_854, &schema) + .unwrap() + .unwrap() + .to_string(); + assert_eq!( + filter, + "timestamp <= TimestampNanosecond(-9223372036854000000, None)" + ); + } + #[tokio::test] async fn test_non_ms_precision() { let catalog_list = MemoryCatalogManager::with_default_setup(); @@ -12205,12 +12412,7 @@ mod test { .unwrap(); assert_eq!( plan.display_indent_schema().to_string(), - "PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [field:Float64;N, tag:Utf8, timestamp:Timestamp(ms)]\ - \n PromSeriesDivide: tags=[\"tag\"] [field:Float64;N, tag:Utf8, timestamp:Timestamp(ms)]\ - \n Sort: metrics.tag ASC NULLS FIRST, metrics.timestamp ASC NULLS FIRST [field:Float64;N, tag:Utf8, timestamp:Timestamp(ms)]\ - \n Filter: metrics.tag = Utf8(\"1\") AND metrics.timestamp >= TimestampMillisecond(-999, None) AND metrics.timestamp <= TimestampMillisecond(100000000, None) [field:Float64;N, tag:Utf8, timestamp:Timestamp(ms)]\ - \n Projection: metrics.field, metrics.tag, CAST(metrics.timestamp AS Timestamp(ms)) AS timestamp [field:Float64;N, tag:Utf8, timestamp:Timestamp(ms)]\ - \n TableScan: metrics [tag:Utf8, timestamp:Timestamp(ns), field:Float64;N]" + "PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [field:Float64;N, tag:Utf8, timestamp:Timestamp(ms)]\n PromSeriesDivide: tags=[\"tag\"] [field:Float64;N, tag:Utf8, timestamp:Timestamp(ns)]\n Sort: metrics.tag ASC NULLS FIRST, metrics.timestamp ASC NULLS FIRST [field:Float64;N, tag:Utf8, timestamp:Timestamp(ns)]\n Filter: metrics.tag = Utf8(\"1\") AND metrics.timestamp > TimestampNanosecond(-1000000000, None) AND metrics.timestamp <= TimestampNanosecond(100000000000000, None) [field:Float64;N, tag:Utf8, timestamp:Timestamp(ns)]\n Projection: metrics.field, metrics.tag, metrics.timestamp [field:Float64;N, tag:Utf8, timestamp:Timestamp(ns)]\n TableScan: metrics [tag:Utf8, timestamp:Timestamp(ns), field:Float64;N]" ); let plan = PromPlanner::stmt_to_plan( DfTableSourceProvider::new( @@ -12235,15 +12437,7 @@ mod test { .unwrap(); assert_eq!( plan.display_indent_schema().to_string(), - "Filter: prom_avg_over_time(timestamp_range,field) IS NOT NULL [timestamp:Timestamp(ms), prom_avg_over_time(timestamp_range,field):Float64;N, tag:Utf8]\ - \n Projection: metrics.timestamp, prom_avg_over_time(timestamp_range, field) AS prom_avg_over_time(timestamp_range,field), metrics.tag [timestamp:Timestamp(ms), prom_avg_over_time(timestamp_range,field):Float64;N, tag:Utf8]\ - \n PromRangeManipulate: req range=[0..100000000], interval=[5000], eval range=[5000], time index=[timestamp], values=[\"field\"] [field:Dictionary(Int64, Float64);N, tag:Utf8, timestamp:Timestamp(ms), timestamp_range:Dictionary(Int64, Timestamp(ms))]\ - \n PromSeriesNormalize: offset=[0], time index=[timestamp], filter NaN: [true] [field:Float64;N, tag:Utf8, timestamp:Timestamp(ms)]\ - \n PromSeriesDivide: tags=[\"tag\"] [field:Float64;N, tag:Utf8, timestamp:Timestamp(ms)]\ - \n Sort: metrics.tag ASC NULLS FIRST, metrics.timestamp ASC NULLS FIRST [field:Float64;N, tag:Utf8, timestamp:Timestamp(ms)]\ - \n Filter: metrics.tag = Utf8(\"1\") AND metrics.timestamp >= TimestampMillisecond(-4999, None) AND metrics.timestamp <= TimestampMillisecond(100000000, None) [field:Float64;N, tag:Utf8, timestamp:Timestamp(ms)]\ - \n Projection: metrics.field, metrics.tag, CAST(metrics.timestamp AS Timestamp(ms)) AS timestamp [field:Float64;N, tag:Utf8, timestamp:Timestamp(ms)]\ - \n TableScan: metrics [tag:Utf8, timestamp:Timestamp(ns), field:Float64;N]" + "Filter: prom_avg_over_time(timestamp_range,field) IS NOT NULL [timestamp:Timestamp(ms), prom_avg_over_time(timestamp_range,field):Float64;N, tag:Utf8]\n Projection: metrics.timestamp, prom_avg_over_time(timestamp_range, field) AS prom_avg_over_time(timestamp_range,field), metrics.tag [timestamp:Timestamp(ms), prom_avg_over_time(timestamp_range,field):Float64;N, tag:Utf8]\n PromRangeManipulate: req range=[0..100000000], interval=[5000], eval range=[5000], time index=[timestamp], values=[\"field\"] [field:Dictionary(Int64, Float64);N, tag:Utf8, timestamp:Timestamp(ms), timestamp_range:Dictionary(Int64, Timestamp(ms))]\n PromSeriesNormalize: offset=[0], time index=[timestamp], filter NaN: [true] [field:Float64;N, tag:Utf8, timestamp:Timestamp(ns)]\n PromSeriesDivide: tags=[\"tag\"] [field:Float64;N, tag:Utf8, timestamp:Timestamp(ns)]\n Sort: metrics.tag ASC NULLS FIRST, metrics.timestamp ASC NULLS FIRST [field:Float64;N, tag:Utf8, timestamp:Timestamp(ns)]\n Filter: metrics.tag = Utf8(\"1\") AND metrics.timestamp > TimestampNanosecond(-5000000000, None) AND metrics.timestamp <= TimestampNanosecond(100000000000000, None) [field:Float64;N, tag:Utf8, timestamp:Timestamp(ns)]\n Projection: metrics.field, metrics.tag, metrics.timestamp [field:Float64;N, tag:Utf8, timestamp:Timestamp(ns)]\n TableScan: metrics [tag:Utf8, timestamp:Timestamp(ns), field:Float64;N]" ); } diff --git a/src/query/src/promql/planner/test/delta.rs b/src/query/src/promql/planner/test/delta.rs index 93e214d45b5..f6c37bb24a1 100644 --- a/src/query/src/promql/planner/test/delta.rs +++ b/src/query/src/promql/planner/test/delta.rs @@ -14,6 +14,8 @@ use common_query::logical_plan::SubstraitPlanDecoder; use common_query::prelude::set_default_prefix; +use datafusion::arrow::array::{DictionaryArray, UInt32Array}; +use datafusion::arrow::datatypes::UInt32Type; use datafusion::catalog::SchemaProvider; use super::*; @@ -453,6 +455,7 @@ async fn delta_mixed_ranges_drop_and_float_ranges_sum() { 2_000, 2_000, 1_000, + 0, 2_000, "ts".to_string(), planner.ctx.field_columns.clone(), @@ -757,3 +760,394 @@ async fn binary_joins_align_only_the_temporality_marker() { let (_, batches) = execute(set, &build_query_engine_state()).await; assert_eq!(1, batches.iter().map(RecordBatch::num_rows).sum::()); } + +#[tokio::test] +async fn binary_joins_align_dictionary_temporality_marker_with_tagless_vector() { + let marker = OTLP_AGGREGATION_TEMPORALITY_LABEL; + let marker_schema = Arc::new(ArrowSchema::new(vec![ + Field::new( + "ts", + ArrowDataType::Timestamp(ArrowTimeUnit::Millisecond, None), + false, + ), + Field::new("job", ArrowDataType::Utf8, true), + Field::new( + marker, + ArrowDataType::Dictionary( + Box::new(ArrowDataType::UInt32), + Box::new(ArrowDataType::Utf8), + ), + true, + ), + Field::new("v", ArrowDataType::Float64, true), + ])); + let tagless_schema = Arc::new(ArrowSchema::new(vec![ + Field::new( + "ts", + ArrowDataType::Timestamp(ArrowTimeUnit::Millisecond, None), + false, + ), + Field::new("job", ArrowDataType::Utf8, true), + Field::new("v", ArrowDataType::Float64, true), + ])); + let tagless_batch = RecordBatch::try_new( + tagless_schema.clone(), + vec![ + Arc::new(TimestampMillisecondArray::from(vec![1])), + Arc::new(StringArray::from(vec![Some("job")])), + Arc::new(Float64Array::from(vec![10.0])), + ], + ) + .unwrap(); + for (marker_on_left, null_key) in [(true, true), (true, false), (false, true), (false, false)] { + let marker_values: Arc = Arc::new( + DictionaryArray::::try_new( + UInt32Array::from(if null_key { + vec![Some(0), None] + } else { + vec![Some(0), Some(1)] + }), + Arc::new(StringArray::from(vec![ + Some(GREPTIME_TEMPORALITY_DELTA), + None, + ])), + ) + .unwrap(), + ); + let marker_batch = RecordBatch::try_new( + marker_schema.clone(), + vec![ + Arc::new(TimestampMillisecondArray::from(vec![1, 1])), + Arc::new(StringArray::from(vec![Some("job"), Some("job")])), + marker_values, + Arc::new(Float64Array::from(vec![1.0, 2.0])), + ], + ) + .unwrap(); + let marker_table = Arc::new( + MemTable::try_new(marker_schema.clone(), vec![vec![marker_batch], vec![]]).unwrap(), + ); + let tagless_table = Arc::new( + MemTable::try_new( + tagless_schema.clone(), + vec![vec![tagless_batch.clone()], vec![]], + ) + .unwrap(), + ); + let (left, right, left_context, right_context) = if marker_on_left { + ( + marker_table, + tagless_table.clone(), + direct_or_context("lhs", &["job", marker], "v"), + direct_or_context("rhs", &["job"], "v"), + ) + } else { + ( + tagless_table.clone(), + marker_table, + direct_or_context("lhs", &["job"], "v"), + direct_or_context("rhs", &["job", marker], "v"), + ) + }; + let mut planner = PromPlanner { + table_provider: build_test_table_provider_with_fields( + &[(DEFAULT_SCHEMA_NAME.to_string(), "dummy".to_string())], + &[], + ) + .await, + ctx: PromPlannerContext::default(), + promql_annotations: None, + }; + let scan = |name, table: Arc| { + LogicalPlanBuilder::scan(name, provider_as_source(table), None) + .unwrap() + .build() + .unwrap() + }; + let joined = planner + .join_on_non_field_columns( + scan("lhs", left.clone()), + scan("rhs", right.clone()), + TableReference::bare("lhs"), + TableReference::bare("rhs"), + Some("ts".to_string()), + Some("ts".to_string()), + false, + &None, + &left_context, + &right_context, + ) + .unwrap(); + let marker_side = if marker_on_left { "lhs" } else { "rhs" }; + assert!( + joined + .schema() + .qualified_field_with_name(Some(&TableReference::bare(marker_side)), marker) + .is_ok(), + "marker_on_left={marker_on_left}, null_key={null_key}: {joined:?}" + ); + let arithmetic = LogicalPlanBuilder::from(joined) + .project(vec![ + DfExpr::Column(Column::new(Some(TableReference::bare("lhs")), "ts")).alias("ts"), + DfExpr::Column(Column::new(Some(TableReference::bare("lhs")), "job")).alias("job"), + (DfExpr::Column(Column::new(Some(TableReference::bare("lhs")), "v")) + + DfExpr::Column(Column::new(Some(TableReference::bare("rhs")), "v"))) + .alias("v"), + ]) + .unwrap() + .build() + .unwrap(); + let values = |batches: &[RecordBatch]| { + batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("v") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .iter() + .copied() + }) + .collect::>() + }; + let (_, batches) = execute(arithmetic.clone(), &build_query_engine_state()).await; + assert_eq!( + values(&batches), + &[12.0], + "marker_on_left={marker_on_left}, null_key={null_key}" + ); + + if marker_on_left && null_key { + let nested = planner + .join_on_non_field_columns( + arithmetic, + scan("rhs", tagless_table.clone()), + TableReference::bare("nested"), + TableReference::bare("rhs"), + Some("ts".to_string()), + Some("ts".to_string()), + false, + &None, + &direct_or_context("nested", &["job"], "v"), + &direct_or_context("rhs", &["job"], "v"), + ) + .unwrap(); + assert!( + nested + .schema() + .qualified_field_with_name(Some(&TableReference::bare("nested")), "v") + .is_ok(), + "{nested:?}" + ); + let nested = LogicalPlanBuilder::from(nested) + .project(vec![ + (DfExpr::Column(Column::new(Some(TableReference::bare("nested")), "v")) + + DfExpr::Column(Column::new(Some(TableReference::bare("rhs")), "v"))) + .alias("v"), + ]) + .unwrap() + .build() + .unwrap(); + let (_, batches) = execute(nested, &build_query_engine_state()).await; + assert_eq!(values(&batches), &[22.0]); + } + + for (expression, expected_values, expected_marker) in [ + ( + "lhs and rhs", + if marker_on_left { + &[2.0][..] + } else { + &[10.0][..] + }, + marker_on_left.then_some(None), + ), + ( + "lhs unless rhs", + if marker_on_left { &[1.0][..] } else { &[] }, + marker_on_left.then_some(Some(GREPTIME_TEMPORALITY_DELTA)), + ), + ] { + let PromExpr::Binary(binary) = parser::parse(expression).unwrap() else { + unreachable!() + }; + let set = planner + .set_op_on_non_field_columns( + scan("lhs", left.clone()), + scan("rhs", right.clone()), + left_context.clone(), + right_context.clone(), + binary.op, + &binary.modifier, + ) + .unwrap(); + assert_eq!( + expected_marker.is_some(), + set.schema().field_with_unqualified_name(marker).is_ok(), + "{expression}, marker_on_left={marker_on_left}, null_key={null_key}" + ); + let mut output = vec![DfExpr::Column(Column::from_name("v"))]; + if expected_marker.is_some() { + output.push( + DfExpr::Cast(Cast::new( + Box::new(DfExpr::Column(Column::from_name(marker))), + ArrowDataType::Utf8, + )) + .alias(marker), + ); + } + let output = LogicalPlanBuilder::from(set) + .project(output) + .unwrap() + .build() + .unwrap(); + let (_, batches) = execute(output, &build_query_engine_state()).await; + assert_eq!( + expected_values, + values(&batches), + "{expression}, marker_on_left={marker_on_left}, null_key={null_key}" + ); + if let Some(expected_marker) = expected_marker { + let markers = batches + .iter() + .flat_map(|batch| { + batch + .column_by_name(marker) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .map(|value| value.map(str::to_string)) + }) + .collect::>(); + assert_eq!( + vec![expected_marker.map(str::to_string)], + markers, + "{expression}, marker_on_left={marker_on_left}, null_key={null_key}" + ); + } + } + } +} + +#[tokio::test] +async fn delta_offsets_survive_optimized_plan_serialization() { + let eval_time = UNIX_EPOCH.checked_add(Duration::from_secs(120)).unwrap(); + for (name, query, expected) in [ + ( + "selector positive offset", + r#"delta_metric{series="cumulative"} offset 60s"#, + 10.0, + ), + ( + "selector negative offset", + r#"delta_metric{series="cumulative"} offset -60s"#, + 30.0, + ), + ( + "timestamp positive offset", + r#"timestamp(delta_metric{series="cumulative"} offset 60s)"#, + 120.0, + ), + ( + "timestamp negative offset", + r#"timestamp(delta_metric{series="cumulative"} offset -60s)"#, + 120.0, + ), + ( + "range positive offset", + r#"last_over_time(delta_metric{series="cumulative"}[60s] offset 60s)"#, + 10.0, + ), + ( + "range negative offset", + r#"last_over_time(delta_metric{series="cumulative"}[60s] offset -60s)"#, + 30.0, + ), + ( + "subquery positive offset", + r#"last_over_time((delta_metric{series="cumulative"} offset 60s)[60s:60s])"#, + 10.0, + ), + ( + "subquery negative offset", + r#"last_over_time((delta_metric{series="cumulative"} offset -60s)[60s:60s])"#, + 30.0, + ), + ] { + let eval_stmt = EvalStmt { + expr: parser::parse(query).unwrap(), + start: eval_time, + end: eval_time, + interval: Duration::from_secs(60), + lookback_delta: Duration::from_secs(300), + }; + let (provider, state, datafusion_table) = delta_temporality_table_provider(); + let raw = PromPlanner::stmt_to_plan(provider, &eval_stmt, &state) + .await + .unwrap(); + let context = QueryEngineContext::new(state.session_state(), QueryContext::arc()); + let optimized = state.optimize_by_extension_rules(raw, &context).unwrap(); + let optimized = state.optimize_logical_plan(optimized).unwrap(); + + let context = SessionContext::new_with_state(state.session_state()); + let catalog = Arc::new(MemoryCatalogProvider::new()); + let schema = Arc::new(MemorySchemaProvider::new()); + schema + .register_table("delta_metric".to_string(), datafusion_table) + .unwrap(); + catalog + .register_schema(DEFAULT_SCHEMA_NAME, schema) + .unwrap(); + context.register_catalog("datafusion", catalog); + let decoder = DefaultPlanDecoder::new(context.state(), &QueryContext::arc()).unwrap(); + let decoded = decoder + .decode( + DFLogicalSubstraitConvertor + .encode(&optimized, DefaultSerializer) + .unwrap(), + context.state().catalog_list().clone(), + false, + ) + .await + .unwrap(); + + let mut outputs = Vec::new(); + for plan in [optimized, decoded] { + let (_, batches) = execute(plan, &state).await; + let mut output = Vec::new(); + for batch in batches { + let value_field = batch + .schema() + .fields() + .iter() + .find(|field| field.data_type() == &ArrowDataType::Float64) + .unwrap() + .name() + .clone(); + let values = batch + .column_by_name(&value_field) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let timestamps = batch + .column_by_name(greptime_timestamp()) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + output.push((timestamps.value(row), values.value(row))); + } + } + assert_eq!(vec![(120_000, expected)], output, "{name}"); + outputs.push(output); + } + assert_eq!(outputs[0], outputs[1], "{name}"); + } +} diff --git a/src/query/src/query_engine/context.rs b/src/query/src/query_engine/context.rs index c967fc38bd9..3386a38e238 100644 --- a/src/query/src/query_engine/context.rs +++ b/src/query/src/query_engine/context.rs @@ -55,6 +55,7 @@ impl QueryEngineContext { session_id, state.config().clone(), state.scalar_functions().clone(), + state.higher_order_functions().clone(), state.aggregate_functions().clone(), state.window_functions().clone(), state.runtime_env().clone(), diff --git a/src/query/src/query_engine/default_serializer.rs b/src/query/src/query_engine/default_serializer.rs index e009d80954f..f4d6b29fa2c 100644 --- a/src/query/src/query_engine/default_serializer.rs +++ b/src/query/src/query_engine/default_serializer.rs @@ -437,6 +437,7 @@ mod tests { 0, 1000, 1000, + 0, 1000, "timestamp".to_string(), vec!["float".to_string(), "histogram".to_string()], diff --git a/src/query/src/query_engine/state.rs b/src/query/src/query_engine/state.rs index a3d8432421b..68a301adf7d 100644 --- a/src/query/src/query_engine/state.rs +++ b/src/query/src/query_engine/state.rs @@ -29,7 +29,7 @@ use common_function::handlers::{ use common_function::state::FunctionState; use common_stat::get_total_memory_bytes; use common_telemetry::warn; -use datafusion::catalog::TableFunction; +use datafusion::catalog::{Session, TableFunction}; use datafusion::dataframe::DataFrame; use datafusion::error::Result as DfResult; use datafusion::execution::SessionStateBuilder; @@ -65,6 +65,7 @@ use crate::optimizer::const_normalization::ConstNormalizationRule; use crate::optimizer::constant_term::MatchesConstantTermOptimizer; use crate::optimizer::count_nest_aggr::CountNestAggrRule; use crate::optimizer::count_wildcard::CountWildcardToTimeIndexRule; +use crate::optimizer::enforce_sorting::EnforceSorting; use crate::optimizer::global_limit::EnsureGlobalLimitForFetch; use crate::optimizer::json_schema_concretize::JsonSchemaConcretizeRule; use crate::optimizer::json_type_concretize::JsonTypeConcretizeRule; @@ -214,6 +215,14 @@ impl QueryEngineState { } analyzer.rules.push(Arc::new(FixStateUdafOrderingAnalyzer)); + // Note: Postgres oid-alias string coercion (`'x'::regclass`, + // `'public'::regnamespace`, ...) is handled by the + // `PostgresCompatibilityParser`'s built-in `RewriteRegCastToSubquery` + // rule at SQL-parse time, so no analyzer rule is registered here. The + // implicit `oid_col = 'name'` form is not resolved (it needs schema + // awareness the parser lacks); the few client queries that use it are + // handled by the parser's blacklist. + let mut optimizer = Optimizer::new(); optimizer.rules.push(Arc::new(ScanHintRule)); optimizer.rules.push(Arc::new(JsonTypeConcretizeRule)); @@ -232,11 +241,9 @@ impl QueryEngineState { physical_optimizer .rules .insert(7, Arc::new(PromqlTsidNarrowJoin)); - // Enforce sorting AFTER custom rules that modify the plan structure - physical_optimizer.rules.insert( - 8, - Arc::new(datafusion::physical_optimizer::enforce_sorting::EnforceSorting {}), - ); + // Re-enforce sorting after custom rules update scan partitioning and distribution. + // Keep it immediately before DataFusion's default EnsureRequirements. + physical_optimizer.rules.insert(8, Arc::new(EnforceSorting)); // Add rule for windowed sort physical_optimizer .rules @@ -275,9 +282,9 @@ impl QueryEngineState { .with_optimizer_rules(optimizer.rules) .with_physical_optimizer_rules(physical_optimizer.rules) .build(); - let df_context = SessionContext::new_with_state(session_state); register_function_aliases(&df_context); + register_pg_catalog_compat(&df_context); Ok(Self { df_context, @@ -502,10 +509,10 @@ impl QueryPlanner for DfQueryPlanner { async fn create_physical_plan( &self, logical_plan: &DfLogicalPlan, - session_state: &SessionState, + session: &dyn Session, ) -> DfResult> { self.physical_planner - .create_physical_plan(logical_plan, session_state) + .create_physical_plan(logical_plan, session) .await } } @@ -545,6 +552,22 @@ fn register_function_aliases(ctx: &SessionContext) { } } +/// Register the session-level (not function-registry) Postgres-compatibility +/// extension sourced from `datafusion-pg-catalog`: widen Postgres `int4` +/// bounds to `int8` for `generate_series`/`range` (their stock implementations +/// require `int8` literals, and DF invokes them at planning time, before any +/// analyzer rule can run). +/// +/// The oid-alias type planner is supplied per SQL-planner context by +/// [`DfContextProviderAdapter::get_type_planner`](crate::datafusion::planner::DfContextProviderAdapter::get_type_planner). +/// Forward oid-alias name->oid resolution is handled at SQL-parse time by the +/// `PostgresCompatibilityParser`'s built-in rewrite rule, not here. +fn register_pg_catalog_compat(ctx: &SessionContext) { + datafusion_pg_catalog::pg_catalog::generate_series_arg_coercion::CoerceIntArgsToBigInt::widen( + ctx, + ); +} + impl DfQueryPlanner { fn new( catalog_manager: CatalogManagerRef, @@ -611,7 +634,17 @@ impl MetricsMemoryPool { } } +impl fmt::Display for MetricsMemoryPool { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}(inner_pool: {})", self.name(), self.inner) + } +} + impl MemoryPool for MetricsMemoryPool { + fn name(&self) -> &str { + "metrics" + } + fn register(&self, consumer: &MemoryConsumer) { self.inner.register(consumer); } @@ -820,7 +853,7 @@ mod tests { let plugins = Plugins::default(); plugins.insert::(Arc::new(ErrorRuntimeProvider)); - let err = QueryEngineState::try_new( + let err = match QueryEngineState::try_new( catalog::memory::new_memory_catalog_manager().unwrap(), None, None, @@ -830,8 +863,10 @@ mod tests { false, plugins, QueryOptions::default(), - ) - .unwrap_err(); + ) { + Err(err) => err, + Ok(_) => panic!("expected runtime provider error"), + }; assert!( matches!(err, DataFusionError::Execution(message) if message == "runtime provider error") @@ -943,7 +978,9 @@ mod tests { assert!(!env.disk_manager.tmp_files_enabled()); let result = env.disk_manager.create_tmp_file("test spill"); assert!(result.is_err()); - assert!(format!("{}", result.unwrap_err()).contains("DiskManager is disabled")); + if let Err(error) = result { + assert!(format!("{error}").contains("DiskManager is disabled")); + } } #[test] diff --git a/src/query/src/range_select/plan.rs b/src/query/src/range_select/plan.rs index 5cc607249cc..1961735c4af 100644 --- a/src/query/src/range_select/plan.rs +++ b/src/query/src/range_select/plan.rs @@ -21,22 +21,23 @@ use std::sync::Arc; use std::task::{Context, Poll}; use std::time::Duration; -use ahash::RandomState; use arrow::compute::{self, CastOptions, cast_with_options, take_arrays}; use arrow_schema::{DataType, Field, Schema, SchemaRef, SortOptions, TimeUnit}; use common_function::aggrs::aggr_wrapper::get_aggr_func; use common_recordbatch::DfSendableRecordBatchStream; +use datafusion::catalog::Session; use datafusion::common::Result as DataFusionResult; use datafusion::error::Result as DfResult; use datafusion::execution::TaskContext; -use datafusion::execution::context::SessionState; +use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, RecordBatchStream, - SendableRecordBatchStream, + SendableRecordBatchStream, apply_expression_roots, }; -use datafusion_common::hash_utils::create_hashes; +use datafusion_common::hash_utils::{RandomState, create_hashes}; +use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_common::{DFSchema, DFSchemaRef, DataFusionError, ScalarValue}; use datafusion_expr::utils::{COUNT_STAR_EXPANSION, exprlist_to_fields}; use datafusion_expr::{ @@ -535,7 +536,8 @@ impl RangeSelect { is_count_aggr: bool, exprs: &[Expr], df_schema: &Arc, - session_state: &SessionState, + session: &dyn Session, + planning_ctx: &PhysicalPlanningContext, ) -> DfResult>> { exprs .iter() @@ -549,9 +551,15 @@ impl RangeSelect { Expr::Wildcard { .. } if is_count_aggr => create_physical_expr( &lit(COUNT_STAR_EXPANSION), df_schema.as_ref(), - session_state.execution_props(), + session.execution_props(), + planning_ctx, + ), + _ => create_physical_expr( + e, + df_schema.as_ref(), + session.execution_props(), + planning_ctx, ), - _ => create_physical_expr(e, df_schema.as_ref(), session_state.execution_props()), }) .collect::>>() } @@ -560,7 +568,8 @@ impl RangeSelect { &self, logical_input: &LogicalPlan, exec_input: Arc, - session_state: &SessionState, + session: &dyn Session, + planning_ctx: &PhysicalPlanningContext, ) -> DfResult> { let fields: Vec<_> = self .schema_before_project @@ -599,7 +608,8 @@ impl RangeSelect { create_physical_sort_expr( x, input_dfschema.as_ref(), - session_state.execution_props(), + session.execution_props(), + planning_ctx, ) }) .collect::>>()? @@ -608,7 +618,8 @@ impl RangeSelect { let time_index = create_physical_expr( &self.time_expr, input_dfschema.as_ref(), - session_state.execution_props(), + session.execution_props(), + planning_ctx, )?; vec![PhysicalSortExpr { expr: time_index, @@ -622,7 +633,8 @@ impl RangeSelect { false, &aggr.params.args, input_dfschema, - session_state, + session, + planning_ctx, )?; // first_value/last_value has only one param. // The param have been checked by datafusion in logical plan stage. @@ -642,7 +654,8 @@ impl RangeSelect { create_physical_sort_expr( x, input_dfschema.as_ref(), - session_state.execution_props(), + session.execution_props(), + planning_ctx, ) }) .collect::>>()? @@ -656,7 +669,8 @@ impl RangeSelect { aggr.func.name() == "count", &aggr.params.args, input_dfschema, - session_state, + session, + planning_ctx, )?; AggregateExprBuilder::new(aggr.func.clone(), input_phy_exprs) .schema(input_schema.clone()) @@ -688,7 +702,8 @@ impl RangeSelect { } else { schema_before_project.clone() }; - let by = self.create_physical_expr_list(false, &self.by, input_dfschema, session_state)?; + let by = + self.create_physical_expr_list(false, &self.by, input_dfschema, session, planning_ctx)?; let cache = Arc::new(PlanProperties::new( EquivalenceProperties::new(schema.clone()), Partitioning::UnknownPartitioning(1), @@ -790,10 +805,6 @@ impl DisplayAs for RangeSelectExec { } impl ExecutionPlan for RangeSelectExec { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> SchemaRef { self.schema.clone() } @@ -810,6 +821,19 @@ impl ExecutionPlan for RangeSelectExec { vec![&self.input] } + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> DfResult, + ) -> DfResult { + apply_expression_roots( + self.range_exec + .iter() + .flat_map(RangeFnExec::expressions) + .chain(self.by.iter().cloned()), + f, + ) + } + fn with_new_children( self: Arc, children: Vec>, @@ -858,7 +882,7 @@ impl ExecutionPlan for RangeSelectExec { schema: self.schema.clone(), range_exec: self.range_exec.clone(), input, - random_state: RandomState::new(), + random_state: RandomState::default(), time_index, align: self.align, align_to: self.align_to, @@ -1299,7 +1323,7 @@ mod test { }; use datafusion::datasource::memory::MemorySourceConfig; use datafusion::datasource::source::DataSourceExec; - use datafusion::functions_aggregate::min_max; + use datafusion::functions_aggregate::{first_last, min_max}; use datafusion::physical_plan::sorts::sort::SortExec; use datafusion::prelude::SessionContext; use datafusion_physical_expr::PhysicalSortExpr; @@ -1801,6 +1825,89 @@ mod test { .await; } + #[test] + fn range_select_apply_expressions_visits_owned_roots() { + let input = Arc::new(prepare_test_data(true, false)); + let input_schema = input.schema().clone(); + let schema = Arc::new(Schema::new(vec![Field::new( + "FIRST_VALUE(value)", + DataType::Float64, + true, + )])); + let range_select = RangeSelectExec { + input, + range_exec: vec![RangeFnExec { + expr: Arc::new( + AggregateExprBuilder::new( + first_last::first_value_udaf(), + vec![Arc::new(Column::new("value", 1))], + ) + .schema(input_schema) + .order_by(vec![PhysicalSortExpr { + expr: Arc::new(Column::new(TIME_INDEX_COLUMN, 0)), + options: SortOptions::default(), + }]) + .alias("FIRST_VALUE(value)") + .build() + .unwrap(), + ), + range: 10_000, + fill: None, + need_cast: None, + }], + align: 5_000, + align_to: 0, + time_index: TIME_INDEX_COLUMN.to_string(), + by: vec![Arc::new(Column::new("host", 2))], + schema: schema.clone(), + by_schema: Arc::new(Schema::empty()), + metric: ExecutionPlanMetricsSet::new(), + schema_project: None, + schema_before_project: schema.clone(), + cache: Arc::new(PlanProperties::new( + EquivalenceProperties::new(schema), + Partitioning::UnknownPartitioning(1), + EmissionType::Incremental, + Boundedness::Bounded, + )), + }; + assert_eq!(range_select.range_exec[0].expr.order_bys().len(), 1); + + let mut visited = Vec::new(); + assert_eq!( + range_select + .apply_expressions(&mut |expr| { + visited.push(expr.to_string()); + Ok(TreeNodeRecursion::Continue) + }) + .unwrap(), + TreeNodeRecursion::Continue + ); + assert_eq!(visited, ["value@1", "timestamp@0", "host@2"]); + + let mut stopped = Vec::new(); + assert_eq!( + range_select + .apply_expressions(&mut |expr| { + stopped.push(expr.to_string()); + Ok(TreeNodeRecursion::Stop) + }) + .unwrap(), + TreeNodeRecursion::Stop + ); + assert_eq!(stopped, ["value@1"]); + + assert_eq!( + range_select + .apply_expressions(&mut |_| { + Err(DataFusionError::Execution("apply failure".into())) + }) + .unwrap_err() + .to_string(), + "Execution error: apply failure" + ); + } + #[tokio::test] async fn range_select_respects_session_batch_size() { let result = diff --git a/src/query/src/range_select/plan_rewrite.rs b/src/query/src/range_select/plan_rewrite.rs index 59132b85815..a809f85a643 100644 --- a/src/query/src/range_select/plan_rewrite.rs +++ b/src/query/src/range_select/plan_rewrite.rs @@ -167,8 +167,10 @@ fn evaluate_expr_to_millisecond( return Err(dispose_parse_error(Some(expr))); } let info = match scheduled_time { - Some(dt) => SimplifyContext::default().with_query_execution_start_time(Some(dt)), - None => SimplifyContext::default().with_current_time(), + Some(dt) => SimplifyContext::builder() + .with_query_execution_start_time(Some(dt)) + .build(), + None => SimplifyContext::builder().with_current_time().build(), }; let simplify_expr = ExprSimplifier::new(info).simplify(expr.clone())?; match simplify_expr { @@ -597,11 +599,9 @@ impl RangePlanRewriter { } }; let table = table_source - .as_any() .downcast_ref::() .context(UnknownTableSnafu)? .table_provider - .as_any() .downcast_ref::() .context(UnknownTableSnafu)? .table(); @@ -738,27 +738,27 @@ fn interval_only_in_expr(expr: &Expr) -> bool { // A cast expression for an interval. if matches!( expr, - Expr::Cast(Cast{ + Expr::Cast(Cast { expr, - data_type: DataType::Interval(_) - }) if matches!(&**expr, Expr::Literal(ScalarValue::Utf8(_), _)) + field, + }) if matches!(field.data_type(), DataType::Interval(_)) + && matches!(&**expr, Expr::Literal(ScalarValue::Utf8(_), _)) ) { // Stop checking the sub `expr`, // which is a `Utf8` type and has already been tested above. return Ok(TreeNodeRecursion::Stop); } - if !matches!( + if !(matches!( expr, Expr::Literal(ScalarValue::IntervalDayTime(_), _) | Expr::Literal(ScalarValue::IntervalMonthDayNano(_), _) | Expr::Literal(ScalarValue::IntervalYearMonth(_), _) | Expr::BinaryExpr(_) - | Expr::Cast(Cast { - data_type: DataType::Interval(_), - .. - }) - ) { + ) || matches!( + expr, + Expr::Cast(Cast { field, .. }) if matches!(field.data_type(), DataType::Interval(_)) + )) { all_interval = false; Ok(TreeNodeRecursion::Stop) } else { @@ -1373,7 +1373,7 @@ mod test { let query = r#"SELECT sum(avg(field_0 + field_1) RANGE '5m' + 1) RANGE '5m' + 1 FROM test ALIGN '1h' by (tag_0,tag_1);"#; assert_eq!( do_query(query).await.unwrap_err().to_string(), - "Range Query: Nest Range Query is not allowed" + "Failed to plan SQL" ) } @@ -1574,10 +1574,10 @@ mod test { parse_duration("1y4w").unwrap() ); // test cast expression - let args = vec![Expr::Cast(Cast { - expr: Box::new("15 minutes".lit()), - data_type: DataType::Interval(IntervalUnit::MonthDayNano), - })]; + let args = vec![Expr::Cast(Cast::new( + Box::new("15 minutes".lit()), + DataType::Interval(IntervalUnit::MonthDayNano), + ))]; assert_eq!( parse_duration_expr(&args, 0).unwrap(), parse_duration("15m").unwrap() @@ -1711,10 +1711,10 @@ mod test { assert!(interval_only_in_expr(&expr)); let expr = Expr::BinaryExpr(BinaryExpr { - left: Box::new(Expr::Cast(Cast { - expr: Box::new("15 minute".lit()), - data_type: DataType::Interval(IntervalUnit::MonthDayNano), - })), + left: Box::new(Expr::Cast(Cast::new( + Box::new("15 minute".lit()), + DataType::Interval(IntervalUnit::MonthDayNano), + ))), op: Operator::Minus, right: Box::new( ScalarValue::IntervalDayTime(Some(IntervalDayTime::new(10, 0).into())).lit(), @@ -1722,19 +1722,19 @@ mod test { }); assert!(interval_only_in_expr(&expr)); - let expr = Expr::Cast(Cast { - expr: Box::new(Expr::BinaryExpr(BinaryExpr { - left: Box::new(Expr::Cast(Cast { - expr: Box::new("15 minute".lit()), - data_type: DataType::Interval(IntervalUnit::MonthDayNano), - })), + let expr = Expr::Cast(Cast::new( + Box::new(Expr::BinaryExpr(BinaryExpr { + left: Box::new(Expr::Cast(Cast::new( + Box::new("15 minute".lit()), + DataType::Interval(IntervalUnit::MonthDayNano), + ))), op: Operator::Minus, right: Box::new( ScalarValue::IntervalDayTime(Some(IntervalDayTime::new(10, 0).into())).lit(), ), })), - data_type: DataType::Interval(IntervalUnit::MonthDayNano), - }); + DataType::Interval(IntervalUnit::MonthDayNano), + )); assert!(interval_only_in_expr(&expr)); } diff --git a/src/query/src/range_select/planner.rs b/src/query/src/range_select/planner.rs index 73b3bfb4ef3..5e30eabd5d3 100644 --- a/src/query/src/range_select/planner.rs +++ b/src/query/src/range_select/planner.rs @@ -15,8 +15,9 @@ use std::sync::Arc; use async_trait::async_trait; +use datafusion::catalog::Session; use datafusion::error::Result as DfResult; -use datafusion::execution::context::SessionState; +use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext; use datafusion::logical_expr::{LogicalPlan, UserDefinedLogicalNode}; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_planner::{ExtensionPlanner, PhysicalPlanner}; @@ -33,13 +34,15 @@ impl ExtensionPlanner for RangeSelectPlanner { node: &dyn UserDefinedLogicalNode, logical_inputs: &[&LogicalPlan], physical_inputs: &[Arc], - session_state: &SessionState, + session: &dyn Session, + planning_ctx: &PhysicalPlanningContext, ) -> DfResult>> { if let Some(node) = node.as_any().downcast_ref::() { Ok(Some(node.to_execution_plan( logical_inputs[0], physical_inputs[0].clone(), - session_state, + session, + planning_ctx, )?)) } else { Ok(None) diff --git a/src/query/src/test_util.rs b/src/query/src/test_util.rs index 954e249550f..fd404b65c5b 100644 --- a/src/query/src/test_util.rs +++ b/src/query/src/test_util.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; @@ -27,7 +26,8 @@ use datafusion::execution::{RecordBatchStream, TaskContext}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; -use datafusion_physical_expr::{EquivalenceProperties, Partitioning}; +use datafusion_common::tree_node::TreeNodeRecursion; +use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr}; use futures::Stream; pub fn new_ts_array(unit: TimeUnit, arr: Vec) -> ArrayRef { @@ -80,10 +80,6 @@ impl ExecutionPlan for MockInputExec { "MockInputExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.properties } @@ -92,6 +88,13 @@ impl ExecutionPlan for MockInputExec { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> datafusion_common::Result, + ) -> datafusion_common::Result { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, _children: Vec>, diff --git a/src/query/src/window_sort.rs b/src/query/src/window_sort.rs index 7267de3dab9..5e47711f75a 100644 --- a/src/query/src/window_sort.rs +++ b/src/query/src/window_sort.rs @@ -15,7 +15,6 @@ //! A physical plan for window sort(Which is sorting multiple sorted ranges according to input `PartitionRange`). //! -use std::any::Any; use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::pin::Pin; use std::slice::from_ref; @@ -38,10 +37,12 @@ use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSe use datafusion::physical_plan::sorts::streaming_merge::StreamingMergeBuilder; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, + apply_expression_roots, }; +use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_common::utils::bisect; use datafusion_common::{DataFusionError, internal_err}; -use datafusion_physical_expr::PhysicalSortExpr; +use datafusion_physical_expr::{PhysicalExpr, PhysicalSortExpr}; use datatypes::value::Value; use futures::Stream; use itertools::Itertools; @@ -195,10 +196,6 @@ impl DisplayAs for WindowedSortExec { } impl ExecutionPlan for WindowedSortExec { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.input.schema() } @@ -211,6 +208,13 @@ impl ExecutionPlan for WindowedSortExec { vec![&self.input] } + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> datafusion_common::Result, + ) -> datafusion_common::Result { + apply_expression_roots([&self.expression.expr], f) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml index dbb956c0d5e..7eb008369ba 100644 --- a/src/servers/Cargo.toml +++ b/src/servers/Cargo.toml @@ -24,7 +24,7 @@ api.workspace = true arrow.workspace = true arrow-flight.workspace = true arrow-ipc.workspace = true -arrow-pg = "0.14" +arrow-pg = "0.15" arrow-schema.workspace = true async-trait.workspace = true auth.workspace = true @@ -36,6 +36,7 @@ bytes.workspace = true catalog.workspace = true chrono.workspace = true common-base.workspace = true +common-batcher.workspace = true common-catalog.workspace = true common-decimal.workspace = true common-error.workspace = true diff --git a/src/servers/benches/to_http_output.rs b/src/servers/benches/to_http_output.rs index 2f2708a2636..f3a9149083a 100644 --- a/src/servers/benches/to_http_output.rs +++ b/src/servers/benches/to_http_output.rs @@ -13,15 +13,34 @@ // limitations under the License. use std::sync::Arc; -use std::time::Instant; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; -use arrow::array::StringArray; +use arrow::array::{ArrayRef, StringArray, UInt64Array}; use arrow_schema::{DataType, Field, Schema}; -use common_recordbatch::RecordBatch; -use criterion::{Criterion, criterion_group, criterion_main}; +use async_trait::async_trait; +use axum::extract::{Query, State}; +use axum::response::IntoResponse; +use axum::{Extension, Form}; +use common_query::Output; +use common_recordbatch::{RecordBatch, RecordBatchStreamWrapper}; +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group}; +use datafusion_expr::LogicalPlan; use datatypes::schema::SchemaRef; use datatypes::vectors::StringVector; -use servers::http::HttpRecordsOutput; +use futures::stream; +use query::parser::PromQuery; +use query::query_engine::DescribeResult; +use servers::error::Result; +use servers::http::handler::{SqlQuery, sql}; +use servers::http::{ApiState, HttpRecordsOutput}; +use servers::query_handler::sql::SqlQueryHandler; +use session::context::{QueryContext, QueryContextRef}; +use sql::statements::statement::Statement; + +#[cfg(not(windows))] +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; fn mock_schema() -> SchemaRef { let mut fields = Vec::with_capacity(10); @@ -70,12 +89,227 @@ fn bench_convert_record_batch_to_http_output(c: &mut Criterion) { }); } +struct StreamingSqlHandler { + rows: usize, + schema: SchemaRef, + consumed: Arc, +} + +impl StreamingSqlHandler { + fn new(rows: usize) -> Self { + let fields = std::iter::once(Field::new("id", DataType::UInt64, false)) + .chain((1..10).map(|i| Field::new(format!("field{i}"), DataType::Utf8, false))); + let schema = Arc::new(Schema::new(fields.collect::>())); + Self { + rows, + schema: Arc::new(schema.try_into().unwrap()), + consumed: Arc::new(AtomicUsize::new(0)), + } + } +} + +#[async_trait] +impl SqlQueryHandler for StreamingSqlHandler { + async fn do_query(&self, _: &str, _: QueryContextRef) -> Vec> { + let schema = self.schema.clone(); + let consumed = self.consumed.clone(); + let rows = self.rows; + consumed.store(0, Ordering::Relaxed); + // Allocate independent buffers lazily, as a query stream would. Prebuilding + // all batches or sharing one batch's buffers would hide retention costs. + let batches = (0..rows).step_by(4096).map(move |offset| { + let end = (offset + 4096).min(rows); + let mut columns: Vec = vec![Arc::new(UInt64Array::from_iter_values( + offset as u64..end as u64, + ))]; + columns.extend((1..10).map(|_| { + Arc::new(StringArray::from_iter_values(std::iter::repeat_n( + "0123456789abcdef0123456789abcdef", + end - offset, + ))) as ArrayRef + })); + let batch = + arrow::record_batch::RecordBatch::try_new(schema.arrow_schema().clone(), columns) + .unwrap(); + consumed.fetch_add(end - offset, Ordering::Relaxed); + Ok(RecordBatch::from_df_record_batch(schema.clone(), batch)) + }); + vec![Ok(Output::new_with_stream(Box::pin( + RecordBatchStreamWrapper::new(self.schema.clone(), stream::iter(batches)), + )))] + } + + async fn do_analyze_stream_query(&self, _: &str, _: QueryContextRef) -> Result { + unimplemented!() + } + + async fn do_exec_plan( + &self, + _: LogicalPlan, + _: Option, + _: QueryContextRef, + ) -> Result { + unimplemented!() + } + + async fn do_promql_query(&self, _: &PromQuery, _: QueryContextRef) -> Vec> { + unimplemented!() + } + + async fn do_describe( + &self, + _: Statement, + _: QueryContextRef, + ) -> Result> { + unimplemented!() + } + + async fn is_valid_schema(&self, _: &str, _: &str) -> Result { + Ok(true) + } +} + +#[derive(Clone, Copy)] +enum LimitPlacement { + Late, + Early, +} + +async fn limited_response( + handler: Arc, + limit: usize, + placement: LimitPlacement, +) -> bytes::Bytes { + let response = sql( + State(ApiState { + sql_handler: handler.clone(), + }), + Query(SqlQuery { + sql: Some("select * from benchmark_input".to_string()), + limit: matches!(placement, LimitPlacement::Early).then_some(limit), + ..Default::default() + }), + Extension(QueryContext::with_db_name(None)), + Form(SqlQuery::default()), + ) + .await; + // Reconstruct the old path with the existing post-conversion truncation API. + let response = match placement { + LimitPlacement::Late => response.with_limit(limit), + LimitPlacement::Early => response, + }; + let response = response.with_execution_time(0).into_response(); + assert!(response.status().is_success()); + assert_eq!(handler.consumed.load(Ordering::Relaxed), handler.rows); + axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap() +} + +fn bench_http_sql_limit(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let mut group = c.benchmark_group("http_sql_limit"); + group.sample_size(10); + group.warm_up_time(Duration::from_millis(500)); + group.measurement_time(Duration::from_secs(2)); + for (rows, limit) in [ + (10_000, 100), + (100_000, 100), + (1_000_000, 100), + (1_000_000, 0), + (100_000, 100_000), + ] { + let handler = Arc::new(StreamingSqlHandler::new(rows)); + { + let late = runtime.block_on(limited_response( + handler.clone(), + limit, + LimitPlacement::Late, + )); + let early = runtime.block_on(limited_response( + handler.clone(), + limit, + LimitPlacement::Early, + )); + assert_eq!(late, early); + let json: serde_json::Value = serde_json::from_slice(&early).unwrap(); + assert_eq!( + json["output"][0]["records"]["rows"] + .as_array() + .unwrap() + .len(), + rows.min(limit) + ); + } + group.throughput(Throughput::Elements(rows as u64)); + for (name, placement) in [ + ("late", LimitPlacement::Late), + ("early", LimitPlacement::Early), + ] { + group.bench_function( + BenchmarkId::new(name, format!("rows_{rows}_limit_{limit}")), + |b| { + b.iter(|| { + runtime.block_on(limited_response(handler.clone(), limit, placement)) + }); + }, + ); + } + } + group.finish(); +} + #[cfg(not(windows))] criterion_group! { name = benches; config = Criterion::default().with_profiler(pprof::criterion::PProfProfiler::new(101, pprof::criterion::Output::Flamegraph(None))); - targets = bench_convert_record_batch_to_http_output + targets = bench_convert_record_batch_to_http_output, bench_http_sql_limit } #[cfg(windows)] -criterion_group!(benches, bench_convert_record_batch_to_http_output); -criterion_main!(benches); +criterion_group!( + benches, + bench_convert_record_batch_to_http_output, + bench_http_sql_limit +); + +fn main() { + // Run each memory case in a fresh process, e.g. with `/usr/bin/time -v`: + // GREPTIME_HTTP_LIMIT_MEMORY_CASE=late,1000000,100 + if let Ok(case) = std::env::var("GREPTIME_HTTP_LIMIT_MEMORY_CASE") { + let parts: Vec<_> = case.split(',').collect(); + assert_eq!(parts.len(), 3, "expected late|early,rows,limit"); + let placement = match parts[0] { + "late" => LimitPlacement::Late, + "early" => LimitPlacement::Early, + _ => panic!("expected late or early"), + }; + let rows = parts[1].parse().unwrap(); + let limit = parts[2].parse().unwrap(); + let handler = Arc::new(StreamingSqlHandler::new(rows)); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + #[cfg(not(windows))] + let allocated = tikv_jemalloc_ctl::thread::allocatedp::read().unwrap(); + #[cfg(not(windows))] + let before = allocated.get(); + let start = Instant::now(); + let response = runtime.block_on(limited_response(handler, limit, placement)); + let elapsed = start.elapsed(); + // Memory mode writes measurements to stdout for external benchmark tools. + #[allow(clippy::print_stdout)] + { + #[cfg(not(windows))] + println!("allocated_bytes={}", allocated.get() - before); + println!( + "case={case} elapsed={elapsed:?} response_bytes={}", + response.len() + ); + } + return; + } + benches(); + Criterion::default().configure_from_args().final_summary(); +} diff --git a/src/servers/src/addrs.rs b/src/servers/src/addrs.rs index 07422028f97..d68568726fa 100644 --- a/src/servers/src/addrs.rs +++ b/src/servers/src/addrs.rs @@ -25,7 +25,7 @@ pub fn resolve_addr(bind_addr: &str, server_addr: Option<&str>) -> String { } else { // otherwise, resolve port from bind_addr // should be safe to unwrap here because bind_addr is already validated - let port = bind_addr.split(':').nth(1).unwrap(); + let port = bind_addr.rsplit_once(':').unwrap().1; format!("{server_addr}:{port}") } } @@ -50,6 +50,16 @@ mod tests { super::resolve_addr("127.0.0.1:3001", Some("tomcat:3002")) ); + assert_eq!( + "tomcat:3001", + super::resolve_addr("[::]:3001", Some("tomcat")) + ); + + assert_eq!( + "tomcat:3001", + super::resolve_addr("[2001:db8::10]:3001", Some("tomcat")) + ); + assert_eq!( "127.0.0.1:3001", super::resolve_addr("127.0.0.1:3001", None) diff --git a/src/servers/src/elasticsearch.rs b/src/servers/src/elasticsearch.rs index 94c04300cc6..5ccda5ed7f4 100644 --- a/src/servers/src/elasticsearch.rs +++ b/src/servers/src/elasticsearch.rs @@ -260,11 +260,12 @@ fn write_bulk_response(took_ms: i64, n: usize, status_code: u32, error_reason: & }) }) .collect(); - json!({ + let mut response = json!({ "took": took_ms, "errors": false, - "items": items, - }) + }); + response["items"] = Value::Array(items); + response } else { json!({ "took": took_ms, diff --git a/src/servers/src/error.rs b/src/servers/src/error.rs index fa5957759c7..fe1d226a10c 100644 --- a/src/servers/src/error.rs +++ b/src/servers/src/error.rs @@ -745,7 +745,7 @@ impl ErrorExt for Error { #[cfg(not(windows))] UpdateJemallocMetrics { .. } => StatusCode::Internal, - CollectRecordbatch { .. } => StatusCode::EngineExecuteQuery, + CollectRecordbatch { source, .. } => source.status_code(), ExecuteQuery { source, .. } | ExecutePlan { source, .. } @@ -986,3 +986,37 @@ pub fn status_code_to_http_status(status_code: &StatusCode) -> HttpStatusCode { | StatusCode::EngineExecuteQuery => HttpStatusCode::INTERNAL_SERVER_ERROR, } } + +#[cfg(test)] +mod tests { + use common_error::GREPTIME_DB_HEADER_ERROR_CODE; + use common_error::ext::PlainError; + + use super::*; + + #[test] + fn collect_recordbatch_preserves_poll_stream_status_in_tonic_status() { + let error = Error::CollectRecordbatch { + source: common_recordbatch::error::Error::PollStream { + error: DataFusionError::External(Box::new(BoxedError::new(PlainError::new( + "neutral error".to_string(), + StatusCode::RequestOutdated, + )))), + location: Location::default(), + }, + location: Location::default(), + }; + + let status: tonic::Status = error.into(); + assert_eq!(status.code(), tonic::Code::InvalidArgument); + assert_eq!( + status + .metadata() + .get(GREPTIME_DB_HEADER_ERROR_CODE) + .unwrap() + .to_str() + .unwrap(), + (StatusCode::RequestOutdated as u32).to_string() + ); + } +} diff --git a/src/servers/src/grpc/flight.rs b/src/servers/src/grpc/flight.rs index 0b7a9dd5704..f0035ed3f8c 100644 --- a/src/servers/src/grpc/flight.rs +++ b/src/servers/src/grpc/flight.rs @@ -632,7 +632,10 @@ fn output_to_flight_record_batch_source( #[cfg(test)] mod tests { - use query::options::FLOW_SCHEDULED_TIME_MILLIS; + use query::options::{ + FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE, FLOW_RETURN_REGION_SEQ, + FLOW_SCHEDULED_TIME_MILLIS, FLOW_SINK_TABLE_ID, FlowIncrementalMode, + }; use tonic::metadata::{AsciiMetadataValue, MetadataMap}; use super::*; @@ -680,6 +683,47 @@ mod tests { ); } + #[test] + fn test_flow_extensions_forward_sequence_range_to_query_context() { + let mut metadata = MetadataMap::new(); + metadata.insert( + FLOW_EXTENSIONS_METADATA_KEY, + AsciiMetadataValue::try_from( + r#"[["flow.return_region_seq","true"],["flow.incremental_mode","sequence_range"],["flow.incremental_after_seqs","{\"1\":10,\"2\":20}"],["flow.sink_table_id","42"]]"#, + ) + .unwrap(), + ); + + let flow_extensions = extract_flow_extensions(&metadata).unwrap(); + let query_ctx = + create_query_context(Channel::Grpc, None, flow_extensions, HashMap::new()).unwrap(); + let parsed = + query::options::FlowQueryExtensions::parse_flow_extensions(&query_ctx.extensions()) + .unwrap() + .unwrap(); + + assert_eq!( + parsed.incremental_mode, + Some(FlowIncrementalMode::SequenceRange) + ); + assert_eq!( + parsed.incremental_after_seqs, + Some(HashMap::from([(1, 10), (2, 20)])) + ); + assert!(parsed.return_region_seq); + assert_eq!(parsed.sink_table_id, Some(42)); + assert_eq!( + query_ctx.extension(FLOW_INCREMENTAL_MODE), + Some("sequence_range") + ); + assert_eq!( + query_ctx.extension(FLOW_INCREMENTAL_AFTER_SEQS), + Some(r#"{"1":10,"2":20}"#) + ); + assert_eq!(query_ctx.extension(FLOW_RETURN_REGION_SEQ), Some("true")); + assert_eq!(query_ctx.extension(FLOW_SINK_TABLE_ID), Some("42")); + } + #[test] fn test_extract_flow_extensions_rejects_invalid_json() { let mut metadata = MetadataMap::new(); diff --git a/src/servers/src/http/handler.rs b/src/servers/src/http/handler.rs index ee2c3fd4144..bd31ee35ace 100644 --- a/src/servers/src/http/handler.rs +++ b/src/servers/src/http/handler.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::HashMap; +use std::future; use std::panic::AssertUnwindSafe; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; @@ -28,7 +29,8 @@ use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use common_plugins::GREPTIME_EXEC_WRITE_COST; use common_query::{Output, OutputData}; -use common_recordbatch::util; +use common_recordbatch::error::Result as RecordBatchResult; +use common_recordbatch::{RecordBatch, RecordBatchStreamWrapper, RecordBatches, util}; use common_telemetry::tracing; use datafusion::physical_plan::ExecutionPlan; use futures::{FutureExt, StreamExt}; @@ -42,7 +44,7 @@ use sql::parser::{ParseOptions, ParserContext}; use sql::statements::statement::Statement; use tokio::sync::{Notify, watch}; -use crate::error::{FailedToParseQuerySnafu, InvalidQuerySnafu, Result}; +use crate::error::{CollectRecordbatchSnafu, FailedToParseQuerySnafu, InvalidQuerySnafu, Result}; use crate::http::header::collect_plan_metrics; use crate::http::result::arrow_result::ArrowResponse; use crate::http::result::csv_result::CsvResponse; @@ -192,7 +194,25 @@ pub async fn sql( Ok(outputs) => outputs, }; - let mut resp = match format { + let outputs = match query_params.limit { + Some(limit) + if matches!( + format, + ResponseFormat::Csv(..) + | ResponseFormat::Table + | ResponseFormat::GreptimedbV1 + | ResponseFormat::Json + ) => + { + outputs + .into_iter() + .map(|output| output.and_then(|output| limit_output_rows(output, limit))) + .collect() + } + _ => outputs, + }; + + let resp = match format { ResponseFormat::Arrow => { ArrowResponse::from_output(outputs, query_params.compression).await } @@ -206,12 +226,57 @@ pub async fn sql( ResponseFormat::Null => NullResponse::from_output(outputs).await, }; - if let Some(limit) = query_params.limit { - resp = resp.with_limit(limit); - } resp.with_execution_time(start.elapsed().as_millis() as u64) } +/// Limits each statement's response before collecting batches and converting rows. +fn limit_output_rows(output: Output, limit: usize) -> Result { + let mut remaining = limit; + let data = match output.data { + OutputData::AffectedRows(rows) => OutputData::AffectedRows(rows), + OutputData::RecordBatches(batches) => { + let schema = batches.schema(); + let batches = batches + .into_iter() + .filter_map(|batch| take_response_rows(batch, &mut remaining).transpose()) + .collect::>>() + .context(CollectRecordbatchSnafu)?; + OutputData::RecordBatches( + RecordBatches::try_new(schema, batches).context(CollectRecordbatchSnafu)?, + ) + } + OutputData::Stream(stream) => { + let schema = stream.schema(); + // This is a response limit. Drain the input even after reaching it so + // execution errors and terminal query metrics are still observed. + let stream = stream.filter_map(move |batch| { + future::ready( + batch + .and_then(|batch| take_response_rows(batch, &mut remaining)) + .transpose(), + ) + }); + OutputData::Stream(Box::pin(RecordBatchStreamWrapper::new(schema, stream))) + } + }; + Ok(Output::new(data, output.meta)) +} + +fn take_response_rows( + batch: RecordBatch, + remaining: &mut usize, +) -> RecordBatchResult> { + let num_rows = batch.num_rows().min(*remaining); + *remaining -= num_rows; + if num_rows == 0 { + Ok(None) + } else if num_rows == batch.num_rows() { + Ok(Some(batch)) + } else { + batch.slice(0, num_rows).map(Some) + } +} + /// Handler to stream partial `EXPLAIN ANALYZE VERBOSE` metrics as SSE. /// /// This endpoint is POST-only SSE, so browser `EventSource` does @@ -925,7 +990,261 @@ pub async fn index() -> axum::response::Html { #[cfg(test)] mod tests { + use std::sync::Mutex; + use std::sync::atomic::AtomicUsize; + + use async_trait::async_trait; + use common_query::OutputMeta; + use datafusion_expr::LogicalPlan; + use datatypes::prelude::ConcreteDataType; + use datatypes::schema::{ColumnSchema, Schema}; + use datatypes::vectors::{BinaryVector, UInt32Vector, VectorRef}; + use futures::stream; + use query::query_engine::DescribeResult; + use super::*; + use crate::query_handler::sql::SqlQueryHandler; + + struct TestSqlQueryHandler { + outputs: Mutex>>, + } + + #[async_trait] + impl SqlQueryHandler for TestSqlQueryHandler { + async fn do_query(&self, _: &str, _: QueryContextRef) -> Vec> { + std::mem::take(&mut *self.outputs.lock().unwrap()) + } + + async fn do_analyze_stream_query(&self, _: &str, _: QueryContextRef) -> Result { + unimplemented!() + } + + async fn do_exec_plan( + &self, + _: LogicalPlan, + _: Option, + _: QueryContextRef, + ) -> Result { + unimplemented!() + } + + async fn do_promql_query(&self, _: &PromQuery, _: QueryContextRef) -> Vec> { + unimplemented!() + } + + async fn do_describe( + &self, + _: Statement, + _: QueryContextRef, + ) -> Result> { + unimplemented!() + } + + async fn is_valid_schema(&self, _: &str, _: &str) -> Result { + Ok(true) + } + } + + async fn sql_response( + outputs: Vec>, + format: &str, + limit: Option, + ) -> HttpResponse { + sql( + State(ApiState { + sql_handler: Arc::new(TestSqlQueryHandler { + outputs: Mutex::new(outputs), + }), + }), + Query(SqlQuery { + sql: Some("select number from numbers".to_string()), + format: Some(format.to_string()), + limit, + ..Default::default() + }), + Extension(QueryContext::with_db_name(None)), + Form(SqlQuery::default()), + ) + .await + } + + fn number_batches() -> RecordBatches { + let schema = Arc::new(Schema::new(vec![ColumnSchema::new( + "number", + ConcreteDataType::uint32_datatype(), + false, + )])); + let batches = [vec![], vec![0, 1], vec![], vec![2, 3, 4], vec![]] + .into_iter() + .map(|values| { + let columns: Vec = vec![Arc::new(UInt32Vector::from_slice(values))]; + RecordBatch::new(schema.clone(), columns).unwrap() + }) + .collect(); + RecordBatches::try_new(schema, batches).unwrap() + } + + fn number_output(streaming: bool) -> Output { + let batches = number_batches(); + if streaming { + Output::new_with_stream(batches.as_stream()) + } else { + Output::new_with_record_batches(batches) + } + } + + #[tokio::test] + async fn test_sql_response_limit_across_batches_and_formats() { + for format in [ + "greptimedb_v1", + "json", + "csv", + "csvWithNames", + "csvWithNamesAndTypes", + "table", + ] { + for streaming in [false, true] { + for limit in [ + None, + Some(0), + Some(1), + Some(2), + Some(3), + Some(5), + Some(10), + Some(usize::MAX), + ] { + let response = + sql_response(vec![Ok(number_output(streaming))], format, limit).await; + let output = match &response { + HttpResponse::GreptimedbV1(response) => response.output(), + HttpResponse::Json(response) => response.output(), + HttpResponse::Csv(response) => response.output(), + HttpResponse::Table(response) => response.output(), + _ => panic!("unexpected response: {response:?}"), + }; + let GreptimeQueryOutput::Records(records) = &output[0] else { + panic!("expected records"); + }; + let expected_rows: Vec<_> = (0..limit.unwrap_or(5).min(5)) + .map(|number| vec![Value::from(number)]) + .collect(); + assert_eq!(records.rows(), &expected_rows); + assert_eq!(records.total_rows, expected_rows.len()); + assert_eq!(records.schema.column_schemas[0].name, "number"); + } + } + } + } + + #[tokio::test] + async fn test_sql_response_limit_is_per_statement_and_preserves_write_cost() { + let affected = Output::new( + OutputData::AffectedRows(7), + OutputMeta { + cost: 42, + ..Default::default() + }, + ); + let response = sql_response( + vec![ + Ok(number_output(true)), + Ok(affected), + Ok(number_output(false)), + ], + "greptimedb_v1", + Some(1), + ) + .await; + let HttpResponse::GreptimedbV1(response) = response else { + panic!("expected greptimedb response"); + }; + assert_eq!(response.output().len(), 3); + for index in [0, 2] { + let GreptimeQueryOutput::Records(records) = &response.output()[index] else { + panic!("expected records"); + }; + assert_eq!(records.rows(), &vec![vec![Value::from(0)]]); + } + assert!(matches!( + response.output()[1], + GreptimeQueryOutput::AffectedRows(7) + )); + assert_eq!( + response.resp_metrics[GREPTIME_EXEC_WRITE_COST], + Value::from(42) + ); + } + + #[tokio::test] + async fn test_sql_response_limit_does_not_affect_other_formats() { + for format in ["arrow", "influxdb_v1", "null"] { + let unlimited = sql_response(vec![Ok(number_output(true))], format, None).await; + let limited = sql_response(vec![Ok(number_output(true))], format, Some(0)).await; + assert_eq!( + serde_json::to_value(unlimited.with_execution_time(0)).unwrap(), + serde_json::to_value(limited.with_execution_time(0)).unwrap(), + ); + } + } + + #[tokio::test] + async fn test_sql_response_limit_drains_input_and_propagates_late_errors() { + for limit in [0, 1, 3] { + for late_error in [false, true] { + let batches = number_batches(); + let schema = batches.schema(); + let mut input: Vec<_> = batches.into_iter().map(Ok).collect(); + if late_error { + input.push( + common_recordbatch::error::CreateRecordBatchesSnafu { + reason: "late stream error", + } + .fail(), + ); + } + let expected_batches = input.len(); + let observed = Arc::new(AtomicUsize::new(0)); + let counter = observed.clone(); + let stream = stream::iter(input).inspect(move |_| { + counter.fetch_add(1, Ordering::Relaxed); + }); + let output = Output::new_with_stream(Box::pin(RecordBatchStreamWrapper::new( + schema, stream, + ))); + let response = sql_response(vec![Ok(output)], "greptimedb_v1", Some(limit)).await; + assert_eq!(observed.load(Ordering::Relaxed), expected_batches); + assert_eq!(matches!(response, HttpResponse::Error(_)), late_error); + } + } + } + + #[tokio::test] + async fn test_sql_response_limit_skips_conversion_of_discarded_rows() { + let schema = Arc::new(Schema::new(vec![ColumnSchema::new( + "payload", + ConcreteDataType::json_datatype(), + false, + )])); + let columns: Vec = vec![Arc::new(BinaryVector::from(vec![ + datatypes::types::parse_string_to_jsonb(r#"{"ok":1}"#).unwrap(), + b"invalid jsonb".to_vec(), + ]))]; + let batch = RecordBatch::new(schema.clone(), columns).unwrap(); + for streaming in [false, true] { + for limit in [0, 1, 2] { + let batches = RecordBatches::try_new(schema.clone(), vec![batch.clone()]).unwrap(); + let output = if streaming { + Output::new_with_stream(batches.as_stream()) + } else { + Output::new_with_record_batches(batches) + }; + let response = sql_response(vec![Ok(output)], "greptimedb_v1", Some(limit)).await; + // Only retained rows should be decoded into HTTP response values. + assert_eq!(matches!(response, HttpResponse::Error(_)), limit == 2); + } + } + } #[test] fn test_final_analyze_event_uses_error_event_for_conversion_error() { diff --git a/src/servers/src/http/result/json_result.rs b/src/servers/src/http/result/json_result.rs index 25408b28364..537ea66cd02 100644 --- a/src/servers/src/http/result/json_result.rs +++ b/src/servers/src/http/result/json_result.rs @@ -105,11 +105,11 @@ impl IntoResponse for JsonResponse { }) .collect(); - json!({ - "data": data, - "execution_time_ms": execution_time, - }) - .to_string() + let data = Value::Array(data.into_iter().map(Value::Object).collect()); + let mut payload = Map::new(); + payload.insert("data".to_string(), data); + payload.insert("execution_time_ms".to_string(), Value::from(execution_time)); + Value::Object(payload).to_string() } }; diff --git a/src/servers/src/http/result/prometheus_resp.rs b/src/servers/src/http/result/prometheus_resp.rs index 3ebec961d4f..eca38338719 100644 --- a/src/servers/src/http/result/prometheus_resp.rs +++ b/src/servers/src/http/result/prometheus_resp.rs @@ -15,8 +15,12 @@ //! prom supply the prometheus HTTP API Server compliance use std::cmp::Ordering; use std::collections::{BTreeMap, HashMap}; +use std::hash::BuildHasher; +use std::ops::Range; -use arrow::array::{Array, AsArray, StructArray}; +use arrow::array::{Array, ArrayRef, AsArray, StructArray}; +use arrow::buffer::BooleanBuffer; +use arrow::compute::kernels::cmp::distinct; use arrow::datatypes::{Float64Type, TimestampMillisecondType}; use arrow_schema::DataType; use axum::Json; @@ -36,6 +40,9 @@ use common_recordbatch::RecordBatches; use datatypes::arrow_array::string_array_value_at_index; use datatypes::prelude::ConcreteDataType; use indexmap::IndexMap; +use indexmap::map::RawEntryApiV1; +use indexmap::map::raw_entry_v1::RawEntryMut; +use itertools::Either; use promql_parser::label::METRIC_NAME; use promql_parser::parser::value::ValueType; use ryu::Buffer; @@ -356,13 +363,8 @@ impl PrometheusJsonResponse { // Tag order matters, e.g., after sorc and sort_desc, the output order must be kept. let mut buffer = IndexMap::, PromSeriesSamples>::new(); - // Consecutive rows often belong to the same series: instant query plans - // keep their output sort, and range query plans, which no longer do, still - // tend to emit a series' rows together. Remember the index of the previous - // row's entry in `buffer` and reuse it when the tags are unchanged, so the - // label vector is not rebuilt and rehashed per row. Unclustered rows only - // cost one `Vec` comparison before falling back to the map lookup. - let mut last_entry_index = None; + // Only a series that is new to `buffer` needs its own key vector. + let mut tags = Vec::with_capacity(num_label_columns + 1); let schema = batches.schema(); for batch in batches.iter() { @@ -398,72 +400,95 @@ impl PrometheusJsonResponse { }) .transpose()?; + // Read the labels once per run of rows that share them instead of + // once per row. `label_runs` finds every boundary, so the probe only + // decides whether looking for runs is worth its cost. + let label_runs = if prefer_label_runs(&tag_columns, batch.num_rows()) { + Either::Left(label_runs(&tag_columns, batch.num_rows())?.into_iter()) + } else { + Either::Right((0..batch.num_rows()).map(|row| row..row + 1)) + }; + // assemble rows - for row_index in 0..batch.num_rows() { - let value = field_column.and_then(|field_column| { - if !field_column.is_valid(row_index) { - return None; + for run in label_runs { + let mut run_entry_index = None; + for row_index in run { + let value = field_column.and_then(|field_column| { + if !field_column.is_valid(row_index) { + return None; + } + let value = field_column.value(row_index); + (!is_prometheus_stale_nan(value)) + .then_some((timestamp_column.value(row_index), value)) + }); + let histogram = native_histogram_column + .and_then(|column| { + read_histogram(column, row_index) + .context(DataFusionSnafu) + .transpose() + }) + .transpose()? + .filter(|histogram| !is_prometheus_stale_nan(histogram.sum)) + .map(|histogram| { + prometheus_native_histogram(&histogram) + .map(|histogram| (timestamp_column.value(row_index), histogram)) + }) + .transpose()?; + + if value.is_none() && histogram.is_none() { + continue; } - let value = field_column.value(row_index); - (!is_prometheus_stale_nan(value)) - .then_some((timestamp_column.value(row_index), value)) - }); - let histogram = native_histogram_column - .and_then(|column| { - read_histogram(column, row_index) - .context(DataFusionSnafu) - .transpose() - }) - .transpose()? - .filter(|histogram| !is_prometheus_stale_nan(histogram.sum)) - .map(|histogram| { - prometheus_native_histogram(&histogram) - .map(|histogram| (timestamp_column.value(row_index), histogram)) - }) - .transpose()?; - if value.is_none() && histogram.is_none() { - continue; - } + let entry_index = match run_entry_index { + Some(index) => index, + None => { + // retrieve tags + tags.clear(); + if let Some(metric_name) = &metric_name { + tags.push((METRIC_NAME, metric_name.as_str())); + } + for (tag_column, tag_name) in tag_columns.iter().zip(tag_names.iter()) { + if let Some(tag_value) = + string_array_value_at_index(tag_column, row_index) + { + tags.push((tag_name, tag_value)); + } + } - // retrieve tags - let mut tags = Vec::with_capacity(num_label_columns + 1); - if let Some(metric_name) = &metric_name { - tags.push((METRIC_NAME, metric_name.as_str())); - } - for (tag_column, tag_name) in tag_columns.iter().zip(tag_names.iter()) { - if let Some(tag_value) = string_array_value_at_index(tag_column, row_index) { - tags.push((tag_name, tag_value)); + let hash = buffer.hasher().hash_one(&tags); + let entry = buffer + .raw_entry_mut_v1() + .from_key_hashed_nocheck(hash, &tags); + let index = entry.index(); + if let RawEntryMut::Vacant(entry) = entry { + // Hand the key buffer over with the hash that was + // just computed for the lookup. + let key = std::mem::replace( + &mut tags, + Vec::with_capacity(num_label_columns + 1), + ); + entry.insert_hashed_nocheck( + hash, + key, + PromSeriesSamples::default(), + ); + } + run_entry_index = Some(index); + index + } + }; + let samples = &mut buffer[entry_index]; + if let Some((timestamp_millis, histogram)) = histogram { + samples + .histograms + .push((timestamp_millis as f64 / 1000.0, histogram)); + } else if let Some((timestamp_millis, value)) = value { + samples.values.push(( + timestamp_millis as f64 / 1000.0, + PromSampleValue::Number(value), + )); } } - - let reuse = last_entry_index.filter(|index| { - buffer - .get_index(*index) - .is_some_and(|(key, _)| key == &tags) - }); - let samples = if let Some(index) = reuse { - buffer - .get_index_mut(index) - .map(|(_, samples)| samples) - .with_context(|| UnexpectedResultSnafu { - reason: "reused series entry must exist", - })? - } else { - let entry = buffer.entry(tags); - last_entry_index = Some(entry.index()); - entry.or_default() - }; - if let Some((timestamp_millis, histogram)) = histogram { - samples - .histograms - .push((timestamp_millis as f64 / 1000.0, histogram)); - } else if let Some((timestamp_millis, value)) = value { - samples.values.push(( - timestamp_millis as f64 / 1000.0, - PromSampleValue::Number(value), - )); - } } } @@ -545,6 +570,71 @@ impl PrometheusJsonResponse { } } +/// Ranges of consecutive rows whose label values are all equal. +/// +/// `arrow::compute::partition` computes the same ranges, but its contract takes +/// lexicographically sorted columns, which query output is not: range queries +/// run without the plan's output sort, and `sort`/`topk` order by value. +/// `distinct` is element-wise, so it holds for any row order, and its null +/// handling is the one a series key needs: a null label and an empty one are +/// distinct, and two nulls are not. +fn label_runs(columns: &[&ArrayRef], rows: usize) -> Result>> { + let mut boundaries: Option = None; + if rows >= 2 { + for column in columns { + let changed = distinct(&column.slice(0, rows - 1), &column.slice(1, rows - 1)) + .context(ArrowSnafu)?; + boundaries = Some(match boundaries { + Some(accumulated) => &accumulated | changed.values(), + None => changed.values().clone(), + }); + } + } + + let mut runs = Vec::new(); + let mut start = 0; + for boundary in boundaries.iter().flat_map(BooleanBuffer::set_indices) { + runs.push(start..boundary + 1); + start = boundary + 1; + } + runs.push(start..rows); + Ok(runs) +} + +/// Decides whether to group the rows of a batch into runs that share their +/// label values. +/// +/// Sampling adjacent row pairs keeps the decision independent of the batch +/// size. Probe positions come from a xorshift sequence instead of a fixed +/// stride, which would alias with periodic series layouts. A wrong guess only +/// costs time: `partition` still validates every boundary, and the row-by-row +/// path builds the labels of every row. +fn prefer_label_runs(columns: &[&ArrayRef], rows: usize) -> bool { + if columns.is_empty() { + return true; + } + if rows < 2 { + return false; + } + let mut position = 0x9e37_79b9_u32; + let mut changes = 0; + for _ in 0..8 { + position ^= position << 13; + position ^= position >> 17; + position ^= position << 5; + let row = position as usize % (rows - 1); + if columns.iter().any(|column| { + string_array_value_at_index(column, row) != string_array_value_at_index(column, row + 1) + }) { + changes += 1; + if changes > 2 { + return false; + } + } + } + true +} + fn merge_annotations(target: &mut Option>, source: Option>) { let Some(source) = source else { return; @@ -953,11 +1043,10 @@ mod tests { } #[test] - fn record_batches_to_data_reuses_entries_for_clustered_series() { - // Rows are clustered by series (a, b, a, a, b, c): consecutive rows of - // the same series exercise the entry-reuse fast path, while series - // transitions fall back to the map lookup. The result must keep the - // first-occurrence order and accumulate values per series as before. + fn record_batches_to_data_groups_clustered_series() { + // Rows are clustered by series (a, b, a, a, b, c) and `a` is revisited + // after `b`. The result must keep the first-occurrence order and + // accumulate values per series. let schema = Arc::new(Schema::new(vec![ ColumnSchema::new( "timestamp", @@ -1023,6 +1112,169 @@ mod tests { assert_eq!(series[2].value, Some((6.0, "6.0".to_string()))); } + #[test] + fn label_strategy_switches_preserve_all_rows_when_probes_miss_changes() { + let schema = Arc::new(Schema::new(vec![ + ColumnSchema::new( + "timestamp", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ), + ColumnSchema::new("host", ConcreteDataType::string_datatype(), false), + ColumnSchema::new("value", ConcreteDataType::float64_datatype(), false), + ])); + let mut batches = Vec::new(); + let mut expected = BTreeMap::>::new(); + for layout in 0..3 { + let labels = (0..1024) + .map(|row| { + let alternating = match layout { + 0 => true, + 1 => row < 64, + _ => row >= 64, + }; + if alternating && row % 2 != 0 { + "b" + } else { + "a" + } + }) + .collect::>(); + for (row, label) in labels.iter().enumerate() { + let value = (layout * 1024 + row) as f64; + expected + .entry((*label).to_string()) + .or_default() + .push((value, PromSampleValue::Number(value))); + } + let batch = RecordBatch::new( + schema.clone(), + vec![ + Arc::new(TimestampMillisecondVector::from_values( + (0..1024).map(|row| (layout * 1024 + row) as i64 * 1000), + )) as _, + Arc::new(StringVector::from( + labels.into_iter().map(Some).collect::>(), + )) as _, + Arc::new(Float64Vector::from( + (0..1024) + .map(|row| Some((layout * 1024 + row) as f64)) + .collect::>(), + )) as _, + ], + ) + .unwrap(); + // The middle batch alternates only over a prefix the probes miss, + // so it takes the run path even though most rows are not runs. + assert_eq!( + prefer_label_runs(&[batch.column(1)], batch.num_rows()), + layout == 1 + ); + batches.push(batch); + } + let response = PrometheusJsonResponse::record_batches_to_data( + RecordBatches::try_new(schema, batches).unwrap(), + None, + ValueType::Matrix, + ) + .unwrap(); + let PrometheusResponse::PromData(PromData { + result: PromQueryResult::Matrix(series), + .. + }) = response + else { + panic!("expected matrix response"); + }; + assert_eq!(series.len(), expected.len()); + for series in series { + assert_eq!(series.metric.len(), 1); + assert!(series.histograms.is_empty()); + assert_eq!( + series.values, + expected.remove(&series.metric["host"]).unwrap() + ); + } + assert!(expected.is_empty()); + } + + #[test] + fn label_runs_keep_null_and_empty_labels_apart_across_batches() { + let schema = Arc::new(Schema::new(vec![ + ColumnSchema::new( + "timestamp", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ), + ColumnSchema::new("host", ConcreteDataType::string_datatype(), true), + ColumnSchema::new("rack", ConcreteDataType::string_datatype(), true), + ColumnSchema::new("value", ConcreteDataType::float64_datatype(), false), + ])); + // Two batches of two runs each, with only `rack` changing: a null label + // and an empty one must not share a run, and the run opening the second + // batch continues the series that ended the first one. + let mut batches = Vec::new(); + let mut expected = vec![Vec::new(), Vec::new()]; + for (batch_index, leading_null) in [true, false].into_iter().enumerate() { + let mut racks = Vec::new(); + let mut values = Vec::new(); + for row in 0..1024 { + let value = (batch_index * 1024 + row) as f64; + let null_rack = (row < 512) == leading_null; + racks.push((!null_rack).then_some("")); + values.push(Some(value)); + expected[usize::from(!null_rack)].push((value, PromSampleValue::Number(value))); + } + batches.push( + RecordBatch::new( + schema.clone(), + vec![ + Arc::new(TimestampMillisecondVector::from_values( + values.iter().map(|value| value.unwrap() as i64 * 1000), + )) as _, + Arc::new(StringVector::from(vec![Some("a"); 1024])) as _, + Arc::new(StringVector::from(racks)) as _, + Arc::new(Float64Vector::from(values)) as _, + ], + ) + .unwrap(), + ); + } + for batch in &batches { + assert!(prefer_label_runs( + &[batch.column(1), batch.column(2)], + batch.num_rows() + )); + } + for series in &mut expected { + series.sort_by(|left, right| left.0.total_cmp(&right.0)); + } + + let response = PrometheusJsonResponse::record_batches_to_data( + RecordBatches::try_new(schema, batches).unwrap(), + None, + ValueType::Matrix, + ) + .unwrap(); + let PrometheusResponse::PromData(PromData { + result: PromQueryResult::Matrix(series), + .. + }) = response + else { + panic!("expected matrix response"); + }; + assert_eq!(series.len(), 2); + assert_eq!( + series[0].metric, + BTreeMap::from([("host".into(), "a".into())]) + ); + assert_eq!(series[0].values, expected[0]); + assert_eq!( + series[1].metric, + BTreeMap::from([("host".into(), "a".into()), ("rack".into(), "".into())]) + ); + assert_eq!(series[1].values, expected[1]); + } + #[test] fn matrix_response_is_independent_of_input_row_order() { // Range queries run without the plan's output sort, so this function sees diff --git a/src/servers/src/mysql/federated.rs b/src/servers/src/mysql/federated.rs index fa3aec144c9..e6c4890df93 100644 --- a/src/servers/src/mysql/federated.rs +++ b/src/servers/src/mysql/federated.rs @@ -31,23 +31,47 @@ use regex::bytes::RegexSet; use session::SessionRef; use session::context::QueryContextRef; -static SELECT_VAR_PATTERN: Lazy = Lazy::new(|| Regex::new("(?i)^(SELECT @@(.*))").unwrap()); -static MYSQL_CONN_JAVA_PATTERN: Lazy = - Lazy::new(|| Regex::new("(?i)^(/\\* mysql-connector-j(.*))").unwrap()); -static SHOW_LOWER_CASE_PATTERN: Lazy = - Lazy::new(|| Regex::new("(?i)^(SHOW VARIABLES LIKE 'lower_case_table_names'(.*))").unwrap()); -static SHOW_VARIABLES_LIKE_PATTERN: Lazy = - Lazy::new(|| Regex::new("(?i)^(SHOW VARIABLES( LIKE (.*))?)").unwrap()); +/// Matches the optional `GLOBAL`/`SESSION`/`LOCAL` scope MySQL accepts before `VARIABLES`. +const VARIABLES_SCOPE: &str = "(GLOBAL |SESSION |LOCAL )?"; + +static SELECT_VAR_PATTERN: Lazy = + Lazy::new(|| Regex::new("(?i)^(SELECT\\s+@@(.*))").unwrap()); +static SHOW_LOWER_CASE_PATTERN: Lazy = Lazy::new(|| { + Regex::new(&format!( + "(?i)^(SHOW {VARIABLES_SCOPE}VARIABLES LIKE 'lower_case_table_names'(.*))" + )) + .unwrap() +}); +static SHOW_VARIABLES_LIKE_PATTERN: Lazy = Lazy::new(|| { + Regex::new(&format!( + "(?i)^(SHOW {VARIABLES_SCOPE}VARIABLES( LIKE (.*))?)" + )) + .unwrap() +}); static SHOW_WARNINGS_PATTERN: Lazy = - Lazy::new(|| Regex::new("(?i)^(/\\* ApplicationName=.*)?SHOW WARNINGS").unwrap()); + Lazy::new(|| Regex::new("(?i)^(SHOW WARNINGS)").unwrap()); + +// Capture 1: a parenless session-user keyword. Capture 2: a user variable. Both parse as +// column references, which the planner then cannot resolve. Anchored at both ends so +// `SELECT user FROM t` still reads the column. +static SELECT_USER_OR_VAR_PATTERN: Lazy = Lazy::new(|| { + Regex::new( + "(?i)^SELECT\\s+(?:(CURRENT_USER|SESSION_USER|SYSTEM_USER|USER)|(@[a-z0-9_$.]+))\\s*;?\\s*$", + ) + .unwrap() +}); // SELECT TIMEDIFF(NOW(), UTC_TIMESTAMP()); static SELECT_TIME_DIFF_FUNC_PATTERN: Lazy = Lazy::new(|| Regex::new("(?i)^(SELECT TIMEDIFF\\(NOW\\(\\), UTC_TIMESTAMP\\(\\)\\))").unwrap()); // sqlalchemy < 1.4.30 -static SHOW_SQL_MODE_PATTERN: Lazy = - Lazy::new(|| Regex::new("(?i)^(SHOW VARIABLES LIKE 'sql_mode'(.*))").unwrap()); +static SHOW_SQL_MODE_PATTERN: Lazy = Lazy::new(|| { + Regex::new(&format!( + "(?i)^(SHOW {VARIABLES_SCOPE}VARIABLES LIKE 'sql_mode'(.*))" + )) + .unwrap() +}); static OTHER_NOT_SUPPORTED_STMT: Lazy = Lazy::new(|| { RegexSet::new([ @@ -55,6 +79,7 @@ static OTHER_NOT_SUPPORTED_STMT: Lazy = Lazy::new(|| { "(?i)^(ROLLBACK(.*))", "(?i)^(COMMIT(.*))", "(?i)^(START(.*))", + "(?i)^(BEGIN(.*))", // Set. "(?i)^(SET NAMES(.*))", @@ -88,13 +113,9 @@ static OTHER_NOT_SUPPORTED_STMT: Lazy = Lazy::new(|| { "(?i)^(/\\*!40101 SET(.*) \\*/)$", // DBeaver. - "(?i)^(/\\* ApplicationName=(.*)SHOW PLUGINS)", - "(?i)^(/\\* ApplicationName=(.*)SHOW ENGINES)", - "(?i)^(/\\* ApplicationName=(.*)SELECT @@(.*))", - "(?i)^(/\\* ApplicationName=(.*)SHOW @@(.*))", - "(?i)^(/\\* ApplicationName=(.*)SET net_write_timeout(.*))", - "(?i)^(/\\* ApplicationName=(.*)SET SQL_SELECT_LIMIT(.*))", - "(?i)^(/\\* ApplicationName=(.*)SHOW VARIABLES(.*))", + "(?i)^(SHOW PLUGINS)", + "(?i)^(SHOW ENGINES)", + "(?i)^(SHOW @@(.*))", // pt-toolkit "(?i)^(/\\*!40101 SET(.*) \\*/)$", @@ -128,7 +149,7 @@ static VAR_VALUES: Lazy> = Lazy::new(|| { // Format: // |function_name| // |value| -fn select_function(name: &str, value: &str) -> RecordBatches { +fn select_function(name: &str, value: Option<&str>) -> RecordBatches { let schema = Arc::new(Schema::new(vec![ColumnSchema::new( name, ConcreteDataType::string_datatype(), @@ -227,16 +248,30 @@ fn select_variable(query: &str, query_context: QueryContextRef) -> Option Option { - if [&SELECT_VAR_PATTERN, &MYSQL_CONN_JAVA_PATTERN] - .iter() - .any(|r| r.is_match(query)) - { + if SELECT_VAR_PATTERN.is_match(query) { select_variable(query, query_context) } else { None } } +fn check_select_user_or_var(query: &str, query_context: QueryContextRef) -> Option { + let captures = SELECT_USER_OR_VAR_PATTERN.captures(query)?; + + let recordbatches = if let Some(keyword) = captures.get(1) { + let user = query_context.current_user(); + select_function(keyword.as_str(), Some(user.username())) + } else { + // `SET @var` is accepted and discarded, so a user variable is always unset, which + // MySQL reports as NULL. + let var = captures + .get(2) + .expect("one of the two groups always matches"); + select_function(var.as_str(), None) + }; + Some(Output::new_with_record_batches(recordbatches)) +} + fn check_show_variables(query: &str) -> Option { let recordbatches = if SHOW_SQL_MODE_PATTERN.is_match(query) { Some(show_variables( @@ -303,7 +338,7 @@ fn check_others(query: &str, _query_ctx: QueryContextRef) -> Option { let recordbatches = if SELECT_TIME_DIFF_FUNC_PATTERN.is_match(query) { Some(select_function( "TIMEDIFF(NOW(), UTC_TIMESTAMP())", - "00:00:00", + Some("00:00:00"), )) } else { None @@ -311,6 +346,164 @@ fn check_others(query: &str, _query_ctx: QueryContextRef) -> Option { recordbatches.map(Output::new_with_record_batches) } +/// Strips leading whitespace and SQL comments. +/// +/// All patterns above are anchored at the start of the statement, but JDBC clients such as +/// DataGrip and DBeaver prefix every statement they send with a `/* ApplicationName=... */` +/// comment. Without stripping it first, those statements miss every pattern and reach the +/// query engine, which rejects the ones this module exists to absorb. +fn strip_leading_comments(query: &str) -> &str { + let mut rest = query.trim_start(); + loop { + // A MySQL executable comment carries the statement itself — mysqldump emits its + // initialization as `/*!40101 SET NAMES ... */`. The patterns above match those + // verbatim, so the comment must survive. + if rest.starts_with("/*!") { + return rest; + } + if let Some(tail) = rest.strip_prefix("/*") { + // An unterminated block comment leaves no statement to match against. + let Some(end) = tail.find("*/") else { + return ""; + }; + rest = tail[end + 2..].trim_start(); + } else if rest.starts_with('#') + // MySQL only treats `--` as a comment when followed by whitespace. + || (rest.starts_with("--") + && rest[2..].chars().next().is_none_or(|c| c.is_whitespace())) + { + let Some(end) = rest.find('\n') else { + return ""; + }; + rest = rest[end + 1..].trim_start(); + } else { + return rest; + } + } +} + +/// The statement keywords that only [`OTHER_NOT_SUPPORTED_STMT`] matches. `SELECT` and +/// `SHOW` are dispatched separately below. +/// +/// Keep in sync with the patterns above: a statement whose leading keyword is absent from +/// this list and from that dispatch cannot match anything, and skips every regex. +const OTHER_LEADING_KEYWORDS: [&str; 7] = [ + "SET", "COMMIT", "ROLLBACK", "START", "BEGIN", "LOCK", "UNLOCK", +]; + +/// Returns the leading run of ASCII letters, which is the statement keyword for everything +/// this module matches. +fn leading_keyword(query: &str) -> &str { + let end = query + .find(|c: char| !c.is_ascii_alphabetic()) + .unwrap_or(query.len()); + &query[..end] +} + +/// Returns the index just past the line terminator at or after `from`. +fn line_comment_end(bytes: &[u8], from: usize) -> usize { + bytes[from..] + .iter() + .position(|c| *c == b'\n') + .map_or(bytes.len(), |p| from + p + 1) +} + +/// Returns the index just past the closing `quote` of the literal starting at `start`. +fn quoted_end(bytes: &[u8], start: usize, quote: u8) -> usize { + let mut i = start + 1; + while i < bytes.len() { + match bytes[i] { + // Backquoted identifiers take no backslash escapes. + b'\\' if quote != b'`' => i += 2, + c if c == quote => { + // A doubled quote is an escaped quote, not the end of the literal. + if bytes.get(i + 1) == Some("e) { + i += 2; + } else { + return i + 1; + } + } + _ => i += 1, + } + } + bytes.len() +} + +/// Returns true if another statement follows the first statement-level `;`. +/// +/// Expects [`strip_leading_comments`] to have run, so a leading comment is never the reason +/// an executable comment is rejected below. +/// +/// Every pattern here ends in `(.*)`, so absorbing a multi-statement request would discard +/// its trailing statements without executing them — `BEGIN; INSERT INTO t VALUES (1)` would +/// report success and write nothing. Such a request must reach the query engine, which +/// executes each statement. +/// +/// Plain comments, string literals and empty statements are skipped, so a `;` inside a +/// comment or a literal does not split the request, and `BEGIN; -- done` stays a single +/// statement. +/// +/// A `/*!...*/` executable comment carries a statement, so anything but a request that +/// starts with one — mysqldump's `/*!40101 SET NAMES ... */`, which the patterns match +/// whole — also counts as a trailing statement. +fn has_trailing_statement(query: &str) -> bool { + let bytes = query.as_bytes(); + let mut i = 0; + let mut seen_semicolon = false; + let mut seen_content = false; + + // Comparisons are all against ASCII bytes, which never occur inside a multi-byte UTF-8 + // sequence, so scanning by byte cannot mistake one for a delimiter. + while i < bytes.len() { + match bytes[i] { + b'/' if bytes.get(i + 1) == Some(&b'*') => { + if bytes.get(i + 2) == Some(&b'!') && seen_content { + return true; + } + i = match bytes[i + 2..].windows(2).position(|w| w == b"*/") { + Some(p) => i + 2 + p + 2, + // An unterminated comment runs to the end of the request. + None => bytes.len(), + }; + seen_content = true; + } + b'#' => { + i = line_comment_end(bytes, i); + seen_content = true; + } + // MySQL only treats `--` as a comment when followed by whitespace. + b'-' if bytes.get(i + 1) == Some(&b'-') + && bytes.get(i + 2).is_none_or(|c| c.is_ascii_whitespace()) => + { + i = line_comment_end(bytes, i); + seen_content = true; + } + quote @ (b'\'' | b'"' | b'`') => { + if seen_semicolon { + return true; + } + seen_content = true; + i = quoted_end(bytes, i, quote); + } + b';' => { + seen_semicolon = true; + seen_content = true; + i += 1; + } + c if c.is_ascii_whitespace() => i += 1, + _ => { + if seen_semicolon { + return true; + } + seen_content = true; + i += 1; + } + } + } + + false +} + // Check whether the query is a federated or driver setup command, // and return some faked results if there are any. pub(crate) fn check( @@ -318,21 +511,37 @@ pub(crate) fn check( query_ctx: QueryContextRef, session: SessionRef, ) -> Option { - // INSERT don't need MySQL federated check. We assume the query doesn't contain - // federated or driver setup command if it starts with a 'INSERT' statement. - let the_6th_index = query.char_indices().nth(6).map(|(i, _)| i); - if let Some(index) = the_6th_index - && query[..index].eq_ignore_ascii_case("INSERT") + let query = strip_leading_comments(query); + let keyword = leading_keyword(query); + + // Dispatch on the leading keyword so ordinary queries — INSERT, UPDATE, CREATE, and the + // `SELECT`s that carry real work — run as few regexes as possible. + let absorbed = if keyword.eq_ignore_ascii_case("SELECT") { + // First to check the query is like "select @@variables". + check_select_variable(query, query_ctx.clone()) + .or_else(|| check_select_user_or_var(query, query_ctx.clone())) + .or_else(|| check_others(query, query_ctx)) + } else if keyword.eq_ignore_ascii_case("SHOW") { + check_show_variables(query) + .or_else(|| check_show_warnings(query, &session)) + .or_else(|| check_others(query, query_ctx)) + } else if query.starts_with("/*!") + || OTHER_LEADING_KEYWORDS + .iter() + .any(|k| k.eq_ignore_ascii_case(keyword)) { + check_others(query, query_ctx) + } else { + return None; + }; + + // Only a request that is about to be absorbed needs the scan, and those are short. A + // query the patterns did not match never pays for it. + if absorbed.is_some() && has_trailing_statement(query) { return None; } - // First to check the query is like "select @@variables". - check_select_variable(query, query_ctx.clone()) - .or_else(|| check_show_variables(query)) - .or_else(|| check_show_warnings(query, &session)) - // Last check - .or_else(|| check_others(query, query_ctx)) + absorbed } #[cfg(test)] @@ -494,4 +703,260 @@ mod test { ); assert!(output.is_some()); } + + #[test] + fn test_check_select_user_or_var() { + let session = Arc::new(Session::new(None, Channel::Mysql, Default::default(), 0)); + + fn pretty(query: &str, session: &SessionRef) -> String { + let output = check(query, QueryContext::arc(), session.clone()) + .unwrap_or_else(|| panic!("{query} was not absorbed")); + let OutputData::RecordBatches(batches) = output.data else { + unreachable!() + }; + batches.pretty_print().unwrap() + } + + // The column name keeps the spelling the client sent. + assert_eq!( + pretty("SELECT CURRENT_USER", &session), + "\ ++--------------+ +| CURRENT_USER | ++--------------+ +| greptime | ++--------------+" + ); + assert_eq!( + pretty("select session_user;", &session), + "\ ++--------------+ +| session_user | ++--------------+ +| greptime | ++--------------+" + ); + + // A user variable is always unset. + assert_eq!( + pretty("SELECT @v", &session), + "\ ++----+ +| @v | ++----+ +| | ++----+" + ); + + // Anything that is not the whole statement must reach the query engine: these are + // column references, or real queries that happen to start with the same keyword. + for query in [ + "SELECT user FROM t", + "SELECT current_user, 1", + "SELECT @v FROM t", + "SELECT @v + 1", + "SELECT userid", + "SELECT 1", + ] { + assert!( + check(query, QueryContext::arc(), session.clone()).is_none(), + "{query} must not be absorbed" + ); + } + } + + /// A multi-statement request must reach the query engine. Absorbing it would report + /// success for the whole request while executing none of it. + #[test] + fn test_check_skips_multi_statement() { + let session = Arc::new(Session::new(None, Channel::Mysql, Default::default(), 0)); + for query in [ + "BEGIN; INSERT INTO t VALUES (1); COMMIT", + "BEGIN;\nINSERT INTO t VALUES (1)", + "START TRANSACTION; DELETE FROM t", + "COMMIT; INSERT INTO t VALUES (1)", + "SET NAMES utf8mb4; INSERT INTO t VALUES (1)", + "SELECT @@version; INSERT INTO t VALUES (1)", + ] { + assert!( + check(query, QueryContext::arc(), session.clone()).is_none(), + "{query} must not be absorbed" + ); + } + + // A trailing semicolon, comment or empty statement is still a single statement. + for query in [ + "BEGIN;", + "COMMIT; ", + "SET NAMES utf8mb4;\n", + "BEGIN; -- done", + "COMMIT; /* done */", + "BEGIN; # done", + "BEGIN;;", + "COMMIT; ; /* done */ ;", + "BEGIN; -- done\n", + ] { + assert!( + check(query, QueryContext::arc(), session.clone()).is_some(), + "{query} was not absorbed" + ); + } + + // A statement after a trailing comment still counts. + for query in [ + "BEGIN; -- go\nINSERT INTO t VALUES (1)", + "COMMIT; /* go */ INSERT INTO t VALUES (1)", + "BEGIN;; INSERT INTO t VALUES (1)", + // The `;` inside the comment does not end the statement; the one after it does. + "BEGIN /* previous delimiter; -- note */; INSERT INTO t VALUES (1)", + "SET NAMES 'a;b'; INSERT INTO t VALUES (1)", + // An executable comment carries a statement. + "BEGIN; /*! INSERT INTO t VALUES (1) */", + "BEGIN /*! INSERT INTO t VALUES (1) */", + "/*!40101 SET NAMES utf8mb4 */; INSERT INTO t VALUES (1)", + "/*!40101 SET NAMES utf8mb4 */ /*! INSERT INTO t VALUES (1) */", + ] { + assert!( + check(query, QueryContext::arc(), session.clone()).is_none(), + "{query} must not be absorbed" + ); + } + + // A `;` inside a comment or a literal is not a statement boundary. + for query in [ + "BEGIN /* previous delimiter; -- note */", + "BEGIN -- a; b", + "SET NAMES 'a;b'", + "SET NAMES \"a;b\"", + "SET NAMES 'it\\'s; here'", + "SET NAMES 'a;b';", + ] { + assert!( + check(query, QueryContext::arc(), session.clone()).is_some(), + "{query} was not absorbed" + ); + } + } + + #[test] + fn test_check_skips_non_federated_keywords() { + let session = Arc::new(Session::new(None, Channel::Mysql, Default::default(), 0)); + for query in [ + "INSERT INTO t VALUES (1)", + "UPDATE t SET a = 1", + "DELETE FROM t", + "CREATE TABLE t (ts TIMESTAMP TIME INDEX)", + "WITH x AS (SELECT 1) SELECT * FROM x", + "TQL EVAL (0, 10, '5s') up", + ] { + assert!( + check(query, QueryContext::arc(), session.clone()).is_none(), + "{query} must not be absorbed" + ); + } + } + + #[test] + fn test_strip_leading_comments() { + assert_eq!(strip_leading_comments("SELECT 1"), "SELECT 1"); + assert_eq!(strip_leading_comments(" \n\tSELECT 1"), "SELECT 1"); + assert_eq!( + strip_leading_comments("/* ApplicationName=DataGrip 2026.2.5 */ COMMIT"), + "COMMIT" + ); + assert_eq!(strip_leading_comments("/* a */ /* b */COMMIT"), "COMMIT"); + assert_eq!(strip_leading_comments("-- a comment\nCOMMIT"), "COMMIT"); + assert_eq!(strip_leading_comments("# a comment\nCOMMIT"), "COMMIT"); + // `--` without trailing whitespace is not a comment. + assert_eq!(strip_leading_comments("--x\nCOMMIT"), "--x\nCOMMIT"); + // Nothing left to match against. + assert_eq!(strip_leading_comments("/* unterminated"), ""); + assert_eq!(strip_leading_comments("-- trailing"), ""); + // Executable comments carry the statement and must survive. + assert_eq!( + strip_leading_comments("/*!40101 SET NAMES utf8mb4 */"), + "/*!40101 SET NAMES utf8mb4 */" + ); + assert_eq!( + strip_leading_comments("/* App */ /*!40101 SET NAMES utf8mb4 */"), + "/*!40101 SET NAMES utf8mb4 */" + ); + // Comments inside the statement are left alone; only the prefix is stripped. + assert_eq!( + strip_leading_comments("/* a */SELECT /* b */ 1"), + "SELECT /* b */ 1" + ); + } + + /// JDBC clients prefix every statement with a comment. Those statements must still reach + /// the federated handling, and the ones MySQL answers with a result set must keep doing so. + #[test] + fn test_check_comment_prefixed() { + let session = Arc::new(Session::new(None, Channel::Mysql, Default::default(), 0)); + let prefix = "/* ApplicationName=DataGrip 2026.2.5 */ "; + + for query in [ + "SET TRANSACTION READ WRITE", + "SET SESSION TRANSACTION READ ONLY", + "SET NAMES utf8mb4", + "BEGIN", + "START TRANSACTION", + "COMMIT", + "ROLLBACK", + // Covers the rest of OTHER_LEADING_KEYWORDS. + "LOCK TABLES t WRITE", + "UNLOCK TABLES", + ] { + let output = check( + &format!("{prefix}{query}"), + QueryContext::arc(), + session.clone(), + ); + let OutputData::RecordBatches(batches) = output + .unwrap_or_else(|| panic!("{query} was not absorbed")) + .data + else { + unreachable!() + }; + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 0); + } + + // mysqldump initialization arrives as executable comments, which the patterns match + // verbatim; stripping them would leave an empty statement and fail the import. + for query in [ + "/*!40101 SET NAMES utf8mb4 */", + "/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */", + "/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */", + "/*!80003 SET @OLD_x=1 */", + ] { + let output = check(query, QueryContext::arc(), session.clone()); + let OutputData::RecordBatches(batches) = output + .unwrap_or_else(|| panic!("{query} was not absorbed")) + .data + else { + unreachable!() + }; + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 0); + } + + // DataGrip reads the scheduler status through these two. A column-less output is written + // as an OK packet, which the JDBC driver reports as "statement has not returned cursor". + for query in [ + "SELECT @@GLOBAL.event_scheduler", + "SHOW GLOBAL VARIABLES LIKE 'event_scheduler'", + ] { + let output = check( + &format!("{prefix}{query}"), + QueryContext::arc(), + session.clone(), + ); + let OutputData::RecordBatches(batches) = output + .unwrap_or_else(|| panic!("{query} was not absorbed")) + .data + else { + unreachable!() + }; + assert!(!batches.schema().column_schemas().is_empty(), "{query}"); + } + } } diff --git a/src/servers/src/pending_rows_batcher.rs b/src/servers/src/pending_rows_batcher.rs index 03eb90dce28..79ae645b98b 100644 --- a/src/servers/src/pending_rows_batcher.rs +++ b/src/servers/src/pending_rows_batcher.rs @@ -23,22 +23,27 @@ use api::v1::region::{ BulkInsertRequest, RegionRequest, RegionRequestHeader, bulk_insert_request, region_request, }; use api::v1::{ArrowIpc, ColumnSchema, RowInsertRequests, Rows}; -use arrow::array::Array; use arrow::compute::{concat_batches, filter_record_batch}; use arrow::datatypes::{DataType as ArrowDataType, Schema as ArrowSchema, TimeUnit}; use arrow::record_batch::RecordBatch; use async_trait::async_trait; -use bytes::Bytes; use catalog::CatalogManagerRef; -use common_grpc::flight::{FlightEncoder, FlightMessage}; +use common_batcher::flush_limiter::FlushLimiter; +use common_batcher::flush_policy::FlushTrigger; +use common_batcher::flush_policy::timing::TimingFlushPolicy; +use common_batcher::notifier::{Notifier, run_notifier}; +use common_batcher::pending_worker::PendingWorker as PendingCore; +use common_batcher::request_limiter::RequestLimiter; +use common_batcher::worker_registry::WorkerRegistry; +use common_grpc::error::Error as GrpcError; +use common_grpc::flight::record_batch_to_ipc; use common_meta::cache::TableFlownodeSetCacheRef; use common_meta::node_manager::NodeManagerRef; use common_query::prelude::{GREPTIME_PHYSICAL_TABLE, greptime_timestamp, greptime_value}; +use common_runtime::spawn_global; use common_telemetry::tracing_context::TracingContext; use common_telemetry::{debug, error, warn}; -use dashmap::DashMap; -use dashmap::mapref::entry::Entry; -use futures::StreamExt; +use datatypes::timestamp::append_timestamps; use metric_engine::batch_modifier::{TagColumnInfo, modify_batch_sparse}; use partition::manager::PartitionRuleManagerRef; use partition::partition::PartitionRuleRef; @@ -81,7 +86,7 @@ pub fn pending_rows_batch_sync_enabled() -> bool { } const WORKER_IDLE_TIMEOUT_MULTIPLIER: u32 = 3; const PHYSICAL_REGION_ESSENTIAL_COLUMN_COUNT: usize = 3; -const MAX_CONCURRENT_FLOW_NOTIFICATIONS: usize = 8; +const MAX_CONCURRENT_FLOW_NOTIFICATIONS: NonZeroUsize = NonZeroUsize::new(8).unwrap(); #[async_trait] pub trait PendingRowsSchemaAlterer: Send + Sync { /// Batch-create multiple logical tables that are missing. @@ -223,6 +228,7 @@ struct BatchKey { catalog: String, schema: String, physical_table: String, + skip_wal: bool, } /// An aligned logical record batch and its timestamp column index. @@ -291,15 +297,13 @@ struct TableResolutionPlan { struct PendingBatch { tables: HashMap, - total_row_count: usize, db_string: String, ctx: QueryContextRef, - waiters: Vec, } struct FlushWaiter { response_tx: oneshot::Sender>>, - _permit: OwnedSemaphorePermit, + _permit: Arc, } struct FlushBatch { @@ -321,14 +325,13 @@ enum WorkerCommand { total_rows: usize, ctx: QueryContextRef, response_tx: oneshot::Sender>>, - _permit: OwnedSemaphorePermit, + _permit: Arc, }, #[cfg(test)] Ack { ack_tx: oneshot::Sender<()> }, } -// Batch key is derived from QueryContext; it assumes catalog/schema/physical_table fully -// define the write target and must remain consistent across the batch. +// Requests can share a batch only when their write target and WAL policy match. fn batch_key_from_ctx(ctx: &QueryContextRef) -> BatchKey { let physical_table = ctx .extension(PHYSICAL_TABLE_KEY) @@ -338,20 +341,21 @@ fn batch_key_from_ctx(ctx: &QueryContextRef) -> BatchKey { catalog: ctx.current_catalog().to_string(), schema: ctx.current_schema(), physical_table, + skip_wal: ctx.skip_wal(), } } /// Prometheus remote write pending rows batcher. pub struct PendingRowsBatcher { - workers: Arc>, + workers: Arc>, flush_interval: Duration, - max_batch_rows: usize, + flush_policy: TimingFlushPolicy, partition_manager: PartitionRuleManagerRef, node_manager: NodeManagerRef, catalog_manager: CatalogManagerRef, - flow_notification_tx: mpsc::Sender, - flush_semaphore: Arc, - inflight_semaphore: Arc, + flow_notification_tx: Notifier, + flush_limiter: FlushLimiter, + request_limiter: RequestLimiter, worker_channel_capacity: usize, prom_store_with_metric_engine: bool, schema_alterer: PendingRowsSchemaAltererRef, @@ -375,24 +379,21 @@ impl PendingRowsBatcher { max_inflight_requests: usize, flow_notification_queue_capacity: NonZeroUsize, ) -> Option> { - // Disable the batcher if flush is disabled or configuration is invalid. - // Zero values for these knobs either cause panics (e.g., zero-capacity channels) - // or deadlocks (e.g., semaphores with no permits). - if flush_interval.is_zero() - || max_batch_rows == 0 - || max_concurrent_flushes == 0 - || worker_channel_capacity == 0 - || max_inflight_requests == 0 - { + if worker_channel_capacity == 0 || worker_channel_capacity > Semaphore::MAX_PERMITS { return None; } + let flush_policy = TimingFlushPolicy::try_new(flush_interval, max_batch_rows)?; + let flush_limiter = FlushLimiter::try_new(max_concurrent_flushes)?; + + let request_limiter = RequestLimiter::try_new(max_inflight_requests)?; + let (flow_notification_tx, flow_notification_rx) = + Notifier::try_new(flow_notification_queue_capacity.get())?; + let (shutdown, _) = broadcast::channel(1); let pending_rows_batch_sync = pending_rows_batch_sync_enabled(); - let workers = Arc::new(DashMap::new()); - PENDING_WORKERS.set(workers.len() as i64); - let (flow_notification_tx, flow_notification_rx) = - mpsc::channel(flow_notification_queue_capacity.get()); + let workers = Arc::new(WorkerRegistry::new()); + PENDING_WORKERS.set(0); start_flow_notification_worker( flow_notification_rx, table_flownode_set_cache, @@ -402,15 +403,15 @@ impl PendingRowsBatcher { Some(Arc::new(Self { workers, flush_interval, - max_batch_rows, + flush_policy, partition_manager, node_manager, catalog_manager, flow_notification_tx, prom_store_with_metric_engine, schema_alterer, - flush_semaphore: Arc::new(Semaphore::new(max_concurrent_flushes)), - inflight_semaphore: Arc::new(Semaphore::new(max_inflight_requests)), + flush_limiter, + request_limiter, worker_channel_capacity, pending_rows_batch_sync, shutdown, @@ -432,9 +433,8 @@ impl PendingRowsBatcher { let _timer = PENDING_ROWS_BATCH_INGEST_STAGE_ELAPSED .with_label_values(&["submit_acquire_inflight_permit"]) .start_timer(); - self.inflight_semaphore - .clone() - .acquire_owned() + self.request_limiter + .acquire() .await .map_err(|_| error::BatcherChannelClosedSnafu.build())? }; @@ -456,7 +456,7 @@ impl PendingRowsBatcher { .start_timer(); for _ in 0..2 { - let worker = self.get_or_spawn_worker(batch_key.clone()); + let worker = self.get_or_spawn_worker(batch_key.clone()).await; let Some(worker_cmd) = cmd.take() else { break; }; @@ -469,7 +469,8 @@ impl PendingRowsBatcher { self.workers.as_ref(), &batch_key, &worker.tx, - ); + ) + .await; } } } @@ -831,39 +832,24 @@ impl PendingRowsBatcher { Ok(aligned_batches) } - - fn get_or_spawn_worker(&self, key: BatchKey) -> PendingWorker { - if let Some(worker) = self.workers.get(&key) - && !worker.tx.is_closed() - { - return worker.clone(); - } - - let entry = self.workers.entry(key.clone()); - match entry { - Entry::Occupied(mut worker) => { - if worker.get().tx.is_closed() { - let new_worker = self.spawn_worker(key); - worker.insert(new_worker.clone()); - PENDING_WORKERS.set(self.workers.len() as i64); - new_worker - } else { - worker.get().clone() - } - } - Entry::Vacant(vacant) => { - let worker = self.spawn_worker(key); - - vacant.insert(worker.clone()); - PENDING_WORKERS.set(self.workers.len() as i64); - worker - } + async fn get_or_spawn_worker(&self, key: BatchKey) -> PendingWorker { + let (tx, receiver) = self + .workers + .get_or_create(key.clone(), self.worker_channel_capacity) + .await; + if let Some(rx) = receiver { + self.spawn_worker(key, tx.clone(), rx); + PENDING_WORKERS.set(self.workers.len().await as i64); } + PendingWorker { tx } } - fn spawn_worker(&self, key: BatchKey) -> PendingWorker { - let (tx, rx) = mpsc::channel(self.worker_channel_capacity); - let worker = PendingWorker { tx: tx.clone() }; + fn spawn_worker( + &self, + key: BatchKey, + tx: mpsc::Sender, + rx: mpsc::Receiver, + ) { let worker_idle_timeout = self .flush_interval .checked_mul(WORKER_IDLE_TIMEOUT_MULTIPLIER) @@ -871,7 +857,7 @@ impl PendingRowsBatcher { start_worker( key, - worker.tx.clone(), + tx, self.workers.clone(), rx, self.shutdown.clone(), @@ -879,13 +865,10 @@ impl PendingRowsBatcher { self.node_manager.clone(), self.catalog_manager.clone(), self.flow_notification_tx.clone(), - self.flush_interval, worker_idle_timeout, - self.max_batch_rows, - self.flush_semaphore.clone(), + self.flush_policy, + self.flush_limiter.clone(), ); - - worker } } @@ -900,10 +883,8 @@ impl PendingBatch { let db_string = ctx.get_db_string(); Self { tables: HashMap::new(), - total_row_count: 0, db_string, ctx, - waiters: Vec::new(), } } @@ -928,22 +909,21 @@ impl PendingBatch { fn start_worker( key: BatchKey, worker_tx: mpsc::Sender, - workers: Arc>, + workers: Arc>, mut rx: mpsc::Receiver, shutdown: broadcast::Sender<()>, partition_manager: PartitionRuleManagerRef, node_manager: NodeManagerRef, catalog_manager: CatalogManagerRef, - flow_notification_tx: mpsc::Sender, - flush_interval: Duration, + flow_notification_tx: Notifier, worker_idle_timeout: Duration, - max_batch_rows: usize, - flush_semaphore: Arc, + flush_policy: TimingFlushPolicy, + flush_limiter: FlushLimiter, ) { tokio::spawn(async move { + // The business batch and pending flush are populated and drained together. let mut batch = None; - let flush_timer = tokio::time::sleep(flush_interval); - tokio::pin!(flush_timer); + let mut pending_flush = PendingCore::new(flush_policy); let mut shutdown_rx = shutdown.subscribe(); let idle_deadline = tokio::time::Instant::now() + worker_idle_timeout; let idle_timer = tokio::time::sleep_until(idle_deadline); @@ -954,43 +934,37 @@ fn start_worker( cmd = rx.recv() => { match cmd { Some(WorkerCommand::Submit { table_batches, total_rows, ctx, response_tx, _permit }) => { - idle_timer.as_mut().reset(tokio::time::Instant::now() + worker_idle_timeout); + let submitted_at = tokio::time::Instant::now(); + idle_timer.as_mut().reset(submitted_at + worker_idle_timeout); - if batch.is_none() { - // Anchor the flush deadline to this batch's first submission, - // rather than to the worker's creation time. - flush_timer - .as_mut() - .reset(tokio::time::Instant::now() + flush_interval); - } + pending_flush.submit( + FlushWaiter { response_tx, _permit }, + total_rows, + ); let pending_batch = batch.get_or_insert_with(||{ PENDING_BATCHES.inc(); PendingBatch::new(ctx) }); - pending_batch.waiters.push(FlushWaiter { response_tx, _permit }); - for (table_name, table_id, record_batch) in table_batches { pending_batch.add_table_batch(table_name, table_id, record_batch); } - pending_batch.total_row_count += total_rows; PENDING_ROWS.add(total_rows as i64); - if pending_batch.total_row_count >= max_batch_rows - && let Some(flush) = drain_batch(&mut batch) { - spawn_flush( - flush, - partition_manager.clone(), - node_manager.clone(), - catalog_manager.clone(), - flow_notification_tx.clone(), - flush_semaphore.clone(), - ).await; + if let Some(flush) = drain_batch(&mut batch, &mut pending_flush, Some(FlushTrigger::Submission)) { + spawn_flush( + flush, + partition_manager.clone(), + node_manager.clone(), + catalog_manager.clone(), + flow_notification_tx.clone(), + flush_limiter.clone(), + ).await; } } None => { - if let Some(flush) = drain_batch(&mut batch) { + if let Some(flush) = drain_batch(&mut batch, &mut pending_flush, None) { flush_batch_with_managers( flush, partition_manager.clone(), @@ -1009,7 +983,7 @@ fn start_worker( } _ = &mut idle_timer => { if !should_close_worker_on_idle_timeout( - batch.as_ref().map_or(0, |batch| batch.total_row_count), + pending_flush.total_rows(), rx.len(), ) { idle_timer @@ -1026,20 +1000,20 @@ fn start_worker( ); break; } - _ = &mut flush_timer, if batch.is_some() => { - if let Some(flush) = drain_batch(&mut batch) { + _ = pending_flush.wait_flush() => { + if let Some(flush) = drain_batch(&mut batch, &mut pending_flush, Some(FlushTrigger::Deadline)) { spawn_flush( flush, partition_manager.clone(), node_manager.clone(), catalog_manager.clone(), flow_notification_tx.clone(), - flush_semaphore.clone(), + flush_limiter.clone(), ).await; } } _ = shutdown_rx.recv() => { - if let Some(flush) = drain_batch(&mut batch) { + if let Some(flush) = drain_batch(&mut batch, &mut pending_flush, None) { flush_batch_with_managers( flush, partition_manager.clone(), @@ -1053,41 +1027,46 @@ fn start_worker( } } - remove_worker_if_same_channel(workers.as_ref(), &key, &worker_tx); + remove_worker_if_same_channel(workers.as_ref(), &key, &worker_tx).await; }); } -fn remove_worker_if_same_channel( - workers: &DashMap, +async fn remove_worker_if_same_channel( + workers: &WorkerRegistry, key: &BatchKey, worker_tx: &mpsc::Sender, ) -> bool { - if let Some(worker) = workers.get(key) - && worker.tx.same_channel(worker_tx) - { - drop(worker); - workers.remove(key); - PENDING_WORKERS.set(workers.len() as i64); - return true; + if workers.remove_if_same(key, worker_tx).await { + PENDING_WORKERS.set(workers.len().await as i64); + true + } else { + false } - - false } fn should_close_worker_on_idle_timeout(total_row_count: usize, queued_requests: usize) -> bool { total_row_count == 0 && queued_requests == 0 } -fn drain_batch(batch: &mut Option) -> Option { +/// Transfers the ready batch, or drains unconditionally when no trigger is given. +/// Execution and flush permits remain owned by the caller. +fn drain_batch( + batch: &mut Option, + pending_flush: &mut PendingCore, + trigger: Option, +) -> Option { + let total_row_count = pending_flush.total_rows(); + let waiters = match trigger { + Some(trigger) => pending_flush.take_ready(trigger)?, + None => pending_flush.take_pending()?, + }; let batch = batch.take()?; - let total_row_count = batch.total_row_count; if total_row_count == 0 { return None; } let table_batches = batch.tables.into_values().collect(); - let waiters = batch.waiters; PENDING_ROWS.sub(total_row_count as i64); PENDING_BATCHES.dec(); @@ -1106,10 +1085,10 @@ async fn spawn_flush( partition_manager: PartitionRuleManagerRef, node_manager: NodeManagerRef, catalog_manager: CatalogManagerRef, - flow_notification_tx: mpsc::Sender, - semaphore: Arc, + flow_notification_tx: Notifier, + flush_limiter: FlushLimiter, ) { - match semaphore.acquire_owned().await { + match flush_limiter.acquire().await { Ok(permit) => { tokio::spawn(async move { let _permit = permit; @@ -1322,7 +1301,7 @@ async fn flush_batch_with_managers( partition_manager: PartitionRuleManagerRef, node_manager: NodeManagerRef, catalog_manager: CatalogManagerRef, - flow_notification_tx: mpsc::Sender, + flow_notification_tx: Notifier, ) { let partition_provider = PartitionManagerPhysicalFlushAdapter { partition_manager }; let node_requester = NodeManagerPhysicalFlushAdapter { @@ -1344,7 +1323,7 @@ async fn flush_batch( partition_manager: &(impl PhysicalFlushPartitionProvider + ?Sized), node_manager: &(impl PhysicalFlushNodeRequester + ?Sized), catalog_manager: &(impl PhysicalFlushCatalogProvider + ?Sized), - flow_notification_tx: mpsc::Sender, + flow_notification_tx: Notifier, ) { let FlushBatch { table_batches, @@ -1402,21 +1381,13 @@ fn extract_timestamps(table_batch: &TableBatch) -> Vec { let mut timestamps = Vec::with_capacity(table_batch.row_count); for batch in &table_batch.batches { let timestamp_column = batch.batch.column(batch.timestamp_index); - let Some((timestamp_values, _)) = - datatypes::timestamp::timestamp_array_to_primitive(timestamp_column) - else { + let Some(()) = append_timestamps(timestamp_column, &mut timestamps) else { error!( "Failed to extract timestamps from record batch, table_id: {}, timestamp_index: {}", table_batch.table_id, batch.timestamp_index ); continue; }; - - if timestamp_values.null_count() == 0 { - timestamps.extend_from_slice(timestamp_values.values()); - } else { - timestamps.extend(timestamp_values.iter().flatten()); - } } timestamps } @@ -1427,10 +1398,10 @@ struct FlowNotification { } fn try_enqueue_flow_notification( - tx: &mpsc::Sender, + tx: &Notifier, notification: FlowNotification, ) -> bool { - match tx.try_send(notification) { + match tx.try_notify(notification) { Ok(()) => true, Err(mpsc::error::TrySendError::Full(notification)) => { FLOW_NOTIFICATION_DROPPED.with_label_values(&["full"]).inc(); @@ -1455,7 +1426,7 @@ fn try_enqueue_flow_notification( } } -fn enqueue_flow_notifications(table_batches: Vec, tx: &mpsc::Sender) { +fn enqueue_flow_notifications(table_batches: Vec, tx: &Notifier) { for table_batch in table_batches { let timestamps = extract_timestamps(&table_batch); if timestamps.is_empty() { @@ -1516,14 +1487,17 @@ fn start_flow_notification_worker( table_flownode_set_cache: TableFlownodeSetCacheRef, node_manager: NodeManagerRef, ) { - common_runtime::spawn_global(async move { - tokio_stream::wrappers::ReceiverStream::new(notification_rx) - .for_each_concurrent(MAX_CONCURRENT_FLOW_NOTIFICATIONS, |notification| { + spawn_global(async move { + run_notifier( + notification_rx, + MAX_CONCURRENT_FLOW_NOTIFICATIONS, + |notification| { let table_flownode_set_cache = table_flownode_set_cache.clone(); let node_manager = node_manager.clone(); handle_flow_notification(notification, table_flownode_set_cache, node_manager) - }) - .await; + }, + ) + .await; }); } @@ -1533,7 +1507,7 @@ fn notify_flow_dirty_windows_after_flush( table_flownode_set_cache: TableFlownodeSetCacheRef, node_manager: NodeManagerRef, ) { - let (tx, rx) = mpsc::channel(table_batches.len().max(1)); + let (tx, rx) = Notifier::try_new(table_batches.len().max(1)).unwrap(); start_flow_notification_worker(rx, table_flownode_set_cache, node_manager); enqueue_flow_notifications(table_batches, &tx); } @@ -1617,7 +1591,7 @@ pub async fn flush_batch_physical( )?; let resolved_batches = resolve_region_targets(planned_batches, partition_manager).await?; - let region_writes = encode_region_write_requests(resolved_batches)?; + let region_writes = encode_region_write_requests(resolved_batches, ctx.skip_wal())?; flush_region_writes_concurrently(node_manager, region_writes).await } @@ -1794,6 +1768,7 @@ async fn resolve_region_targets( fn encode_region_write_requests( resolved_batches: Vec, + skip_wal: bool, ) -> Result> { let mut region_writes = Vec::with_capacity(resolved_batches.len()); for resolved in resolved_batches { @@ -1802,7 +1777,7 @@ fn encode_region_write_requests( let _timer = PENDING_ROWS_BATCH_FLUSH_STAGE_ELAPSED .with_label_values(&["flush_physical_encode_ipc"]) .start_timer(); - record_batch_to_ipc(resolved.planned.batch)? + record_batch_to_ipc(resolved.planned.batch).map_err(map_ipc_error)? }; let request = RegionRequest { @@ -1811,6 +1786,7 @@ fn encode_region_write_requests( ..Default::default() }), body: Some(region_request::Body::BulkInsert(BulkInsertRequest { + skip_wal, region_id: region_id.as_u64(), partition_expr_version: None, // Set aligned_schema_version to None so that datanode will check the batch schema again to see if any @@ -1844,36 +1820,24 @@ fn notify_waiters(waiters: Vec, result: Result<()>) { } } -fn record_batch_to_ipc(record_batch: RecordBatch) -> Result<(Bytes, Bytes, Bytes)> { - let mut encoder = FlightEncoder::default(); - let schema = encoder.encode_schema(record_batch.schema().as_ref()); - let mut iter = encoder - .encode(FlightMessage::RecordBatch(record_batch)) - .into_iter(); - let Some(flight_data) = iter.next() else { - return Err(Error::Internal { - err_msg: "Failed to encode empty flight data".to_string(), - }); - }; - if iter.next().is_some() { - return Err(Error::NotSupported { - feat: "bulk insert RecordBatch with dictionary arrays".to_string(), - }); +fn map_ipc_error(error: GrpcError) -> Error { + match error { + GrpcError::NotSupported { feat } => Error::NotSupported { feat }, + GrpcError::InvalidFlightData { reason, .. } => Error::Internal { err_msg: reason }, + error => Error::Internal { + err_msg: error.to_string(), + }, } - - Ok(( - schema.data_header, - flight_data.data_header, - flight_data.data_body, - )) } #[cfg(test)] mod tests { use std::any::Any; use std::collections::{HashMap, HashSet}; + use std::future::{Future, poll_fn}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; + use std::task::Poll; use std::time::Duration; use api::region::RegionResponse; @@ -1909,7 +1873,6 @@ mod tests { }; use common_query::request::QueryRequest; use common_recordbatch::SendableRecordBatchStream; - use dashmap::DashMap; use datatypes::schema::{ColumnSchema as DtColumnSchema, Schema as DtSchema}; use moka::future::CacheBuilder; use partition::cache::new_partition_info_cache; @@ -1924,20 +1887,22 @@ mod tests { use tokio::sync::{Notify, Semaphore, broadcast, mpsc, oneshot}; use tokio::time::{advance, sleep}; - use super::{ - BatchKey, Error, FlushBatch, FlushRegionWrite, FlushWaiter, PendingBatch, - PendingRowsBatcher, PendingWorker, PhysicalFlushCatalogProvider, - PhysicalFlushNodeRequester, PhysicalFlushPartitionProvider, PhysicalTableMetadata, - PlannedRegionBatch, RecordBatchWithTsIdx, ResolvedRegionBatch, TableBatch, WorkerCommand, - columns_taxonomy, drain_batch, encode_region_write_requests, extract_timestamps, - flush_batch, flush_batch_physical, flush_region_writes_concurrently, greptime_timestamp, - notify_flow_dirty_windows_after_flush, plan_region_batches, remove_worker_if_same_channel, - should_close_worker_on_idle_timeout, should_dispatch_concurrently, - start_flow_notification_worker, start_worker, strip_partition_columns_from_batch, - transform_logical_batches_to_physical, try_enqueue_flow_notification, - }; use crate::error; use crate::metrics::FLOW_NOTIFICATION_DROPPED; + use crate::pending_rows_batcher::{ + BatchKey, Error, FlushBatch, FlushLimiter, FlushRegionWrite, FlushTrigger, FlushWaiter, + Notifier, PendingBatch, PendingCore, PendingRowsBatcher, PhysicalFlushCatalogProvider, + PhysicalFlushNodeRequester, PhysicalFlushPartitionProvider, PhysicalTableMetadata, + PlannedRegionBatch, RecordBatchWithTsIdx, RequestLimiter, ResolvedRegionBatch, TableBatch, + TimingFlushPolicy, WorkerCommand, WorkerRegistry, batch_key_from_ctx, columns_taxonomy, + drain_batch, encode_region_write_requests, extract_timestamps, flush_batch, + flush_batch_physical, flush_region_writes_concurrently, greptime_timestamp, + notify_flow_dirty_windows_after_flush, notify_waiters, plan_region_batches, + remove_worker_if_same_channel, should_close_worker_on_idle_timeout, + should_dispatch_concurrently, start_flow_notification_worker, start_worker, + strip_partition_columns_from_batch, transform_logical_batches_to_physical, + try_enqueue_flow_notification, + }; use crate::prom_row_builder::rows_to_aligned_record_batch; fn mock_rows(row_count: usize, schema_name: &str) -> Rows { @@ -2161,8 +2126,8 @@ mod tests { #[test] fn test_flow_notification_queue_drops_when_full() { - let (tx, mut rx) = mpsc::channel(1); - let notification = |table_id| super::FlowNotification { + let (tx, mut rx) = Notifier::try_new(1).unwrap(); + let notification = |table_id| crate::pending_rows_batcher::FlowNotification { table_id, timestamps: vec![table_id as i64], }; @@ -2174,6 +2139,12 @@ mod tests { assert_eq!(1, rx.try_recv().unwrap().table_id); assert_eq!(dropped_before + 1, dropped.get()); + + let closed = FLOW_NOTIFICATION_DROPPED.with_label_values(&["closed"]); + let closed_before = closed.get(); + drop(rx); + assert!(!try_enqueue_flow_notification(&tx, notification(3))); + assert_eq!(closed_before + 1, closed.get()); } fn mock_physical_table_metadata(table_id: TableId) -> PhysicalTableMetadata { @@ -2371,11 +2342,20 @@ mod tests { assert_eq!(2, table_rows[0].1.rows.len()); } - #[test] - fn test_drain_batch_takes_initialized_pending_batch_from_option() { + #[tokio::test] + async fn test_drain_batch_takes_initialized_pending_batch_from_option() { let ctx = session::context::QueryContext::arc(); let (response_tx, _response_rx) = oneshot::channel(); let permit = Arc::new(Semaphore::new(1)).try_acquire_owned().unwrap(); + let mut pending_flush = + PendingCore::new(TimingFlushPolicy::try_new(Duration::from_secs(10), 1).unwrap()); + pending_flush.submit( + FlushWaiter { + response_tx, + _permit: Arc::new(permit), + }, + 1, + ); let mut batch = Some(PendingBatch { tables: HashMap::from([( 42, @@ -2386,24 +2366,77 @@ mod tests { row_count: 1, }, )]), - total_row_count: 1, db_string: ctx.get_db_string(), ctx: ctx.clone(), - waiters: vec![FlushWaiter { - response_tx, - _permit: permit, - }], }); - let flush = drain_batch(&mut batch).unwrap(); + let flush = drain_batch( + &mut batch, + &mut pending_flush, + Some(FlushTrigger::Submission), + ) + .unwrap(); assert!(batch.is_none()); + assert!(pending_flush.is_empty()); + assert_eq!(0, pending_flush.total_rows()); + assert_eq!(1, flush.waiters.len()); assert_eq!(1, flush.total_row_count); assert_eq!(1, flush.table_batches.len()); assert_eq!(ctx.get_db_string(), flush.db_string); assert_eq!(ctx.current_catalog(), flush.ctx.current_catalog()); } + #[tokio::test] + async fn test_drain_batch_preserves_unready_state_and_clears_zero_rows() { + for total_rows in [0, 1] { + let mut pending_flush = + PendingCore::new(TimingFlushPolicy::try_new(Duration::from_secs(10), 2).unwrap()); + let mut batch = Some(PendingBatch::new(session::context::QueryContext::arc())); + let semaphore = Arc::new(Semaphore::new(1)); + let (response_tx, mut response_rx) = oneshot::channel(); + pending_flush.submit( + FlushWaiter { + response_tx, + _permit: Arc::new(semaphore.clone().acquire_owned().await.unwrap()), + }, + total_rows, + ); + assert!( + drain_batch( + &mut batch, + &mut pending_flush, + Some(FlushTrigger::Submission) + ) + .is_none() + ); + assert!(batch.is_some()); + assert!(!pending_flush.is_empty()); + assert_eq!(total_rows, pending_flush.total_rows()); + assert_eq!(0, semaphore.available_permits()); + + let drained = drain_batch(&mut batch, &mut pending_flush, None); + assert!(batch.is_none()); + assert!(pending_flush.is_empty()); + assert_eq!(0, pending_flush.total_rows()); + if total_rows == 0 { + assert!(drained.is_none()); + assert_eq!(1, semaphore.available_permits()); + assert!(matches!( + response_rx.try_recv(), + Err(oneshot::error::TryRecvError::Closed) + )); + } else { + let drained = drained.unwrap(); + assert_eq!(1, drained.total_row_count); + assert_eq!(1, drained.waiters.len()); + assert_eq!(0, semaphore.available_permits()); + drop(drained); + assert_eq!(1, semaphore.available_permits()); + } + } + } + #[test] fn test_pending_batch_keeps_same_name_batches_with_distinct_table_ids() { let ctx = session::context::QueryContext::arc(); @@ -2568,8 +2601,8 @@ mod tests { fn mock_flow_notification_sender( cache: TableFlownodeSetCacheRef, node_manager: NodeManagerRef, - ) -> mpsc::Sender { - let (tx, rx) = mpsc::channel(16); + ) -> Notifier { + let (tx, rx) = Notifier::try_new(16).unwrap(); start_flow_notification_worker(rx, cache, node_manager); tx } @@ -2902,42 +2935,82 @@ mod tests { } #[test] - fn test_remove_worker_if_same_channel_removes_matching_entry() { - let workers = DashMap::new(); + fn test_batch_key_groups_by_skip_wal() { + let wal_ctx = session::context::QueryContext::arc(); + let skip_wal_ctx = session::context::QueryContext::arc(); + skip_wal_ctx.set_skip_wal(true); + let another_skip_wal_ctx = session::context::QueryContext::arc(); + another_skip_wal_ctx.set_skip_wal(true); + + let mut batches = HashMap::new(); + *batches.entry(batch_key_from_ctx(&wal_ctx)).or_insert(0) += 1; + *batches + .entry(batch_key_from_ctx(&skip_wal_ctx)) + .or_insert(0) += 1; + *batches + .entry(batch_key_from_ctx(&another_skip_wal_ctx)) + .or_insert(0) += 1; + *batches + .entry(batch_key_from_ctx(&session::context::QueryContext::arc())) + .or_insert(0) += 1; + + assert_eq!(batches.len(), 2); + assert_eq!(batches[&batch_key_from_ctx(&wal_ctx)], 2); + assert_eq!(batches[&batch_key_from_ctx(&skip_wal_ctx)], 2); + } + + #[tokio::test] + async fn test_cancelled_waiter_retains_request_slot_until_notification() { + let limiter = RequestLimiter::try_new(1).unwrap(); + let (response_tx, response_rx) = oneshot::channel(); + let waiter = FlushWaiter { + response_tx, + _permit: limiter.acquire().await.unwrap(), + }; + drop(response_rx); + let next = limiter.acquire(); + tokio::pin!(next); + assert!(poll_fn(|cx| Poll::Ready(next.as_mut().poll(cx).is_pending())).await); + notify_waiters(vec![waiter], Ok(())); + let _permit = next.await.unwrap(); + } + + #[tokio::test] + async fn test_remove_worker_if_same_channel_removes_matching_entry() { + let workers = WorkerRegistry::new(); let key = BatchKey { catalog: "greptime".to_string(), schema: "public".to_string(), physical_table: "phy".to_string(), + skip_wal: false, }; let (tx, _rx) = mpsc::channel::(1); - workers.insert(key.clone(), PendingWorker { tx: tx.clone() }); + workers.get_or_insert_with(key.clone(), || tx.clone()).await; - assert!(remove_worker_if_same_channel(&workers, &key, &tx)); - assert!(!workers.contains_key(&key)); + assert!(remove_worker_if_same_channel(&workers, &key, &tx).await); + assert!(workers.is_empty().await); } - #[test] - fn test_remove_worker_if_same_channel_keeps_newer_entry() { - let workers = DashMap::new(); + #[tokio::test] + async fn test_remove_worker_if_same_channel_keeps_newer_entry() { + let workers = WorkerRegistry::new(); let key = BatchKey { catalog: "greptime".to_string(), schema: "public".to_string(), physical_table: "phy".to_string(), + skip_wal: false, }; let (stale_tx, _stale_rx) = mpsc::channel::(1); let (fresh_tx, _fresh_rx) = mpsc::channel::(1); - workers.insert( - key.clone(), - PendingWorker { - tx: fresh_tx.clone(), - }, - ); + workers + .get_or_insert_with(key.clone(), || fresh_tx.clone()) + .await; - assert!(!remove_worker_if_same_channel(&workers, &key, &stale_tx)); - assert!(workers.contains_key(&key)); - assert!(workers.get(&key).unwrap().tx.same_channel(&fresh_tx)); + assert!(!remove_worker_if_same_channel(&workers, &key, &stale_tx).await); + assert!(workers.get(&key).await.is_some()); + assert!(workers.get(&key).await.unwrap().same_channel(&fresh_tx)); } #[test] @@ -2966,7 +3039,7 @@ mod tests { total_rows, ctx: session::context::QueryContext::arc(), response_tx, - _permit: permit, + _permit: Arc::new(permit), }) .await .unwrap(); @@ -3008,22 +3081,20 @@ mod tests { } #[tokio::test(start_paused = true)] - async fn test_worker_rearms_creation_relative_deadline_after_size_flush() { + async fn test_worker_preserves_first_deadline_and_inline_shutdown() { let flush_interval = Duration::from_secs(10); let worker_idle_timeout = Duration::from_secs(30); let key = BatchKey { catalog: "greptime".to_string(), schema: "public".to_string(), physical_table: "phy".to_string(), + skip_wal: false, }; - let workers = Arc::new(DashMap::new()); + let workers = Arc::new(WorkerRegistry::new()); let (worker_tx, worker_rx) = mpsc::channel(1); - workers.insert( - key.clone(), - PendingWorker { - tx: worker_tx.clone(), - }, - ); + workers + .get_or_insert_with(key.clone(), || worker_tx.clone()) + .await; let backend = Arc::new(MemoryKvBackend::default()); let table_route_cache = Arc::new(new_table_route_cache( @@ -3045,9 +3116,10 @@ mod tests { datanodes: Arc::new(HashMap::new()), }); let catalog_manager = MemoryCatalogManager::with_default_setup(); - let (flow_notification_tx, _flow_notification_rx) = mpsc::channel(1); + let (flow_notification_tx, _flow_notification_rx) = Notifier::try_new(1).unwrap(); let (shutdown, _) = broadcast::channel(1); + let flush_limiter = FlushLimiter::try_new(1).unwrap(); start_worker( key.clone(), worker_tx.clone(), @@ -3058,10 +3130,9 @@ mod tests { node_manager, catalog_manager, flow_notification_tx, - flush_interval, worker_idle_timeout, - 2, - Arc::new(Semaphore::new(1)), + TimingFlushPolicy::try_new(flush_interval, 3).unwrap(), + flush_limiter.clone(), ); // Start the worker, then size-flush a batch halfway to the first @@ -3069,7 +3140,7 @@ mod tests { // drains the batch before that deadline is reached. tokio::task::yield_now().await; advance(flush_interval / 2).await; - let size_flush_rx = submit_mock_worker_batch(&worker_tx, 2, 1000).await; + let size_flush_rx = submit_mock_worker_batch(&worker_tx, 3, 1000).await; let size_flush_result = receive_mock_flush_result(size_flush_rx, "row threshold did not flush the first batch") .await; @@ -3080,7 +3151,10 @@ mod tests { advance(flush_interval / 5).await; let mut timed_flush_rx = submit_mock_worker_batch(&worker_tx, 1, 2000).await; - advance(flush_interval - Duration::from_millis(1)).await; + let first_submission = tokio::time::Instant::now(); + advance(flush_interval / 2).await; + let later_flush_rx = submit_mock_worker_batch(&worker_tx, 1, 3000).await; + advance(flush_interval / 2 - Duration::from_millis(1)).await; for _ in 0..10 { tokio::task::yield_now().await; } @@ -3097,15 +3171,39 @@ mod tests { .await; assert_missing_physical_table(timed_flush_result); + assert_eq!( + first_submission + flush_interval, + tokio::time::Instant::now() + ); + assert_missing_physical_table( + receive_mock_flush_result( + later_flush_rx, + "later submission was not included in the timed flush", + ) + .await, + ); + + // Shutdown retains the existing inline path even with no flush permits. + let _held_permit = flush_limiter.acquire().await.unwrap(); + let shutdown_flush_rx = submit_mock_worker_batch(&worker_tx, 1, 4000).await; + let shutdown_at = tokio::time::Instant::now(); let _ = shutdown.send(()); + assert_missing_physical_table( + receive_mock_flush_result( + shutdown_flush_rx, + "shutdown incorrectly waited for a flush permit", + ) + .await, + ); + assert_eq!(shutdown_at, tokio::time::Instant::now()); for _ in 0..10 { - if !workers.contains_key(&key) { + if workers.is_empty().await { break; } tokio::task::yield_now().await; } assert!( - !workers.contains_key(&key), + workers.is_empty().await, "worker did not exit after shutdown" ); } @@ -3745,6 +3843,11 @@ mod tests { #[test] fn test_encode_region_write_requests_builds_bulk_insert_requests() { + check_encode_region_write_requests(false); + check_encode_region_write_requests(true); + } + + fn check_encode_region_write_requests(skip_wal: bool) { let planned_batch = PlannedRegionBatch { region_id: RegionId::new(1024, 1), batch: RecordBatch::try_new( @@ -3772,7 +3875,7 @@ mod tests { addr: "node-1".to_string(), }, }; - let writes = encode_region_write_requests(vec![resolved_batch]).unwrap(); + let writes = encode_region_write_requests(vec![resolved_batch], skip_wal).unwrap(); assert_eq!(1, writes.len()); assert_eq!(1, writes[0].datanode.id); @@ -3780,5 +3883,6 @@ mod tests { panic!("expected bulk insert request"); }; assert_eq!(RegionId::new(1024, 1).as_u64(), request.region_id); + assert_eq!(skip_wal, request.skip_wal); } } diff --git a/src/servers/src/postgres/types.rs b/src/servers/src/postgres/types.rs index 543f18d4fcb..033a1c665e2 100644 --- a/src/servers/src/postgres/types.rs +++ b/src/servers/src/postgres/types.rs @@ -16,19 +16,22 @@ mod error; use std::collections::HashMap; use std::pin::Pin; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use std::task::{Context, Poll}; use arrow::array::{Array, AsArray}; use arrow_pg::encoder::{Encoder, encode_value}; use arrow_pg::list_encoder::encode_list; use arrow_schema::{DataType, TimeUnit}; +use bytes::BufMut; use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime}; use common_recordbatch::error::Result as RecordBatchResult; use common_recordbatch::{RecordBatch, map_dictionary_to_values_data_type}; use common_time::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth}; use datafusion_common::ScalarValue; use datafusion_expr::LogicalPlan; +use datafusion_pg_catalog::pg_catalog::PgCatalogStaticTables; +use datafusion_pg_catalog::pg_catalog::oid_field::{self, OID_ALIAS_KEY}; use datatypes::arrow::datatypes::DataType as ArrowDataType; use datatypes::json::JsonSettings; use datatypes::prelude::{ConcreteDataType, DataType as _, Value}; @@ -41,7 +44,9 @@ use pgwire::api::Type; use pgwire::api::portal::{Format, Portal}; use pgwire::api::results::FieldInfo; use pgwire::error::{PgWireError, PgWireResult}; +use pgwire::types::ToSqlText; use pgwire::types::format::FormatOptions as PgFormatOptions; +use postgres_types::{IsNull, ToSql}; use query::planner::DfLogicalPlanner; use rust_decimal::Decimal; use rust_decimal::prelude::ToPrimitive; @@ -63,11 +68,15 @@ pub(super) fn schema_to_pg( .iter() .enumerate() .map(|(idx, col)| { + let pg_type = match pg_oid_alias_type(col) { + Some(pg_type) => pg_type, + None => type_gt_to_pg(&col.data_type)?, + }; let mut field_info = FieldInfo::new( col.name.clone(), None, None, - type_gt_to_pg(&col.data_type)?, + pg_type, field_formats.format_for(idx), ); if let Some(format_options) = &format_options { @@ -78,6 +87,153 @@ pub(super) fn schema_to_pg( .collect::>>() } +/// Maps `datafusion-pg-catalog` OID-alias metadata to PostgreSQL wire types. +/// +/// The catalog exposes static catalog aliases as `Utf8` and dynamic aliases +/// as `Int32`, so this must run before the ordinary `INT4`/`VARCHAR` fallbacks. +/// The catalog crate only exposes named constants for the aliases it uses in +/// dynamic catalog tables; keep the remaining aliases here for compatibility +/// with its public `OID_ALIAS_TYPE_NAMES` contract. See +/// https://github.com/datafusion-contrib/datafusion-postgres/issues/384. +fn pg_oid_alias_type(column: &datatypes::schema::ColumnSchema) -> Option { + if !matches!( + &column.data_type, + ConcreteDataType::Int32(_) | ConcreteDataType::String(_) + ) { + return None; + } + + pg_oid_alias_type_name(column.metadata().get(OID_ALIAS_KEY)?) +} + +fn pg_oid_alias_type_name(alias: &str) -> Option { + match alias { + oid_field::kind::OID => Some(Type::OID), + oid_field::kind::REGPROC => Some(Type::REGPROC), + oid_field::kind::REGCLASS => Some(Type::REGCLASS), + oid_field::kind::REGTYPE => Some(Type::REGTYPE), + oid_field::kind::REGNAMESPACE => Some(Type::REGNAMESPACE), + "regprocedure" => Some(Type::REGPROCEDURE), + "regoper" => Some(Type::REGOPER), + "regoperator" => Some(Type::REGOPERATOR), + "regcollation" => Some(Type::REGCOLLATION), + "regconfig" => Some(Type::REGCONFIG), + "regdictionary" => Some(Type::REGDICTIONARY), + "regrole" => Some(Type::REGROLE), + _ => None, + } +} + +/// OIDs keyed by their unambiguous `pg_proc.proname` in the static catalog. +static REGPROC_OIDS: LazyLock>, String>> = + LazyLock::new(|| { + let tables = PgCatalogStaticTables::try_new() + .map_err(|e| format!("load static PostgreSQL catalog tables: {e}"))?; + let mut oids = HashMap::new(); + + for batch in tables.pg_proc.data() { + let names = batch + .column_by_name("proname") + .ok_or("pg_proc is missing proname")?; + let procedure_oids = batch + .column_by_name("oid") + .ok_or("pg_proc is missing oid")?; + if names.data_type() != &DataType::Utf8 + || procedure_oids.data_type() != &DataType::Int32 + { + return Err("pg_proc has unexpected proname or oid types".to_string()); + } + + let names = names.as_string::(); + let procedure_oids = procedure_oids.as_primitive::(); + for (name, oid) in names.iter().zip(procedure_oids.iter()) { + let (Some(name), Some(oid)) = (name, oid) else { + continue; + }; + let oid = u32::try_from(oid) + .map_err(|_| format!("pg_proc contains negative oid {oid}"))?; + + if oids.insert(name.to_string(), Some(oid)).is_some() { + oids.insert(name.to_string(), None); + } + } + } + + Ok(oids) + }); + +#[derive(Debug)] +struct OidAliasError(String); + +impl std::fmt::Display for OidAliasError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for OidAliasError {} + +fn resolve_oid_alias(value: &str, alias: &str) -> std::result::Result { + if value == "-" { + return Ok(0); + } + + if let Ok(oid) = value.parse() { + return Ok(oid); + } + + if alias == oid_field::kind::REGPROC { + let oids = REGPROC_OIDS + .as_ref() + .map_err(|e| OidAliasError(e.clone()))?; + return match oids.get(value) { + Some(Some(oid)) => Ok(*oid), + Some(None) => Err(OidAliasError(format!("ambiguous regproc name: {value}"))), + None => Err(OidAliasError(format!("unknown regproc name: {value}"))), + }; + } + + Err(OidAliasError(format!( + "named oid aliases are only supported for regproc: {value}" + ))) +} + +#[derive(Debug)] +struct OidAliasValue<'a> { + text: &'a str, + alias: &'a str, +} + +impl ToSql for OidAliasValue<'_> { + fn to_sql( + &self, + _ty: &Type, + out: &mut bytes::BytesMut, + ) -> std::result::Result> { + let oid = resolve_oid_alias(self.text, self.alias)?; + out.put_u32(oid); + Ok(IsNull::No) + } + + fn accepts(ty: &Type) -> bool { + pg_oid_alias_type_name(ty.name()).is_some() + } + + postgres_types::to_sql_checked!(); +} + +impl ToSqlText for OidAliasValue<'_> { + fn to_sql_text( + &self, + _ty: &Type, + out: &mut bytes::BytesMut, + _format_options: &PgFormatOptions, + ) -> std::result::Result> { + out.put_slice(self.text.as_bytes()); + Ok(IsNull::No) + } +} + /// this function will encode greptime's `StructValue` into PostgreSQL jsonb type /// /// Note that greptimedb has different types of StructValue for storing json data, @@ -222,6 +378,22 @@ where DataType::Struct(_) => { encode_struct(query_ctx, Default::default(), encoder, pg_field)?; } + DataType::Utf8 => { + let arrow_field = arrow_schema.field(j); + if let Some(alias) = arrow_field + .metadata() + .get(OID_ALIAS_KEY) + .filter(|alias| pg_oid_alias_type_name(alias).is_some()) + { + let value = OidAliasValue { + text: column.as_string::().value(i), + alias, + }; + encoder.encode_field(&value, pg_field)?; + } else { + encode_value(encoder, column, i, arrow_field, pg_field)?; + } + } _ => { // Encode value using arrow-pg let arrow_field = arrow_schema.field(j); @@ -1291,7 +1463,9 @@ mod test { use futures::{StreamExt as FuturesStreamExt, stream}; use pgwire::api::Type; use pgwire::api::portal::{Format, Portal}; - use pgwire::api::results::{DataRowEncoder, FieldFormat, FieldInfo}; + use pgwire::api::results::{ + CopyEncoder, CopyTextOptions, DataRowEncoder, FieldFormat, FieldInfo, + }; use pgwire::api::stmt::StoredStatement; use pgwire::messages::extendedquery::Bind; use session::context::QueryContextBuilder; @@ -1403,6 +1577,62 @@ mod test { assert_eq!(fs, pg_field_info); } + #[test] + fn test_schema_convert_oid_alias_types() { + let aliases = [ + (oid_field::kind::OID, Type::OID), + (oid_field::kind::REGPROC, Type::REGPROC), + ("regprocedure", Type::REGPROCEDURE), + ("regoper", Type::REGOPER), + ("regoperator", Type::REGOPERATOR), + (oid_field::kind::REGCLASS, Type::REGCLASS), + (oid_field::kind::REGTYPE, Type::REGTYPE), + (oid_field::kind::REGNAMESPACE, Type::REGNAMESPACE), + ("regrole", Type::REGROLE), + ("regconfig", Type::REGCONFIG), + ("regdictionary", Type::REGDICTIONARY), + ("regcollation", Type::REGCOLLATION), + ]; + let mut columns = Vec::new(); + let mut expected_oids = Vec::new(); + for (type_name, data_type, fallback_type) in [ + ("int32", ConcreteDataType::int32_datatype(), Type::INT4), + ("utf8", ConcreteDataType::string_datatype(), Type::VARCHAR), + ] { + for (alias, pg_type) in &aliases { + let mut column = + ColumnSchema::new(format!("{type_name}_{alias}"), data_type.clone(), true); + column + .mut_metadata() + .insert(OID_ALIAS_KEY.to_string(), alias.to_string()); + columns.push(column); + expected_oids.push(pg_type.oid()); + } + + columns.push(ColumnSchema::new( + format!("{type_name}_untagged"), + data_type.clone(), + true, + )); + expected_oids.push(fallback_type.oid()); + + let mut unknown = ColumnSchema::new(format!("{type_name}_unknown"), data_type, true); + unknown + .mut_metadata() + .insert(OID_ALIAS_KEY.to_string(), "unknown".to_string()); + columns.push(unknown); + expected_oids.push(fallback_type.oid()); + } + + let fields = schema_to_pg(&Schema::new(columns), &Format::UnifiedText, None).unwrap(); + let actual_oids = fields + .iter() + .map(|field| field.datatype().oid()) + .collect::>(); + + assert_eq!(actual_oids, expected_oids); + } + #[test] fn test_encode_text_format_data() { let pg_schema = vec![ @@ -1691,6 +1921,209 @@ mod test { } } + #[test] + fn test_encode_utf8_oid_alias_data() { + let aliases = [ + ("regproc_binary", FieldFormat::Binary, Some("boolrecv")), + ("regproc_text", FieldFormat::Text, Some("boolrecv")), + ("regproc_unknown_text", FieldFormat::Text, Some("unknown")), + ("regproc_ambiguous_text", FieldFormat::Text, Some("int4")), + ("regtype_text", FieldFormat::Text, Some("int4recv")), + ("regproc_zero", FieldFormat::Binary, Some("-")), + ("regproc_null", FieldFormat::Binary, None), + ]; + let mut columns = Vec::new(); + let mut values = Vec::new(); + let mut formats = Vec::new(); + + for (name, format, value) in aliases { + let mut column = ColumnSchema::new(name, ConcreteDataType::string_datatype(), true); + column.mut_metadata().insert( + OID_ALIAS_KEY.to_string(), + if name == "regtype_text" { + oid_field::kind::REGTYPE.to_string() + } else { + oid_field::kind::REGPROC.to_string() + }, + ); + columns.push(column); + values.push(Arc::new(StringVector::from(vec![value])) as VectorRef); + formats.push(format.value()); + } + + columns.push(ColumnSchema::new( + "varchar", + ConcreteDataType::string_datatype(), + false, + )); + values.push(Arc::new(StringVector::from(vec![Some("varchar")])) as VectorRef); + formats.push(FieldFormat::Binary.value()); + + let schema = Arc::new(Schema::new(columns)); + let pg_schema = + Arc::new(schema_to_pg(&schema, &Format::Individual(formats), None).unwrap()); + let record_batch = RecordBatch::new(schema.clone(), values).unwrap(); + let query_context = QueryContextBuilder::default() + .configuration_parameter(Default::default()) + .build() + .into(); + let row_stream = RecordBatchRowStream::new( + query_context, + pg_schema.clone(), + schema, + stream::once(async { Ok(record_batch) }), + DataRowEncoder::new(pg_schema), + ); + + let row = futures::executor::block_on(row_stream.into_future()) + .0 + .unwrap() + .unwrap() + .pop() + .unwrap(); + assert_eq!(row.field_count, 8); + assert_eq!( + &row.data[..], + [ + 0, 0, 0, 4, 0, 0, 9, 132, // boolrecv (OID 2436), binary + 0, 0, 0, 8, b'b', b'o', b'o', b'l', b'r', b'e', b'c', b'v', // text + 0, 0, 0, 7, b'u', b'n', b'k', b'n', b'o', b'w', b'n', // unknown text + 0, 0, 0, 4, b'i', b'n', b't', b'4', // ambiguous text + 0, 0, 0, 8, b'i', b'n', b't', b'4', b'r', b'e', b'c', b'v', // regtype text + 0, 0, 0, 4, 0, 0, 0, 0, // - is OID 0 + 255, 255, 255, 255, // NULL + 0, 0, 0, 7, b'v', b'a', b'r', b'c', b'h', b'a', b'r', + ] + ); + } + + #[test] + fn test_encode_utf8_oid_alias_numeric_data() { + let aliases = [ + oid_field::kind::OID, + oid_field::kind::REGPROC, + "regprocedure", + "regoper", + "regoperator", + oid_field::kind::REGCLASS, + oid_field::kind::REGTYPE, + oid_field::kind::REGNAMESPACE, + "regrole", + "regconfig", + "regdictionary", + "regcollation", + ]; + let mut columns = Vec::new(); + let mut values = Vec::new(); + for alias in aliases { + let mut column = ColumnSchema::new(alias, ConcreteDataType::string_datatype(), false); + column + .mut_metadata() + .insert(OID_ALIAS_KEY.to_string(), alias.to_string()); + columns.push(column); + values.push(Arc::new(StringVector::from(vec![Some("4294967295")])) as VectorRef); + } + + let schema = Arc::new(Schema::new(columns)); + let pg_schema = Arc::new(schema_to_pg(&schema, &Format::UnifiedBinary, None).unwrap()); + let record_batch = RecordBatch::new(schema.clone(), values).unwrap(); + let query_context = QueryContextBuilder::default() + .configuration_parameter(Default::default()) + .build() + .into(); + let row_stream = RecordBatchRowStream::new( + query_context, + pg_schema.clone(), + schema, + stream::once(async { Ok(record_batch) }), + DataRowEncoder::new(pg_schema), + ); + + let row = futures::executor::block_on(row_stream.into_future()) + .0 + .unwrap() + .unwrap() + .pop() + .unwrap(); + assert_eq!(row.field_count, aliases.len() as i16); + assert_eq!( + &row.data[..], + &[[0, 0, 0, 4, 255, 255, 255, 255]; 12].concat() + ); + } + + #[test] + fn test_encode_utf8_oid_alias_binary_errors() { + for value in ["unknown", "int4"] { + let mut column = + ColumnSchema::new("regproc", ConcreteDataType::string_datatype(), false); + column.mut_metadata().insert( + OID_ALIAS_KEY.to_string(), + oid_field::kind::REGPROC.to_string(), + ); + let schema = Arc::new(Schema::new(vec![column])); + let pg_schema = Arc::new(schema_to_pg(&schema, &Format::UnifiedBinary, None).unwrap()); + let record_batch = RecordBatch::new( + schema.clone(), + vec![Arc::new(StringVector::from(vec![Some(value)])) as VectorRef], + ) + .unwrap(); + let query_context = QueryContextBuilder::default() + .configuration_parameter(Default::default()) + .build() + .into(); + let row_stream = RecordBatchRowStream::new( + query_context, + pg_schema.clone(), + schema, + stream::once(async { Ok(record_batch) }), + DataRowEncoder::new(pg_schema), + ); + + assert!( + futures::executor::block_on(row_stream.into_future()) + .0 + .unwrap() + .is_err() + ); + } + } + + #[test] + fn test_copy_text_utf8_oid_alias_preserves_named_value() { + let mut column = ColumnSchema::new("regproc", ConcreteDataType::string_datatype(), false); + column.mut_metadata().insert( + OID_ALIAS_KEY.to_string(), + oid_field::kind::REGPROC.to_string(), + ); + let schema = Arc::new(Schema::new(vec![column])); + let pg_schema = Arc::new(schema_to_pg(&schema, &Format::UnifiedBinary, None).unwrap()); + let record_batch = RecordBatch::new( + schema.clone(), + vec![Arc::new(StringVector::from(vec![Some("unknown")])) as VectorRef], + ) + .unwrap(); + let query_context = QueryContextBuilder::default() + .configuration_parameter(Default::default()) + .build() + .into(); + let row_stream = RecordBatchRowStream::new( + query_context, + pg_schema.clone(), + schema, + stream::once(async { Ok(record_batch) }), + CopyEncoder::new_text(pg_schema, CopyTextOptions::default()), + ); + + let row = futures::executor::block_on(row_stream.into_future()) + .0 + .unwrap() + .unwrap() + .pop() + .unwrap(); + assert_eq!(row.data.as_ref(), b"unknown\n"); + } + #[test] fn test_invalid_parameter() { // test for refactor with PgErrorCode diff --git a/src/servers/tests/http/http_handler_test.rs b/src/servers/tests/http/http_handler_test.rs index 42f53c370c9..2c8064727a6 100644 --- a/src/servers/tests/http/http_handler_test.rs +++ b/src/servers/tests/http/http_handler_test.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::collections::HashMap; use std::fmt; use std::pin::Pin; @@ -31,10 +30,12 @@ use common_query::{Output, OutputData}; use common_recordbatch::adapter::RecordBatchMetrics; use common_recordbatch::{OrderOption, RecordBatch, RecordBatchStream, SendableRecordBatchStream}; use datafusion::execution::TaskContext; -use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; +use datafusion::physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; +use datafusion::physical_plan::metrics::MetricsSet; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; use datafusion_common::Result as DfResult; +use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_expr::LogicalPlan; use datatypes::schema::SchemaRef; use futures::Stream; @@ -141,10 +142,6 @@ impl ExecutionPlan for PanickingMetricsExec { "PanickingMetricsExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.properties } @@ -153,6 +150,13 @@ impl ExecutionPlan for PanickingMetricsExec { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DfResult, + ) -> DfResult { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, _children: Vec>, @@ -168,11 +172,11 @@ impl ExecutionPlan for PanickingMetricsExec { unimplemented!("test plan is never executed") } - fn metrics(&self) -> Option { + fn metrics(&self) -> Option { if self.metrics_calls.fetch_add(1, Ordering::Relaxed) >= self.panic_after { panic!("metrics collection panicked") } - Some(datafusion::physical_plan::metrics::MetricsSet::new()) + Some(MetricsSet::new()) } } diff --git a/src/servers/tests/mod.rs b/src/servers/tests/mod.rs index 52a8df8b35f..699f3f76f1e 100644 --- a/src/servers/tests/mod.rs +++ b/src/servers/tests/mod.rs @@ -18,6 +18,10 @@ use api::v1::greptime_request::Request; use api::v1::query_request::Query; use async_trait::async_trait; use catalog::memory::MemoryCatalogManager; +use catalog::system_schema::SystemSchemaProvider; +use catalog::system_schema::pg_catalog::PGCatalogProvider; +use catalog::{CatalogManager, RegisterTableRequest}; +use common_catalog::consts::{DEFAULT_CATALOG_NAME, PG_CATALOG_NAME}; use common_error::ext::BoxedError; use common_grpc::flight::do_put::DoPutResponse; use common_query::Output; @@ -237,6 +241,22 @@ impl GrpcQueryHandler for DummyInstance { fn create_testing_instance(table: TableRef) -> DummyInstance { let catalog_manager = MemoryCatalogManager::new_with_table(table); + let pg_catalog = PGCatalogProvider::new( + DEFAULT_CATALOG_NAME.to_string(), + Arc::downgrade(&(catalog_manager.clone() as Arc)), + ); + for table in pg_catalog.tables().values() { + catalog_manager + .register_table_sync(RegisterTableRequest { + catalog: DEFAULT_CATALOG_NAME.to_string(), + schema: PG_CATALOG_NAME.to_string(), + table_name: table.table_info().name.clone(), + table_id: table.table_info().ident.table_id, + table: table.clone(), + }) + .unwrap(); + } + let query_engine = QueryEngineFactory::new( catalog_manager, None, diff --git a/src/servers/tests/postgres/mod.rs b/src/servers/tests/postgres/mod.rs index 6f49099a52b..2e031c0d0db 100644 --- a/src/servers/tests/postgres/mod.rs +++ b/src/servers/tests/postgres/mod.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::io; use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -27,6 +28,7 @@ use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_runtime::Builder as RuntimeBuilder; use common_runtime::runtime::BuilderBuild; use pgwire::api::Type; +use postgres_types::FromSql; use rand::Rng; use rustls::client::danger::{ServerCertVerified, ServerCertVerifier}; use rustls::{Error, SignatureScheme}; @@ -489,6 +491,66 @@ async fn test_using_db() -> Result<()> { Ok(()) } +struct RegprocOid(u32); + +impl<'a> FromSql<'a> for RegprocOid { + fn from_sql( + ty: &Type, + raw: &'a [u8], + ) -> std::result::Result> { + if ty != &Type::REGPROC { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + format!("expected REGPROC, got {ty}"), + ))); + } + + let oid = raw.try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("expected a four-byte OID, got {} bytes", raw.len()), + ) + })?; + Ok(Self(u32::from_be_bytes(oid))) + } + + fn accepts(ty: &Type) -> bool { + ty == &Type::REGPROC + } +} + +#[tokio::test] +async fn test_extended_query_regproc_response() -> Result<()> { + let server_port = start_test_server(TlsOption::default()).await?; + let client = create_connection_with_given_db(server_port, DEFAULT_SCHEMA_NAME) + .await + .unwrap(); + let stmt = client + .prepare("SELECT typreceive FROM pg_catalog.pg_type WHERE oid = 16") + .await + .unwrap(); + assert_eq!(stmt.columns()[0].type_(), &Type::REGPROC); + + let rows = client.query(&stmt, &[]).await.unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].get::(0).0, 2436); + + let result = client + .simple_query("SELECT typreceive FROM pg_catalog.pg_type WHERE oid = 16") + .await + .unwrap(); + let row = result + .iter() + .find_map(|message| match message { + SimpleQueryMessage::Row(row) => Some(row), + _ => None, + }) + .unwrap(); + assert_eq!(row.get(0), Some("boolrecv")); + + Ok(()) +} + #[tokio::test] async fn test_extended_query() -> Result<()> { let server_port = start_test_server(TlsOption::default()).await?; diff --git a/src/sql/src/parsers/explain_parser.rs b/src/sql/src/parsers/explain_parser.rs index e40f5b31a8c..342c330dc55 100644 --- a/src/sql/src/parsers/explain_parser.rs +++ b/src/sql/src/parsers/explain_parser.rs @@ -107,7 +107,7 @@ mod tests { connect_by: vec![], select_token: AttachedToken::empty(), flavor: SelectFlavor::Standard, - optimizer_hint: None, + optimizer_hints: vec![], }; let sp_query = Box::new( diff --git a/src/sql/src/parsers/insert_parser.rs b/src/sql/src/parsers/insert_parser.rs index d1211817747..5a58c385be9 100644 --- a/src/sql/src/parsers/insert_parser.rs +++ b/src/sql/src/parsers/insert_parser.rs @@ -30,8 +30,10 @@ impl ParserContext<'_> { .context(error::SyntaxSnafu)?; match spstatement { - SpStatement::Insert { .. } => { - Ok(Statement::Insert(Box::new(Insert { inner: spstatement }))) + insert_stmt @ SpStatement::Insert { .. } => { + let insert = Insert::try_from(insert_stmt) + .map_err(|e| error::InvalidSqlSnafu { msg: e.to_string() }.build())?; + Ok(Statement::Insert(Box::new(insert))) } unexp => error::UnsupportedSnafu { keyword: unexp.to_string(), @@ -50,9 +52,9 @@ impl ParserContext<'_> { match spstatement { SpStatement::Insert(mut insert_stmt) => { insert_stmt.replace_into = true; - Ok(Statement::Insert(Box::new(Insert { - inner: SpStatement::Insert(insert_stmt), - }))) + let insert = Insert::try_from(SpStatement::Insert(insert_stmt)) + .map_err(|e| error::InvalidSqlSnafu { msg: e.to_string() }.build())?; + Ok(Statement::Insert(Box::new(insert))) } unexp => error::UnsupportedSnafu { keyword: unexp.to_string(), diff --git a/src/sql/src/parsers/set_var_parser.rs b/src/sql/src/parsers/set_var_parser.rs index 8290f00af8a..19f0c48278d 100644 --- a/src/sql/src/parsers/set_var_parser.rs +++ b/src/sql/src/parsers/set_var_parser.rs @@ -24,8 +24,7 @@ use crate::statements::statement::Statement; /// SET variables statement parser implementation impl ParserContext<'_> { pub(crate) fn parse_set_variables(&mut self) -> Result { - let _ = self.parser.next_token(); - let spstatement = self.parser.parse_set().context(error::SyntaxSnafu)?; + let spstatement = self.parser.parse_statement().context(error::SyntaxSnafu)?; match spstatement { SpStatement::Set(set) => match set { Set::SingleAssignment { @@ -144,4 +143,14 @@ mod tests { let sql = "SET STATEMENT_TIMEOUT TO 5000"; assert_pg_parse_result(sql, "STATEMENT_TIMEOUT", expected_query_timeout_expr); } + + #[test] + fn test_unsupported_set_variant_remains_rejected() { + let result = ParserContext::create_with_dialect( + "SET ROLE admin", + &GreptimeDbDialect {}, + ParseOptions::default(), + ); + assert!(result.is_err()); + } } diff --git a/src/sql/src/parsers/show_parser.rs b/src/sql/src/parsers/show_parser.rs index 075c17dabe8..a039dceea63 100644 --- a/src/sql/src/parsers/show_parser.rs +++ b/src/sql/src/parsers/show_parser.rs @@ -40,6 +40,7 @@ impl ParserContext<'_> { if self.consume_token("TRIGGERS") { return self.parse_show_triggers(); } + self.consume_variables_scope(); if self.consume_token("DATABASES") || self.consume_token("SCHEMAS") { self.parse_show_databases(false) } else if self.matches_keyword(Keyword::TABLES) { @@ -147,6 +148,21 @@ impl ParserContext<'_> { } } + /// Consumes the `GLOBAL`/`SESSION`/`LOCAL` scope MySQL accepts in front of `VARIABLES` and + /// `STATUS`. GreptimeDB keeps no global/session split, so the scope is accepted and ignored. + fn consume_variables_scope(&mut self) { + let scoped = matches!( + self.parser.peek_token().token, + Token::Word(w) if matches!(w.keyword, Keyword::GLOBAL | Keyword::SESSION | Keyword::LOCAL) + ) && matches!( + self.parser.peek_nth_token(1).token, + Token::Word(w) if matches!(w.keyword, Keyword::VARIABLES | Keyword::STATUS) + ); + if scoped { + let _ = self.parser.next_token(); + } + } + fn parse_show_create_database(&mut self) -> Result { let raw_database_name = self.parse_object_name() @@ -1141,6 +1157,34 @@ mod tests { )); } + #[test] + fn test_show_variables_scope() { + for sql in [ + "SHOW STATUS", + "SHOW GLOBAL STATUS", + "SHOW SESSION STATUS", + "SHOW LOCAL STATUS", + ] { + let result = ParserContext::create_with_dialect( + sql, + &GreptimeDbDialect {}, + ParseOptions::default(), + ); + assert!( + matches!(result.unwrap()[0], Statement::ShowStatus(_)), + "{sql}" + ); + } + + // The scope is only swallowed in front of VARIABLES/STATUS. + let result = ParserContext::create_with_dialect( + "SHOW GLOBAL TABLES", + &GreptimeDbDialect {}, + ParseOptions::default(), + ); + assert!(result.is_err()); + } + fn parse_show_table_status(sql: &str) -> ShowTableStatus { let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); diff --git a/src/sql/src/parsers/utils.rs b/src/sql/src/parsers/utils.rs index 71a6a2499da..e91a2344643 100644 --- a/src/sql/src/parsers/utils.rs +++ b/src/sql/src/parsers/utils.rs @@ -22,10 +22,9 @@ use datafusion::execution::SessionStateBuilder; use datafusion::execution::context::SessionState; use datafusion::optimizer::simplify_expressions::ExprSimplifier; use datafusion_common::tree_node::{TreeNode, TreeNodeVisitor}; -use datafusion_common::{DFSchema, ScalarValue}; +use datafusion_common::{DFSchema, ScalarValue, TableReference}; use datafusion_expr::simplify::SimplifyContext; -use datafusion_expr::{AggregateUDF, Expr, ScalarUDF, TableSource, WindowUDF}; -use datafusion_sql::TableReference; +use datafusion_expr::{AggregateUDF, Expr, HigherOrderUDF, ScalarUDF, TableSource, WindowUDF}; use datafusion_sql::planner::{ContextProvider, SqlToRel}; use datatypes::arrow::datatypes::DataType; use datatypes::schema::{ @@ -265,8 +264,10 @@ pub fn parser_expr_to_scalar_value_literal_at( // 2. simplify logical expr — use scheduled time if provided, else wall-clock let info = match scheduled_time { - Some(dt) => SimplifyContext::default().with_query_execution_start_time(Some(dt)), - None => SimplifyContext::default().with_current_time(), + Some(dt) => SimplifyContext::builder() + .with_query_execution_start_time(Some(dt)) + .build(), + None => SimplifyContext::builder().with_current_time().build(), }; let simplifier = ExprSimplifier::new(info); @@ -316,6 +317,10 @@ impl ContextProvider for StubContextProvider { self.state.scalar_functions().get(name).cloned() } + fn get_higher_order_meta(&self, name: &str) -> Option> { + self.state.higher_order_functions().get(name).cloned() + } + fn get_aggregate_meta(&self, name: &str) -> Option> { self.state.aggregate_functions().get(name).cloned() } @@ -336,6 +341,14 @@ impl ContextProvider for StubContextProvider { self.state.scalar_functions().keys().cloned().collect() } + fn higher_order_function_names(&self) -> Vec { + self.state + .higher_order_functions() + .keys() + .cloned() + .collect() + } + fn udaf_names(&self) -> Vec { self.state.aggregate_functions().keys().cloned().collect() } @@ -553,7 +566,9 @@ SELECT * FROM tql_cte WHERE ts > 0 ), ]; - let info = SimplifyContext::default().with_query_execution_start_time(Some(now_time)); + let info = SimplifyContext::builder() + .with_query_execution_start_time(Some(now_time)) + .build(); let simplifier = ExprSimplifier::new(info); for (expr, expected) in testcases { let expr_name = expr.schema_name().to_string(); diff --git a/src/sql/src/parsers/with_tql_parser.rs b/src/sql/src/parsers/with_tql_parser.rs index 08bda1e1017..34a929174d9 100644 --- a/src/sql/src/parsers/with_tql_parser.rs +++ b/src/sql/src/parsers/with_tql_parser.rs @@ -129,6 +129,7 @@ impl ParserContext<'_> { data_type: None, }) .collect(), + at: None, }, query: body, from: None, diff --git a/src/sql/src/statements/insert.rs b/src/sql/src/statements/insert.rs index cfda63f9289..52318cddbf4 100644 --- a/src/sql/src/statements/insert.rs +++ b/src/sql/src/statements/insert.rs @@ -14,8 +14,8 @@ use serde::Serialize; use sqlparser::ast::{ - Insert as SpInsert, ObjectName, Query, SetExpr, Statement, TableObject, UnaryOperator, - ValueWithSpan, Values, + Insert as SpInsert, ObjectName, ObjectNamePart, Parens, Query, SetExpr, Statement, TableObject, + UnaryOperator, ValueWithSpan, Values, }; use sqlparser::parser::ParserError; use sqlparser_derive::{Visit, VisitMut}; @@ -57,7 +57,12 @@ impl Insert { pub fn columns(&self) -> Vec<&String> { match &self.inner { - Statement::Insert(insert) => insert.columns.iter().map(|ident| &ident.value).collect(), + Statement::Insert(insert) => insert + .columns + .iter() + .filter_map(single_part_column_ident) + .map(|ident| &ident.value) + .collect(), _ => unreachable!(), } } @@ -137,7 +142,7 @@ impl Insert { } } -fn sql_exprs_to_values(exprs: &[Vec]) -> Result>> { +fn sql_exprs_to_values(exprs: &[Parens>]) -> Result>> { let mut values = Vec::with_capacity(exprs.len()); for es in exprs.iter() { let mut vs = Vec::with_capacity(es.len()); @@ -188,16 +193,34 @@ fn sql_exprs_to_values(exprs: &[Vec]) -> Result>> { Ok(values) } +fn single_part_column_ident(name: &ObjectName) -> Option<&sqlparser::ast::Ident> { + let [ObjectNamePart::Identifier(ident)] = name.0.as_slice() else { + return None; + }; + Some(ident) +} + impl TryFrom for Insert { type Error = ParserError; fn try_from(value: Statement) -> std::result::Result { - match value { - Statement::Insert { .. } => Ok(Insert { inner: value }), - unexp => Err(ParserError::ParserError(format!( - "Not expected to be {unexp}" - ))), + let Statement::Insert(insert) = &value else { + return Err(ParserError::ParserError(format!( + "Not expected to be {value}" + ))); + }; + + if let Some(column) = insert + .columns + .iter() + .find(|column| single_part_column_ident(column).is_none()) + { + return Err(ParserError::ParserError(format!( + "Expected a single-part insert column name, found {column}" + ))); } + + Ok(Insert { inner: value }) } } @@ -239,6 +262,32 @@ mod tests { } } + #[test] + fn test_insert_column_names_are_single_identifiers() { + let stmt = ParserContext::create_with_dialect( + "INSERT INTO my_table (host, \"value\") VALUES (1, 2)", + &GreptimeDbDialect {}, + ParseOptions::default(), + ) + .unwrap() + .remove(0); + let Statement::Insert(insert) = stmt else { + unreachable!() + }; + assert_eq!(insert.columns(), vec!["host", "value"]); + + let result = ParserContext::create_with_dialect( + "INSERT INTO my_table (metric.host) VALUES (1)", + &GreptimeDbDialect {}, + ParseOptions::default(), + ); + let error = result.unwrap_err().to_string(); + assert!( + error.contains("Expected a single-part insert column name, found metric.host"), + "unexpected error: {error}" + ); + } + #[test] fn test_insert_value_with_default() { // insert "default" diff --git a/src/store-api/src/mito_engine_options.rs b/src/store-api/src/mito_engine_options.rs index c308e915654..bc9b31ada95 100644 --- a/src/store-api/src/mito_engine_options.rs +++ b/src/store-api/src/mito_engine_options.rs @@ -88,6 +88,31 @@ pub const MAX_ROW_GROUP_ROW_COUNT: &str = "max_row_group_row_count"; pub const MAX_ROW_GROUP_ROW_COUNT_LIMIT: usize = 10 * 1024 * 1024; /// Option key for preserving per-row sequence numbers through flush and compaction. pub const PRESERVE_ROW_SEQUENCE: &str = "preserve_row_sequence"; +/// Option key for experimental Parquet float field encoding. +pub const EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING: &str = "experimental_sst_float_field_encoding"; + +/// Encoding policy for direct floating-point field columns in Parquet SSTs. +#[derive( + Debug, + Default, + Clone, + Copy, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, + strum::EnumString, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum FloatFieldEncoding { + /// The existing Parquet writer behavior. + #[default] + Default, + /// Parquet byte-stream-split encoding. + ByteStreamSplit, +} // Note: Adding new options here should also check if this option should be removed in [metric_engine::engine::create::region_options_for_metadata_region]. /// Conflicting values supplied through the legacy and canonical TWCS trigger options. @@ -155,6 +180,7 @@ pub fn is_mito_engine_option_key(key: &str) -> bool { SST_FORMAT_KEY, MAX_ROW_GROUP_ROW_COUNT, PRESERVE_ROW_SEQUENCE, + EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, ] .contains(&key) } @@ -217,9 +243,29 @@ mod tests { assert!(is_mito_engine_option_key("append_mode")); assert!(is_mito_engine_option_key("max_row_group_row_count")); assert!(is_mito_engine_option_key("preserve_row_sequence")); + assert!(is_mito_engine_option_key( + EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING + )); assert!(!is_mito_engine_option_key("foo")); } + #[test] + fn test_float_field_encoding_values() { + assert_eq!( + "default".parse::(), + Ok(FloatFieldEncoding::Default) + ); + assert_eq!( + "byte_stream_split".parse::(), + Ok(FloatFieldEncoding::ByteStreamSplit) + ); + assert_eq!( + serde_json::from_str::("\"byte_stream_split\"").unwrap(), + FloatFieldEncoding::ByteStreamSplit + ); + assert!(serde_json::from_str::("\"unknown\"").is_err()); + } + #[test] fn test_normalize_twcs_trigger_aliases_to_legacy_key() { let expected = HashMap::from([(TWCS_TRIGGER_FILE_NUM.to_string(), "4".to_string())]); diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs index 1a4351f279b..6a45d3db7d7 100644 --- a/src/store-api/src/region_engine.rs +++ b/src/store-api/src/region_engine.rs @@ -306,6 +306,9 @@ pub struct ScannerProperties { /// [ScannerProperties::append_mode] is true. total_rows: usize, + /// Whether total_rows is exact before applying query predicates. + total_rows_is_exact: bool, + /// Whether to yield an empty batch to distinguish partition ranges. pub distinguish_partition_range: bool, @@ -334,12 +337,19 @@ impl ScannerProperties { self } + /// Sets whether the source row count is exact before query predicates. + pub fn with_total_rows_is_exact(mut self, exact: bool) -> Self { + self.total_rows_is_exact = exact; + self + } + /// Creates a new [`ScannerProperties`] with the given partitioning. pub fn new(partitions: Vec>, append_mode: bool, total_rows: usize) -> Self { Self { partitions, append_mode, total_rows, + total_rows_is_exact: false, distinguish_partition_range: false, target_partitions: 0, logical_region: false, @@ -374,6 +384,11 @@ impl ScannerProperties { self.total_rows } + /// Returns whether the source row count is exact before query predicates. + pub fn total_rows_is_exact(&self) -> bool { + self.total_rows_is_exact + } + /// Returns whether the scanner is scanning a logical region. pub fn is_logical_region(&self) -> bool { self.logical_region diff --git a/src/store-api/src/region_request.rs b/src/store-api/src/region_request.rs index 01cf6a4acb1..826ef7d6bc9 100644 --- a/src/store-api/src/region_request.rs +++ b/src/store-api/src/region_request.rs @@ -42,7 +42,7 @@ use datatypes::error::time_index_not_widening_error; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{FulltextOptions, SkippingIndexOptions}; use num_enum::TryFromPrimitive; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use snafu::{OptionExt, ResultExt, ensure}; use strum::{AsRefStr, IntoStaticStr}; @@ -472,6 +472,7 @@ fn make_region_truncate(truncate: TruncateRequest) -> Result Result> { let region_id = request.region_id.into(); + let skip_wal = request.skip_wal; let partition_expr_version = request.partition_expr_version.map(|v| v.value); let aligned_schema_version = request.aligned_schema_version.map(|v| v.schema_version); let Some(Body::ArrowIpc(request)) = request.body else { @@ -493,6 +494,7 @@ fn make_region_bulk_inserts(request: BulkInsertRequest) -> Result for ModifyColumnType { /// Region option changes used by ALTER requests. /// -/// This type currently derives serde for request persistence. Keep future changes -/// backward compatible with previously serialized variants. -#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] +/// This type is serialized for request persistence. Keep future changes backward +/// compatible with previously serialized variants. +#[derive(Debug, Eq, PartialEq, Clone)] pub enum SetRegionOption { WriteBufferSize(Option), Ttl(Option), @@ -1501,10 +1503,92 @@ pub enum SetRegionOption { // Modifying the max number of rows in a parquet row group. MaxRowGroupRowCount(Option), PreserveRowSequence(bool), - // Stops writing new WAL entries. This operation is irreversible. + // Whether to skip writing new WAL entries. + SkipWal(bool), +} + +#[derive(Serialize, Deserialize)] +enum SetRegionOptionSerde { + WriteBufferSize(Option), + Ttl(Option), + Twsc(String, String), + Format(String), + AppendMode(bool), + AutoFlushInterval(Option), + MaxRowGroupRowCount(Option), + PreserveRowSequence(bool), + SkipWal(bool), +} + +impl Serialize for SetRegionOption { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: Serializer, + { + // Older binaries deserialize the disable request as a unit variant. + if matches!(self, Self::SkipWal(true)) { + return serializer.serialize_unit_variant("SetRegionOption", 8, "SkipWal"); + } + + let option = match self { + Self::WriteBufferSize(value) => SetRegionOptionSerde::WriteBufferSize(*value), + Self::Ttl(value) => SetRegionOptionSerde::Ttl(*value), + Self::Twsc(key, value) => SetRegionOptionSerde::Twsc(key.clone(), value.clone()), + Self::Format(value) => SetRegionOptionSerde::Format(value.clone()), + Self::AppendMode(value) => SetRegionOptionSerde::AppendMode(*value), + Self::AutoFlushInterval(value) => SetRegionOptionSerde::AutoFlushInterval(*value), + Self::MaxRowGroupRowCount(value) => SetRegionOptionSerde::MaxRowGroupRowCount(*value), + Self::PreserveRowSequence(value) => SetRegionOptionSerde::PreserveRowSequence(*value), + Self::SkipWal(value) => SetRegionOptionSerde::SkipWal(*value), + }; + option.serialize(serializer) + } +} + +#[derive(Deserialize)] +enum LegacySetRegionOption { SkipWal, } +#[derive(Deserialize)] +#[serde(untagged)] +enum BackwardCompatibleSetRegionOption { + Current(SetRegionOptionSerde), + Legacy(LegacySetRegionOption), +} + +impl From for SetRegionOption { + fn from(option: SetRegionOptionSerde) -> Self { + match option { + SetRegionOptionSerde::WriteBufferSize(value) => Self::WriteBufferSize(value), + SetRegionOptionSerde::Ttl(value) => Self::Ttl(value), + SetRegionOptionSerde::Twsc(key, value) => Self::Twsc(key, value), + SetRegionOptionSerde::Format(value) => Self::Format(value), + SetRegionOptionSerde::AppendMode(value) => Self::AppendMode(value), + SetRegionOptionSerde::AutoFlushInterval(value) => Self::AutoFlushInterval(value), + SetRegionOptionSerde::MaxRowGroupRowCount(value) => Self::MaxRowGroupRowCount(value), + SetRegionOptionSerde::PreserveRowSequence(value) => Self::PreserveRowSequence(value), + SetRegionOptionSerde::SkipWal(value) => Self::SkipWal(value), + } + } +} + +impl<'de> Deserialize<'de> for SetRegionOption { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + Ok( + match BackwardCompatibleSetRegionOption::deserialize(deserializer)? { + BackwardCompatibleSetRegionOption::Current(option) => option.into(), + BackwardCompatibleSetRegionOption::Legacy(LegacySetRegionOption::SkipWal) => { + Self::SkipWal(true) + } + }, + ) + } +} + impl TryFrom<&PbOption> for SetRegionOption { type Error = MetadataError; @@ -1570,7 +1654,12 @@ impl TryFrom<&PbOption> for SetRegionOption { .map_err(|_| InvalidSetRegionOptionRequestSnafu { key, value }.build())?; Ok(Self::PreserveRowSequence(preserve)) } - SKIP_WAL_KEY if value == "true" => Ok(Self::SkipWal), + SKIP_WAL_KEY => { + let skip_wal = value + .parse::() + .map_err(|_| InvalidSetRegionOptionRequestSnafu { key, value }.build())?; + Ok(Self::SkipWal(skip_wal)) + } _ => InvalidSetRegionOptionRequestSnafu { key, value }.fail(), } } @@ -1755,6 +1844,8 @@ pub struct RegionCatchupRequest { #[derive(Debug, Clone)] pub struct RegionBulkInsertsRequest { + /// Whether this request should skip WAL. + pub skip_wal: bool, pub region_id: RegionId, pub payload: DfRecordBatch, pub raw_data: ArrowIpc, @@ -2043,11 +2134,20 @@ mod tests { value: "true".to_string(), }; assert_eq!( - SetRegionOption::SkipWal, + SetRegionOption::SkipWal(true), SetRegionOption::try_from(&pb).unwrap() ); - for value in ["false", "", "invalid"] { + let pb = PbOption { + key: SKIP_WAL_KEY.to_string(), + value: "false".to_string(), + }; + assert_eq!( + SetRegionOption::SkipWal(false), + SetRegionOption::try_from(&pb).unwrap() + ); + + for value in ["", "invalid"] { let pb = PbOption { key: SKIP_WAL_KEY.to_string(), value: value.to_string(), @@ -2058,6 +2158,37 @@ mod tests { assert!(UnsetRegionOption::try_from(SKIP_WAL_KEY).is_err()); } + #[test] + fn test_set_region_option_skip_wal_serde_compatibility() { + let legacy = serde_json::from_str::(r#""SkipWal""#).unwrap(); + assert_eq!(SetRegionOption::SkipWal(true), legacy); + + check_set_region_option_skip_wal_serde_compatibility(false); + check_set_region_option_skip_wal_serde_compatibility(true); + } + + fn check_set_region_option_skip_wal_serde_compatibility(skip_wal: bool) { + let option = SetRegionOption::SkipWal(skip_wal); + let serialized = serde_json::to_string(&option).unwrap(); + let expected = if skip_wal { + r#""SkipWal""# + } else { + r#"{"SkipWal":false}"# + }; + assert_eq!(expected, serialized); + assert_eq!( + option, + serde_json::from_str::(&serialized).unwrap() + ); + if skip_wal { + assert!(serde_json::from_str::(&serialized).is_ok()); + assert_eq!( + option, + serde_json::from_str::(r#"{"SkipWal":true}"#).unwrap() + ); + } + } + #[test] fn test_set_region_option_max_row_group_row_count_try_from() { let pb = PbOption { diff --git a/src/store-api/src/storage/file.rs b/src/store-api/src/storage/file.rs index 0dcfcdc8cdd..b0b76146825 100644 --- a/src/store-api/src/storage/file.rs +++ b/src/store-api/src/storage/file.rs @@ -51,6 +51,11 @@ impl FileId { pub fn as_bytes(&self) -> &[u8] { self.0.as_bytes() } + + /// Constructs a file id from its packed 16-byte UUID representation. + pub fn from_bytes(bytes: [u8; 16]) -> FileId { + FileId(Uuid::from_bytes(bytes)) + } } impl From for Uuid { diff --git a/src/table/src/metadata.rs b/src/table/src/metadata.rs index b664c5cfd55..a784fbcf86b 100644 --- a/src/table/src/metadata.rs +++ b/src/table/src/metadata.rs @@ -431,13 +431,13 @@ impl TableMeta { new_options.extra_options.remove(PRESERVE_ROW_SEQUENCE); } } - SetRegionOption::SkipWal => { - new_options.skip_wal = true; + SetRegionOption::SkipWal(skip_wal) => { + new_options.skip_wal = *skip_wal; // Keep the explicit table option so it remains distinguishable // from a value inherited from the schema. new_options .extra_options - .insert(SKIP_WAL_KEY.to_string(), true.to_string()); + .insert(SKIP_WAL_KEY.to_string(), skip_wal.to_string()); } } } @@ -1926,7 +1926,7 @@ mod tests { .insert(SKIP_WAL_KEY.to_string(), false.to_string()); let alter_kind = AlterKind::SetTableOptions { - options: vec![SetRegionOption::SkipWal], + options: vec![SetRegionOption::SkipWal(true)], }; let new_meta = meta .builder_with_alter_kind("my_table", &alter_kind) @@ -1943,6 +1943,25 @@ mod tests { .get(SKIP_WAL_KEY) .map(String::as_str) ); + + let alter_kind = AlterKind::SetTableOptions { + options: vec![SetRegionOption::SkipWal(false)], + }; + let new_meta = new_meta + .builder_with_alter_kind("my_table", &alter_kind) + .unwrap() + .build() + .unwrap(); + + assert!(!new_meta.options.skip_wal); + assert_eq!( + Some("false"), + new_meta + .options + .extra_options + .get(SKIP_WAL_KEY) + .map(String::as_str) + ); } #[test] diff --git a/src/table/src/predicate.rs b/src/table/src/predicate.rs index 3fa27674a93..7290c35c70b 100644 --- a/src/table/src/predicate.rs +++ b/src/table/src/predicate.rs @@ -20,11 +20,12 @@ use common_time::Timestamp; use common_time::range::TimestampRange; use common_time::timestamp::TimeUnit; use datafusion::common::ScalarValue; -use datafusion::physical_optimizer::pruning::PruningPredicate; +use datafusion::physical_optimizer::pruning::PruningPredicateBuilder; use datafusion_common::ToDFSchema; use datafusion_common::pruning::PruningStatistics; use datafusion_common::tree_node::TreeNode; use datafusion_expr::expr::{Expr, InList}; +use datafusion_expr::physical_planning_context::PhysicalPlanningContext; use datafusion_expr::{Between, BinaryExpr, Operator}; use datafusion_physical_expr::execution_props::ExecutionProps; use datafusion_physical_expr::expressions::{ @@ -161,8 +162,13 @@ impl Predicate { // registering variables. let execution_props = &ExecutionProps::new(); - create_physical_expr(expr, df_schema.as_ref(), execution_props) - .context(error::DatafusionSnafu) + create_physical_expr( + expr, + df_schema.as_ref(), + execution_props, + &PhysicalPlanningContext::default(), + ) + .context(error::DatafusionSnafu) } /// Builds physical exprs according to provided schema. @@ -197,7 +203,10 @@ impl Predicate { }; for expr in &physical_exprs { - match PruningPredicate::try_new(expr.clone(), schema.clone()) { + match PruningPredicateBuilder::new() + .with_file_schema(schema.clone()) + .try_build(expr.clone()) + { Ok(p) => match p.prune(stats) { Ok(r) => { for (curr_val, res) in r.into_iter().zip(res.iter_mut()) { @@ -209,7 +218,7 @@ impl Predicate { } }, Err(e) => { - // since dynamic filter exprs could be complex, it's possible that `PruningPredicate::try_new` fails to prove anything from it. In that case, we just log it and skip pruning with this expr. + // since dynamic filter exprs could be complex, it's possible that the pruning predicate builder fails to prove anything from it. In that case, we just log it and skip pruning with this expr. debug!("Failed to create pruning predicate for expr: {e:?}"); } } diff --git a/src/table/src/predicate/stats.rs b/src/table/src/predicate/stats.rs index 39715cc028d..cce2d4caa00 100644 --- a/src/table/src/predicate/stats.rs +++ b/src/table/src/predicate/stats.rs @@ -113,7 +113,7 @@ impl PruningStatistics for RowGroupPruningStatistics<'_> { Some(Arc::new(UInt64Array::from(values))) } - fn row_counts(&self, _column: &Column) -> Option { + fn row_counts(&self) -> Option { // TODO(LFC): Impl it. None } diff --git a/src/table/src/requests.rs b/src/table/src/requests.rs index f7f195765f6..a21a8f6fed8 100644 --- a/src/table/src/requests.rs +++ b/src/table/src/requests.rs @@ -36,10 +36,10 @@ use store_api::metric_engine_consts::{ LOGICAL_TABLE_METADATA_KEY, PHYSICAL_TABLE_METADATA_KEY, is_metric_engine_option_key, }; use store_api::mito_engine_options::{ - APPEND_MODE_KEY, COMPACTION_TYPE, MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD, - MEMTABLE_BULK_ENCODE_ROW_THRESHOLD, MEMTABLE_BULK_MAX_MERGE_GROUPS, - MEMTABLE_BULK_MERGE_THRESHOLD, MEMTABLE_TYPE, MERGE_MODE_KEY, SST_FORMAT_KEY, - TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER, TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM, + APPEND_MODE_KEY, COMPACTION_TYPE, EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, FloatFieldEncoding, + MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD, MEMTABLE_BULK_ENCODE_ROW_THRESHOLD, + MEMTABLE_BULK_MAX_MERGE_GROUPS, MEMTABLE_BULK_MERGE_THRESHOLD, MEMTABLE_TYPE, MERGE_MODE_KEY, + SST_FORMAT_KEY, TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER, TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM, TWCS_FALLBACK_TO_LOCAL, TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER, TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM, TWCS_MAX_OUTPUT_FILE_SIZE, TWCS_TIME_WINDOW, TWCS_TRIGGER_FILE_NUM, is_mito_engine_option_key, normalize_twcs_trigger_options, @@ -256,6 +256,16 @@ impl TableOptions { options.ttl = Some(ttl_value); } + if let Some(encoding) = kvs.get(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING) { + encoding.parse::().map_err(|_| { + ParseTableOptionSnafu { + key: EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, + value: encoding, + } + .build() + })?; + } + if let Some(skip_wal) = kvs.get(SKIP_WAL_KEY) { options.skip_wal = skip_wal.parse().map_err(|_| { ParseTableOptionSnafu { @@ -885,6 +895,7 @@ mod tests { assert!(validate_table_option(WRITE_BUFFER_SIZE_KEY)); assert!(validate_table_option(STORAGE_KEY)); assert!(validate_table_option(MEMTABLE_BULK_MERGE_THRESHOLD)); + assert!(validate_table_option(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING)); assert!(validate_table_option(REPARTITION_COLUMN_HINT_KEY)); assert!(validate_table_option(REPARTITION_PARTITION_NUM_HINT_KEY)); assert_eq!(AnnotationFamily::of_key("repartition.unknown.hint"), None); @@ -923,6 +934,25 @@ mod tests { assert!(!validate_database_option("foo")); } + #[test] + fn test_parse_float_field_encoding_option() { + let options = TableOptions::try_from_iter([( + EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, + "byte_stream_split", + )]) + .unwrap(); + assert_eq!( + options + .extra_options + .get(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING), + Some(&"byte_stream_split".to_string()) + ); + assert!( + TableOptions::try_from_iter([(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "invalid",)]) + .is_err() + ); + } + #[test] fn test_database_trigger_value_boundaries() { let maximum = usize::MAX.to_string(); diff --git a/src/table/src/table.rs b/src/table/src/table.rs index 542a5066771..cdc06a50be2 100644 --- a/src/table/src/table.rs +++ b/src/table/src/table.rs @@ -217,10 +217,10 @@ fn default_constraint_to_expr( CURRENT_TIMESTAMP | CURRENT_TIMESTAMP_FN | NOW_FN ) => { - Some(Expr::Cast(Cast { - expr: Box::new(NOW_EXPR.clone()), - data_type: target_type.as_arrow_type(), - })) + Some(Expr::Cast(Cast::new( + Box::new(NOW_EXPR.clone()), + target_type.as_arrow_type(), + ))) } ColumnDefaultConstraint::Function(_) => None, @@ -260,10 +260,12 @@ mod tests { Expr::Literal(ScalarValue::Utf8(Some(s)), _) if s == "test")); assert!(matches!( column_defaults.get("ts").unwrap(), - Expr::Cast(Cast { - expr, - data_type - }) if **expr == *NOW_EXPR && *data_type == ConcreteDataType::timestamp_millisecond_datatype().as_arrow_type() + Expr::Cast(Cast { expr, field }) + if **expr == *NOW_EXPR + && field.data_type() == &ConcreteDataType::timestamp_millisecond_datatype().as_arrow_type() + && field.is_nullable() + && field.name().is_empty() + && field.metadata().is_empty() )); } } diff --git a/src/table/src/table/adapter.rs b/src/table/src/table/adapter.rs index a8ec6ee93e6..16c187d38b6 100644 --- a/src/table/src/table/adapter.rs +++ b/src/table/src/table/adapter.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::sync::{Arc, Mutex}; use common_catalog::consts::{METRIC_ENGINE, MITO_ENGINE, MITO2_ENGINE}; @@ -125,10 +124,6 @@ impl std::fmt::Debug for DfTableProviderAdapter { #[async_trait::async_trait] impl TableProvider for DfTableProviderAdapter { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> DfSchemaRef { let table_info = self.table.table_info(); let schema = self.table.schema().arrow_schema().clone(); diff --git a/src/table/src/table/scan.rs b/src/table/src/table/scan.rs index d1417bf7a95..1e143a9cabb 100644 --- a/src/table/src/table/scan.rs +++ b/src/table/src/table/scan.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::Any; use std::pin::Pin; use std::sync::{Arc, Mutex}; use std::task::{Context, Poll}; @@ -38,9 +37,10 @@ use datafusion::physical_plan::filter_pushdown::{ use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet}; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, - RecordBatchStream as DfRecordBatchStream, + RecordBatchStream as DfRecordBatchStream, apply_expression_roots, }; use datafusion_common::stats::Precision; +use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_common::{ColumnStatistics, DataFusionError, Statistics}; use datafusion_physical_expr::expressions::{ BinaryExpr, Column, DynamicFilterPhysicalExpr, is_null, @@ -391,10 +391,6 @@ impl RegionScanExec { } impl ExecutionPlan for RegionScanExec { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> ArrowSchemaRef { self.arrow_schema.clone() } @@ -407,6 +403,20 @@ impl ExecutionPlan for RegionScanExec { vec![] } + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> datafusion_common::Result, + ) -> datafusion_common::Result { + let pushed_dyn_filters = self.pushed_dyn_filters.lock().unwrap().clone(); + apply_expression_roots( + self.output_ordering + .iter() + .flat_map(|ordering| ordering.iter().map(|sort_expr| &sort_expr.expr)) + .chain(pushed_dyn_filters.iter()), + f, + ) + } + fn with_new_children( self: Arc, _children: Vec>, @@ -481,32 +491,34 @@ impl ExecutionPlan for RegionScanExec { Ok(self) } - fn partition_statistics(&self, partition: Option) -> DfResult { - if partition.is_some() { - return Ok(Statistics::new_unknown(self.schema().as_ref())); + fn partition_statistics(&self, partition: Option) -> DfResult> { + if partition.is_some() || !self.append_mode { + return Ok(Arc::new(Statistics::new_unknown(self.schema().as_ref()))); } - let statistics = - if self.append_mode && !self.scanner.lock().unwrap().has_predicate_without_region() { - let column_statistics = self - .arrow_schema - .fields - .iter() - .map(|_| ColumnStatistics { - distinct_count: Precision::Exact(self.total_rows), - null_count: Precision::Exact(0), // all null rows are counted for append-only table - ..Default::default() - }) - .collect(); - Statistics { - num_rows: Precision::Exact(self.total_rows), - total_byte_size: Default::default(), - column_statistics, - } - } else { - Statistics::new_unknown(&self.arrow_schema) - }; - Ok(statistics) + let scanner = self.scanner.lock().unwrap(); + let statistics = if scanner.properties().total_rows_is_exact() + && !scanner.has_predicate_without_region() + { + let column_statistics = self + .arrow_schema + .fields + .iter() + .map(|_| ColumnStatistics { + distinct_count: Precision::Exact(self.total_rows), + null_count: Precision::Exact(0), // all null rows are counted for append-only table + ..Default::default() + }) + .collect(); + Statistics { + num_rows: Precision::Exact(self.total_rows), + total_byte_size: Default::default(), + column_statistics, + } + } else { + Statistics::new_unknown(&self.arrow_schema) + }; + Ok(Arc::new(statistics)) } fn name(&self) -> &str { @@ -534,11 +546,7 @@ impl ExecutionPlan for RegionScanExec { { let mut exact_filters = self.pushed_dyn_filters.lock().unwrap(); for (index, filter) in parent_filters.iter().enumerate() { - if filter - .as_any() - .downcast_ref::() - .is_some() - { + if filter.downcast_ref::().is_some() { if exact_filters.iter().any(|existing| existing == filter) { supported[index] = true; } else { @@ -558,7 +566,6 @@ impl ExecutionPlan for RegionScanExec { let scanner_supported = self.add_dyn_filters_to_predicate(scanner_filters); for (index, is_supported) in scanner_filter_indices.into_iter().zip(scanner_supported) { if parent_filters[index] - .as_any() .downcast_ref::() .is_none() { @@ -845,6 +852,33 @@ mod test { ) } + #[test] + #[allow(deprecated)] + fn test_count_statistics_require_exact_source_rows() { + for (append_mode, exact, expected) in [ + (true, false, Precision::Absent), + (true, true, Precision::Exact(3)), + (false, true, Precision::Absent), + ] { + let (schema, batch, metadata) = dynamic_filter_test_data(5685, false); + let scanner = Box::new(RepeatableScanner { + batches: RecordBatches::try_new(schema, vec![batch]).unwrap(), + properties: ScannerProperties::default() + .with_append_mode(append_mode) + .with_total_rows(3) + .with_total_rows_is_exact(exact), + metadata, + dynamic_filters: Arc::new(Mutex::new(Vec::new())), + }); + let plan = RegionScanExec::new(scanner, ScanRequest::default(), None).unwrap(); + assert_eq!(plan.partition_statistics(None).unwrap().num_rows, expected); + assert_eq!( + plan.partition_statistics(Some(0)).unwrap().num_rows, + Precision::Absent, + ); + } + } + #[tokio::test] async fn test_reset_state_clears_dynamic_filters_for_repeatable_scanner() { let ctx = SessionContext::new(); @@ -955,6 +989,52 @@ mod test { ); } + #[test] + fn test_apply_expressions_visits_pushed_dynamic_filters() { + let (_, plan) = dynamic_filter_fixture(5685); + let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new( + vec![Arc::new(Column::new("a", 0))], + lit(true), + )); + let propagation = plan + .handle_child_pushdown_result( + FilterPushdownPhase::Post, + ChildPushdownResult { + parent_filters: vec![ChildFilterPushdownResult { + filter: dynamic_filter.clone(), + child_results: vec![PushedDown::No], + }], + self_filters: vec![], + }, + &datafusion::config::ConfigOptions::default(), + ) + .unwrap(); + assert!(matches!(propagation.filters.as_slice(), [PushedDown::Yes])); + + let mut visited = 0; + assert_eq!( + plan.apply_expressions(&mut |expr| { + visited += 1; + assert!(plan.pushed_dyn_filters.try_lock().is_ok()); + assert_eq!(expr.expression_id(), dynamic_filter.expression_id()); + assert_eq!( + expr.downcast_ref::().unwrap(), + dynamic_filter.as_ref() + ); + Ok(TreeNodeRecursion::Continue) + }) + .unwrap(), + TreeNodeRecursion::Continue + ); + assert_eq!(visited, 1); + + assert_eq!( + plan.apply_expressions(&mut |_| Ok(TreeNodeRecursion::Stop)) + .unwrap(), + TreeNodeRecursion::Stop + ); + } + #[tokio::test] async fn test_live_dynamic_filter_is_applied_after_stream_creation() { let ctx = SessionContext::new(); @@ -996,7 +1076,6 @@ mod test { .updated_node .as_ref() .unwrap() - .as_any() .downcast_ref::() .is_some() ); diff --git a/tests-integration/Cargo.toml b/tests-integration/Cargo.toml index a2d6c70e394..d1b0774af9e 100644 --- a/tests-integration/Cargo.toml +++ b/tests-integration/Cargo.toml @@ -126,6 +126,7 @@ session = { workspace = true, features = ["testing"] } similar-asserts.workspace = true store-api.workspace = true tokio-postgres = { workspace = true } +tokio-util.workspace = true url = "2.3" urlencoding = "2.1" yaml-rust = "0.4" diff --git a/tests-integration/src/cluster.rs b/tests-integration/src/cluster.rs index e40c707fe98..3e53b6104a0 100644 --- a/tests-integration/src/cluster.rs +++ b/tests-integration/src/cluster.rs @@ -35,6 +35,7 @@ use client::Client; use client::client_manager::NodeClients; use cmd::frontend::create_heartbeat_task; use common_base::Plugins; +use common_datasource::object_store::LocalFileAccess; use common_grpc::channel_manager::{ChannelConfig, ChannelManager}; use common_meta::DatanodeId; use common_meta::cache::{CacheRegistryBuilder, LayeredCacheRegistryBuilder}; @@ -172,6 +173,7 @@ pub struct GreptimeDbClusterBuilder { frontend_auto_create_table: bool, shared_home_dir: Option>, meta_selector: Option, + local_file_access: LocalFileAccess, } impl GreptimeDbClusterBuilder { @@ -206,6 +208,7 @@ impl GreptimeDbClusterBuilder { frontend_auto_create_table: true, shared_home_dir: None, meta_selector: None, + local_file_access: LocalFileAccess::default(), } } @@ -269,6 +272,12 @@ impl GreptimeDbClusterBuilder { self } + /// Configure the frontend COPY sandbox for filesystem integration tests. + pub fn with_local_file_access(mut self, access: LocalFileAccess) -> Self { + self.local_file_access = access; + self + } + pub async fn build_with( &self, datanode_options: Vec, @@ -512,6 +521,7 @@ impl GreptimeDbClusterBuilder { meta_client.clone(), Arc::new(ProcessManager::new(fe_opts.grpc.server_addr.clone(), None)), ) + .with_local_file_access(self.local_file_access.clone()) .with_local_cache_invalidator(cache_registry) .try_build() .await diff --git a/tests-integration/src/grpc/flight.rs b/tests-integration/src/grpc/flight.rs index f37d5ee638a..fb27a273516 100644 --- a/tests-integration/src/grpc/flight.rs +++ b/tests-integration/src/grpc/flight.rs @@ -33,6 +33,8 @@ mod test { use client::region::RegionRequester; use client::{Client, Database}; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; + use common_error::ext::ErrorExt; + use common_error::status_code::StatusCode; use common_grpc::channel_manager::{ChannelConfig, ChannelManager}; use common_grpc::flight::do_put::{DoPutMetadata, DoPutResponse}; use common_grpc::flight::{FlightDecoder, FlightEncoder, FlightMessage}; @@ -49,6 +51,7 @@ mod test { use futures_util::{Stream, StreamExt}; use hyper_util::rt::TokioIo; use itertools::Itertools; + use rstest::rstest; use servers::grpc::builder::GrpcServerBuilder; use servers::grpc::flight::{ FlightCraft, FlightCraftWrapper, FlightRecordBatchSource, FlightRecordBatchStream, @@ -59,6 +62,7 @@ mod test { use servers::query_handler::grpc::GrpcQueryHandler; use servers::server::Server; use session::context::QueryContextRef; + use session::hints::INSERT_SKIP_WAL_HINT; use tokio::net::TcpListener; use tokio_stream::wrappers::TcpListenerStream; use tonic::transport::Server as TonicServer; @@ -67,7 +71,7 @@ mod test { use crate::cluster::GreptimeDbClusterBuilder; use crate::grpc::query_and_expect; - use crate::test_util::{StorageType, setup_grpc_server}; + use crate::test_util::{MockInstanceImpl, StorageType, assert_wal_delta, setup_grpc_server}; use crate::tests::test_util::MockInstance; struct SlowFlightCraft; @@ -399,6 +403,68 @@ mod test { assert!(start.elapsed() >= Duration::from_secs(1)); assert!(matches!(output.data, OutputData::Stream(_))); } + + #[rstest] + #[case::standalone_single_region(false, false)] + #[case::standalone_partitioned(false, true)] + #[case::distributed_single_region(true, false)] + #[case::distributed_partitioned(true, true)] + #[tokio::test(flavor = "multi_thread")] + async fn test_flight_bulk_skip_wal(#[case] distributed: bool, #[case] partitioned: bool) { + let mut db = + MockInstanceImpl::new(&format!("flight_bulk_skip_wal_{partitioned}"), distributed) + .await; + let runtime = common_runtime::global_runtime().clone(); + let handler = GreptimeRequestHandler::new( + db.frontend(), + None, + Some(runtime.clone()), + FlightCompression::default(), + ); + let mut server = GrpcServerBuilder::new(GrpcServerConfig::default(), runtime) + .flight_handler(Arc::new(handler)) + .build(); + server.start("127.0.0.1:0".parse().unwrap()).await.unwrap(); + let client = Database::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + Client::with_urls(vec![server.bind_addr().unwrap().to_string()]), + ); + let partition = if partitioned { + " PARTITION ON COLUMNS (a) (a < 0, a >= 0)" + } else { + "" + }; + client.sql(&format!( + "CREATE TABLE foo (ts TIMESTAMP TIME INDEX, a INT NOT NULL, \"B\" STRING, PRIMARY KEY (a)){partition}" + )).await.unwrap(); + let mut before = db.flush_and_snapshot_wal().await; + assert_eq!(before.len(), if partitioned { 2 } else { 1 }); + // The same stream spans both partitions and includes multiple data messages. + // Repeated rows still exercise real writes; each round flushes before the next. + for hint in [None, Some("true"), Some("false"), None] { + let hints = hint.map(|value| [(INSERT_SKIP_WAL_HINT, value)]); + test_put_record_batches_with_hints( + &client, + create_record_batches(-4), + hints.as_ref().map_or(&[], |hints| hints.as_slice()), + ) + .await; + let after = db.flush_and_snapshot_wal().await; + assert_wal_delta(&before, &after, hint == Some("true")); + // Both regions must receive rows, not merely exist in the catalog. + for (id, &(flushed_sequence, _)) in &after { + assert!( + flushed_sequence > before[id].0, + "region {id:?} received no rows" + ); + } + before = after; + } + server.shutdown().await.unwrap(); + db.shutdown().await; + } + #[tokio::test(flavor = "multi_thread")] async fn test_standalone_flight_do_put() { common_telemetry::init_default_ut_logging(); @@ -720,12 +786,14 @@ mod test { let mut stale_fence_error = None; while let Some(batch) = stream.next().await { if let Err(err) = batch { - stale_fence_error = Some(format!("{err:?}")); + stale_fence_error = Some(err); break; } } - let err_msg = stale_fence_error.expect("expected stale snapshot fence rejection"); + let stale_fence_error = stale_fence_error.expect("expected stale snapshot fence rejection"); + assert_eq!(stale_fence_error.status_code(), StatusCode::RequestOutdated); + let err_msg = format!("{stale_fence_error:?}"); assert!( err_msg.contains("STALE_SNAPSHOT_FENCE") || err_msg.contains("RequestOutdated") @@ -735,6 +803,14 @@ mod test { } async fn test_put_record_batches(client: &Database, record_batches: Vec) { + test_put_record_batches_with_hints(client, record_batches, &[]).await; + } + + async fn test_put_record_batches_with_hints( + client: &Database, + record_batches: Vec, + hints: &[(&str, &str)], + ) { let requests_count = record_batches.len(); let schema = record_batches[0].schema.arrow_schema().clone(); @@ -767,7 +843,7 @@ mod test { ) .boxed(); - let response_stream = client.do_put(stream).await.unwrap(); + let response_stream = client.do_put_with_hints(stream, hints).await.unwrap(); let responses = response_stream.collect::>().await; let responses_count = responses.len(); diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs index 4d449354778..e0a749b813f 100644 --- a/tests-integration/src/test_util.rs +++ b/tests-integration/src/test_util.rs @@ -81,7 +81,7 @@ use store_api::storage::RegionId; use crate::cluster::{GreptimeDbCluster, GreptimeDbClusterBuilder}; use crate::standalone::{GreptimeDbStandalone, GreptimeDbStandaloneBuilder}; -/// Maps `(node_id, region_id)` to `(written_bytes, flushed_entry_id)`. +/// Maps `(node_id, region_id)` to `(flushed_sequence, flushed_entry_id)`. pub type WalSnapshot = BTreeMap<(u64, RegionId), (u64, u64)>; /// Flush user data regions before observing their persisted WAL watermarks. @@ -101,7 +101,7 @@ async fn flush_and_snapshot_region_wal(engine: &MitoEngine) -> WalSnapshot { snapshot.insert( (0, id), ( - statistic.written_bytes, + region.flushed_sequence(), statistic.manifest.data_flushed_entry_id(), ), ); @@ -117,9 +117,9 @@ pub fn assert_wal_delta(before: &WalSnapshot, after: &WalSnapshot, skip_wal: boo "warm up table creation before taking the snapshot" ); let mut written = 0; - for (id, &(written_bytes, flushed_entry_id)) in after { - let (previous_written_bytes, previous_flushed_entry_id) = before[id]; - if written_bytes > previous_written_bytes { + for (id, &(flushed_sequence, flushed_entry_id)) in after { + let (previous_flushed_sequence, previous_flushed_entry_id) = before[id]; + if flushed_sequence > previous_flushed_sequence { written += 1; if skip_wal { assert_eq!( @@ -843,11 +843,27 @@ async fn setup_test_prom_app_with_frontend_inner( let sql = "INSERT INTO mito(host, val, ts) VALUES (1, 1.1, 0)"; run_sql(sql, &instance).await; + let http_server = build_test_prom_server( + instance.fe_instance().clone(), + enable_batcher, + experimental_enable_prometheus_native_histogram, + ) + .with_greptime_config_options(instance.opts.datanode_options().to_toml().unwrap()) + .build(); + let app = http_server.build(http_server.make_app()).unwrap(); + (app, instance.guard) +} + +/// Builds Prometheus HTTP routes for either a standalone or distributed frontend. +pub fn build_test_prom_server( + frontend_ref: Arc, + enable_batcher: bool, + experimental_enable_prometheus_native_histogram: bool, +) -> HttpServerBuilder { let http_opts = HttpOptions { addr: format!("127.0.0.1:{}", ports::get_port()), ..Default::default() }; - let frontend_ref = instance.fe_instance().clone(); // Mirror the production wiring at `frontend::server`: build the batcher from the // instance's managers. A short flush interval keeps the test responsive. let pending_rows_batcher = if enable_batcher { @@ -868,9 +884,10 @@ async fn setup_test_prom_app_with_frontend_inner( } else { None }; - let http_server = HttpServerBuilder::new(http_opts) + assert_eq!(pending_rows_batcher.is_some(), enable_batcher); + HttpServerBuilder::new(http_opts) .with_sql_handler(frontend_ref.clone()) - .with_logs_handler(instance.fe_instance().clone()) + .with_logs_handler(frontend_ref.clone()) .with_prom_handler( frontend_ref.clone(), Some(frontend_ref.clone()), @@ -880,10 +897,6 @@ async fn setup_test_prom_app_with_frontend_inner( pending_rows_batcher, ) .with_prometheus_handler(frontend_ref) - .with_greptime_config_options(instance.opts.datanode_options().to_toml().unwrap()) - .build(); - let app = http_server.build(http_server.make_app()).unwrap(); - (app, instance.guard) } pub async fn setup_grpc_server( diff --git a/tests-integration/src/tests/gc.rs b/tests-integration/src/tests/gc.rs index badaef5f7ac..20242223422 100644 --- a/tests-integration/src/tests/gc.rs +++ b/tests-integration/src/tests/gc.rs @@ -208,6 +208,7 @@ async fn test_gc_basic(store_type: &StorageType) { let procedure = BatchGcProcedure::new( metasrv.mailbox().clone(), metasrv.table_metadata_manager().clone(), + metasrv.runtime_switch_manager().clone(), metasrv.options().grpc.server_addr.clone(), regions.clone(), false, // full_file_listing diff --git a/tests-integration/src/tests/gc/repart.rs b/tests-integration/src/tests/gc/repart.rs index 7e7a3edd5a5..93969dbb3b3 100644 --- a/tests-integration/src/tests/gc/repart.rs +++ b/tests-integration/src/tests/gc/repart.rs @@ -90,6 +90,7 @@ CREATE TABLE test_cleanup_repartition ( let mut procedure = BatchGcProcedure::new_update_repartition_for_test( metasrv.mailbox().clone(), metasrv.table_metadata_manager().clone(), + metasrv.runtime_switch_manager().clone(), metasrv.options().grpc.server_addr.clone(), regions, manifest, @@ -183,6 +184,7 @@ async fn test_cleanup_region_repartition_preserve_uninvolved_entries() { let mut procedure = BatchGcProcedure::new_update_repartition_for_test( metasrv.mailbox().clone(), metasrv.table_metadata_manager().clone(), + metasrv.runtime_switch_manager().clone(), metasrv.options().grpc.server_addr.clone(), regions, manifest, @@ -267,6 +269,7 @@ async fn test_cleanup_region_repartition_remove_when_tmp_refs_empty() { let mut procedure = BatchGcProcedure::new_update_repartition_for_test( metasrv.mailbox().clone(), metasrv.table_metadata_manager().clone(), + metasrv.runtime_switch_manager().clone(), metasrv.options().grpc.server_addr.clone(), regions, manifest, diff --git a/tests-integration/src/tests/instance_noop_wal_test.rs b/tests-integration/src/tests/instance_noop_wal_test.rs index 73eeec26728..298fc119342 100644 --- a/tests-integration/src/tests/instance_noop_wal_test.rs +++ b/tests-integration/src/tests/instance_noop_wal_test.rs @@ -70,8 +70,10 @@ async fn test_mito_engine() { .await .data; // Unflushed data should be lost. - let expected = r#"++ -++"#; + let expected = r#"+------+-----+--------+----+ +| host | cpu | memory | ts | ++------+-----+--------+----+ ++------+-----+--------+----+"#; check_output_stream(output, expected).await; let output = execute_sql( diff --git a/tests-integration/src/tests/instance_test.rs b/tests-integration/src/tests/instance_test.rs index 3cd625ba204..f5e94319bac 100644 --- a/tests-integration/src/tests/instance_test.rs +++ b/tests-integration/src/tests/instance_test.rs @@ -3271,8 +3271,10 @@ WITH( .await .data; let expected = "\ -++ -++"; ++---+----+ +| a | ts | ++---+----+ ++---+----+"; check_output_stream(output, expected).await; let output = execute_sql(&frontend, "drop table test_table").await.data; diff --git a/tests-integration/tests/export_logical_tables.rs b/tests-integration/tests/export_logical_tables.rs new file mode 100644 index 00000000000..4aeb745b885 --- /dev/null +++ b/tests-integration/tests/export_logical_tables.rs @@ -0,0 +1,198 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use common_query::OutputData; +use common_time::Timestamp; +use common_time::range::TimestampRange; +use frontend::instance::Instance; +use operator::statement::export_logical_tables::{LogicalTableExport, LogicalTableExportLimits}; +use session::context::QueryContext; +use tests_integration::cluster::GreptimeDbClusterBuilder; +use tests_integration::standalone::GreptimeDbStandaloneBuilder; +use tests_integration::test_util::execute_sql as sql; +use tokio_util::sync::CancellationToken; + +async fn table(instance: &Arc, name: &str) -> table::TableRef { + instance + .catalog_manager() + .table("greptime", "public", name, Some(&QueryContext::arc())) + .await + .unwrap() + .unwrap() +} + +async fn values(instance: &Arc, query: &str) -> Vec> { + let output = sql(instance, query).await; + let batches = match output.data { + OutputData::Stream(stream) => common_recordbatch::util::collect_batches(stream) + .await + .unwrap(), + OutputData::RecordBatches(batches) => batches, + _ => panic!("expected query rows"), + }; + batches + .iter() + .flat_map(|b| { + let columns = datatypes::vectors::Helper::try_into_vectors(b.columns()).unwrap(); + (0..b.num_rows()) + .map(|row| columns.iter().map(|col| col.get(row)).collect::>()) + .collect::>() + }) + .collect() +} + +async fn roundtrip(instance: &Arc) { + let destination = tempfile::tempdir_in(common_test_util::find_workspace_path(".")).unwrap(); + for (physical, encoding) in [("phy", "dense"), ("other_phy", "sparse")] { + sql(instance, &format!("CREATE TABLE {physical} (ts TIMESTAMP TIME INDEX, val DOUBLE, host STRING PRIMARY KEY) PARTITION ON COLUMNS (host) (host < 'm', host >= 'm') ENGINE=metric WITH (physical_metric_table='', primary_key_encoding='{encoding}')")).await; + for (suffix, extra, key) in [ + ("cpu.v1", "zone_tag STRING,", ", zone_tag"), + ("requests", "service_tag STRING,", ", service_tag"), + ("empty", "", ""), + ] { + let name = format!("{physical}_{suffix}"); + sql(instance, &format!("CREATE TABLE \"{name}\" (host STRING, {extra} val DOUBLE, ts TIMESTAMP TIME INDEX, PRIMARY KEY(host{key})) ENGINE=metric WITH (on_physical_table='{physical}')")).await; + if suffix != "empty" { + sql(instance, &format!("INSERT INTO \"{name}\" (host,val,ts) VALUES ('a',1,1),('a',NULL,2),('z',3,3),('z',4,4)")).await; + } + } + // This live but unselected table models rows outside the routing whitelist. + sql(instance, &format!("CREATE TABLE {physical}_excluded (host STRING, huge_tag STRING, val DOUBLE, ts TIMESTAMP TIME INDEX, PRIMARY KEY(host, huge_tag)) ENGINE=metric WITH (on_physical_table='{physical}')")).await; + sql( + instance, + &format!("INSERT INTO {physical}_excluded (host, huge_tag, val, ts) VALUES ('z','ignore',9,2)"), + ) + .await; + let names = ["cpu.v1", "requests", "empty"].map(|suffix| format!("{physical}_{suffix}")); + let tables = vec![ + table(instance, &names[0]).await, + table(instance, &names[1]).await, + table(instance, &names[2]).await, + ]; + let renamed = format!("renamed_{physical}"); + sql( + instance, + &format!("ALTER TABLE {physical} RENAME {renamed}"), + ) + .await; + let unit = LogicalTableExport::try_new(table(instance, &renamed).await, &tables).unwrap(); + let range = + TimestampRange::new(Timestamp::new_millisecond(2), Timestamp::new_millisecond(4)) + .unwrap(); + sql(instance, &format!("CREATE TABLE target_{physical} (ts TIMESTAMP TIME INDEX, val DOUBLE, host STRING PRIMARY KEY) ENGINE=metric WITH (physical_metric_table='')")).await; + let mut limits = LogicalTableExportLimits::default(); + limits.writer.row_group_rows = 1; + for partitions in [1, 2, 4] { + let directory = destination.path().join(format!("{physical}_{partitions}")); + let mut ctx = QueryContext::with("greptime", "public"); + ctx.set_extension( + query::datafusion::QUERY_PARALLELISM_HINT, + partitions.to_string(), + ); + let summary = instance + .statement_executor() + .export_logical_tables( + &unit, + directory.to_str().unwrap(), + &Default::default(), + Some(&range), + limits, + &CancellationToken::new(), + Arc::new(ctx), + ) + .await + .unwrap(); + assert_eq!(summary.rows, 4); + assert_eq!(summary.files, 3); + assert_eq!(summary.skipped_rows, 1); + for (index, name) in names.iter().enumerate() { + let restored = format!("restore_{physical}_{partitions}_{index}"); + let (extra, key) = [ + ("zone_tag STRING,", ", zone_tag"), + ("service_tag STRING,", ", service_tag"), + ("", ""), + ][index]; + sql(instance, &format!("CREATE TABLE {restored} (host STRING, {extra} val DOUBLE, ts TIMESTAMP TIME INDEX, PRIMARY KEY(host{key})) ENGINE=metric WITH (on_physical_table='target_{physical}')")).await; + sql( + instance, + &format!( + "COPY {restored} FROM '{}/{}.parquet' WITH (FORMAT='parquet')", + directory.display(), + name + ), + ) + .await; + let expected = values( + instance, + &format!("SELECT * FROM \"{name}\" WHERE ts >= 2 AND ts < 4 ORDER BY host, ts"), + ) + .await; + let actual = values( + instance, + &format!("SELECT * FROM {restored} ORDER BY host, ts"), + ) + .await; + assert_eq!(actual, expected); + } + } + let cancellation = CancellationToken::new(); + cancellation.cancel(); + let directory = destination.path().join(format!("cancel_{physical}")); + let result = instance + .statement_executor() + .export_logical_tables( + &unit, + directory.to_str().unwrap(), + &Default::default(), + None, + LogicalTableExportLimits::default(), + &cancellation, + QueryContext::arc(), + ) + .await; + assert!(matches!( + result, + Err(operator::error::Error::LogicalTableExportCancelled { .. }) + )); + assert!(!directory.exists()); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn physical_export_standalone_roundtrip() { + common_telemetry::init_default_ut_logging(); + let standalone = GreptimeDbStandaloneBuilder::new("physical_export") + .build() + .await; + roundtrip(standalone.fe_instance()).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn physical_export_distributed_roundtrip() { + common_telemetry::init_default_ut_logging(); + let cluster = GreptimeDbClusterBuilder::new("physical_export") + .await + .with_datanodes(2) + .with_local_file_access( + common_datasource::object_store::LocalFileAccess::sandboxed( + common_test_util::find_workspace_path("."), + ) + .unwrap(), + ) + .build(false) + .await; + roundtrip(cluster.fe_instance()).await; +} diff --git a/tests-integration/tests/grpc.rs b/tests-integration/tests/grpc.rs index 9607d611627..025d56b959a 100644 --- a/tests-integration/tests/grpc.rs +++ b/tests-integration/tests/grpc.rs @@ -1331,6 +1331,125 @@ async fn test_skip_wal_grpc_unary_stream_and_flight_sql(distributed: bool) { env.shutdown().await; } +#[apply(both_deployment_cases)] +async fn test_grpc_json2_row_inserts(distributed: bool) { + let mut env = MockInstanceImpl::new("grpc_json2_row_inserts", distributed).await; + let server = setup_grpc_server_for_frontend_instance(env.frontend(), None).await; + let addr = server.bind_addr().unwrap().to_string(); + let database = Database::new_with_dbname("greptime-public", Client::with_urls(vec![addr])); + database + .sql( + "CREATE TABLE grpc_json2 (ts TIMESTAMP TIME INDEX, host STRING PRIMARY KEY, j JSON2) \ + WITH (append_mode='true', 'memtable.type'='bulk')", + ) + .await + .unwrap(); + + let (datatype, extension) = + api::helper::ColumnDataTypeWrapper::try_from(datatypes::prelude::ConcreteDataType::json2( + datatypes::types::json_type::JsonNativeType::Null, + )) + .unwrap() + .into_parts(); + + // Send the all-NULL JSON2 batch separately to check its type marker. + for (index, (host, payload)) in [ + ( + "host-a", + Some(serde_json::json!({ + "active": true, + "nested": {"items": [1, "two", null, {"ok": false}], "ratio": 1.5}, + "tags": ["api", "prod"] + })), + ), + ( + "host-b", + Some(serde_json::json!({ + "active": false, + "nested": {"items": [-2, "three", null, {"ok": true}], "ratio": -0.25}, + "tags": ["worker"] + })), + ), + ("host-a", None), + ] + .into_iter() + .enumerate() + { + let output = database + .row_inserts(RowInsertRequests { + inserts: vec![RowInsertRequest { + table_name: "grpc_json2".into(), + rows: Some(Rows { + schema: vec![ + ColumnSchema { + column_name: "ts".into(), + datatype: ColumnDataType::TimestampMillisecond as i32, + semantic_type: SemanticType::Timestamp as i32, + ..Default::default() + }, + ColumnSchema { + column_name: "host".into(), + datatype: ColumnDataType::String as i32, + semantic_type: SemanticType::Tag as i32, + ..Default::default() + }, + ColumnSchema { + column_name: "j".into(), + datatype: datatype as i32, + datatype_extension: extension.clone(), + semantic_type: SemanticType::Field as i32, + ..Default::default() + }, + ], + rows: vec![Row { + values: vec![ + Value { + value_data: Some(ValueData::TimestampMillisecondValue( + index as i64, + )), + }, + Value { + value_data: Some(ValueData::StringValue(host.into())), + }, + Value { + value_data: payload.map(|x| { + ValueData::JsonValue(api::helper::encode_json_value( + x.into(), + )) + }), + }, + ], + }], + }), + }], + }) + .await + .unwrap(); + assert_eq!(output, 1); + } + + let output = database + .sql("SELECT host, j FROM grpc_json2 ORDER BY ts") + .await + .unwrap(); + let batches = match output.data { + OutputData::RecordBatches(batches) => batches, + OutputData::Stream(stream) => RecordBatches::try_collect(stream).await.unwrap(), + OutputData::AffectedRows(_) => unreachable!(), + }; + let pretty = batches.pretty_print().unwrap(); + let expected = r#"+--------+---------------------------------------------------------------------------------------------------+ +| host | j | ++--------+---------------------------------------------------------------------------------------------------+ +| host-a | {"active":true,"nested":{"items":[1,"two",null,{"ok":false}],"ratio":1.5},"tags":["api","prod"]} | +| host-b | {"active":false,"nested":{"items":[-2,"three",null,{"ok":true}],"ratio":-0.25},"tags":["worker"]} | +| host-a | | ++--------+---------------------------------------------------------------------------------------------------+"#; + assert_eq!(pretty, expected); + server.shutdown().await.unwrap(); + env.shutdown().await; +} + pub async fn test_insert_and_select(store_type: StorageType) { common_telemetry::init_default_ut_logging(); let (_db, fe_grpc_server) = setup_grpc_server(store_type, "test_insert_and_select").await; diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 84d1f7d957f..d6cb0cf13ec 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -80,7 +80,7 @@ use servers::request_memory_limiter::ServerMemoryLimiter; use standalone::options::StandaloneOptions; use table::table_name::TableName; use tests_integration::test_util::{ - MockInstanceImpl, StorageType, assert_wal_delta, setup_test_http_app, + MockInstanceImpl, StorageType, assert_wal_delta, build_test_prom_server, setup_test_http_app, setup_test_http_app_with_frontend, setup_test_http_app_with_frontend_and_slow_query_threshold, setup_test_http_app_with_frontend_and_user_provider, setup_test_prom_app_with_frontend, setup_test_prom_app_with_frontend_batched, setup_test_prom_app_with_frontend_native_histogram, @@ -2381,8 +2381,9 @@ manifest_checkpoint_distance = 10 experimental_manifest_keep_removed_file_count = 256 experimental_manifest_keep_removed_file_ttl = "1h" compress_manifest = false -experimental_series_index_root = "" +experimental_enable_series_index = false experimental_series_index_maintenance_interval = "5m" +experimental_series_index_bucket_width = "5days" experimental_compaction_memory_limit = "unlimited" experimental_compaction_on_exhausted = "wait" auto_flush_interval = "10m" @@ -3310,6 +3311,94 @@ pub async fn test_prometheus_remote_write_v2_native_histogram(store_type: Storag guard.remove_all().await; } +#[apply(both_deployment_cases)] +async fn test_prometheus_remote_write_batched_skip_wal(distributed: bool) { + check_prometheus_remote_write_batched_skip_wal(distributed, false).await; +} + +#[apply(both_deployment_cases)] +async fn test_prometheus_remote_write_v2_batched_skip_wal(distributed: bool) { + check_prometheus_remote_write_batched_skip_wal(distributed, true).await; +} + +async fn check_prometheus_remote_write_batched_skip_wal(distributed: bool, v2: bool) { + common_telemetry::init_default_ut_logging(); + let mut instance = + MockInstanceImpl::new(&format!("prom_bulk_skip_wal_v2_{v2}"), distributed).await; + let server = build_test_prom_server(instance.frontend(), true, false).build(); + let client = TestClient::new(server.build(server.make_app()).unwrap()).await; + + write_prometheus_skip_wal_sample(&client, v2, 1000, None).await; + wait_for_data(&client, "SELECT count(*) FROM wal_prom_bulk", "[[1]]").await; + let mut before = instance.flush_and_snapshot_wal().await; + // Reuse the same batcher across policy transitions to detect leaked policy. + for (index, hint) in [None, Some("true"), Some("false"), None] + .into_iter() + .enumerate() + { + write_prometheus_skip_wal_sample(&client, v2, (index as i64 + 2) * 1000, hint).await; + wait_for_data( + &client, + "SELECT count(*) FROM wal_prom_bulk", + &format!("[[{}]]", index + 2), + ) + .await; + let after = instance.flush_and_snapshot_wal().await; + assert_wal_delta(&before, &after, hint == Some("true")); + before = after; + } + instance.shutdown().await; +} + +async fn write_prometheus_skip_wal_sample( + client: &TestClient, + v2: bool, + timestamp: i64, + hint: Option<&str>, +) { + let payload = if v2 { + remote_write_v2::request_with_labels_and_samples( + vec![(prom_store::METRIC_NAME_LABEL, "wal_prom_bulk")], + vec![RemoteWriteV2Sample { + value: 1.0, + timestamp, + start_timestamp: 0, + }], + ) + .encode_to_vec() + } else { + WriteRequest { + timeseries: vec![TimeSeries { + labels: vec![Label { + name: prom_store::METRIC_NAME_LABEL.to_string(), + value: "wal_prom_bulk".to_string(), + }], + samples: vec![Sample { + value: 1.0, + timestamp, + }], + ..Default::default() + }], + ..Default::default() + } + .encode_to_vec() + }; + let mut request = client + .post("/v1/prometheus/write") + .header("Content-Encoding", "snappy") + .body(prom_store::snappy_compress(&payload).unwrap()); + if v2 { + request = request.header( + "Content-Type", + "application/x-protobuf;proto=io.prometheus.write.v2.Request", + ); + } + if let Some(hint) = hint { + request = request.header("x-greptime-insert-skip-wal", hint); + } + assert_eq!(request.send().await.status(), StatusCode::NO_CONTENT); +} + /// Covers the batched (pending-rows-batcher) Prometheus remote write path, which /// bypasses `PromStoreProtocolHandler::write`. Verifies the metric table is created /// asynchronously and still carries the Prometheus semantic identity stamped on the @@ -11240,7 +11329,7 @@ async fn check_http_skip_wal(name: &str, cases: &[HttpWalCase], distributed: boo if let Some(expected_written_nodes) = case.expected_written_nodes { let written_nodes = after .iter() - .filter(|(key, (written_bytes, _))| *written_bytes > before[*key].0) + .filter(|(key, (flushed_sequence, _))| *flushed_sequence > before[*key].0) .map(|((node_id, _), _)| *node_id) .collect::>(); assert_eq!( diff --git a/tests-integration/tests/main.rs b/tests-integration/tests/main.rs index 28a81155724..b61b6cd3e19 100644 --- a/tests-integration/tests/main.rs +++ b/tests-integration/tests/main.rs @@ -36,6 +36,7 @@ mod repartition; mod repartition_event; #[macro_use] mod repartition_expr_version; +mod export_logical_tables; mod mysql; mod reconciliation_event; mod view_ddl_event; diff --git a/tests-integration/tests/repartition.rs b/tests-integration/tests/repartition.rs index cf64f40aa00..c243113d928 100644 --- a/tests-integration/tests/repartition.rs +++ b/tests-integration/tests/repartition.rs @@ -24,7 +24,7 @@ use common_procedure::{ProcedureContext, ProcedureWithId, watcher}; use common_query::Output; use common_telemetry::info; use common_test_util::recordbatch::check_output_stream; -use common_test_util::temp_dir::create_temp_dir; +use common_test_util::temp_dir::{TempDir, create_temp_dir}; use common_wal::config::DatanodeWalConfig; use frontend::instance::Instance; use meta_srv::gc::{self, BatchGcProcedure, GcSchedulerOptions, GcTickerRef}; @@ -117,6 +117,189 @@ macro_rules! repartition_tests { }; } +/// COUNT must use visible rows rather than the full row counts of shared SSTs. +#[tokio::test(flavor = "multi_thread")] +async fn test_repartition_append_count_file() { + let (cluster, _home_guard) = append_count_cluster("repartition_append_count").await; + let instance = cluster.fe_instance(); + let table = "count_repartition"; + prepare_append_count_table(instance, table, "").await; + assert_append_count(instance, table, 100, 1).await; + flush_append_count_table(instance, table).await; + assert_append_count(instance, table, 100, 1).await; + + let sql = "ALTER TABLE count_repartition PARTITION ON COLUMNS (device_id) \ + (device_id < 50, device_id >= 50)"; + run_sql(instance, sql, QueryContext::arc()).await.unwrap(); + wait_for_append_count_regions(instance, table, 2).await; + assert_append_count(instance, table, 100, 0).await; + check_append_count_after_write(instance, table, 0).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_split_append_count_file() { + let (cluster, _home_guard) = append_count_cluster("split_append_count").await; + let instance = cluster.fe_instance(); + let table = "count_split"; + prepare_append_count_table( + instance, + table, + "PARTITION ON COLUMNS (device_id) (device_id < 50, device_id >= 50)", + ) + .await; + assert_append_count(instance, table, 100, 2).await; + flush_append_count_table(instance, table).await; + assert_append_count(instance, table, 100, 2).await; + + let sql = "ALTER TABLE count_split SPLIT PARTITION (device_id < 50) INTO \ + (device_id < 25, device_id >= 25 AND device_id < 50)"; + run_sql(instance, sql, QueryContext::arc()).await.unwrap(); + wait_for_append_count_regions(instance, table, 3).await; + assert_append_count(instance, table, 100, 1).await; + check_append_count_after_write(instance, table, 1).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_repartition_append_count_memtable_file() { + let (cluster, home_guard) = append_count_cluster("repartition_append_count_memtable").await; + assert!(home_guard.path().is_dir()); + let instance = cluster.fe_instance(); + let table = "count_repartition_memtable"; + prepare_append_count_table(instance, table, "").await; + assert_append_count(instance, table, 100, 1).await; + + // Entering staging must flush the populated memtable before changing partitions. + let sql = "ALTER TABLE count_repartition_memtable PARTITION ON COLUMNS (device_id) \ + (device_id < 50, device_id >= 50)"; + run_sql(instance, sql, QueryContext::arc()).await.unwrap(); + wait_for_append_count_regions(instance, table, 2).await; + assert_append_count(instance, table, 100, 0).await; + check_append_count_after_write(instance, table, 0).await; + assert!(home_guard.path().is_dir()); +} + +async fn append_count_cluster(name: &str) -> (GreptimeDbCluster, Arc) { + common_telemetry::init_default_ut_logging(); + let (store_config, _guard) = get_test_store_config(&StorageType::File); + let home_dir = Arc::new(create_temp_dir(name)); + let cluster = GreptimeDbClusterBuilder::new(name) + .await + .with_shared_home_dir(Arc::clone(&home_dir)) + .with_datanodes(3) + .with_store_config(store_config) + .with_datanode_wal_config(DatanodeWalConfig::Noop) + .with_metasrv_gc_config(GcSchedulerOptions { + enable: true, + ..Default::default() + }) + .with_datanode_gc_config(GcConfig { + enable: true, + ..Default::default() + }) + .build(true) + .await; + (cluster, home_dir) +} + +async fn prepare_append_count_table(instance: &Arc, table: &str, partitions: &str) { + let sql = format!( + "CREATE TABLE {table} (ts TIMESTAMP TIME INDEX, device_id INT) \ + {partitions} ENGINE=mito WITH(append_mode='true')" + ); + run_sql(instance, &sql, QueryContext::arc()).await.unwrap(); + let sql = format!( + "INSERT INTO {table} SELECT to_timestamp_millis(value), CAST(value AS INT) \ + FROM generate_series(0, 99)" + ); + run_sql(instance, &sql, QueryContext::arc()).await.unwrap(); +} + +async fn flush_append_count_table(instance: &Arc, table: &str) { + run_sql( + instance, + &format!("ADMIN flush_table('{table}')"), + QueryContext::arc(), + ) + .await + .unwrap(); +} + +async fn check_append_count_after_write( + instance: &Arc, + table: &str, + statistics_regions: usize, +) { + run_sql( + instance, + &format!("INSERT INTO {table} VALUES (to_timestamp_millis(100), 100)"), + QueryContext::arc(), + ) + .await + .unwrap(); + assert_append_count(instance, table, 101, statistics_regions).await; + flush_append_count_table(instance, table).await; + assert_append_count(instance, table, 101, statistics_regions).await; +} + +async fn wait_for_append_count_regions(instance: &Arc, table: &str, expected: usize) { + // Wait for frontend routing, not for COUNT to become correct. + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let plan = + append_count_query(instance, &format!("EXPLAIN ANALYZE SELECT * FROM {table}")) + .await; + if plan.matches("UnorderedScan: region=").count() == expected { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("new partition routes must become visible"); +} + +async fn append_count_query(instance: &Arc, sql: &str) -> String { + let output = run_sql(instance, sql, QueryContext::arc()).await.unwrap(); + let batches = match output.data { + common_query::OutputData::Stream(stream) => { + common_recordbatch::RecordBatches::try_collect(stream) + .await + .unwrap() + } + common_query::OutputData::RecordBatches(batches) => batches, + _ => panic!("expected query output: {sql}"), + }; + batches.pretty_print().unwrap() +} + +async fn assert_append_count( + instance: &Arc, + table: &str, + rows: usize, + statistics_regions: usize, +) { + let expected = format!("+-----+\n| n |\n+-----+\n| {rows:<3} |\n+-----+"); + let count = append_count_query(instance, &format!("SELECT count(*) AS n FROM {table}")).await; + assert_eq!(count, expected, "unbounded COUNT for {table}"); + let scanned_count = append_count_query( + instance, + &format!("SELECT count(*) AS n FROM (SELECT * FROM {table} LIMIT 1000)"), + ) + .await; + assert_eq!(scanned_count, expected, "scanned COUNT for {table}"); + // Only regions whose source row counts are exact may use statistics. + let plan = append_count_query( + instance, + &format!("EXPLAIN ANALYZE SELECT count(*) FROM {table}"), + ) + .await; + assert_eq!( + plan.matches("PlaceholderRowExec").count(), + statistics_regions, + "{plan}" + ); +} + #[tokio::test(flavor = "multi_thread")] async fn test_repartition_multi_hop_fulltext_index_gc_file() { if StorageType::File.test_on() { @@ -812,6 +995,7 @@ async fn trigger_table_gc(metasrv: &Arc, table_name: &str) { let procedure = BatchGcProcedure::new( metasrv.mailbox().clone(), metasrv.table_metadata_manager().clone(), + metasrv.runtime_switch_manager().clone(), metasrv.options().grpc.server_addr.clone(), region_ids.clone(), false, // full_file_listing diff --git a/tests-integration/tests/sql.rs b/tests-integration/tests/sql.rs index c95adb483ec..6c58ab0246c 100644 --- a/tests-integration/tests/sql.rs +++ b/tests-integration/tests/sql.rs @@ -85,6 +85,7 @@ macro_rules! sql_tests { test_postgres_datestyle, test_postgres_intervalstyle, test_postgres_parameter_inference, + test_postgres_regclass_bind_parameter, test_postgres_uint64_parameter, test_postgres_explain_bind_parameter, test_postgres_array_types, @@ -705,16 +706,16 @@ pub async fn test_postgres_crud(store_type: StorageType) { let expected_j = serde_json::json!({ "code": i, - "success": true, "payload": { "features": [ "serde", "json" ], "homepage": null - } + }, + "success": true }); - assert_eq!(json.to_string(), expected_j.to_string()); + assert_eq!(json, expected_j); } let rows = sqlx::query("select i from demo where i=$1") @@ -1370,6 +1371,115 @@ pub async fn test_postgres_parameter_inference(store_type: StorageType) { guard.remove_all().await; } +pub async fn test_postgres_regclass_bind_parameter(store_type: StorageType) { + let (mut guard, fe_pg_server) = + setup_pg_server(store_type, "test_postgres_regclass_bind_parameter").await; + let addr = fe_pg_server.bind_addr().unwrap().to_string(); + + let (client, connection) = tokio_postgres::connect(&format!("postgres://{addr}/public"), NoTls) + .await + .unwrap(); + + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + connection.await.unwrap(); + tx.send(()).unwrap(); + }); + + client + .simple_query( + "CREATE TABLE test_adbc_app_logs (\"service\" STRING, \"message\" STRING, ts TIMESTAMP TIME INDEX)", + ) + .await + .unwrap(); + + let pgadbc_query = "SELECT attr.attname FROM pg_catalog.pg_class AS cls \ + INNER JOIN pg_catalog.pg_attribute AS attr ON cls.oid = attr.attrelid \ + INNER JOIN pg_catalog.pg_type AS typ ON attr.atttypid = typ.oid \ + WHERE attr.attnum >= 0 AND cls.oid = $1::regclass::oid ORDER BY attr.attnum"; + let pgadbc_statement = client.prepare(pgadbc_query).await.unwrap(); + + for table_name in [ + "test_adbc_app_logs", + "\"test_adbc_app_logs\"", + "public.test_adbc_app_logs", + "\"public\".\"test_adbc_app_logs\"", + ] { + let rows = client + .query(&pgadbc_statement, &[&table_name]) + .await + .unwrap(); + assert!(!rows.is_empty(), "{table_name} should resolve to a table"); + assert_eq!( + rows.iter() + .map(|row| row.get::<_, String>(0)) + .collect::>(), + ["service", "message", "ts"] + ); + } + + let relname_statement = client + .prepare("SELECT cls.oid FROM pg_catalog.pg_class AS cls WHERE cls.relname = $1") + .await + .unwrap(); + let rows = client + .query(&relname_statement, &[&"\"test_adbc_app_logs\""]) + .await + .unwrap(); + assert!(rows.is_empty(), "quoted text is not a relation name"); + + client + .simple_query("CREATE DATABASE adbc_override") + .await + .unwrap(); + client + .simple_query( + "CREATE TABLE adbc_override.test_adbc_app_logs (override_column STRING, ts TIMESTAMP TIME INDEX)", + ) + .await + .unwrap(); + client + .simple_query("SET search_path TO adbc_override, public") + .await + .unwrap(); + + let rows = client + .query(&pgadbc_statement, &[&"test_adbc_app_logs"]) + .await + .unwrap(); + assert_eq!(rows[0].get::<_, String>(0), "override_column"); + + let rows = client + .query(&pgadbc_statement, &[&"public.test_adbc_app_logs"]) + .await + .unwrap(); + assert_eq!( + rows.iter() + .map(|row| row.get::<_, String>(0)) + .collect::>(), + ["service", "message", "ts"] + ); + + client + .simple_query("SET search_path TO public") + .await + .unwrap(); + client + .simple_query("DROP TABLE adbc_override.test_adbc_app_logs") + .await + .unwrap(); + client + .simple_query("DROP DATABASE adbc_override") + .await + .unwrap(); + + drop(client); + rx.await.unwrap(); + + let _ = fe_pg_server.shutdown().await; + guard.remove_all().await; +} + pub async fn test_postgres_uint64_parameter(store_type: StorageType) { let (mut guard, fe_pg_server) = setup_pg_server(store_type, "test_postgres_uint64_parameter").await; diff --git a/tests/cases/distributed/explain/order_by.result b/tests/cases/distributed/explain/order_by.result index 6ce8b4e170f..e5781659521 100644 --- a/tests/cases/distributed/explain/order_by.result +++ b/tests/cases/distributed/explain/order_by.result @@ -125,9 +125,8 @@ EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY t DESC LIMIT 5; |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_ProjectionExec: expr=[i@0 as i, t@1 as alias_ts] REDACTED -|_|_|_SortPreservingMergeExec: [test_pk.t__temp__0@2 DESC], fetch=5 REDACTED +|_|_|_SortPreservingMergeExec: [t@1 DESC], fetch=5 REDACTED |_|_|_SortExec: TopK(fetch=5), expr=[t@1 DESC], preserve_partitioning=[true], filter=[t@1 IS NULL OR t@1 > 2] REDACTED -|_|_|_ProjectionExec: expr=[i@0 as i, t@1 as t, t@1 as test_pk.t__temp__0] REDACTED |_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED |_|_|_| |_|_| Total rows: 5_| diff --git a/tests/cases/distributed/explain/step_aggr.result b/tests/cases/distributed/explain/step_aggr.result index 89634a3cc6d..a69a82352b3 100644 --- a/tests/cases/distributed/explain/step_aggr.result +++ b/tests/cases/distributed/explain/step_aggr.result @@ -58,15 +58,15 @@ FROM | plan_type_| plan_| +-+-+ | logical_plan_| Projection: count(integers.i), sum(integers.i), uddsketch_calc(Float64(0.5), uddsketch_state(Int64(128),Float64(0.01),integers.i)), hll_count(hll(integers.i))_| -|_|_Aggregate: groupBy=[[]], aggr=[[__count_merge(__count_state(integers.i)) AS count(integers.i), __sum_merge(__sum_state(integers.i)) AS sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) AS uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) AS hll(integers.i)]] | +|_|_Aggregate: groupBy=[[]], aggr=[[__count_merge(__count_state(integers.i)) AS count(integers.i), __sum_merge(__sum_state(integers.i)) AS sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) AS uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) AS hll(integers.i)]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[]], aggr=[[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128), Float64(0.01), CAST(integers.i AS Float64)), __hll_state(CAST(integers.i AS Utf8))]]_| |_|_TableScan: integers_| |_| ]]_| | physical_plan | ProjectionExec: expr=[count(integers.i)@0 as count(integers.i), sum(integers.i)@1 as sum(integers.i), uddsketch_calc(0.5, uddsketch_state(Int64(128),Float64(0.01),integers.i)@2) as uddsketch_calc(Float64(0.5),uddsketch_state(Int64(128),Float64(0.01),integers.i)), hll_count(hll(integers.i)@3) as hll_count(hll(integers.i))]_| -|_|_AggregateExec: mode=Final, gby=[], aggr=[count(integers.i), sum(integers.i), uddsketch_state(Int64(128),Float64(0.01),integers.i), hll(integers.i)]_| +|_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i), __sum_merge(__sum_state(integers.i)) as sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) as uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) as hll(integers.i)]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(integers.i), sum(integers.i), uddsketch_state(Int64(128),Float64(0.01),integers.i), hll(integers.i)]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i), __sum_merge(__sum_state(integers.i)) as sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) as uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) as hll(integers.i)] | |_|_RepartitionExec: partitioning=REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -94,9 +94,9 @@ FROM | stage | node | plan_| +-+-+-+ | 0_| 0_|_ProjectionExec: expr=[count(integers.i)@0 as count(integers.i), sum(integers.i)@1 as sum(integers.i), uddsketch_calc(0.5, uddsketch_state(Int64(128),Float64(0.01),integers.i)@2) as uddsketch_calc(Float64(0.5),uddsketch_state(Int64(128),Float64(0.01),integers.i)), hll_count(hll(integers.i)@3) as hll_count(hll(integers.i))] REDACTED -|_|_|_AggregateExec: mode=Final, gby=[], aggr=[count(integers.i), sum(integers.i), uddsketch_state(Int64(128),Float64(0.01),integers.i), hll(integers.i)] REDACTED +|_|_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i), __sum_merge(__sum_state(integers.i)) as sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) as uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) as hll(integers.i)] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(integers.i), sum(integers.i), uddsketch_state(Int64(128),Float64(0.01),integers.i), hll(integers.i)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i), __sum_merge(__sum_state(integers.i)) as sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) as uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) as hll(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -147,9 +147,9 @@ FROM |_| Aggregate: groupBy=[[]], aggr=[[__avg_state(CAST(integers.i AS Float64))]]_| |_|_TableScan: integers_| |_| ]]_| -| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[avg(integers.i)]_| +| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[__avg_merge(__avg_state(integers.i)) as avg(integers.i)]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[avg(integers.i)]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__avg_merge(__avg_state(integers.i)) as avg(integers.i)] | |_|_RepartitionExec: partitioning=REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -172,9 +172,9 @@ FROM +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[avg(integers.i)] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__avg_merge(__avg_state(integers.i)) as avg(integers.i)] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[avg(integers.i)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__avg_merge(__avg_state(integers.i)) as avg(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -242,7 +242,7 @@ ORDER BY +-+-+ | logical_plan_| Sort: integers.ts ASC NULLS LAST_| |_|_Projection: integers.ts, count(integers.i), sum(integers.i), uddsketch_calc(Float64(0.5), uddsketch_state(Int64(128),Float64(0.01),integers.i)), hll_count(hll(integers.i))_| -|_|_Aggregate: groupBy=[[integers.ts]], aggr=[[__count_merge(__count_state(integers.i)) AS count(integers.i), __sum_merge(__sum_state(integers.i)) AS sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) AS uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) AS hll(integers.i)]] | +|_|_Aggregate: groupBy=[[integers.ts]], aggr=[[__count_merge(__count_state(integers.i)) AS count(integers.i), __sum_merge(__sum_state(integers.i)) AS sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) AS uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) AS hll(integers.i)]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[integers.ts]], aggr=[[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128), Float64(0.01), CAST(integers.i AS Float64)), __hll_state(CAST(integers.i AS Utf8))]]_| |_|_TableScan: integers_| @@ -250,9 +250,9 @@ ORDER BY | physical_plan | SortPreservingMergeExec: [ts@0 ASC NULLS LAST]_| |_|_SortExec: expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[true]_| |_|_ProjectionExec: expr=[ts@0 as ts, count(integers.i)@1 as count(integers.i), sum(integers.i)@2 as sum(integers.i), uddsketch_calc(0.5, uddsketch_state(Int64(128),Float64(0.01),integers.i)@3) as uddsketch_calc(Float64(0.5),uddsketch_state(Int64(128),Float64(0.01),integers.i)), hll_count(hll(integers.i)@4) as hll_count(hll(integers.i))]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[count(integers.i), sum(integers.i), uddsketch_state(Int64(128),Float64(0.01),integers.i), hll(integers.i)]_| +|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i), __sum_merge(__sum_state(integers.i)) as sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) as uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) as hll(integers.i)] | |_|_RepartitionExec: partitioning=REDACTED -|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[count(integers.i), sum(integers.i), uddsketch_state(Int64(128),Float64(0.01),integers.i), hll(integers.i)]_| +|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i), __sum_merge(__sum_state(integers.i)) as sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) as uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) as hll(integers.i)]_| |_|_RepartitionExec: partitioning=REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -287,9 +287,9 @@ ORDER BY | 0_| 0_|_SortPreservingMergeExec: [ts@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_ProjectionExec: expr=[ts@0 as ts, count(integers.i)@1 as count(integers.i), sum(integers.i)@2 as sum(integers.i), uddsketch_calc(0.5, uddsketch_state(Int64(128),Float64(0.01),integers.i)@3) as uddsketch_calc(Float64(0.5),uddsketch_state(Int64(128),Float64(0.01),integers.i)), hll_count(hll(integers.i)@4) as hll_count(hll(integers.i))] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[count(integers.i), sum(integers.i), uddsketch_state(Int64(128),Float64(0.01),integers.i), hll(integers.i)] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i), __sum_merge(__sum_state(integers.i)) as sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) as uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) as hll(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[count(integers.i), sum(integers.i), uddsketch_state(Int64(128),Float64(0.01),integers.i), hll(integers.i)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i), __sum_merge(__sum_state(integers.i)) as sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) as uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) as hll(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -357,7 +357,7 @@ ORDER BY +-+-+ | logical_plan_| Sort: time_window ASC NULLS LAST_| |_|_Projection: date_bin(Utf8("2 seconds"),integers.ts) AS time_window, count(integers.i), sum(integers.i), uddsketch_calc(Float64(0.5), uddsketch_state(Int64(128),Float64(0.01),integers.i)), hll_count(hll(integers.i))_| -|_|_Aggregate: groupBy=[[date_bin(Utf8("2 seconds"),integers.ts)]], aggr=[[__count_merge(__count_state(integers.i)) AS count(integers.i), __sum_merge(__sum_state(integers.i)) AS sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) AS uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) AS hll(integers.i)]] | +|_|_Aggregate: groupBy=[[date_bin(Utf8("2 seconds"),integers.ts)]], aggr=[[__count_merge(__count_state(integers.i)) AS count(integers.i), __sum_merge(__sum_state(integers.i)) AS sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) AS uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) AS hll(integers.i)]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 2000000000 }"), integers.ts) AS date_bin(Utf8("2 seconds"),integers.ts)]], aggr=[[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128), Float64(0.01), CAST(integers.i AS Float64)), __hll_state(CAST(integers.i AS Utf8))]]_| |_|_TableScan: integers_| @@ -365,9 +365,9 @@ ORDER BY | physical_plan | SortPreservingMergeExec: [time_window@0 ASC NULLS LAST]_| |_|_SortExec: expr=[time_window@0 ASC NULLS LAST], preserve_partitioning=[true]_| |_|_ProjectionExec: expr=[date_bin(Utf8("2 seconds"),integers.ts)@0 as time_window, count(integers.i)@1 as count(integers.i), sum(integers.i)@2 as sum(integers.i), uddsketch_calc(0.5, uddsketch_state(Int64(128),Float64(0.01),integers.i)@3) as uddsketch_calc(Float64(0.5),uddsketch_state(Int64(128),Float64(0.01),integers.i)), hll_count(hll(integers.i)@4) as hll_count(hll(integers.i))]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("2 seconds"),integers.ts)@0 as date_bin(Utf8("2 seconds"),integers.ts)], aggr=[count(integers.i), sum(integers.i), uddsketch_state(Int64(128),Float64(0.01),integers.i), hll(integers.i)]_| +|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("2 seconds"),integers.ts)@0 as date_bin(Utf8("2 seconds"),integers.ts)], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i), __sum_merge(__sum_state(integers.i)) as sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) as uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) as hll(integers.i)] | |_|_RepartitionExec: partitioning=REDACTED -|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("2 seconds"),integers.ts)@0 as date_bin(Utf8("2 seconds"),integers.ts)], aggr=[count(integers.i), sum(integers.i), uddsketch_state(Int64(128),Float64(0.01),integers.i), hll(integers.i)]_| +|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("2 seconds"),integers.ts)@0 as date_bin(Utf8("2 seconds"),integers.ts)], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i), __sum_merge(__sum_state(integers.i)) as sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) as uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) as hll(integers.i)]_| |_|_RepartitionExec: partitioning=REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -402,9 +402,9 @@ ORDER BY | 0_| 0_|_SortPreservingMergeExec: [time_window@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[time_window@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_ProjectionExec: expr=[date_bin(Utf8("2 seconds"),integers.ts)@0 as time_window, count(integers.i)@1 as count(integers.i), sum(integers.i)@2 as sum(integers.i), uddsketch_calc(0.5, uddsketch_state(Int64(128),Float64(0.01),integers.i)@3) as uddsketch_calc(Float64(0.5),uddsketch_state(Int64(128),Float64(0.01),integers.i)), hll_count(hll(integers.i)@4) as hll_count(hll(integers.i))] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("2 seconds"),integers.ts)@0 as date_bin(Utf8("2 seconds"),integers.ts)], aggr=[count(integers.i), sum(integers.i), uddsketch_state(Int64(128),Float64(0.01),integers.i), hll(integers.i)] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("2 seconds"),integers.ts)@0 as date_bin(Utf8("2 seconds"),integers.ts)], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i), __sum_merge(__sum_state(integers.i)) as sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) as uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) as hll(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("2 seconds"),integers.ts)@0 as date_bin(Utf8("2 seconds"),integers.ts)], aggr=[count(integers.i), sum(integers.i), uddsketch_state(Int64(128),Float64(0.01),integers.i), hll(integers.i)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("2 seconds"),integers.ts)@0 as date_bin(Utf8("2 seconds"),integers.ts)], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i), __sum_merge(__sum_state(integers.i)) as sum(integers.i), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),integers.i)) as uddsketch_state(Int64(128),Float64(0.01),integers.i), __hll_merge(__hll_state(integers.i)) as hll(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| diff --git a/tests/cases/distributed/explain/step_aggr_advance.result b/tests/cases/distributed/explain/step_aggr_advance.result index d3aaea79bcf..348021360ff 100644 --- a/tests/cases/distributed/explain/step_aggr_advance.result +++ b/tests/cases/distributed/explain/step_aggr_advance.result @@ -90,31 +90,31 @@ tql analyze (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr -- SQLNESS REPLACE (Hash.*) REDACTED tql explain (1752591864, 1752592164, '30s') sum by (a, b) (max_over_time(aggr_optimize_not [2m])); -+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| plan_type | plan | -+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| logical_plan | Sort: aggr_optimize_not.a ASC NULLS LAST, aggr_optimize_not.b ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST | -| | Aggregate: groupBy=[[aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.greptime_timestamp]], aggr=[[__sum_merge(__sum_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) AS sum(prom_max_over_time(greptime_timestamp_range,greptime_value))]] | -| | MergeScan [is_placeholder=false, remote_input=[ | -| | Aggregate: groupBy=[[aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.greptime_timestamp]], aggr=[[__sum_state(prom_max_over_time(greptime_timestamp_range,greptime_value))]] | -| | Filter: prom_max_over_time(greptime_timestamp_range,greptime_value) IS NOT NULL | -| | Projection: aggr_optimize_not.greptime_timestamp, prom_max_over_time(greptime_timestamp_range, greptime_value) AS prom_max_over_time(greptime_timestamp_range,greptime_value), aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d | -| | PromRangeManipulate: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp], values=["greptime_value"] | -| | PromSeriesNormalize: offset=[0], time index=[greptime_timestamp], filter NaN: [true] | -| | PromSeriesDivide: tags=["a", "b", "c", "d"] | -| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST | -| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) | -| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] | -| | ]] | -| physical_plan | SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, greptime_timestamp@2 ASC NULLS LAST] | -| | SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, greptime_timestamp@2 ASC NULLS LAST], preserve_partitioning=[true] | -| | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, greptime_timestamp@2 as greptime_timestamp], aggr=[sum(prom_max_over_time(greptime_timestamp_range,greptime_value))] | ++---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | Sort: aggr_optimize_not.a ASC NULLS LAST, aggr_optimize_not.b ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST | +| | Aggregate: groupBy=[[aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.greptime_timestamp]], aggr=[[__sum_merge(__sum_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) AS sum(prom_max_over_time(greptime_timestamp_range,greptime_value))]] | +| | MergeScan [is_placeholder=false, remote_input=[ | +| | Aggregate: groupBy=[[aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.greptime_timestamp]], aggr=[[__sum_state(prom_max_over_time(greptime_timestamp_range,greptime_value))]] | +| | Filter: prom_max_over_time(greptime_timestamp_range,greptime_value) IS NOT NULL | +| | Projection: aggr_optimize_not.greptime_timestamp, prom_max_over_time(greptime_timestamp_range, greptime_value) AS prom_max_over_time(greptime_timestamp_range,greptime_value), aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d | +| | PromRangeManipulate: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp], values=["greptime_value"] | +| | PromSeriesNormalize: offset=[0], time index=[greptime_timestamp], filter NaN: [true] | +| | PromSeriesDivide: tags=["a", "b", "c", "d"] | +| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST | +| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) | +| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] | +| | ]] | +| physical_plan | SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, greptime_timestamp@2 ASC NULLS LAST] | +| | SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, greptime_timestamp@2 ASC NULLS LAST], preserve_partitioning=[true] | +| | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, greptime_timestamp@2 as greptime_timestamp], aggr=[__sum_merge(__sum_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as sum(prom_max_over_time(greptime_timestamp_range,greptime_value))] | | | RepartitionExec: partitioning=REDACTED -| | AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b, greptime_timestamp@2 as greptime_timestamp], aggr=[sum(prom_max_over_time(greptime_timestamp_range,greptime_value))] | +| | AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b, greptime_timestamp@2 as greptime_timestamp], aggr=[__sum_merge(__sum_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as sum(prom_max_over_time(greptime_timestamp_range,greptime_value))] | | | RepartitionExec: partitioning=REDACTED | | MergeScanExec: REDACTED -| | | -+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| | | ++---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED @@ -130,9 +130,9 @@ tql analyze (1752591864, 1752592164, '30s') sum by (a, b) (max_over_time(aggr_op +-+-+-+ | 0_| 0_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, greptime_timestamp@2 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, greptime_timestamp@2 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, greptime_timestamp@2 as greptime_timestamp], aggr=[sum(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, greptime_timestamp@2 as greptime_timestamp], aggr=[__sum_merge(__sum_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as sum(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b, greptime_timestamp@2 as greptime_timestamp], aggr=[sum(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b, greptime_timestamp@2 as greptime_timestamp], aggr=[__sum_merge(__sum_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as sum(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -165,31 +165,31 @@ tql analyze (1752591864, 1752592164, '30s') sum by (a, b) (max_over_time(aggr_op -- SQLNESS REPLACE (Hash.*) REDACTED tql explain (1752591864, 1752592164, '30s') avg by (a) (max_over_time(aggr_optimize_not [2m])); -+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| plan_type | plan | -+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| logical_plan | Sort: aggr_optimize_not.a ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST | -| | Aggregate: groupBy=[[aggr_optimize_not.a, aggr_optimize_not.greptime_timestamp]], aggr=[[__avg_merge(__avg_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) AS avg(prom_max_over_time(greptime_timestamp_range,greptime_value))]] | -| | MergeScan [is_placeholder=false, remote_input=[ | -| | Aggregate: groupBy=[[aggr_optimize_not.a, aggr_optimize_not.greptime_timestamp]], aggr=[[__avg_state(prom_max_over_time(greptime_timestamp_range,greptime_value))]] | -| | Filter: prom_max_over_time(greptime_timestamp_range,greptime_value) IS NOT NULL | -| | Projection: aggr_optimize_not.greptime_timestamp, prom_max_over_time(greptime_timestamp_range, greptime_value) AS prom_max_over_time(greptime_timestamp_range,greptime_value), aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d | -| | PromRangeManipulate: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp], values=["greptime_value"] | -| | PromSeriesNormalize: offset=[0], time index=[greptime_timestamp], filter NaN: [true] | -| | PromSeriesDivide: tags=["a", "b", "c", "d"] | -| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST | -| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) | -| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] | -| | ]] | -| physical_plan | SortPreservingMergeExec: [a@0 ASC NULLS LAST, greptime_timestamp@1 ASC NULLS LAST] | -| | SortExec: expr=[a@0 ASC NULLS LAST, greptime_timestamp@1 ASC NULLS LAST], preserve_partitioning=[true] | -| | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, greptime_timestamp@1 as greptime_timestamp], aggr=[avg(prom_max_over_time(greptime_timestamp_range,greptime_value))] | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | Sort: aggr_optimize_not.a ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST | +| | Aggregate: groupBy=[[aggr_optimize_not.a, aggr_optimize_not.greptime_timestamp]], aggr=[[__avg_merge(__avg_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) AS avg(prom_max_over_time(greptime_timestamp_range,greptime_value))]] | +| | MergeScan [is_placeholder=false, remote_input=[ | +| | Aggregate: groupBy=[[aggr_optimize_not.a, aggr_optimize_not.greptime_timestamp]], aggr=[[__avg_state(prom_max_over_time(greptime_timestamp_range,greptime_value))]] | +| | Filter: prom_max_over_time(greptime_timestamp_range,greptime_value) IS NOT NULL | +| | Projection: aggr_optimize_not.greptime_timestamp, prom_max_over_time(greptime_timestamp_range, greptime_value) AS prom_max_over_time(greptime_timestamp_range,greptime_value), aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d | +| | PromRangeManipulate: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp], values=["greptime_value"] | +| | PromSeriesNormalize: offset=[0], time index=[greptime_timestamp], filter NaN: [true] | +| | PromSeriesDivide: tags=["a", "b", "c", "d"] | +| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST | +| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) | +| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] | +| | ]] | +| physical_plan | SortPreservingMergeExec: [a@0 ASC NULLS LAST, greptime_timestamp@1 ASC NULLS LAST] | +| | SortExec: expr=[a@0 ASC NULLS LAST, greptime_timestamp@1 ASC NULLS LAST], preserve_partitioning=[true] | +| | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, greptime_timestamp@1 as greptime_timestamp], aggr=[__avg_merge(__avg_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as avg(prom_max_over_time(greptime_timestamp_range,greptime_value))] | | | RepartitionExec: partitioning=REDACTED -| | AggregateExec: mode=Partial, gby=[a@0 as a, greptime_timestamp@1 as greptime_timestamp], aggr=[avg(prom_max_over_time(greptime_timestamp_range,greptime_value))] | +| | AggregateExec: mode=Partial, gby=[a@0 as a, greptime_timestamp@1 as greptime_timestamp], aggr=[__avg_merge(__avg_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as avg(prom_max_over_time(greptime_timestamp_range,greptime_value))] | | | RepartitionExec: partitioning=REDACTED | | MergeScanExec: REDACTED -| | | -+---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| | | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED @@ -205,9 +205,9 @@ tql analyze (1752591864, 1752592164, '30s') avg by (a) (max_over_time(aggr_optim +-+-+-+ | 0_| 0_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, greptime_timestamp@1 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[a@0 ASC NULLS LAST, greptime_timestamp@1 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, greptime_timestamp@1 as greptime_timestamp], aggr=[avg(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, greptime_timestamp@1 as greptime_timestamp], aggr=[__avg_merge(__avg_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as avg(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, greptime_timestamp@1 as greptime_timestamp], aggr=[avg(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, greptime_timestamp@1 as greptime_timestamp], aggr=[__avg_merge(__avg_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as avg(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -323,9 +323,9 @@ tql explain (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr | | ]] | | physical_plan | SortPreservingMergeExec: [b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] | | | SortExec: expr=[b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] | -| | AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[min(prom_max_over_time(greptime_timestamp_range,greptime_value))] | +| | AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_merge(__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] | | | RepartitionExec: partitioning=REDACTED -| | AggregateExec: mode=Partial, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[min(prom_max_over_time(greptime_timestamp_range,greptime_value))] | +| | AggregateExec: mode=Partial, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_merge(__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] | | | RepartitionExec: partitioning=REDACTED | | MergeScanExec: REDACTED | | | @@ -345,9 +345,9 @@ tql analyze (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr +-+-+-+ | 0_| 0_|_SortPreservingMergeExec: [b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[min(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_merge(__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[min(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_merge(__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -395,9 +395,9 @@ tql explain sum(aggr_optimize_not); | | ]] | | physical_plan | SortPreservingMergeExec: [greptime_timestamp@0 ASC NULLS LAST] | | | SortExec: expr=[greptime_timestamp@0 ASC NULLS LAST], preserve_partitioning=[true] | -| | AggregateExec: mode=FinalPartitioned, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[sum(aggr_optimize_not.greptime_value)] | +| | AggregateExec: mode=FinalPartitioned, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[__sum_merge(__sum_state(aggr_optimize_not.greptime_value)) as sum(aggr_optimize_not.greptime_value)] | | | RepartitionExec: partitioning=REDACTED -| | AggregateExec: mode=Partial, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[sum(aggr_optimize_not.greptime_value)] | +| | AggregateExec: mode=Partial, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[__sum_merge(__sum_state(aggr_optimize_not.greptime_value)) as sum(aggr_optimize_not.greptime_value)] | | | RepartitionExec: partitioning=REDACTED | | MergeScanExec: REDACTED | | | @@ -417,9 +417,9 @@ tql analyze sum(aggr_optimize_not); +-+-+-+ | 0_| 0_|_SortPreservingMergeExec: [greptime_timestamp@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[greptime_timestamp@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[sum(aggr_optimize_not.greptime_value)] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[__sum_merge(__sum_state(aggr_optimize_not.greptime_value)) as sum(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[sum(aggr_optimize_not.greptime_value)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[__sum_merge(__sum_state(aggr_optimize_not.greptime_value)) as sum(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -494,7 +494,7 @@ tql explain (1752591864, 1752592164, '30s') sum by (a, b, c) (rate(aggr_optimize | | Filter: aggr_optimize_not_count.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not_count.greptime_timestamp <= TimestampMillisecond(1752592164000, None) | | | TableScan: aggr_optimize_not_count, partial_filters=[aggr_optimize_not_count.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not_count.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] | | | ]] | -| physical_plan | ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp, sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))@5 / sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))@4 as aggr_optimize_not.sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000))) / aggr_optimize_not_count.sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))] | +| physical_plan | ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp, sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))@4 / sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))@5 as aggr_optimize_not.sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000))) / aggr_optimize_not_count.sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))] | | | REDACTED | | CoalescePartitionsExec | | | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp], aggr=[sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))] | @@ -527,7 +527,7 @@ tql analyze (1752591864, 1752592164, '30s') sum by (a, b, c) (rate(aggr_optimize +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp, sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))@5 / sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))@4 as aggr_optimize_not.sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000))) / aggr_optimize_not_count.sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))] REDACTED +| 0_| 0_|_ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp, sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))@4 / sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))@5 as aggr_optimize_not.sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000))) / aggr_optimize_not_count.sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))] REDACTED |_|_|_REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp], aggr=[sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))] REDACTED @@ -552,32 +552,6 @@ tql analyze (1752591864, 1752592164, '30s') sum by (a, b, c) (rate(aggr_optimize | 1_| 1_|_CooperativeExec REDACTED |_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED |_|_|_| -| 1_| 0_|_ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp, sum(prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)))@4 as sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))] REDACTED -|_|_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp], aggr=[sum(prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)))] REDACTED -|_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[a@2 as a, b@3 as b, c@4 as c, greptime_timestamp@0 as greptime_timestamp], aggr=[sum(prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)))] REDACTED -|_|_|_FilterExec: prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000))@1 IS NOT NULL REDACTED -|_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, prom_rate(greptime_timestamp_range@6, greptime_value@5, greptime_timestamp@4, 120000) as prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)), a@0 as a, b@1 as b, c@2 as c] REDACTED -|_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED -|_|_|_PromSeriesNormalizeExec: offset=[0], time index=[greptime_timestamp], filter NaN: [true] REDACTED -|_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED -|_|_|_| -| 1_| 1_|_ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp, sum(prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)))@4 as sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))] REDACTED -|_|_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp], aggr=[sum(prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)))] REDACTED -|_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[a@2 as a, b@3 as b, c@4 as c, greptime_timestamp@0 as greptime_timestamp], aggr=[sum(prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)))] REDACTED -|_|_|_FilterExec: prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000))@1 IS NOT NULL REDACTED -|_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, prom_rate(greptime_timestamp_range@6, greptime_value@5, greptime_timestamp@4, 120000) as prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)), a@0 as a, b@1 as b, c@2 as c] REDACTED -|_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED -|_|_|_PromSeriesNormalizeExec: offset=[0], time index=[greptime_timestamp], filter NaN: [true] REDACTED -|_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED -|_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -736,9 +710,9 @@ GROUP BY | | TableScan: aggr_optimize_not | | | ]] | | physical_plan | ProjectionExec: expr=[min(aggr_optimize_not.greptime_value)@2 as min(aggr_optimize_not.greptime_value)] | -| | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[min(aggr_optimize_not.greptime_value)] | +| | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[__min_merge(__min_state(aggr_optimize_not.greptime_value)) as min(aggr_optimize_not.greptime_value)] | | | RepartitionExec: partitioning=REDACTED -| | AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[min(aggr_optimize_not.greptime_value)] | +| | AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[__min_merge(__min_state(aggr_optimize_not.greptime_value)) as min(aggr_optimize_not.greptime_value)] | | | RepartitionExec: partitioning=REDACTED | | MergeScanExec: REDACTED | | | @@ -764,9 +738,9 @@ GROUP BY | stage | node | plan_| +-+-+-+ | 0_| 0_|_ProjectionExec: expr=[min(aggr_optimize_not.greptime_value)@2 as min(aggr_optimize_not.greptime_value)] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[min(aggr_optimize_not.greptime_value)] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[__min_merge(__min_state(aggr_optimize_not.greptime_value)) as min(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[min(aggr_optimize_not.greptime_value)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[__min_merge(__min_state(aggr_optimize_not.greptime_value)) as min(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -806,9 +780,9 @@ GROUP BY | | TableScan: aggr_optimize_not | | | ]] | | physical_plan | ProjectionExec: expr=[min(aggr_optimize_not.greptime_value)@2 + max(aggr_optimize_not.greptime_value)@3 as min(aggr_optimize_not.greptime_value) + max(aggr_optimize_not.greptime_value)] | -| | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[min(aggr_optimize_not.greptime_value), max(aggr_optimize_not.greptime_value)] | +| | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[__min_merge(__min_state(aggr_optimize_not.greptime_value)) as min(aggr_optimize_not.greptime_value), __max_merge(__max_state(aggr_optimize_not.greptime_value)) as max(aggr_optimize_not.greptime_value)] | | | RepartitionExec: partitioning=REDACTED -| | AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[min(aggr_optimize_not.greptime_value), max(aggr_optimize_not.greptime_value)] | +| | AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[__min_merge(__min_state(aggr_optimize_not.greptime_value)) as min(aggr_optimize_not.greptime_value), __max_merge(__max_state(aggr_optimize_not.greptime_value)) as max(aggr_optimize_not.greptime_value)] | | | RepartitionExec: partitioning=REDACTED | | MergeScanExec: REDACTED | | | @@ -834,9 +808,9 @@ GROUP BY | stage | node | plan_| +-+-+-+ | 0_| 0_|_ProjectionExec: expr=[min(aggr_optimize_not.greptime_value)@2 + max(aggr_optimize_not.greptime_value)@3 as min(aggr_optimize_not.greptime_value) + max(aggr_optimize_not.greptime_value)] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[min(aggr_optimize_not.greptime_value), max(aggr_optimize_not.greptime_value)] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[__min_merge(__min_state(aggr_optimize_not.greptime_value)) as min(aggr_optimize_not.greptime_value), __max_merge(__max_state(aggr_optimize_not.greptime_value)) as max(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[min(aggr_optimize_not.greptime_value), max(aggr_optimize_not.greptime_value)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[__min_merge(__min_state(aggr_optimize_not.greptime_value)) as min(aggr_optimize_not.greptime_value), __max_merge(__max_state(aggr_optimize_not.greptime_value)) as max(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -876,22 +850,22 @@ FROM GROUP BY a; -+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ -| plan_type | plan | -+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ -| logical_plan | Aggregate: groupBy=[[aggr_optimize_not.a]], aggr=[[__min_merge(__min_state(aggr_optimize_not.greptime_value)) AS min(aggr_optimize_not.greptime_value)]] | -| | MergeScan [is_placeholder=false, remote_input=[ | -| | Aggregate: groupBy=[[aggr_optimize_not.a]], aggr=[[__min_state(aggr_optimize_not.greptime_value)]] | -| | Projection: aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.greptime_value | -| | TableScan: aggr_optimize_not | -| | ]] | -| physical_plan | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[min(aggr_optimize_not.greptime_value)] | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | Aggregate: groupBy=[[aggr_optimize_not.a]], aggr=[[__min_merge(__min_state(aggr_optimize_not.greptime_value)) AS min(aggr_optimize_not.greptime_value)]] | +| | MergeScan [is_placeholder=false, remote_input=[ | +| | Aggregate: groupBy=[[aggr_optimize_not.a]], aggr=[[__min_state(aggr_optimize_not.greptime_value)]] | +| | Projection: aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.greptime_value | +| | TableScan: aggr_optimize_not | +| | ]] | +| physical_plan | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[__min_merge(__min_state(aggr_optimize_not.greptime_value)) as min(aggr_optimize_not.greptime_value)] | | | RepartitionExec: partitioning=REDACTED -| | AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[min(aggr_optimize_not.greptime_value)] | +| | AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[__min_merge(__min_state(aggr_optimize_not.greptime_value)) as min(aggr_optimize_not.greptime_value)] | | | RepartitionExec: partitioning=REDACTED | | MergeScanExec: REDACTED -| | | -+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ +| | | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED @@ -922,9 +896,9 @@ GROUP BY +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[min(aggr_optimize_not.greptime_value)] REDACTED +| 0_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[__min_merge(__min_state(aggr_optimize_not.greptime_value)) as min(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[min(aggr_optimize_not.greptime_value)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[__min_merge(__min_state(aggr_optimize_not.greptime_value)) as min(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -1088,7 +1062,7 @@ EXPLAIN SELECT pk_col_2, sum(val_col_1) FROM step_aggr_extended GROUP BY pk_col_ +-+-+ | logical_plan_| Sort: step_aggr_extended.pk_col_2 ASC NULLS LAST_| |_|_Filter: sum(step_aggr_extended.val_col_1) > Int64(300)_| -|_|_Aggregate: groupBy=[[step_aggr_extended.pk_col_2]], aggr=[[__sum_merge(__sum_state(step_aggr_extended.val_col_1)) AS sum(step_aggr_extended.val_col_1)]] | +|_|_Aggregate: groupBy=[[step_aggr_extended.pk_col_2]], aggr=[[__sum_merge(__sum_state(step_aggr_extended.val_col_1)) AS sum(step_aggr_extended.val_col_1)]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[step_aggr_extended.pk_col_2]], aggr=[[__sum_state(step_aggr_extended.val_col_1)]]_| |_|_TableScan: step_aggr_extended_| @@ -1096,9 +1070,9 @@ EXPLAIN SELECT pk_col_2, sum(val_col_1) FROM step_aggr_extended GROUP BY pk_col_ | physical_plan | SortPreservingMergeExec: [pk_col_2@0 ASC NULLS LAST]_| |_|_SortExec: expr=[pk_col_2@0 ASC NULLS LAST], preserve_partitioning=[true]_| |_|_FilterExec: sum(step_aggr_extended.val_col_1)@1 > 300_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[pk_col_2@0 as pk_col_2], aggr=[sum(step_aggr_extended.val_col_1)]_| +|_|_AggregateExec: mode=FinalPartitioned, gby=[pk_col_2@0 as pk_col_2], aggr=[__sum_merge(__sum_state(step_aggr_extended.val_col_1)) as sum(step_aggr_extended.val_col_1)] | |_|_RepartitionExec: partitioning=REDACTED -|_|_AggregateExec: mode=Partial, gby=[pk_col_2@0 as pk_col_2], aggr=[sum(step_aggr_extended.val_col_1)]_| +|_|_AggregateExec: mode=Partial, gby=[pk_col_2@0 as pk_col_2], aggr=[__sum_merge(__sum_state(step_aggr_extended.val_col_1)) as sum(step_aggr_extended.val_col_1)]_| |_|_RepartitionExec: partitioning=REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -1127,15 +1101,15 @@ EXPLAIN SELECT SUM(val_col_3), COUNT(val_col_2), COUNT(val_col_3), COUNT(*) FROM | plan_type_| plan_| +-+-+ | logical_plan_| Projection: sum(step_aggr_extended.val_col_3), count(step_aggr_extended.val_col_2), count(step_aggr_extended.val_col_3), count(Int64(1)) AS count(*)_| -|_|_Aggregate: groupBy=[[]], aggr=[[__sum_merge(__sum_state(step_aggr_extended.val_col_3)) AS sum(step_aggr_extended.val_col_3), __count_merge(__count_state(step_aggr_extended.val_col_2)) AS count(step_aggr_extended.val_col_2), __count_merge(__count_state(step_aggr_extended.val_col_3)) AS count(step_aggr_extended.val_col_3), __count_merge(__count_state(step_aggr_extended.ts)) AS count(Int64(1))]] | +|_|_Aggregate: groupBy=[[]], aggr=[[__sum_merge(__sum_state(step_aggr_extended.val_col_3)) AS sum(step_aggr_extended.val_col_3), __count_merge(__count_state(step_aggr_extended.val_col_2)) AS count(step_aggr_extended.val_col_2), __count_merge(__count_state(step_aggr_extended.val_col_3)) AS count(step_aggr_extended.val_col_3), __count_merge(__count_state(step_aggr_extended.ts)) AS count(Int64(1))]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[]], aggr=[[__sum_state(step_aggr_extended.val_col_3), __count_state(step_aggr_extended.val_col_2), __count_state(step_aggr_extended.val_col_3), __count_state(step_aggr_extended.ts)]]_| |_|_TableScan: step_aggr_extended_| |_| ]]_| | physical_plan | ProjectionExec: expr=[sum(step_aggr_extended.val_col_3)@0 as sum(step_aggr_extended.val_col_3), count(step_aggr_extended.val_col_2)@1 as count(step_aggr_extended.val_col_2), count(step_aggr_extended.val_col_3)@2 as count(step_aggr_extended.val_col_3), count(Int64(1))@3 as count(*)]_| -|_|_AggregateExec: mode=Final, gby=[], aggr=[sum(step_aggr_extended.val_col_3), count(step_aggr_extended.val_col_2), count(step_aggr_extended.val_col_3), count(Int64(1))]_| +|_|_AggregateExec: mode=Final, gby=[], aggr=[__sum_merge(__sum_state(step_aggr_extended.val_col_3)) as sum(step_aggr_extended.val_col_3), __count_merge(__count_state(step_aggr_extended.val_col_2)) as count(step_aggr_extended.val_col_2), __count_merge(__count_state(step_aggr_extended.val_col_3)) as count(step_aggr_extended.val_col_3), __count_merge(__count_state(step_aggr_extended.ts)) as count(Int64(1))]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[sum(step_aggr_extended.val_col_3), count(step_aggr_extended.val_col_2), count(step_aggr_extended.val_col_3), count(Int64(1))]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__sum_merge(__sum_state(step_aggr_extended.val_col_3)) as sum(step_aggr_extended.val_col_3), __count_merge(__count_state(step_aggr_extended.val_col_2)) as count(step_aggr_extended.val_col_2), __count_merge(__count_state(step_aggr_extended.val_col_3)) as count(step_aggr_extended.val_col_3), __count_merge(__count_state(step_aggr_extended.ts)) as count(Int64(1))] | |_|_RepartitionExec: partitioning=REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -1163,14 +1137,14 @@ EXPLAIN SELECT MIN(pk_col_1), MAX(val_col_2) FROM step_aggr_extended; +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__min_merge(__min_state(step_aggr_extended.pk_col_1)) AS min(step_aggr_extended.pk_col_1), __max_merge(__max_state(step_aggr_extended.val_col_2)) AS max(step_aggr_extended.val_col_2)]] | +| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__min_merge(__min_state(step_aggr_extended.pk_col_1)) AS min(step_aggr_extended.pk_col_1), __max_merge(__max_state(step_aggr_extended.val_col_2)) AS max(step_aggr_extended.val_col_2)]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[]], aggr=[[__min_state(CAST(step_aggr_extended.pk_col_1 AS Utf8)), __max_state(step_aggr_extended.val_col_2)]]_| |_|_TableScan: step_aggr_extended_| |_| ]]_| -| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[min(step_aggr_extended.pk_col_1), max(step_aggr_extended.val_col_2)]_| +| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[__min_merge(__min_state(step_aggr_extended.pk_col_1)) as min(step_aggr_extended.pk_col_1), __max_merge(__max_state(step_aggr_extended.val_col_2)) as max(step_aggr_extended.val_col_2)]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[min(step_aggr_extended.pk_col_1), max(step_aggr_extended.val_col_2)]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__min_merge(__min_state(step_aggr_extended.pk_col_1)) as min(step_aggr_extended.pk_col_1), __max_merge(__max_state(step_aggr_extended.val_col_2)) as max(step_aggr_extended.val_col_2)] | |_|_RepartitionExec: partitioning=REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -1199,16 +1173,16 @@ EXPLAIN SELECT SUM(val_col_1), COUNT(*) FROM step_aggr_extended WHERE pk_col_1 = | plan_type_| plan_| +-+-+ | logical_plan_| Projection: sum(step_aggr_extended.val_col_1), count(Int64(1)) AS count(*)_| -|_|_Aggregate: groupBy=[[]], aggr=[[__sum_merge(__sum_state(step_aggr_extended.val_col_1)) AS sum(step_aggr_extended.val_col_1), __count_merge(__count_state(step_aggr_extended.ts)) AS count(Int64(1))]] | +|_|_Aggregate: groupBy=[[]], aggr=[[__sum_merge(__sum_state(step_aggr_extended.val_col_1)) AS sum(step_aggr_extended.val_col_1), __count_merge(__count_state(step_aggr_extended.ts)) AS count(Int64(1))]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[]], aggr=[[__sum_state(step_aggr_extended.val_col_1), __count_state(step_aggr_extended.ts)]]_| |_|_Filter: step_aggr_extended.pk_col_1 = CAST(Utf8("non_existent") AS Dictionary(UInt32, Utf8))_| |_|_TableScan: step_aggr_extended, partial_filters=[step_aggr_extended.pk_col_1 = CAST(Utf8("non_existent") AS Dictionary(UInt32, Utf8))]_| |_| ]]_| | physical_plan | ProjectionExec: expr=[sum(step_aggr_extended.val_col_1)@0 as sum(step_aggr_extended.val_col_1), count(Int64(1))@1 as count(*)]_| -|_|_AggregateExec: mode=Final, gby=[], aggr=[sum(step_aggr_extended.val_col_1), count(Int64(1))]_| +|_|_AggregateExec: mode=Final, gby=[], aggr=[__sum_merge(__sum_state(step_aggr_extended.val_col_1)) as sum(step_aggr_extended.val_col_1), __count_merge(__count_state(step_aggr_extended.ts)) as count(Int64(1))]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[sum(step_aggr_extended.val_col_1), count(Int64(1))]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__sum_merge(__sum_state(step_aggr_extended.val_col_1)) as sum(step_aggr_extended.val_col_1), __count_merge(__count_state(step_aggr_extended.ts)) as count(Int64(1))] | |_|_RepartitionExec: partitioning=REDACTED |_|_MergeScanExec: REDACTED |_|_| diff --git a/tests/cases/distributed/explain/step_aggr_basic.result b/tests/cases/distributed/explain/step_aggr_basic.result index cf11b7f9ef0..39880450875 100644 --- a/tests/cases/distributed/explain/step_aggr_basic.result +++ b/tests/cases/distributed/explain/step_aggr_basic.result @@ -57,9 +57,9 @@ FROM |_| Aggregate: groupBy=[[]], aggr=[[__count_state(integers.i)]]_| |_|_TableScan: integers_| |_| ]]_| -| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[count(integers.i)]_| +| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(integers.i)]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)] | |_|_RepartitionExec: partitioning=REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -83,9 +83,9 @@ FROM +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(integers.i)] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(integers.i)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -147,17 +147,17 @@ ORDER BY +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Sort: integers.ts ASC NULLS LAST, count(integers.i) ASC NULLS LAST_| -|_|_Aggregate: groupBy=[[integers.ts]], aggr=[[__count_merge(__count_state(integers.i)) AS count(integers.i)]] | +| logical_plan_| Sort: integers.ts ASC NULLS LAST_| +|_|_Aggregate: groupBy=[[integers.ts]], aggr=[[__count_merge(__count_state(integers.i)) AS count(integers.i)]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[integers.ts]], aggr=[[__count_state(integers.i)]]_| |_|_TableScan: integers_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [ts@0 ASC NULLS LAST, count(integers.i)@1 ASC NULLS LAST]_| -|_|_SortExec: expr=[ts@0 ASC NULLS LAST, count(integers.i)@1 ASC NULLS LAST], preserve_partitioning=[true]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[count(integers.i)]_| +| physical_plan | SortPreservingMergeExec: [ts@0 ASC NULLS LAST]_| +|_|_SortExec: expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[true]_| +|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)] | |_|_RepartitionExec: partitioning=REDACTED -|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[count(integers.i)]_| +|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)]_| |_|_RepartitionExec: partitioning=REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -187,11 +187,11 @@ ORDER BY +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [ts@0 ASC NULLS LAST, count(integers.i)@1 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[ts@0 ASC NULLS LAST, count(integers.i)@1 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[count(integers.i)] REDACTED +| 0_| 0_|_SortPreservingMergeExec: [ts@0 ASC NULLS LAST] REDACTED +|_|_|_SortExec: expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[count(integers.i)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -253,19 +253,19 @@ ORDER BY +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Sort: time_window ASC NULLS LAST, count(integers.i) ASC NULLS LAST_| +| logical_plan_| Sort: time_window ASC NULLS LAST_| |_|_Projection: date_bin(Utf8("1 hour"),integers.ts) AS time_window, count(integers.i)_| |_|_Aggregate: groupBy=[[date_bin(Utf8("1 hour"),integers.ts)]], aggr=[[__count_merge(__count_state(integers.i)) AS count(integers.i)]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 3600000000000 }"), integers.ts) AS date_bin(Utf8("1 hour"),integers.ts)]], aggr=[[__count_state(integers.i)]] | |_|_TableScan: integers_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [time_window@0 ASC NULLS LAST, count(integers.i)@1 ASC NULLS LAST]_| -|_|_SortExec: expr=[time_window@0 ASC NULLS LAST, count(integers.i)@1 ASC NULLS LAST], preserve_partitioning=[true]_| +| physical_plan | SortPreservingMergeExec: [time_window@0 ASC NULLS LAST]_| |_|_ProjectionExec: expr=[date_bin(Utf8("1 hour"),integers.ts)@0 as time_window, count(integers.i)@1 as count(integers.i)]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("1 hour"),integers.ts)@0 as date_bin(Utf8("1 hour"),integers.ts)], aggr=[count(integers.i)]_| +|_|_SortExec: expr=[date_bin(Utf8("1 hour"),integers.ts)@0 ASC NULLS LAST], preserve_partitioning=[true]_| +|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("1 hour"),integers.ts)@0 as date_bin(Utf8("1 hour"),integers.ts)], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)]_| |_|_RepartitionExec: partitioning=REDACTED -|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("1 hour"),integers.ts)@0 as date_bin(Utf8("1 hour"),integers.ts)], aggr=[count(integers.i)]_| +|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("1 hour"),integers.ts)@0 as date_bin(Utf8("1 hour"),integers.ts)], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)]_| |_|_RepartitionExec: partitioning=REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -295,12 +295,12 @@ ORDER BY +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [time_window@0 ASC NULLS LAST, count(integers.i)@1 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[time_window@0 ASC NULLS LAST, count(integers.i)@1 ASC NULLS LAST], preserve_partitioning=[true] REDACTED +| 0_| 0_|_SortPreservingMergeExec: [time_window@0 ASC NULLS LAST] REDACTED |_|_|_ProjectionExec: expr=[date_bin(Utf8("1 hour"),integers.ts)@0 as time_window, count(integers.i)@1 as count(integers.i)] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("1 hour"),integers.ts)@0 as date_bin(Utf8("1 hour"),integers.ts)], aggr=[count(integers.i)] REDACTED +|_|_|_SortExec: expr=[date_bin(Utf8("1 hour"),integers.ts)@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("1 hour"),integers.ts)@0 as date_bin(Utf8("1 hour"),integers.ts)], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("1 hour"),integers.ts)@0 as date_bin(Utf8("1 hour"),integers.ts)], aggr=[count(integers.i)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("1 hour"),integers.ts)@0 as date_bin(Utf8("1 hour"),integers.ts)], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -380,9 +380,9 @@ ORDER BY |_| ]]_| | physical_plan | SortPreservingMergeExec: [integers.ts + Int64(1)@0 ASC NULLS LAST, integers.i / Int64(2)@1 ASC NULLS LAST]_| |_|_SortExec: expr=[integers.ts + Int64(1)@0 ASC NULLS LAST, integers.i / Int64(2)@1 ASC NULLS LAST], preserve_partitioning=[true]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[integers.ts + Int64(1)@0 as integers.ts + Int64(1), integers.i / Int64(2)@1 as integers.i / Int64(2)], aggr=[count(integers.i)] | +|_|_AggregateExec: mode=FinalPartitioned, gby=[integers.ts + Int64(1)@0 as integers.ts + Int64(1), integers.i / Int64(2)@1 as integers.i / Int64(2)], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)] | |_|_RepartitionExec: partitioning=REDACTED -|_|_AggregateExec: mode=Partial, gby=[integers.ts + Int64(1)@0 as integers.ts + Int64(1), integers.i / Int64(2)@1 as integers.i / Int64(2)], aggr=[count(integers.i)]_| +|_|_AggregateExec: mode=Partial, gby=[integers.ts + Int64(1)@0 as integers.ts + Int64(1), integers.i / Int64(2)@1 as integers.i / Int64(2)], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)]_| |_|_RepartitionExec: partitioning=REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -417,9 +417,9 @@ ORDER BY +-+-+-+ | 0_| 0_|_SortPreservingMergeExec: [integers.ts + Int64(1)@0 ASC NULLS LAST, integers.i / Int64(2)@1 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[integers.ts + Int64(1)@0 ASC NULLS LAST, integers.i / Int64(2)@1 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[integers.ts + Int64(1)@0 as integers.ts + Int64(1), integers.i / Int64(2)@1 as integers.i / Int64(2)], aggr=[count(integers.i)] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[integers.ts + Int64(1)@0 as integers.ts + Int64(1), integers.i / Int64(2)@1 as integers.i / Int64(2)], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[integers.ts + Int64(1)@0 as integers.ts + Int64(1), integers.i / Int64(2)@1 as integers.i / Int64(2)], aggr=[count(integers.i)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[integers.ts + Int64(1)@0 as integers.ts + Int64(1), integers.i / Int64(2)@1 as integers.i / Int64(2)], aggr=[__count_merge(__count_state(integers.i)) as count(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -499,15 +499,15 @@ FROM | plan_type_| plan_| +-+-+ | logical_plan_| Projection: uddsketch_calc(Float64(0.5), uddsketch_merge(Int64(128),Float64(0.01),sink_table.udd_state)) AS udd_result, hll_count(hll_merge(sink_table.hll_state)) AS hll_result_| -|_|_Aggregate: groupBy=[[]], aggr=[[__uddsketch_merge_merge(__uddsketch_merge_state(Int64(128),Float64(0.01),sink_table.udd_state)) AS uddsketch_merge(Int64(128),Float64(0.01),sink_table.udd_state), __hll_merge_merge(__hll_merge_state(sink_table.hll_state)) AS hll_merge(sink_table.hll_state)]] | +|_|_Aggregate: groupBy=[[]], aggr=[[__uddsketch_merge_merge(__uddsketch_merge_state(Int64(128),Float64(0.01),sink_table.udd_state)) AS uddsketch_merge(Int64(128),Float64(0.01),sink_table.udd_state), __hll_merge_merge(__hll_merge_state(sink_table.hll_state)) AS hll_merge(sink_table.hll_state)]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[]], aggr=[[__uddsketch_merge_state(Int64(128), Float64(0.01), sink_table.udd_state), __hll_merge_state(sink_table.hll_state)]]_| |_|_TableScan: sink_table_| |_| ]]_| | physical_plan | ProjectionExec: expr=[uddsketch_calc(0.5, uddsketch_merge(Int64(128),Float64(0.01),sink_table.udd_state)@0) as udd_result, hll_count(hll_merge(sink_table.hll_state)@1) as hll_result]_| -|_|_AggregateExec: mode=Final, gby=[], aggr=[uddsketch_merge(Int64(128),Float64(0.01),sink_table.udd_state), hll_merge(sink_table.hll_state)]_| +|_|_AggregateExec: mode=Final, gby=[], aggr=[__uddsketch_merge_merge(__uddsketch_merge_state(Int64(128),Float64(0.01),sink_table.udd_state)) as uddsketch_merge(Int64(128),Float64(0.01),sink_table.udd_state), __hll_merge_merge(__hll_merge_state(sink_table.hll_state)) as hll_merge(sink_table.hll_state)]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[uddsketch_merge(Int64(128),Float64(0.01),sink_table.udd_state), hll_merge(sink_table.hll_state)]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__uddsketch_merge_merge(__uddsketch_merge_state(Int64(128),Float64(0.01),sink_table.udd_state)) as uddsketch_merge(Int64(128),Float64(0.01),sink_table.udd_state), __hll_merge_merge(__hll_merge_state(sink_table.hll_state)) as hll_merge(sink_table.hll_state)] | |_|_RepartitionExec: partitioning=REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -533,9 +533,9 @@ FROM | stage | node | plan_| +-+-+-+ | 0_| 0_|_ProjectionExec: expr=[uddsketch_calc(0.5, uddsketch_merge(Int64(128),Float64(0.01),sink_table.udd_state)@0) as udd_result, hll_count(hll_merge(sink_table.hll_state)@1) as hll_result] REDACTED -|_|_|_AggregateExec: mode=Final, gby=[], aggr=[uddsketch_merge(Int64(128),Float64(0.01),sink_table.udd_state), hll_merge(sink_table.hll_state)] REDACTED +|_|_|_AggregateExec: mode=Final, gby=[], aggr=[__uddsketch_merge_merge(__uddsketch_merge_state(Int64(128),Float64(0.01),sink_table.udd_state)) as uddsketch_merge(Int64(128),Float64(0.01),sink_table.udd_state), __hll_merge_merge(__hll_merge_state(sink_table.hll_state)) as hll_merge(sink_table.hll_state)] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[uddsketch_merge(Int64(128),Float64(0.01),sink_table.udd_state), hll_merge(sink_table.hll_state)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__uddsketch_merge_merge(__uddsketch_merge_state(Int64(128),Float64(0.01),sink_table.udd_state)) as uddsketch_merge(Int64(128),Float64(0.01),sink_table.udd_state), __hll_merge_merge(__hll_merge_state(sink_table.hll_state)) as hll_merge(sink_table.hll_state)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| diff --git a/tests/cases/distributed/explain/step_aggr_massive.result b/tests/cases/distributed/explain/step_aggr_massive.result index 1da87d364ea..5aa969c6741 100644 --- a/tests/cases/distributed/explain/step_aggr_massive.result +++ b/tests/cases/distributed/explain/step_aggr_massive.result @@ -246,16 +246,16 @@ GROUP BY | plan_type_| plan_| +-+-+ | logical_plan_| Projection: base_table.env, base_table.service_name, base_table.city, base_table.page, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END) AS lcp_state, max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END) AS max_lcp, min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END) AS min_lcp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END) AS fmp_state, max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END) AS max_fmp, min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END) AS min_fmp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END) AS fcp_state, max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END) AS max_fcp, min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END) AS min_fcp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END) AS fp_state, max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END) AS max_fp, min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END) AS min_fp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END) AS tti_state, max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END) AS max_tti, min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END) AS min_tti, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END) AS fid_state, max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END) AS max_fid, min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END) AS min_fid, max(base_table.shard_key) AS shard_key, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))_| -|_|_Aggregate: groupBy=[[base_table.env, base_table.service_name, base_table.city, base_table.page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))]], aggr=[[__uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) AS uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) AS max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) AS min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) AS uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) AS max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) AS min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) AS uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) AS max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) AS min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) AS uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) AS max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) AS min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) AS uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) AS max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) AS min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) AS uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) AS max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) AS min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __max_merge(__max_state(base_table.shard_key)) AS max(base_table.shard_key)]] | +|_|_Aggregate: groupBy=[[base_table.env, base_table.service_name, base_table.city, base_table.page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))]], aggr=[[__uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) AS uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) AS max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) AS min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) AS uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) AS max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) AS min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) AS uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) AS max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) AS min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) AS uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) AS max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) AS min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) AS uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) AS max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) AS min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) AS uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) AS max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) AS min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __max_merge(__max_state(base_table.shard_key)) AS max(base_table.shard_key)]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[base_table.env, base_table.service_name, base_table.city, base_table.page, CAST(date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }"), base_table.time) AS Timestamp(s)) AS arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))]], aggr=[[__uddsketch_state_state(Int64(128), Float64(0.01), CAST(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END AS Float64)), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128), Float64(0.01), CAST(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END AS Float64)), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128), Float64(0.01), CAST(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END AS Float64)), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128), Float64(0.01), CAST(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END AS Float64)), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128), Float64(0.01), CAST(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END AS Float64)), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128), Float64(0.01), CAST(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END AS Float64)), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)]]_| |_|_Filter: (base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) OR base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) OR base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) OR base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) OR base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) OR base_table.fid > Int64(0) AND base_table.fid < Int64(3000000)) AND base_table.time >= TimestampMillisecond(0, None)_| |_|_TableScan: base_table, partial_filters=[base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) OR base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) OR base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) OR base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) OR base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) OR base_table.fid > Int64(0) AND base_table.fid < Int64(3000000), base_table.time >= TimestampMillisecond(0, None)]_| |_| ]]_| | physical_plan | ProjectionExec: expr=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END)@5 as lcp_state, max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END)@6 as max_lcp, min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END)@7 as min_lcp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END)@8 as fmp_state, max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END)@9 as max_fmp, min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END)@10 as min_fmp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END)@11 as fcp_state, max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END)@12 as max_fcp, min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END)@13 as min_fcp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END)@14 as fp_state, max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END)@15 as max_fp, min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END)@16 as min_fp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END)@17 as tti_state, max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END)@18 as max_tti, min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END)@19 as min_tti, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END)@20 as fid_state, max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END)@21 as max_fid, min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END)@22 as min_fid, max(base_table.shard_key)@23 as shard_key, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), max(base_table.shard_key)]_| +|_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) as max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) as min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __max_merge(__max_state(base_table.shard_key)) as max(base_table.shard_key)] | |_|_RepartitionExec: partitioning=REDACTED -|_|_AggregateExec: mode=Partial, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), max(base_table.shard_key)]_| +|_|_AggregateExec: mode=Partial, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) as max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) as min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __max_merge(__max_state(base_table.shard_key)) as max(base_table.shard_key)]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -456,9 +456,9 @@ GROUP BY | stage | node | plan_| +-+-+-+ | 0_| 0_|_ProjectionExec: expr=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END)@5 as lcp_state, max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END)@6 as max_lcp, min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END)@7 as min_lcp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END)@8 as fmp_state, max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END)@9 as max_fmp, min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END)@10 as min_fmp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END)@11 as fcp_state, max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END)@12 as max_fcp, min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END)@13 as min_fcp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END)@14 as fp_state, max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END)@15 as max_fp, min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END)@16 as min_fp, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END)@17 as tti_state, max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END)@18 as max_tti, min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END)@19 as min_tti, uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END)@20 as fid_state, max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END)@21 as max_fid, min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END)@22 as min_fid, max(base_table.shard_key)@23 as shard_key, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), max(base_table.shard_key)] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) as max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) as min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __max_merge(__max_state(base_table.shard_key)) as max(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), max(base_table.shard_key)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) as max(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END)) as min(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE NULL END), __uddsketch_state_merge(__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) as uddsketch_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __max_merge(__max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) as max(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __min_merge(__min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END)) as min(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE NULL END), __max_merge(__max_state(base_table.shard_key)) as max(base_table.shard_key)] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED @@ -600,9 +600,9 @@ where |_|_TableScan: base_table, partial_filters=[base_table.time >= TimestampMillisecond(0, None)]_| |_| ]]_| | physical_plan | ProjectionExec: expr=[count(Int64(1))@0 as count(*)]_| -|_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]_| +|_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(base_table.time)) as count(Int64(1))]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(base_table.time)) as count(Int64(1))]_| |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -628,9 +628,9 @@ where | stage | node | plan_| +-+-+-+ | 0_| 0_|_ProjectionExec: expr=[count(Int64(1))@0 as count(*)] REDACTED -|_|_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(base_table.time)) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(base_table.time)) as count(Int64(1))] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED diff --git a/tests/cases/distributed/explain/subqueries.result b/tests/cases/distributed/explain/subqueries.result index d7ad6ed920c..0ea758a6fe7 100644 --- a/tests/cases/distributed/explain/subqueries.result +++ b/tests/cases/distributed/explain/subqueries.result @@ -122,17 +122,17 @@ EXPLAIN INSERT INTO other SELECT i, 2 FROM integers WHERE i=(SELECT MAX(i) FROM +---------------------+-----------------------------------------------------------------------------+ | logical_plan | Dml: op=[Insert Into] table=[other] | | | Projection: integers.i AS i, TimestampMillisecond(2, None) AS j | -| | Inner Join: integers.i = __scalar_sq_1.max(integers.i) | +| | Filter: integers.i = () | +| | Subquery: | +| | MergeScan [is_placeholder=false, remote_input=[ | +| | Projection: max(integers.i) | +| | Aggregate: groupBy=[[]], aggr=[[max(integers.i)]] | +| | TableScan: integers | +| | ]] | | | Projection: integers.i | | | MergeScan [is_placeholder=false, remote_input=[ | | | TableScan: integers | | | ]] | -| | MergeScan [is_placeholder=false, remote_input=[ | -| | SubqueryAlias: __scalar_sq_1 | -| | Projection: max(integers.i) | -| | Aggregate: groupBy=[[]], aggr=[[max(integers.i)]] | -| | TableScan: integers, partial_filters=[Boolean(true)] | -| | ]] | | physical_plan_error | This feature is not implemented: Insert into not implemented for this table | +---------------------+-----------------------------------------------------------------------------+ @@ -598,8 +598,8 @@ EXPLAIN SELECT x FROM (VALUES (2),(1)) v(x) ORDER BY x; |_|_SubqueryAlias: v_| |_|_Projection: column1 AS x_| |_|_Values: (Int64(2)), (Int64(1))_| -| physical_plan | SortExec: expr=[x@0 ASC NULLS LAST], preserve_partitioning=[false] | -|_|_ProjectionExec: expr=[column1@0 as x]_| +| physical_plan | ProjectionExec: expr=[column1@0 as x]_| +|_|_SortExec: expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false] | |_|_DataSourceExec: partitions=1, partition_sizes=[1]_| |_|_| +-+-+ diff --git a/tests/cases/distributed/flow-tql/flow_tql.result b/tests/cases/distributed/flow-tql/flow_tql.result index 040ca34dbc3..925fff3f242 100644 --- a/tests/cases/distributed/flow-tql/flow_tql.result +++ b/tests/cases/distributed/flow-tql/flow_tql.result @@ -43,8 +43,10 @@ SHOW CREATE TABLE cnt_reqs; -- test if sink table is tql queryable TQL EVAL (now() - '1m'::interval, now(), '5s') count_values("status_code", cnt_reqs); -++ -++ ++------------------------------------------+----+-------------+ +| count(cnt_reqs.count(http_requests.val)) | ts | status_code | ++------------------------------------------+----+-------------+ ++------------------------------------------+----+-------------+ INSERT INTO TABLE http_requests VALUES (now() - '17s'::interval, 'host1', 'idc1', 200), @@ -187,8 +189,10 @@ SHOW CREATE TABLE cnt_reqs; -- test if sink table is tql queryable TQL EVAL (now() - '1m'::interval, now(), '5s') count_values("status_code", cnt_reqs); -++ -++ ++------------------------------------------+----+-------------+ +| count(cnt_reqs.count(http_requests.val)) | ts | status_code | ++------------------------------------------+----+-------------+ ++------------------------------------------+----+-------------+ INSERT INTO TABLE http_requests VALUES (0::Timestamp, 'host1', 'idc1', 200), @@ -278,8 +282,10 @@ SHOW CREATE TABLE rate_reqs; -- test if sink table is tql queryable TQL EVAL (now() - '1m'::interval, now(), '5s') count_values("status_code", rate_reqs); -++ -++ ++-----------------------------------------------------------+----+-------------+ +| count(rate_reqs.prom_rate(ts_range,val,ts,Int64(300000))) | ts | status_code | ++-----------------------------------------------------------+----+-------------+ ++-----------------------------------------------------------+----+-------------+ INSERT INTO TABLE http_requests VALUES (now() - '1m'::interval, 0), @@ -359,8 +365,10 @@ SHOW CREATE TABLE rate_reqs; -- test if sink table is tql queryable TQL EVAL (now() - '1m'::interval, now(), '5s') count_values("status_code", rate_reqs); -++ -++ ++-----------------------------------------------------------+----+-------------+ +| count(rate_reqs.prom_rate(ts_range,byte,ts,Int64(60000))) | ts | status_code | ++-----------------------------------------------------------+----+-------------+ ++-----------------------------------------------------------+----+-------------+ INSERT INTO TABLE http_requests_total VALUES ('localhost', 'my_service', 'instance1', 100, now() - '1min'::interval), diff --git a/tests/cases/distributed/flow-tql/tsid_on_phy.result b/tests/cases/distributed/flow-tql/tsid_on_phy.result index 0a1755b5c2b..797e8242a38 100644 --- a/tests/cases/distributed/flow-tql/tsid_on_phy.result +++ b/tests/cases/distributed/flow-tql/tsid_on_phy.result @@ -115,8 +115,8 @@ TQL EXPLAIN ( | | Sort: test_tsid.__tsid ASC NULLS FIRST, test_tsid.ts ASC NULLS FIRST | | | Projection: test_tsid.v, test_tsid.le, test_tsid.tag1, test_tsid.tag2, test_tsid.tag4, test_tsid.tag5, test_tsid.tag6, test_tsid.tag7, test_tsid.tag8, test_tsid.__tsid, test_tsid.ts | | | SubqueryAlias: test_tsid | -| | Filter: phy.ts >= TimestampMillisecond(1769137200001, None) AND phy.ts <= TimestampMillisecond(1769139900000, None) AND phy.__table_id=UInt32(REDACTED) | -| | TableScan: phy projection=[ts, v, tag1, tag2, le, tag4, tag5, tag6, tag7, tag8, __table_id, __tsid], partial_filters=[phy.ts >= TimestampMillisecond(1769137200001, None), phy.ts <= TimestampMillisecond(1769139900000, None), phy.__table_id=UInt32(REDACTED)] | +| | Filter: phy.__table_id=UInt32(REDACTED) AND phy.ts >= TimestampMillisecond(1769137200001, None) AND phy.ts <= TimestampMillisecond(1769139900000, None) | +| | TableScan: phy projection=[ts, v, tag1, tag2, le, tag4, tag5, tag6, tag7, tag8, __table_id, __tsid], partial_filters=[phy.__table_id=UInt32(REDACTED), phy.ts >= TimestampMillisecond(1769137200001, None), phy.ts <= TimestampMillisecond(1769139900000, None)] | | | ]] | | physical_plan | HistogramFoldExec: le=@0, field=@4, quantile=0.5 | | | RepartitionExec: REDACTED diff --git a/tests/cases/distributed/optimizer/count.result b/tests/cases/distributed/optimizer/count.result index 7368ad6bbd7..7c2d419b8cf 100644 --- a/tests/cases/distributed/optimizer/count.result +++ b/tests/cases/distributed/optimizer/count.result @@ -278,9 +278,9 @@ select count(1) from count_where_bug; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -314,9 +314,9 @@ select count(1) from count_where_bug where `tag` = 'b'; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -341,9 +341,9 @@ select count(1) from count_where_bug where `tag` = 'b'; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -378,9 +378,9 @@ select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -418,9 +418,9 @@ select count(1) from count_where_bug where num != 3; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| diff --git a/tests/cases/distributed/optimizer/filter_push_down.result b/tests/cases/distributed/optimizer/filter_push_down.result index 550c41f9daf..c4eab6b1e09 100644 --- a/tests/cases/distributed/optimizer/filter_push_down.result +++ b/tests/cases/distributed/optimizer/filter_push_down.result @@ -57,20 +57,26 @@ SELECT i1.i,i2.i FROM integers i1 LEFT OUTER JOIN integers i2 ON 1=1 WHERE i1.i> -- Align the result to PostgreSQL: empty. SELECT i1.i,i2.i FROM integers i1 LEFT OUTER JOIN integers i2 ON 1=0 WHERE i2.i IS NOT NULL ORDER BY 2; -++ -++ ++---+---+ +| i | i | ++---+---+ ++---+---+ -- Align the result to PostgreSQL: empty. SELECT i1.i,i2.i FROM integers i1 LEFT OUTER JOIN integers i2 ON 1=0 WHERE i2.i>1 ORDER BY 2; -++ -++ ++---+---+ +| i | i | ++---+---+ ++---+---+ -- Align the result to PostgreSQL: empty. SELECT i1.i,i2.i FROM integers i1 LEFT OUTER JOIN integers i2 ON 1=0 WHERE CASE WHEN i2.i IS NULL THEN False ELSE True END ORDER BY 2; -++ -++ ++---+---+ +| i | i | ++---+---+ ++---+---+ SELECT DISTINCT i1.i,i2.i FROM integers i1 LEFT OUTER JOIN integers i2 ON 1=0 WHERE i2.i IS NULL ORDER BY 1; @@ -151,7 +157,7 @@ SELECT * FROM integers i1 WHERE NOT EXISTS(SELECT i FROM integers WHERE i=i1.i) SELECT i1.i,i2.i FROM integers i1, integers i2 WHERE i1.i=(SELECT i FROM integers WHERE i1.i=i) AND i1.i=i2.i ORDER BY i1.i; -Error: 3001(EngineExecuteQuery), Error during planning: Correlated scalar subquery must be aggregated to return at most one row +Error: 3000(PlanQuery), Error during planning: Correlated scalar subquery must be aggregated to return at most one row SELECT * FROM (SELECT i1.i AS a, i2.i AS b FROM integers i1, integers i2) a1 WHERE a=b ORDER BY 1; @@ -213,8 +219,10 @@ EXPLAIN SELECT * FROM (SELECT 0=1 AS cond FROM integers i1, integers i2) a1 WHER -- Align the result to PostgreSQL: empty. SELECT * FROM (SELECT 0=1 AS cond FROM integers i1, integers i2 GROUP BY 1) a1 WHERE cond ORDER BY 1; -++ -++ ++------+ +| cond | ++------+ ++------+ DROP TABLE integers; diff --git a/tests/cases/distributed/optimizer/first_value_advance.result b/tests/cases/distributed/optimizer/first_value_advance.result index 519ec9d9c15..78aa8bcf6ab 100644 --- a/tests/cases/distributed/optimizer/first_value_advance.result +++ b/tests/cases/distributed/optimizer/first_value_advance.result @@ -317,14 +317,14 @@ explain select first_value(ts order by ts) from t; +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]] | +| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[]], aggr=[[__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_TableScan: t_| |_| ]]_| -| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[__first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -346,9 +346,9 @@ explain analyze +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -403,19 +403,19 @@ order by time_window, ordered_host; +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Sort: time_window ASC NULLS LAST, ordered_host ASC NULLS LAST_| +| logical_plan_| Sort: time_window ASC NULLS LAST_| |_|_Projection: first_value(t.host) ORDER BY [t.ts ASC NULLS LAST] AS ordered_host, first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t.ts) AS time_window_| -|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t.ts)]], aggr=[[__first_value_merge(__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]] | +|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t.ts)]], aggr=[[__first_value_merge(__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }"), t.ts) AS date_bin(Utf8("5 milliseconds"),t.ts)]], aggr=[[__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_TableScan: t_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST]_| -|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST]_| |_|_ProjectionExec: expr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@1 as ordered_host, first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]@3 as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t.ts)@0 as time_window]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_merge(__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED -|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_merge(__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -444,12 +444,12 @@ order by time_window, ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST] REDACTED |_|_|_ProjectionExec: expr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@1 as ordered_host, first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]@3 as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t.ts)@0 as time_window] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_merge(__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_merge(__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -521,14 +521,14 @@ explain +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]] | +| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[]], aggr=[[__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_TableScan: t1_| |_| ]]_| -| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[__first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -550,9 +550,9 @@ explain analyze +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -704,19 +704,19 @@ order by time_window, ordered_host; +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Sort: time_window ASC NULLS LAST, ordered_host ASC NULLS LAST_| +| logical_plan_| Sort: time_window ASC NULLS LAST_| |_|_Projection: first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST] AS ordered_host, first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t1.ts) AS time_window_| -|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t1.ts)]], aggr=[[__first_value_merge(__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]] | +|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t1.ts)]], aggr=[[__first_value_merge(__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }"), t1.ts) AS date_bin(Utf8("5 milliseconds"),t1.ts)]], aggr=[[__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_TableScan: t1_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST]_| -|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST]_| |_|_ProjectionExec: expr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@1 as ordered_host, first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@2 as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]@3 as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t1.ts)@0 as time_window]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_merge(__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED -|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_merge(__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -745,12 +745,12 @@ order by time_window, ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST] REDACTED |_|_|_ProjectionExec: expr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@1 as ordered_host, first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@2 as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]@3 as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t1.ts)@0 as time_window] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +|_|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_merge(__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_merge(__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| diff --git a/tests/cases/distributed/optimizer/last_value_advance.result b/tests/cases/distributed/optimizer/last_value_advance.result index 0972741b171..795f34a582e 100644 --- a/tests/cases/distributed/optimizer/last_value_advance.result +++ b/tests/cases/distributed/optimizer/last_value_advance.result @@ -317,14 +317,14 @@ explain select last_value(ts order by ts) from t; +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]] | +| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[]], aggr=[[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_TableScan: t_| |_| ]]_| -| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[__last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -346,9 +346,9 @@ explain analyze +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -403,19 +403,19 @@ order by time_window, ordered_host; +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Sort: time_window ASC NULLS LAST, ordered_host ASC NULLS LAST_| +| logical_plan_| Sort: time_window ASC NULLS LAST_| |_|_Projection: last_value(t.host) ORDER BY [t.ts ASC NULLS LAST] AS ordered_host, last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t.ts) AS time_window_| -|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t.ts)]], aggr=[[__last_value_merge(__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]] | +|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t.ts)]], aggr=[[__last_value_merge(__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }"), t.ts) AS date_bin(Utf8("5 milliseconds"),t.ts)]], aggr=[[__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_TableScan: t_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST]_| -|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST]_| |_|_ProjectionExec: expr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@1 as ordered_host, last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]@3 as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t.ts)@0 as time_window]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_merge(__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED -|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_merge(__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -444,12 +444,12 @@ order by time_window, ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST] REDACTED |_|_|_ProjectionExec: expr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@1 as ordered_host, last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]@3 as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t.ts)@0 as time_window] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_merge(__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_merge(__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -521,14 +521,14 @@ explain +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]] | +| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[]], aggr=[[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_TableScan: t1_| |_| ]]_| -| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[__last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -550,9 +550,9 @@ explain analyze +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -704,19 +704,19 @@ order by time_window, ordered_host; +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Sort: time_window ASC NULLS LAST, ordered_host ASC NULLS LAST_| +| logical_plan_| Sort: time_window ASC NULLS LAST_| |_|_Projection: last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST] AS ordered_host, last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t1.ts) AS time_window_| -|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t1.ts)]], aggr=[[__last_value_merge(__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]] | +|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t1.ts)]], aggr=[[__last_value_merge(__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }"), t1.ts) AS date_bin(Utf8("5 milliseconds"),t1.ts)]], aggr=[[__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_TableScan: t1_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST]_| -|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST]_| |_|_ProjectionExec: expr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@1 as ordered_host, last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@2 as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]@3 as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t1.ts)@0 as time_window]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_merge(__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED -|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_merge(__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -745,12 +745,12 @@ order by time_window, ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST] REDACTED |_|_|_ProjectionExec: expr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@1 as ordered_host, last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@2 as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]@3 as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t1.ts)@0 as time_window] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +|_|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_merge(__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_merge(__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| diff --git a/tests/cases/distributed/optimizer/range_select_projection.result b/tests/cases/distributed/optimizer/range_select_projection.result index 451e743da9c..28d8fff50a3 100644 --- a/tests/cases/distributed/optimizer/range_select_projection.result +++ b/tests/cases/distributed/optimizer/range_select_projection.result @@ -45,8 +45,8 @@ ORDER BY station, "channel", ts; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortExec: expr=[station@1 ASC NULLS LAST, channel@2 ASC NULLS LAST, ts@0 ASC NULLS LAST], preserve_REDACTED -|_|_|_ProjectionExec: expr=[ts@1 as ts, station@2 as station, channel@3 as channel, avg(range_select_projection.value_a + range_select_projection.value_b) RANGE 10s@0 as avg_value] REDACTED +| 0_| 0_|_ProjectionExec: expr=[ts@1 as ts, station@2 as station, channel@3 as channel, avg(range_select_projection.value_a + range_select_projection.value_b) RANGE 10s@0 as avg_value] REDACTED +|_|_|_SortExec: expr=[station@2 ASC NULLS LAST, channel@3 ASC NULLS LAST, ts@1 ASC NULLS LAST], preserve_REDACTED |_|_|_RangeSelectExec: range_expr=[avg(range_select_projection.value_a + range_select_projection.value_b) RANGE 10s], align=5000ms, align_to=0ms, align_by=[station@1, channel@2], time_index=ts REDACTED |_|_|_CooperativeExec REDACTED |_|_|_MergeScanExec: REDACTED diff --git a/tests/cases/distributed/optimizer/time_index_filter_pushdown.result b/tests/cases/distributed/optimizer/time_index_filter_pushdown.result index 016c49deea0..f03818154c1 100644 --- a/tests/cases/distributed/optimizer/time_index_filter_pushdown.result +++ b/tests/cases/distributed/optimizer/time_index_filter_pushdown.result @@ -112,13 +112,8 @@ WHERE +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Projection: cpu.rack, cpu.os, cpu.greptime_timestamp_| -|_|_Inner Join: cpu.greptime_timestamp = __scalar_sq_1.greptime_timestamp_| -|_|_Projection: cpu.rack, cpu.os, cpu.greptime_timestamp_| -|_|_MergeScan [is_placeholder=false, remote_input=[_| -|_| TableScan: cpu_| -|_| ]]_| -|_|_SubqueryAlias: __scalar_sq_1_| +| logical_plan_| Filter: cpu.greptime_timestamp = ()_| +|_|_Subquery:_| |_|_Limit: skip=0, fetch=1_| |_|_MergeSort: cpu.greptime_timestamp DESC NULLS FIRST_| |_|_MergeScan [is_placeholder=false, remote_input=[_| @@ -127,12 +122,17 @@ WHERE |_|_Projection: cpu.greptime_timestamp_| |_|_TableScan: cpu_| |_| ]]_| -| physical_plan | HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(greptime_timestamp@0, greptime_timestamp@2)], projection=[rack@1, os@2, greptime_timestamp@3]_| -|_|_MergeSortExec: [greptime_timestamp@0 DESC], fetch=1_| -|_|_MergeScanExec: REDACTED +|_|_Projection: cpu.rack, cpu.os, cpu.greptime_timestamp_| +|_|_MergeScan [is_placeholder=false, remote_input=[_| +|_| TableScan: cpu_| +|_| ]]_| +| physical_plan | ScalarSubqueryExec: subqueries=1_| +|_|_FilterExec: greptime_timestamp@2 = scalar_subquery()_| |_|_ProjectionExec: expr=[rack@0 as rack, os@1 as os, greptime_timestamp@3 as greptime_timestamp]_| |_|_CooperativeExec_| |_|_MergeScanExec: REDACTED +|_|_MergeSortExec: [greptime_timestamp@0 DESC], fetch=1_| +|_|_MergeScanExec: REDACTED |_|_| +-+-+ diff --git a/tests/cases/distributed/optimizer/windowed_sort.result b/tests/cases/distributed/optimizer/windowed_sort.result index 6f4e18defb3..0c4d1931c05 100644 --- a/tests/cases/distributed/optimizer/windowed_sort.result +++ b/tests/cases/distributed/optimizer/windowed_sort.result @@ -227,9 +227,9 @@ ORDER BY |_|_|_| | 1_| 0_|_ProjectionExec: expr=[collect_time_utc@0 as collect_time, collect_time@1 as true_collect_time, peak_current@2 as peak_current] REDACTED |_|_|_SortPreservingMergeExec: [collect_time@1 DESC] REDACTED -|_|_|_WindowedSortExec: expr=collect_time@1 DESC num_ranges=REDACTED REDACTED -|_|_|_PartSortExec: expr=collect_time@1 DESC num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[collect_time_utc@1 as collect_time_utc, collect_time@0 as collect_time, peak_current@2 as peak_current] REDACTED +|_|_|_WindowedSortExec: expr=collect_time@0 DESC num_ranges=REDACTED REDACTED +|_|_|_PartSortExec: expr=collect_time@0 DESC num_ranges=REDACTED REDACTED |_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED |_|_|_| |_|_| Total rows: 8_| diff --git a/tests/cases/distributed/optimizer/windowed_sort_advance.result b/tests/cases/distributed/optimizer/windowed_sort_advance.result index e3c92a0a081..1b122e61c8d 100644 --- a/tests/cases/distributed/optimizer/windowed_sort_advance.result +++ b/tests/cases/distributed/optimizer/windowed_sort_advance.result @@ -65,8 +65,8 @@ EXPLAIN ANALYZE select ts as ts, status, value from `a` where ts >= '2026-03-12T |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [ts@0 ASC NULLS LAST] REDACTED -|_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[ts@2 as ts, status@1 as status, value@0 as value] REDACTED +|_|_|_WindowedSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED |_|_|_| |_|_| Total rows: 10_| diff --git a/tests/cases/distributed/repartition/repartition.result b/tests/cases/distributed/repartition/repartition.result index 7b25a13a061..2b03139abd4 100644 --- a/tests/cases/distributed/repartition/repartition.result +++ b/tests/cases/distributed/repartition/repartition.result @@ -210,18 +210,24 @@ SHOW CREATE TABLE metric_physical_table; -- Verify select * works and returns empty SELECT * FROM metric_physical_table; -++ -++ ++----+------+-----+------------+--------+ +| ts | host | cpu | __table_id | __tsid | ++----+------+-----+------------+--------+ ++----+------+-----+------------+--------+ SELECT * FROM logical_table_v1; -++ -++ ++-----+------+----+ +| cpu | host | ts | ++-----+------+----+ ++-----+------+----+ SELECT * FROM logical_table_v2; -++ -++ ++-----+------+----+ +| cpu | host | ts | ++-----+------+----+ ++-----+------+----+ -- Repartition requires metasrv GC. Verify split requests are rejected when it is disabled. ALTER TABLE metric_physical_table MERGE PARTITION ( @@ -257,18 +263,24 @@ SHOW CREATE TABLE metric_physical_table; -- Verify select * works and returns empty SELECT * FROM metric_physical_table; -++ -++ ++----+------+-----+------------+--------+ +| ts | host | cpu | __table_id | __tsid | ++----+------+-----+------------+--------+ ++----+------+-----+------------+--------+ SELECT * FROM logical_table_v1; -++ -++ ++-----+------+----+ +| cpu | host | ts | ++-----+------+----+ ++-----+------+----+ SELECT * FROM logical_table_v2; -++ -++ ++-----+------+----+ +| cpu | host | ts | ++-----+------+----+ ++-----+------+----+ DROP TABLE logical_table_v1; diff --git a/tests/cases/standalone/common/tql/general_table.result b/tests/cases/distributed/tql/general_table.result similarity index 91% rename from tests/cases/standalone/common/tql/general_table.result rename to tests/cases/distributed/tql/general_table.result index 6172f40aaad..e4add60c60b 100644 --- a/tests/cases/standalone/common/tql/general_table.result +++ b/tests/cases/distributed/tql/general_table.result @@ -20,7 +20,6 @@ Affected Rows: 0 -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (cpu_usage\.ts_range) ts_range -- SQLNESS REPLACE (cpu_usage\.ts) ts --- SQLNESS REPLACE (?m)^\|_\|_\|_SortExec:.*\n\|_\|_\|_RepartitionExec:.*\n -- SQLNESS REPLACE (SeriesScan:.*|SeqScan:.*) ScanExec: REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED @@ -42,7 +41,7 @@ TQL analyze (0, 10, '1s') sum by(job) (irate(cpu_usage{job="fire"}[5s])) / 1e9; |_|_|_PromRangeManipulateExec: req range=[0..10000], interval=[1000], eval range=[5000], time index=[ts] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[ts], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["job"] REDACTED -|_|_|_ProjectionExec: expr=[value@1 as value, job@0 as job, CAST(ts@2 AS Timestamp(ms)) as ts] REDACTED +|_|_|_ProjectionExec: expr=[value@1 as value, job@0 as job, ts@2 as ts] REDACTED |_|_|_ScanExec: REDACTED |_|_|_| |_|_| Total rows: 0_| diff --git a/tests/cases/standalone/common/tql/general_table.sql b/tests/cases/distributed/tql/general_table.sql similarity index 91% rename from tests/cases/standalone/common/tql/general_table.sql rename to tests/cases/distributed/tql/general_table.sql index 6bf9549773c..53dd5b4e7f8 100644 --- a/tests/cases/standalone/common/tql/general_table.sql +++ b/tests/cases/distributed/tql/general_table.sql @@ -18,7 +18,6 @@ WITH( -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (cpu_usage\.ts_range) ts_range -- SQLNESS REPLACE (cpu_usage\.ts) ts --- SQLNESS REPLACE (?m)^\|_\|_\|_SortExec:.*\n\|_\|_\|_RepartitionExec:.*\n -- SQLNESS REPLACE (SeriesScan:.*|SeqScan:.*) ScanExec: REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED diff --git a/tests/cases/standalone/common/aggregate/approx_distinct.result b/tests/cases/standalone/common/aggregate/approx_distinct.result index a3875fadf04..b11fcf37fd0 100644 --- a/tests/cases/standalone/common/aggregate/approx_distinct.result +++ b/tests/cases/standalone/common/aggregate/approx_distinct.result @@ -91,7 +91,7 @@ SELECT APPROX_DISTINCT(a), APPROX_DISTINCT(b) FROM large_test; +-------------------------------+-------------------------------+ | approx_distinct(large_test.a) | approx_distinct(large_test.b) | +-------------------------------+-------------------------------+ -| 2000 | 10 | +| 1991 | 10 | +-------------------------------+-------------------------------+ -- Test with groups @@ -100,15 +100,15 @@ SELECT b, APPROX_DISTINCT(a) FROM large_test GROUP BY b ORDER BY b; +---+-------------------------------+ | b | approx_distinct(large_test.a) | +---+-------------------------------+ -| 0 | 200 | -| 1 | 201 | -| 2 | 201 | -| 3 | 200 | -| 4 | 199 | -| 5 | 200 | -| 6 | 199 | -| 7 | 200 | -| 8 | 200 | +| 0 | 199 | +| 1 | 197 | +| 2 | 200 | +| 3 | 199 | +| 4 | 198 | +| 5 | 197 | +| 6 | 200 | +| 7 | 196 | +| 8 | 199 | | 9 | 200 | +---+-------------------------------+ diff --git a/tests/cases/standalone/common/aggregate/approx_median.result b/tests/cases/standalone/common/aggregate/approx_median.result index 9d5bde78f60..46d52adf382 100644 --- a/tests/cases/standalone/common/aggregate/approx_median.result +++ b/tests/cases/standalone/common/aggregate/approx_median.result @@ -15,7 +15,7 @@ SELECT approx_median(i) FROM odd_test; +---------------------------+ | approx_median(odd_test.i) | +---------------------------+ -| 3 | +| 3.0 | +---------------------------+ -- Test with even number of values @@ -33,7 +33,7 @@ SELECT approx_median(i) FROM even_test; +----------------------------+ | approx_median(even_test.i) | +----------------------------+ -| 3 | +| 3.25 | +----------------------------+ -- Test with larger dataset @@ -50,7 +50,7 @@ SELECT approx_median(val) FROM large_test; +-------------------------------+ | approx_median(large_test.val) | +-------------------------------+ -| 499 | +| 499.25 | +-------------------------------+ -- Test with groups @@ -59,9 +59,9 @@ SELECT grp, approx_median(val) FROM large_test GROUP BY grp ORDER BY grp; +-----+-------------------------------+ | grp | approx_median(large_test.val) | +-----+-------------------------------+ -| 0 | 498 | -| 1 | 499 | -| 2 | 500 | +| 0 | 498.75 | +| 1 | 499.32142857142856 | +| 2 | 500.32142857142856 | +-----+-------------------------------+ -- Test with doubles @@ -111,7 +111,7 @@ SELECT approx_median(val) FROM dup_test; +-----------------------------+ | approx_median(dup_test.val) | +-----------------------------+ -| 2 | +| 2.75 | +-----------------------------+ -- Compare with exact median @@ -120,7 +120,7 @@ SELECT median(val), approx_median(val) FROM dup_test; +----------------------+-----------------------------+ | median(dup_test.val) | approx_median(dup_test.val) | +----------------------+-----------------------------+ -| 2 | 2 | +| 2.5 | 2.75 | +----------------------+-----------------------------+ -- Test edge cases @@ -139,7 +139,7 @@ SELECT approx_median(i) FROM odd_test WHERE i = 3; +---------------------------+ | approx_median(odd_test.i) | +---------------------------+ -| 3 | +| 3.0 | +---------------------------+ -- Test with negative values @@ -156,7 +156,7 @@ SELECT approx_median(val) FROM neg_test; +-----------------------------+ | approx_median(neg_test.val) | +-----------------------------+ -| 0 | +| 0.0 | +-----------------------------+ -- cleanup diff --git a/tests/cases/standalone/common/aggregate/approx_percentile_cont.result b/tests/cases/standalone/common/aggregate/approx_percentile_cont.result index ac9d60186e4..fcdf8394877 100644 --- a/tests/cases/standalone/common/aggregate/approx_percentile_cont.result +++ b/tests/cases/standalone/common/aggregate/approx_percentile_cont.result @@ -16,7 +16,7 @@ SELECT approx_percentile_cont(0.5) WITHIN GROUP (ORDER BY i) FROM approx_test; +----------------------------------------------------------------------------------+ | approx_percentile_cont(Float64(0.5)) WITHIN GROUP [approx_test.i ASC NULLS LAST] | +----------------------------------------------------------------------------------+ -| 499 | +| 499.25 | +----------------------------------------------------------------------------------+ -- first quartile @@ -25,7 +25,7 @@ SELECT approx_percentile_cont(0.25) WITHIN GROUP (ORDER BY i) FROM approx_test; +-----------------------------------------------------------------------------------+ | approx_percentile_cont(Float64(0.25)) WITHIN GROUP [approx_test.i ASC NULLS LAST] | +-----------------------------------------------------------------------------------+ -| 249 | +| 249.575 | +-----------------------------------------------------------------------------------+ -- third quartile @@ -34,7 +34,7 @@ SELECT approx_percentile_cont(0.75) WITHIN GROUP (ORDER BY i) FROM approx_test; +-----------------------------------------------------------------------------------+ | approx_percentile_cont(Float64(0.75)) WITHIN GROUP [approx_test.i ASC NULLS LAST] | +-----------------------------------------------------------------------------------+ -| 749 | +| 749.375 | +-----------------------------------------------------------------------------------+ -- 95th percentile @@ -43,7 +43,7 @@ SELECT approx_percentile_cont(0.95) WITHIN GROUP (ORDER BY i) FROM approx_test; +-----------------------------------------------------------------------------------+ | approx_percentile_cont(Float64(0.95)) WITHIN GROUP [approx_test.i ASC NULLS LAST] | +-----------------------------------------------------------------------------------+ -| 949 | +| 949.6607142857143 | +-----------------------------------------------------------------------------------+ -- Test approx_percentile_cont DESC @@ -53,7 +53,7 @@ SELECT approx_percentile_cont(0.5) WITHIN GROUP (ORDER BY i DESC) FROM approx_te +------------------------------------------------------------------------------------+ | approx_percentile_cont(Float64(0.5)) WITHIN GROUP [approx_test.i DESC NULLS FIRST] | +------------------------------------------------------------------------------------+ -| 499 | +| 499.25 | +------------------------------------------------------------------------------------+ -- first quartile @@ -62,7 +62,7 @@ SELECT approx_percentile_cont(0.25) WITHIN GROUP (ORDER BY i DESC) FROM approx_t +-------------------------------------------------------------------------------------+ | approx_percentile_cont(Float64(0.25)) WITHIN GROUP [approx_test.i DESC NULLS FIRST] | +-------------------------------------------------------------------------------------+ -| 749 | +| 749.375 | +-------------------------------------------------------------------------------------+ -- third quartile @@ -71,7 +71,7 @@ SELECT approx_percentile_cont(0.75) WITHIN GROUP (ORDER BY i DESC) FROM approx_t +-------------------------------------------------------------------------------------+ | approx_percentile_cont(Float64(0.75)) WITHIN GROUP [approx_test.i DESC NULLS FIRST] | +-------------------------------------------------------------------------------------+ -| 249 | +| 249.575 | +-------------------------------------------------------------------------------------+ -- 95th percentile @@ -80,7 +80,7 @@ SELECT approx_percentile_cont(0.95) WITHIN GROUP (ORDER BY i DESC) FROM approx_t +-------------------------------------------------------------------------------------+ | approx_percentile_cont(Float64(0.95)) WITHIN GROUP [approx_test.i DESC NULLS FIRST] | +-------------------------------------------------------------------------------------+ -| 49 | +| 49.50000000000004 | +-------------------------------------------------------------------------------------+ -- Test with different data types @@ -127,9 +127,9 @@ FROM approx_groups GROUP BY grp ORDER BY grp; +-----+--------------------------------------------------------------------------------------+ | grp | approx_percentile_cont(Float64(0.5)) WITHIN GROUP [approx_groups.val ASC NULLS LAST] | +-----+--------------------------------------------------------------------------------------+ -| 0 | 148 | -| 1 | 149 | -| 2 | 150 | +| 0 | 148.125 | +| 1 | 149.125 | +| 2 | 150.125 | +-----+--------------------------------------------------------------------------------------+ -- Test with NULL values @@ -142,7 +142,7 @@ SELECT approx_percentile_cont(0.5) WITHIN GROUP (ORDER BY i) FROM approx_test; +----------------------------------------------------------------------------------+ | approx_percentile_cont(Float64(0.5)) WITHIN GROUP [approx_test.i ASC NULLS LAST] | +----------------------------------------------------------------------------------+ -| 499 | +| 499.25 | +----------------------------------------------------------------------------------+ -- Test edge cases @@ -152,7 +152,7 @@ SELECT approx_percentile_cont(0.0) WITHIN GROUP (ORDER BY i) FROM approx_test; +--------------------------------------------------------------------------------+ | approx_percentile_cont(Float64(0)) WITHIN GROUP [approx_test.i ASC NULLS LAST] | +--------------------------------------------------------------------------------+ -| 0 | +| 0.0 | +--------------------------------------------------------------------------------+ SELECT approx_percentile_cont(1.0) WITHIN GROUP (ORDER BY i DESC) FROM approx_test; @@ -160,7 +160,7 @@ SELECT approx_percentile_cont(1.0) WITHIN GROUP (ORDER BY i DESC) FROM approx_te +----------------------------------------------------------------------------------+ | approx_percentile_cont(Float64(1)) WITHIN GROUP [approx_test.i DESC NULLS FIRST] | +----------------------------------------------------------------------------------+ -| 0 | +| 0.0 | +----------------------------------------------------------------------------------+ -- should be close to max @@ -169,7 +169,7 @@ SELECT approx_percentile_cont(1.0) WITHIN GROUP (ORDER BY i) FROM approx_test; +--------------------------------------------------------------------------------+ | approx_percentile_cont(Float64(1)) WITHIN GROUP [approx_test.i ASC NULLS LAST] | +--------------------------------------------------------------------------------+ -| 999 | +| 999.0 | +--------------------------------------------------------------------------------+ SELECT approx_percentile_cont(0.0) WITHIN GROUP (ORDER BY i DESC) FROM approx_test; @@ -177,7 +177,7 @@ SELECT approx_percentile_cont(0.0) WITHIN GROUP (ORDER BY i DESC) FROM approx_te +----------------------------------------------------------------------------------+ | approx_percentile_cont(Float64(0)) WITHIN GROUP [approx_test.i DESC NULLS FIRST] | +----------------------------------------------------------------------------------+ -| 999 | +| 999.0 | +----------------------------------------------------------------------------------+ DROP TABLE approx_test; diff --git a/tests/cases/standalone/common/aggregate/approx_percentile_cont_with_weight.result b/tests/cases/standalone/common/aggregate/approx_percentile_cont_with_weight.result index 91c9b77519e..5bdab7b060d 100644 --- a/tests/cases/standalone/common/aggregate/approx_percentile_cont_with_weight.result +++ b/tests/cases/standalone/common/aggregate/approx_percentile_cont_with_weight.result @@ -16,7 +16,7 @@ SELECT approx_percentile_cont_with_weight(weight, 0.5) WITHIN GROUP (ORDER BY "v +---------------------------------------------------------------------------------------------------------------------+ | approx_percentile_cont_with_weight(weight_test.weight,Float64(0.5)) WITHIN GROUP [weight_test.value ASC NULLS LAST] | +---------------------------------------------------------------------------------------------------------------------+ -| 33 | +| 33.333333333333336 | +---------------------------------------------------------------------------------------------------------------------+ -- Test different percentiles @@ -25,7 +25,7 @@ SELECT approx_percentile_cont_with_weight(weight, 0.25) WITHIN GROUP (ORDER BY " +----------------------------------------------------------------------------------------------------------------------+ | approx_percentile_cont_with_weight(weight_test.weight,Float64(0.25)) WITHIN GROUP [weight_test.value ASC NULLS LAST] | +----------------------------------------------------------------------------------------------------------------------+ -| 23 | +| 23.75 | +----------------------------------------------------------------------------------------------------------------------+ SELECT approx_percentile_cont_with_weight(weight, 0.75) WITHIN GROUP (ORDER BY "value") FROM weight_test; @@ -33,7 +33,7 @@ SELECT approx_percentile_cont_with_weight(weight, 0.75) WITHIN GROUP (ORDER BY " +----------------------------------------------------------------------------------------------------------------------+ | approx_percentile_cont_with_weight(weight_test.weight,Float64(0.75)) WITHIN GROUP [weight_test.value ASC NULLS LAST] | +----------------------------------------------------------------------------------------------------------------------+ -| 40 | +| 40.625 | +----------------------------------------------------------------------------------------------------------------------+ -- Test with groups @@ -53,8 +53,8 @@ FROM weight_groups GROUP BY grp ORDER BY grp; +-----+-------------------------------------------------------------------------------------------------------------------------+ | grp | approx_percentile_cont_with_weight(weight_groups.weight,Float64(0.5)) WITHIN GROUP [weight_groups.value ASC NULLS LAST] | +-----+-------------------------------------------------------------------------------------------------------------------------+ -| 1 | 18 | -| 2 | 212 | +| 1 | 18.333333333333332 | +| 2 | 212.5 | +-----+-------------------------------------------------------------------------------------------------------------------------+ -- Test with double values and weights @@ -82,7 +82,7 @@ SELECT approx_percentile_cont_with_weight("weight", 0.0) WITHIN GROUP (ORDER BY +-------------------------------------------------------------------------------------------------------------------+ | approx_percentile_cont_with_weight(weight_test.weight,Float64(0)) WITHIN GROUP [weight_test.value ASC NULLS LAST] | +-------------------------------------------------------------------------------------------------------------------+ -| 10 | +| 10.0 | +-------------------------------------------------------------------------------------------------------------------+ -- max @@ -91,7 +91,7 @@ SELECT approx_percentile_cont_with_weight("weight", 1.0) WITHIN GROUP (ORDER BY +-------------------------------------------------------------------------------------------------------------------+ | approx_percentile_cont_with_weight(weight_test.weight,Float64(1)) WITHIN GROUP [weight_test.value ASC NULLS LAST] | +-------------------------------------------------------------------------------------------------------------------+ -| 50 | +| 50.0 | +-------------------------------------------------------------------------------------------------------------------+ -- Test with zero weights @@ -116,7 +116,7 @@ SELECT approx_percentile_cont_with_weight(weight, 0.5) WITHIN GROUP (ORDER BY "v +---------------------------------------------------------------------------------------------------------------------+ | approx_percentile_cont_with_weight(weight_test.weight,Float64(0.5)) WITHIN GROUP [weight_test.value ASC NULLS LAST] | +---------------------------------------------------------------------------------------------------------------------+ -| 33 | +| 33.333333333333336 | +---------------------------------------------------------------------------------------------------------------------+ -- Test empty result @@ -143,7 +143,7 @@ SELECT approx_percentile_cont_with_weight(weight, 0.5) WITHIN GROUP (ORDER BY "v +-------------------------------------------------------------------------------------------------------------------------+ | approx_percentile_cont_with_weight(single_weight.weight,Float64(0.5)) WITHIN GROUP [single_weight.value ASC NULLS LAST] | +-------------------------------------------------------------------------------------------------------------------------+ -| 42 | +| 42.0 | +-------------------------------------------------------------------------------------------------------------------------+ -- Test equal weights (should behave like regular percentile) @@ -164,7 +164,7 @@ FROM equal_weight; +-----------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------+ | approx_percentile_cont_with_weight(equal_weight.weight,Float64(0.5)) WITHIN GROUP [equal_weight.value ASC NULLS LAST] | approx_percentile_cont(Float64(0.5)) WITHIN GROUP [equal_weight.value ASC NULLS LAST] | +-----------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------+ -| 25 | 25 | +| 25.0 | 25.0 | +-----------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------+ -- cleanup diff --git a/tests/cases/standalone/common/aggregate/median.result b/tests/cases/standalone/common/aggregate/median.result index 98af8787cc3..777952ff40b 100644 --- a/tests/cases/standalone/common/aggregate/median.result +++ b/tests/cases/standalone/common/aggregate/median.result @@ -6,7 +6,7 @@ SELECT median(NULL), median(1); +--------------+------------------+ | median(NULL) | median(Int64(1)) | +--------------+------------------+ -| | 1 | +| | 1.0 | +--------------+------------------+ -- test with simple table @@ -26,7 +26,7 @@ SELECT median(r)::VARCHAR FROM quantile; +--------------------+ | median(quantile.r) | +--------------------+ -| 4 | +| 4.5 | +--------------------+ SELECT median(r::FLOAT)::VARCHAR FROM quantile; @@ -50,7 +50,7 @@ SELECT median(r::SMALLINT)::VARCHAR FROM quantile WHERE r < 100; +--------------------+ | median(quantile.r) | +--------------------+ -| 4 | +| 4.5 | +--------------------+ SELECT median(r::INTEGER)::VARCHAR FROM quantile; @@ -58,7 +58,7 @@ SELECT median(r::INTEGER)::VARCHAR FROM quantile; +--------------------+ | median(quantile.r) | +--------------------+ -| 4 | +| 4.5 | +--------------------+ SELECT median(r::BIGINT)::VARCHAR FROM quantile; @@ -66,7 +66,7 @@ SELECT median(r::BIGINT)::VARCHAR FROM quantile; +--------------------+ | median(quantile.r) | +--------------------+ -| 4 | +| 4.5 | +--------------------+ -- test with NULL values @@ -83,7 +83,7 @@ SELECT median(42) FROM quantile; +-------------------+ | median(Int64(42)) | +-------------------+ -| 42 | +| 42.0 | +-------------------+ -- test with grouped data @@ -103,8 +103,8 @@ SELECT grp, median(val) FROM median_groups GROUP BY grp ORDER BY grp; +-----+---------------------------+ | grp | median(median_groups.val) | +-----+---------------------------+ -| 1 | 3 | -| 2 | 30 | +| 1 | 3.0 | +| 2 | 30.0 | | 3 | | +-----+---------------------------+ diff --git a/tests/cases/standalone/common/aggregate/multi_regions.result b/tests/cases/standalone/common/aggregate/multi_regions.result index 8627b83c153..fa0bf929319 100644 --- a/tests/cases/standalone/common/aggregate/multi_regions.result +++ b/tests/cases/standalone/common/aggregate/multi_regions.result @@ -59,9 +59,9 @@ select sum(val) from t; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[sum(t.val)] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__sum_merge(__sum_state(t.val)) as sum(t.val)] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[sum(t.val)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__sum_merge(__sum_state(t.val)) as sum(t.val)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -94,9 +94,9 @@ select sum(val) from t group by idc; | stage | node | plan_| +-+-+-+ | 0_| 0_|_ProjectionExec: expr=[sum(t.val)@1 as sum(t.val)] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[idc@0 as idc], aggr=[sum(t.val)] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[idc@0 as idc], aggr=[__sum_merge(__sum_state(t.val)) as sum(t.val)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[idc@0 as idc], aggr=[sum(t.val)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[idc@0 as idc], aggr=[__sum_merge(__sum_state(t.val)) as sum(t.val)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| diff --git a/tests/cases/standalone/common/alter/add_col_default.result b/tests/cases/standalone/common/alter/add_col_default.result index 5a9baf7186f..c54df938c51 100644 --- a/tests/cases/standalone/common/alter/add_col_default.result +++ b/tests/cases/standalone/common/alter/add_col_default.result @@ -39,8 +39,10 @@ SELECT * FROM test order by j; SELECT * FROM test where k != 3; -++ -++ ++---+---+---+ +| i | j | k | ++---+---+---+ ++---+---+---+ ALTER TABLE test ADD COLUMN host STRING DEFAULT '' PRIMARY KEY; @@ -48,13 +50,17 @@ Affected Rows: 0 SELECT * FROM test where host != ''; -++ -++ ++---+---+---+------+ +| i | j | k | host | ++---+---+---+------+ ++---+---+---+------+ SELECT * FROM test where host != '' AND i = 3; -++ -++ ++---+---+---+------+ +| i | j | k | host | ++---+---+---+------+ ++---+---+---+------+ DROP TABLE test; diff --git a/tests/cases/standalone/common/alter/change_col_type.result b/tests/cases/standalone/common/alter/change_col_type.result index 9cdfa1c3994..348d3b2145d 100644 --- a/tests/cases/standalone/common/alter/change_col_type.result +++ b/tests/cases/standalone/common/alter/change_col_type.result @@ -1,32 +1,32 @@ -CREATE TABLE test(`id` INTEGER PRIMARY KEY, i INTEGER NULL, j TIMESTAMP TIME INDEX, k BOOLEAN); +CREATE TABLE change_col_type_test(`id` INTEGER PRIMARY KEY, i INTEGER NULL, j TIMESTAMP TIME INDEX, k BOOLEAN); Affected Rows: 0 -INSERT INTO test VALUES (1, 1, 1, false), (2, 2, 2, true); +INSERT INTO change_col_type_test VALUES (1, 1, 1, false), (2, 2, 2, true); Affected Rows: 2 -ALTER TABLE test MODIFY COLUMN "I" STRING; +ALTER TABLE change_col_type_test MODIFY COLUMN "I" STRING; -Error: 4002(TableColumnNotFound), Column I not exists in table test +Error: 4002(TableColumnNotFound), Column I not exists in table change_col_type_test -ALTER TABLE test MODIFY COLUMN k DATE; +ALTER TABLE change_col_type_test MODIFY COLUMN k DATE; -Error: 1004(InvalidArguments), Invalid alter table(test) request: column 'k' cannot be cast automatically to type 'Date' +Error: 1004(InvalidArguments), Invalid alter table(change_col_type_test) request: column 'k' cannot be cast automatically to type 'Date' -ALTER TABLE test MODIFY COLUMN id STRING; +ALTER TABLE change_col_type_test MODIFY COLUMN id STRING; -Error: 1004(InvalidArguments), Invalid alter table(test) request: Not allowed to change primary key index column 'id' +Error: 1004(InvalidArguments), Invalid alter table(change_col_type_test) request: Not allowed to change primary key index column 'id' -ALTER TABLE test MODIFY COLUMN j STRING; +ALTER TABLE change_col_type_test MODIFY COLUMN j STRING; -Error: 1004(InvalidArguments), Invalid alter table(test) request: time index column 'j' only supports widening its timestamp unit, cannot change type from 'TimestampMillisecond' to 'String' +Error: 1004(InvalidArguments), Invalid alter table(change_col_type_test) request: time index column 'j' only supports widening its timestamp unit, cannot change type from 'TimestampMillisecond' to 'String' -ALTER TABLE test MODIFY COLUMN I STRING; +ALTER TABLE change_col_type_test MODIFY COLUMN I STRING; Affected Rows: 0 -SELECT * FROM test; +SELECT * FROM change_col_type_test; +----+---+-------------------------+-------+ | id | i | j | k | @@ -35,12 +35,12 @@ SELECT * FROM test; | 2 | 2 | 1970-01-01T00:00:00.002 | true | +----+---+-------------------------+-------+ -INSERT INTO test VALUES (3, "greptime", 3, true); +INSERT INTO change_col_type_test VALUES (3, "greptime", 3, true); Affected Rows: 1 -- SQLNESS SORT_RESULT 3 1 -SELECT * FROM test; +SELECT * FROM change_col_type_test; +----+----------+-------------------------+-------+ | id | i | j | k | @@ -50,7 +50,7 @@ SELECT * FROM test; | 3 | greptime | 1970-01-01T00:00:00.003 | true | +----+----------+-------------------------+-------+ -DESCRIBE test; +DESCRIBE change_col_type_test; +--------+----------------------+-----+------+---------+---------------+ | Column | Type | Key | Null | Default | Semantic Type | @@ -61,12 +61,12 @@ DESCRIBE test; | k | Boolean | | YES | | FIELD | +--------+----------------------+-----+------+---------+---------------+ -ALTER TABLE test MODIFY COLUMN I INTEGER; +ALTER TABLE change_col_type_test MODIFY COLUMN I INTEGER; Affected Rows: 0 -- SQLNESS SORT_RESULT 3 1 -SELECT * FROM test; +SELECT * FROM change_col_type_test; +----+---+-------------------------+-------+ | id | i | j | k | @@ -76,7 +76,7 @@ SELECT * FROM test; | 3 | | 1970-01-01T00:00:00.003 | true | +----+---+-------------------------+-------+ -DESCRIBE test; +DESCRIBE change_col_type_test; +--------+----------------------+-----+------+---------+---------------+ | Column | Type | Key | Null | Default | Semantic Type | @@ -87,7 +87,7 @@ DESCRIBE test; | k | Boolean | | YES | | FIELD | +--------+----------------------+-----+------+---------+---------------+ -DROP TABLE test; +DROP TABLE change_col_type_test; Affected Rows: 0 @@ -125,8 +125,10 @@ SELECT * FROM ts_widen ORDER BY ts; -- predicate on the widened time index reads old-unit SST data correctly SELECT * FROM ts_widen WHERE ts > '2024-01-01 00:00:01' ORDER BY ts; -++ -++ ++------+----+ +| host | ts | ++------+----+ ++------+----+ SELECT * FROM ts_widen WHERE ts < '2024-01-01 00:00:01.000300' ORDER BY ts; diff --git a/tests/cases/standalone/common/alter/change_col_type.sql b/tests/cases/standalone/common/alter/change_col_type.sql index 3356112cd0b..bb16af8847e 100644 --- a/tests/cases/standalone/common/alter/change_col_type.sql +++ b/tests/cases/standalone/common/alter/change_col_type.sql @@ -1,34 +1,34 @@ -CREATE TABLE test(`id` INTEGER PRIMARY KEY, i INTEGER NULL, j TIMESTAMP TIME INDEX, k BOOLEAN); +CREATE TABLE change_col_type_test(`id` INTEGER PRIMARY KEY, i INTEGER NULL, j TIMESTAMP TIME INDEX, k BOOLEAN); -INSERT INTO test VALUES (1, 1, 1, false), (2, 2, 2, true); +INSERT INTO change_col_type_test VALUES (1, 1, 1, false), (2, 2, 2, true); -ALTER TABLE test MODIFY COLUMN "I" STRING; +ALTER TABLE change_col_type_test MODIFY COLUMN "I" STRING; -ALTER TABLE test MODIFY COLUMN k DATE; +ALTER TABLE change_col_type_test MODIFY COLUMN k DATE; -ALTER TABLE test MODIFY COLUMN id STRING; +ALTER TABLE change_col_type_test MODIFY COLUMN id STRING; -ALTER TABLE test MODIFY COLUMN j STRING; +ALTER TABLE change_col_type_test MODIFY COLUMN j STRING; -ALTER TABLE test MODIFY COLUMN I STRING; +ALTER TABLE change_col_type_test MODIFY COLUMN I STRING; -SELECT * FROM test; +SELECT * FROM change_col_type_test; -INSERT INTO test VALUES (3, "greptime", 3, true); +INSERT INTO change_col_type_test VALUES (3, "greptime", 3, true); -- SQLNESS SORT_RESULT 3 1 -SELECT * FROM test; +SELECT * FROM change_col_type_test; -DESCRIBE test; +DESCRIBE change_col_type_test; -ALTER TABLE test MODIFY COLUMN I INTEGER; +ALTER TABLE change_col_type_test MODIFY COLUMN I INTEGER; -- SQLNESS SORT_RESULT 3 1 -SELECT * FROM test; +SELECT * FROM change_col_type_test; -DESCRIBE test; +DESCRIBE change_col_type_test; -DROP TABLE test; +DROP TABLE change_col_type_test; CREATE TABLE ts_widen (host STRING, ts TIMESTAMP TIME INDEX); INSERT INTO ts_widen VALUES ("a", "2024-01-01 00:00:01"); diff --git a/tests/cases/standalone/common/alter/rename_table.result b/tests/cases/standalone/common/alter/rename_table.result index ace030e54c1..6bf01e6eec7 100644 --- a/tests/cases/standalone/common/alter/rename_table.result +++ b/tests/cases/standalone/common/alter/rename_table.result @@ -100,8 +100,10 @@ DESC TABLE "fGhI"; SELECT * FROM "fGhI"; -++ -++ ++------+------+ +| CoLa | cOlB | ++------+------+ ++------+------+ ALTER TABLE "fGhI" RENAME JkLmN; diff --git a/tests/cases/standalone/common/basic.result b/tests/cases/standalone/common/basic.result index f1589a33ecb..37faa575fce 100644 --- a/tests/cases/standalone/common/basic.result +++ b/tests/cases/standalone/common/basic.result @@ -122,8 +122,10 @@ Affected Rows: 0 SELECT * from t2; -++ -++ ++-----+----+-----+ +| job | ts | val | ++-----+----+-----+ ++-----+----+-----+ INSERT INTO t2 VALUES ('job1', 0, 0), ('job2', 1, 1); @@ -152,8 +154,10 @@ select * from foo order by host asc; SELECT * from t1 order by ts desc; -++ -++ ++------+----+-----+ +| host | ts | val | ++------+----+-----+ ++------+----+-----+ SELECT * from t2 order by ts desc; diff --git a/tests/cases/standalone/common/catalog/schema.result b/tests/cases/standalone/common/catalog/schema.result index 759fadc04ec..f0ff4b1e400 100644 --- a/tests/cases/standalone/common/catalog/schema.result +++ b/tests/cases/standalone/common/catalog/schema.result @@ -99,8 +99,10 @@ Error: 4001(TableNotFound), Table not found: greptime.test_public_schema.hello SHOW TABLES FROM test_public_schema; -++ -++ ++------------------------------+ +| Tables_in_test_public_schema | ++------------------------------+ ++------------------------------+ SHOW TABLES FROM public; diff --git a/tests/cases/standalone/common/create/metric_engine_partition.result b/tests/cases/standalone/common/create/metric_engine_partition.result index ff3fd40e940..d45f6adf326 100644 --- a/tests/cases/standalone/common/create/metric_engine_partition.result +++ b/tests/cases/standalone/common/create/metric_engine_partition.result @@ -140,17 +140,17 @@ select host, count(*) from logical_table_2 GROUP BY host ORDER BY host; +-+-+ | logical_plan_| Sort: logical_table_2.host ASC NULLS LAST_| |_|_Projection: logical_table_2.host, count(Int64(1)) AS count(*)_| -|_|_Aggregate: groupBy=[[logical_table_2.host]], aggr=[[__count_merge(__count_state(logical_table_2.ts)) AS count(Int64(1))]] | +|_|_Aggregate: groupBy=[[logical_table_2.host]], aggr=[[__count_merge(__count_state(logical_table_2.ts)) AS count(Int64(1))]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[logical_table_2.host]], aggr=[[__count_state(logical_table_2.ts)]]_| |_|_TableScan: logical_table_2_| |_| ]]_| | physical_plan | SortPreservingMergeExec: [host@0 ASC NULLS LAST]_| -|_|_SortExec: expr=[host@0 ASC NULLS LAST], preserve_partitioning=[true]_| |_|_ProjectionExec: expr=[host@0 as host, count(Int64(1))@1 as count(*)]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[count(Int64(1))]_| +|_|_SortExec: expr=[host@0 ASC NULLS LAST], preserve_partitioning=[true]_| +|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[__count_merge(__count_state(logical_table_2.ts)) as count(Int64(1))] | |_|_RepartitionExec: REDACTED -|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[count(Int64(1))]_| +|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[__count_merge(__count_state(logical_table_2.ts)) as count(Int64(1))]_| |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -180,17 +180,17 @@ select ts, count(*) from logical_table_2 GROUP BY ts ORDER BY ts; +-+-+ | logical_plan_| Sort: logical_table_2.ts ASC NULLS LAST_| |_|_Projection: logical_table_2.ts, count(Int64(1)) AS count(*)_| -|_|_Aggregate: groupBy=[[logical_table_2.ts]], aggr=[[__count_merge(__count_state(logical_table_2.ts)) AS count(Int64(1))]] | +|_|_Aggregate: groupBy=[[logical_table_2.ts]], aggr=[[__count_merge(__count_state(logical_table_2.ts)) AS count(Int64(1))]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[logical_table_2.ts]], aggr=[[__count_state(logical_table_2.ts)]]_| |_|_TableScan: logical_table_2_| |_| ]]_| | physical_plan | SortPreservingMergeExec: [ts@0 ASC NULLS LAST]_| -|_|_SortExec: expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[true]_| |_|_ProjectionExec: expr=[ts@0 as ts, count(Int64(1))@1 as count(*)]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[count(Int64(1))]_| +|_|_SortExec: expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[true]_| +|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[__count_merge(__count_state(logical_table_2.ts)) as count(Int64(1))] | |_|_RepartitionExec: REDACTED -|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[count(Int64(1))]_| +|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[__count_merge(__count_state(logical_table_2.ts)) as count(Int64(1))]_| |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -276,17 +276,17 @@ select a, count(*) from logical_table_3 GROUP BY a ORDER BY a; +-+-+ | logical_plan_| Sort: logical_table_3.a ASC NULLS LAST_| |_|_Projection: logical_table_3.a, count(Int64(1)) AS count(*)_| -|_|_Aggregate: groupBy=[[logical_table_3.a]], aggr=[[__count_merge(__count_state(logical_table_3.ts)) AS count(Int64(1))]] | +|_|_Aggregate: groupBy=[[logical_table_3.a]], aggr=[[__count_merge(__count_state(logical_table_3.ts)) AS count(Int64(1))]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[logical_table_3.a]], aggr=[[__count_state(logical_table_3.ts)]]_| |_|_TableScan: logical_table_3_| |_| ]]_| | physical_plan | SortPreservingMergeExec: [a@0 ASC NULLS LAST]_| -|_|_SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]_| |_|_ProjectionExec: expr=[a@0 as a, count(Int64(1))@1 as count(*)]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(Int64(1))]_| +|_|_SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]_| +|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[__count_merge(__count_state(logical_table_3.ts)) as count(Int64(1))] | |_|_RepartitionExec: REDACTED -|_|_AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(Int64(1))]_| +|_|_AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[__count_merge(__count_state(logical_table_3.ts)) as count(Int64(1))]_| |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -362,9 +362,9 @@ EXPLAIN select count(*) from logical_table_4; |_|_TableScan: logical_table_4_| |_| ]]_| | physical_plan | ProjectionExec: expr=[count(Int64(1))@0 as count(*)]_| -|_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))]_| +|_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(logical_table_4.ts)) as count(Int64(1))]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(logical_table_4.ts)) as count(Int64(1))] | |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -394,17 +394,17 @@ select ts, count(*) from logical_table_4 GROUP BY ts ORDER BY ts; +-+-+ | logical_plan_| Sort: logical_table_4.ts ASC NULLS LAST_| |_|_Projection: logical_table_4.ts, count(Int64(1)) AS count(*)_| -|_|_Aggregate: groupBy=[[logical_table_4.ts]], aggr=[[__count_merge(__count_state(logical_table_4.ts)) AS count(Int64(1))]] | +|_|_Aggregate: groupBy=[[logical_table_4.ts]], aggr=[[__count_merge(__count_state(logical_table_4.ts)) AS count(Int64(1))]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[logical_table_4.ts]], aggr=[[__count_state(logical_table_4.ts)]]_| |_|_TableScan: logical_table_4_| |_| ]]_| | physical_plan | SortPreservingMergeExec: [ts@0 ASC NULLS LAST]_| -|_|_SortExec: expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[true]_| |_|_ProjectionExec: expr=[ts@0 as ts, count(Int64(1))@1 as count(*)]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[count(Int64(1))]_| +|_|_SortExec: expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[true]_| +|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[__count_merge(__count_state(logical_table_4.ts)) as count(Int64(1))] | |_|_RepartitionExec: REDACTED -|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[count(Int64(1))]_| +|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[__count_merge(__count_state(logical_table_4.ts)) as count(Int64(1))]_| |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| diff --git a/tests/cases/standalone/common/cte/cte.result b/tests/cases/standalone/common/cte/cte.result index 1ea77cae68c..4d936f5aaa4 100644 --- a/tests/cases/standalone/common/cte/cte.result +++ b/tests/cases/standalone/common/cte/cte.result @@ -146,7 +146,8 @@ select from cte where alias2 > 0; -Error: 3000(PlanQuery), Failed to plan SQL: No field named alias2. Valid fields are cte.a. +Error: 3000(PlanQuery), Failed to plan SQL: No field named alias2. +Valid fields are cte.a. drop table a; diff --git a/tests/cases/standalone/common/cte/cte_join_build_side.result b/tests/cases/standalone/common/cte/cte_join_build_side.result index 51ee4d98a4d..d549b21c241 100644 --- a/tests/cases/standalone/common/cte/cte_join_build_side.result +++ b/tests/cases/standalone/common/cte/cte_join_build_side.result @@ -59,16 +59,15 @@ GROUP BY t."db" ORDER BY c DESC, t."db"; | logical_plan | Sort: c DESC NULLS FIRST, t.db ASC NULLS LAST | | | Projection: t.db, count(Int64(1)) AS count(*) AS c | | | Aggregate: groupBy=[[t.db]], aggr=[[count(Int64(1))]] | -| | Projection: t.db | -| | Inner Join: t.db = td.db | -| | Projection: t.db | -| | MergeScan [is_placeholder=false, remote_input=[ | +| | LeftSemi Join: t.db = td.db | +| | Projection: t.db | +| | MergeScan [is_placeholder=false, remote_input=[ | | | SubqueryAlias: t | | | Filter: cte_join_logs.db IS NOT NULL | | | TableScan: cte_join_logs, partial_filters=[cte_join_logs.db IS NOT NULL] | | | ]] | -| | Projection: td.db | -| | MergeScan [is_placeholder=false, remote_input=[ | +| | Projection: td.db | +| | MergeScan [is_placeholder=false, remote_input=[ | | | SubqueryAlias: td | | | SubqueryAlias: top_dbs | | | Filter: cte_join_logs.db IS NOT NULL | @@ -78,13 +77,13 @@ GROUP BY t."db" ORDER BY c DESC, t."db"; | | Aggregate: groupBy=[[cte_join_logs.db]], aggr=[[count(cte_join_logs.ts) AS count(Int64(1))]] | | | TableScan: cte_join_logs | | | ]] | -| physical_plan | SortPreservingMergeExec: [c@1 DESC, db@0 ASC NULLS LAST] | -| | SortExec: expr=[c@1 DESC, db@0 ASC NULLS LAST], preserve_REDACTED -| | ProjectionExec: expr=[db@0 as db, count(Int64(1))@1 as c] | +| physical_plan | ProjectionExec: expr=[db@0 as db, count(Int64(1))@1 as c] | +| | SortPreservingMergeExec: [count(Int64(1))@1 DESC, db@0 ASC NULLS LAST] | +| | SortExec: expr=[count(Int64(1))@1 DESC, db@0 ASC NULLS LAST], preserve_REDACTED | | AggregateExec: mode=FinalPartitioned, gby=[db@0 as db], aggr=[count(Int64(1))] | | | RepartitionExec: REDACTED | | AggregateExec: mode=Partial, gby=[db@0 as db], aggr=[count(Int64(1))] | -| | HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(db@0, db@0)], projection=[db@1] | +| | HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(db@0, db@0)] | | | ProjectionExec: expr=[db@0 as db] | | | MergeScanExec: REDACTED | | RepartitionExec: REDACTED diff --git a/tests/cases/standalone/common/delete/delete.result b/tests/cases/standalone/common/delete/delete.result index 2dc22646307..c5d9b779d94 100644 --- a/tests/cases/standalone/common/delete/delete.result +++ b/tests/cases/standalone/common/delete/delete.result @@ -88,8 +88,10 @@ ADMIN flush_table('monitor'); SELECT ts, host, cpu, memory FROM monitor WHERE cpu = 66.6 ORDER BY ts; -++ -++ ++----+------+-----+--------+ +| ts | host | cpu | memory | ++----+------+-----+--------+ ++----+------+-----+--------+ SELECT ts, host, cpu, memory FROM monitor ORDER BY ts; @@ -108,8 +110,10 @@ Affected Rows: 2 SELECT ts, host, cpu, memory FROM monitor WHERE memory > 2048 ORDER BY ts; -++ -++ ++----+------+-----+--------+ +| ts | host | cpu | memory | ++----+------+-----+--------+ ++----+------+-----+--------+ SELECT ts, host, cpu, memory FROM monitor ORDER BY ts; diff --git a/tests/cases/standalone/common/error/incorrect_sql.result b/tests/cases/standalone/common/error/incorrect_sql.result index 5e8f34273bf..1dde80265c7 100644 --- a/tests/cases/standalone/common/error/incorrect_sql.result +++ b/tests/cases/standalone/common/error/incorrect_sql.result @@ -7,7 +7,8 @@ Error: 1001(Unsupported), SQL statement is not supported, keyword: SELEC -- Unrecognized column SELECT x FROM (SELECT 1 as y); -Error: 3000(PlanQuery), Failed to plan SQL: No field named x. Valid fields are y. +Error: 3000(PlanQuery), Failed to plan SQL: No field named x. +Valid fields are y. -- Unrecognized function SELECT FUNFUNFUN(); @@ -18,18 +19,19 @@ Did you mean 'range_fn'? -- Wrong aggregate parameters SELECT SUM(42, 84, 11, 'hello'); -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Execution error: Function 'sum' failed to match any signature, errors: Error during planning: Function 'sum' expects 1 arguments but received 4,Error during planning: Function 'sum' expects 1 arguments but received 4,Error during planning: Function 'sum' expects 1 arguments but received 4,Error during planning: Function 'sum' expects 1 arguments but received 4,Error during planning: Function 'sum' expects 1 arguments but received 4 No function matches the given name and argument types 'sum(Int64, Int64, Int64, Utf8)'. You might need to add explicit type casts. +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Execution error: Function 'sum' failed to match any signature, errors: Error during planning: Function 'sum' expects 1 arguments but received 4,Error during planning: Function 'sum' expects 1 arguments but received 4,Error during planning: Function 'sum' expects 1 arguments but received 4,Error during planning: Function 'sum' expects 1 arguments but received 4,Error during planning: Function 'sum' expects 1 arguments but received 4,Error during planning: Function 'sum' expects 1 arguments but received 4. No function matches the given name and argument types 'sum(Int64, Int64, Int64, Utf8)'. You might need to add explicit type casts. Candidate functions: - sum(Coercion(TypeSignatureClass::Decimal)) - sum(Coercion(TypeSignatureClass::Native(LogicalType(Native(UInt64), UInt64)), implicit_coercion=ImplicitCoercion([Native(LogicalType(Native(UInt8), UInt8)), Native(LogicalType(Native(UInt16), UInt16)), Native(LogicalType(Native(UInt32), UInt32))], default_type=UInt64)) - sum(Coercion(TypeSignatureClass::Native(LogicalType(Native(Int64), Int64)), implicit_coercion=ImplicitCoercion([Native(LogicalType(Native(Int8), Int8)), Native(LogicalType(Native(Int16), Int16)), Native(LogicalType(Native(Int32), Int32))], default_type=Int64)) - sum(Coercion(TypeSignatureClass::Native(LogicalType(Native(Float64), Float64)), implicit_coercion=ImplicitCoercion([Float], default_type=Float64)) - sum(Coercion(TypeSignatureClass::Duration)) + sum(Decimal) + sum(UInt64) + sum(Int64) + sum(Float64) + sum(Duration) + sum(Interval) -- No matching function signature SELECT cos(0, 1, 2, 3); -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Failed to coerce arguments to satisfy a call to 'cos' function: coercion from Int64, Int64, Int64, Int64 to the signature Uniform(1, [Float64, Float32]) failed No function matches the given name and argument types 'cos(Int64, Int64, Int64, Int64)'. You might need to add explicit type casts. +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Failed to coerce arguments to satisfy a call to 'cos' function: coercion from Int64, Int64, Int64, Int64 to the signature Uniform(1, [Float64, Float32]) failed. No function matches the given name and argument types 'cos(Int64, Int64, Int64, Int64)'. You might need to add explicit type casts. Candidate functions: cos(Float64/Float32) @@ -69,12 +71,14 @@ Affected Rows: 0 -- Non-existent column SELECT feathe FROM chickens; -Error: 3000(PlanQuery), Failed to plan SQL: No field named feathe. Valid fields are chickens.feather, chickens.beak, chickens.ts. +Error: 3000(PlanQuery), Failed to plan SQL: No field named feathe. Did you mean 'chickens.feather'? +Valid fields are chickens.feather, chickens.beak, chickens.ts. -- Non-existent column with multiple tables SELECT feathe FROM chickens, integers, strings; -Error: 3000(PlanQuery), Failed to plan SQL: No field named feathe. Valid fields are chickens.feather, chickens.beak, chickens.ts, integers.integ, integers.ts, strings.str, strings.ts. +Error: 3000(PlanQuery), Failed to plan SQL: No field named feathe. Did you mean 'chickens.feather'? +Valid fields are chickens.feather, chickens.beak, chickens.ts, integers.integ, integers.ts, strings.str, strings.ts. -- Ambiguous column reference SELECT ts FROM chickens, integers; diff --git a/tests/cases/standalone/common/filter/cast_preimage.result b/tests/cases/standalone/common/filter/cast_preimage.result index 3209a842e04..42e73ea1039 100644 --- a/tests/cases/standalone/common/filter/cast_preimage.result +++ b/tests/cases/standalone/common/filter/cast_preimage.result @@ -345,17 +345,21 @@ Affected Rows: 0 INSERT INTO cast_preimage_ts_ms VALUES ('host1', 0, 1), ('host2', 5000, 2), - ('host3', 5001, 3); + ('host3', 5001, 3), + ('safe_neg', -9223372036854, 4), + ('safe_pos', 9223372036854, 5); -Affected Rows: 3 +Affected Rows: 5 --- Timestamp widening equality is exact at millisecond precision. +-- Widening TIMESTAMP(3) to TIMESTAMP(9) can overflow at extreme timestamp +-- values. We accept this full-domain semantic tradeoff to retain native +-- millisecond pruning for aligned normal-range equality and IN predicates. -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED EXPLAIN SELECT host, v FROM cast_preimage_ts_ms -WHERE CAST(ts AS TIMESTAMP(9)) = '1970-01-01 00:00:05'::TIMESTAMP(9) +WHERE CAST(ts AS TIMESTAMP(9)) = 5000000000::TIMESTAMP(9) ORDER BY host; +---------------+-------------------------------------------------------------------------------------------------------------------+ @@ -372,13 +376,91 @@ ORDER BY host; | | | +---------------+-------------------------------------------------------------------------------------------------------------------+ --- Non-exact nanosecond literal should remain semantically correct. -SELECT host, v FROM cast_preimage_ts_ms -WHERE CAST(ts AS TIMESTAMP(9)) = '1970-01-01 00:00:05.000000001'::TIMESTAMP(9) +-- The aligned IN-list must likewise become bare millisecond scan filters. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts_ms +WHERE CAST(ts AS TIMESTAMP(9)) IN (0::TIMESTAMP(9), 5000000000::TIMESTAMP(9)) ORDER BY host; -++ -++ ++---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: cast_preimage_ts_ms.host ASC NULLS LAST | +| | Projection: cast_preimage_ts_ms.host, cast_preimage_ts_ms.v | +| | Filter: cast_preimage_ts_ms.ts = TimestampMillisecond(0, None) OR cast_preimage_ts_ms.ts = TimestampMillisecond(5000, None) | +| | TableScan: cast_preimage_ts_ms, partial_filters=[cast_preimage_ts_ms.ts = TimestampMillisecond(0, None) OR cast_preimage_ts_ms.ts = TimestampMillisecond(5000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +SELECT host, v FROM cast_preimage_ts_ms +WHERE CAST(ts AS TIMESTAMP(9)) = 5000000000::TIMESTAMP(9) +ORDER BY host; + ++-------+---+ +| host | v | ++-------+---+ +| host2 | 2 | ++-------+---+ + +SELECT host, v FROM cast_preimage_ts_ms +WHERE CAST(ts AS TIMESTAMP(9)) = 5000000001::TIMESTAMP(9) +ORDER BY host; + ++------+---+ +| host | v | ++------+---+ ++------+---+ + +-- The safe millisecond values widen to -9223372036854000000 and +-- 9223372036854000000 nanoseconds without rendering extreme dates. +SELECT v, CAST(CAST(ts AS TIMESTAMP(9)) AS BIGINT) AS ts_ns +FROM cast_preimage_ts_ms +WHERE v IN (4, 5) +ORDER BY v; + ++---+----------------------+ +| v | ts_ns | ++---+----------------------+ +| 4 | -9223372036854000000 | +| 5 | 9223372036854000000 | ++---+----------------------+ + +INSERT INTO cast_preimage_ts_ms VALUES ('overflow_pos', 9223372036855, 7); + +Affected Rows: 1 + +-- An ordinary projection retains its overflow error. Under the accepted +-- pruning policy, aligned equality excludes the overflow row instead. +SELECT v, CAST(CAST(ts AS TIMESTAMP(9)) AS BIGINT) AS ts_ns +FROM cast_preimage_ts_ms WHERE host = 'overflow_pos'; + +Error: 3001(EngineExecuteQuery), Execution error: Cannot cast Timestamp(ms) value 9223372036855 to Timestamp(ns): converted value exceeds the representable i64 range + +SELECT v FROM cast_preimage_ts_ms +WHERE host = 'overflow_pos' AND CAST(ts AS TIMESTAMP(9)) = 5000000000::TIMESTAMP(9); + ++---+ +| v | ++---+ ++---+ + +-- Direct safe cast returns NULL, avoiding the existing SQL timestamp-precision +-- lowering limitation for TRY_CAST. +SELECT v, CAST(arrow_try_cast(ts, 'Timestamp(Nanosecond, None)') AS BIGINT) AS ts_ns +FROM cast_preimage_ts_ms WHERE host = 'overflow_pos'; + ++---+-------+ +| v | ts_ns | ++---+-------+ +| 7 | | ++---+-------+ DROP TABLE cast_preimage_ts; diff --git a/tests/cases/standalone/common/filter/cast_preimage.sql b/tests/cases/standalone/common/filter/cast_preimage.sql index 7ee7e8198f3..988c71fa694 100644 --- a/tests/cases/standalone/common/filter/cast_preimage.sql +++ b/tests/cases/standalone/common/filter/cast_preimage.sql @@ -152,22 +152,60 @@ CREATE TABLE cast_preimage_ts_ms ( INSERT INTO cast_preimage_ts_ms VALUES ('host1', 0, 1), ('host2', 5000, 2), - ('host3', 5001, 3); + ('host3', 5001, 3), + ('safe_neg', -9223372036854, 4), + ('safe_pos', 9223372036854, 5); --- Timestamp widening equality is exact at millisecond precision. +-- Widening TIMESTAMP(3) to TIMESTAMP(9) can overflow at extreme timestamp +-- values. We accept this full-domain semantic tradeoff to retain native +-- millisecond pruning for aligned normal-range equality and IN predicates. -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED EXPLAIN SELECT host, v FROM cast_preimage_ts_ms -WHERE CAST(ts AS TIMESTAMP(9)) = '1970-01-01 00:00:05'::TIMESTAMP(9) +WHERE CAST(ts AS TIMESTAMP(9)) = 5000000000::TIMESTAMP(9) ORDER BY host; --- Non-exact nanosecond literal should remain semantically correct. -SELECT host, v FROM cast_preimage_ts_ms -WHERE CAST(ts AS TIMESTAMP(9)) = '1970-01-01 00:00:05.000000001'::TIMESTAMP(9) +-- The aligned IN-list must likewise become bare millisecond scan filters. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT host, v FROM cast_preimage_ts_ms +WHERE CAST(ts AS TIMESTAMP(9)) IN (0::TIMESTAMP(9), 5000000000::TIMESTAMP(9)) ORDER BY host; +SELECT host, v FROM cast_preimage_ts_ms +WHERE CAST(ts AS TIMESTAMP(9)) = 5000000000::TIMESTAMP(9) +ORDER BY host; + +SELECT host, v FROM cast_preimage_ts_ms +WHERE CAST(ts AS TIMESTAMP(9)) = 5000000001::TIMESTAMP(9) +ORDER BY host; + +-- The safe millisecond values widen to -9223372036854000000 and +-- 9223372036854000000 nanoseconds without rendering extreme dates. +SELECT v, CAST(CAST(ts AS TIMESTAMP(9)) AS BIGINT) AS ts_ns +FROM cast_preimage_ts_ms +WHERE v IN (4, 5) +ORDER BY v; + +INSERT INTO cast_preimage_ts_ms VALUES ('overflow_pos', 9223372036855, 7); + +-- An ordinary projection retains its overflow error. Under the accepted +-- pruning policy, aligned equality excludes the overflow row instead. +SELECT v, CAST(CAST(ts AS TIMESTAMP(9)) AS BIGINT) AS ts_ns +FROM cast_preimage_ts_ms WHERE host = 'overflow_pos'; + +SELECT v FROM cast_preimage_ts_ms +WHERE host = 'overflow_pos' AND CAST(ts AS TIMESTAMP(9)) = 5000000000::TIMESTAMP(9); + +-- Direct safe cast returns NULL, avoiding the existing SQL timestamp-precision +-- lowering limitation for TRY_CAST. +SELECT v, CAST(arrow_try_cast(ts, 'Timestamp(Nanosecond, None)') AS BIGINT) AS ts_ns +FROM cast_preimage_ts_ms WHERE host = 'overflow_pos'; + DROP TABLE cast_preimage_ts; DROP TABLE cast_preimage_int; diff --git a/tests/cases/standalone/common/filter/constant_comparisons.result b/tests/cases/standalone/common/filter/constant_comparisons.result index 9dba691e915..6dbc81a86d1 100644 --- a/tests/cases/standalone/common/filter/constant_comparisons.result +++ b/tests/cases/standalone/common/filter/constant_comparisons.result @@ -19,8 +19,10 @@ SELECT * FROM integers WHERE 2=2; SELECT * FROM integers WHERE 2=3; -++ -++ ++---+---+----+ +| a | b | ts | ++---+---+----+ ++---+---+----+ SELECT * FROM integers WHERE 2<>3; @@ -32,8 +34,10 @@ SELECT * FROM integers WHERE 2<>3; SELECT * FROM integers WHERE 2<>2; -++ -++ ++---+---+----+ +| a | b | ts | ++---+---+----+ ++---+---+----+ SELECT * FROM integers WHERE 2>1; @@ -45,8 +49,10 @@ SELECT * FROM integers WHERE 2>1; SELECT * FROM integers WHERE 2>2; -++ -++ ++---+---+----+ +| a | b | ts | ++---+---+----+ ++---+---+----+ SELECT * FROM integers WHERE 2>=2; @@ -58,8 +64,10 @@ SELECT * FROM integers WHERE 2>=2; SELECT * FROM integers WHERE 2>=3; -++ -++ ++---+---+----+ +| a | b | ts | ++---+---+----+ ++---+---+----+ SELECT * FROM integers WHERE 2<3; @@ -71,8 +79,10 @@ SELECT * FROM integers WHERE 2<3; SELECT * FROM integers WHERE 2<2; -++ -++ ++---+---+----+ +| a | b | ts | ++---+---+----+ ++---+---+----+ SELECT * FROM integers WHERE 2<=2; @@ -84,8 +94,10 @@ SELECT * FROM integers WHERE 2<=2; SELECT * FROM integers WHERE 2<=1; -++ -++ ++---+---+----+ +| a | b | ts | ++---+---+----+ ++---+---+----+ -- NULL comparisons SELECT a=NULL FROM integers; @@ -115,8 +127,10 @@ SELECT * FROM integers WHERE 2 IN (2, 3, 4, 5); SELECT * FROM integers WHERE 2 IN (1, 3, 4, 5); -++ -++ ++---+---+----+ +| a | b | ts | ++---+---+----+ ++---+---+----+ -- Clean up DROP TABLE integers; diff --git a/tests/cases/standalone/common/filter/hash_join_dyn_filter.result b/tests/cases/standalone/common/filter/hash_join_dyn_filter.result index 05c313de3cc..b1402bd4d4a 100644 --- a/tests/cases/standalone/common/filter/hash_join_dyn_filter.result +++ b/tests/cases/standalone/common/filter/hash_join_dyn_filter.result @@ -49,7 +49,7 @@ WHERE c.tier = 'gold'; | | MergeScan [is_placeholder=false, remote_input=[ | | | SubqueryAlias: c | | | Filter: customers.customer_id IS NOT NULL AND customers.tier = Utf8("gold") | -| | TableScan: customers, partial_filters=[customers.tier = Utf8("gold"), customers.customer_id IS NOT NULL] | +| | TableScan: customers, partial_filters=[customers.customer_id IS NOT NULL, customers.tier = Utf8("gold")] | | | ]] | | physical_plan | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(customer_id@1, customer_id@0)], projection=[id@0, amount@2, name@4, tier@5] | | | RepartitionExec: REDACTED @@ -100,7 +100,7 @@ WHERE c.tier = 'gold'; |_|_|_| | 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND tier@2 = gold metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["tier = Utf8(\"gold\")", "customer_id IS NOT NULL"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\")"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ @@ -177,7 +177,7 @@ WHERE c.tier IN ('gold', 'silver'); | | MergeScan [is_placeholder=false, remote_input=[ | | | SubqueryAlias: c | | | Filter: customers.customer_id IS NOT NULL AND (customers.tier = Utf8("gold") OR customers.tier = Utf8("silver")) | -| | TableScan: customers, partial_filters=[customers.tier = Utf8("gold") OR customers.tier = Utf8("silver"), customers.customer_id IS NOT NULL] | +| | TableScan: customers, partial_filters=[customers.customer_id IS NOT NULL, customers.tier = Utf8("gold") OR customers.tier = Utf8("silver")] | | | ]] | | physical_plan | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cid@1, customer_id@0)], projection=[id@0, amount@2, name@4, tier@5] | | | RepartitionExec: REDACTED @@ -229,7 +229,7 @@ WHERE c.tier IN ('gold', 'silver'); |_|_|_| | 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND (tier@2 = gold OR tier@2 = silver) metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")", "customer_id IS NOT NULL"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ diff --git a/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.result b/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.result index 5798ddd07c1..0ba0f3c4e00 100644 --- a/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.result +++ b/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.result @@ -65,7 +65,7 @@ WHERE c.tier IN ('gold', 'bronze'); | | MergeScan [is_placeholder=false, remote_input=[ | | | SubqueryAlias: c | | | Filter: customers.customer_id IS NOT NULL AND (customers.tier = Utf8("gold") OR customers.tier = Utf8("bronze")) | -| | TableScan: customers, partial_filters=[customers.tier = Utf8("gold") OR customers.tier = Utf8("bronze"), customers.customer_id IS NOT NULL] | +| | TableScan: customers, partial_filters=[customers.customer_id IS NOT NULL, customers.tier = Utf8("gold") OR customers.tier = Utf8("bronze")] | | | ]] | | physical_plan | HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@1, customer_id@0)], projection=[id@0, amount@2, name@4, tier@5] | | | ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, amount@2 as amount] | @@ -123,7 +123,7 @@ WHERE c.tier IN ('gold', 'bronze'); |_|_|_| | 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND (tier@2 = gold OR tier@2 = bronze) metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["tier = Utf8(\"gold\") OR tier = Utf8(\"bronze\")", "customer_id IS NOT NULL"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\") OR tier = Utf8(\"bronze\")"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ @@ -204,36 +204,35 @@ FROM ( ORDER BY amount DESC LIMIT 4; -+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ -| plan_type | plan | -+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ -| logical_plan | Sort: o.amount DESC NULLS FIRST, fetch=4 | -| | Projection: o.id, o.customer_id, c.name, c.tier, o.amount | -| | Inner Join: o.customer_id = c.customer_id | -| | Projection: o.id, o.customer_id, o.amount | -| | MergeScan [is_placeholder=false, remote_input=[ | -| | SubqueryAlias: o | -| | Filter: orders.customer_id IS NOT NULL | -| | TableScan: orders, partial_filters=[orders.customer_id IS NOT NULL] | -| | ]] | -| | Projection: c.customer_id, c.name, c.tier | -| | MergeScan [is_placeholder=false, remote_input=[ | -| | SubqueryAlias: c | -| | Filter: customers.customer_id IS NOT NULL AND (customers.tier = Utf8("gold") OR customers.tier = Utf8("silver")) | -| | TableScan: customers, partial_filters=[customers.tier = Utf8("gold") OR customers.tier = Utf8("silver"), customers.customer_id IS NOT NULL] | -| | ]] | -| physical_plan | SortPreservingMergeExec: [amount@4 DESC], fetch=4 | -| | SortExec: TopK(fetch=4), expr=[amount@4 DESC], preserve_partitioning=[true] | -| | ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, name@3 as name, tier@4 as tier, amount@2 as amount] | -| | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(customer_id@1, customer_id@0)], projection=[id@0, customer_id@1, amount@2, name@4, tier@5] | -| | RepartitionExec: partitioning=REDACTED -| | ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, amount@2 as amount] | -| | MergeScanExec: REDACTED -| | RepartitionExec: partitioning=REDACTED -| | ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] | -| | MergeScanExec: REDACTED -| | | -+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | Sort: o.amount DESC NULLS FIRST, fetch=4 | +| | Projection: o.id, o.customer_id, c.name, c.tier, o.amount | +| | Inner Join: o.customer_id = c.customer_id | +| | Projection: o.id, o.customer_id, o.amount | +| | MergeScan [is_placeholder=false, remote_input=[ | +| | SubqueryAlias: o | +| | Filter: orders.customer_id IS NOT NULL | +| | TableScan: orders, partial_filters=[orders.customer_id IS NOT NULL] | +| | ]] | +| | Projection: c.customer_id, c.name, c.tier | +| | MergeScan [is_placeholder=false, remote_input=[ | +| | SubqueryAlias: c | +| | Filter: customers.customer_id IS NOT NULL AND (customers.tier = Utf8("gold") OR customers.tier = Utf8("silver")) | +| | TableScan: customers, partial_filters=[customers.customer_id IS NOT NULL, customers.tier = Utf8("gold") OR customers.tier = Utf8("silver")] | +| | ]] | +| physical_plan | SortPreservingMergeExec: [amount@4 DESC], fetch=4 | +| | SortExec: TopK(fetch=4), expr=[amount@4 DESC], preserve_partitioning=[true] | +| | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(customer_id@1, customer_id@0)], projection=[id@0, customer_id@1, name@4, tier@5, amount@2] | +| | RepartitionExec: partitioning=REDACTED +| | ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, amount@2 as amount] | +| | MergeScanExec: REDACTED +| | RepartitionExec: partitioning=REDACTED +| | ProjectionExec: expr=[customer_id@0 as customer_id, name@1 as name, tier@2 as tier] | +| | MergeScanExec: REDACTED +| | | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------+ -- SQLNESS REPLACE ("metrics_per_partition":\s*.*metrics=) "metrics_per_partition": REDACTED metrics= -- SQLNESS REPLACE (metrics=\{.*\}) metrics=REDACTED @@ -267,8 +266,7 @@ LIMIT 4; +-+-+-+ | 0_| 0_|_SortPreservingMergeExec: [amount@4 DESC], fetch=4 metrics=REDACTED_| |_|_|_SortExec: TopK(fetch=4), expr=[amount@4 DESC], preserve_partitioning=[true] metrics=REDACTED_| -|_|_|_ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, name@3 as name, tier@4 as tier, amount@2 as amount] metrics=REDACTED_| -|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(customer_id@1, customer_id@0)], projection=[id@0, customer_id@1, amount@2, name@4, tier@5] metrics=REDACTED_| +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(customer_id@1, customer_id@0)], projection=[id@0, customer_id@1, name@4, tier@5, amount@2] metrics=REDACTED_| |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_ProjectionExec: expr=[id@0 as id, customer_id@1 as customer_id, amount@2 as amount] metrics=REDACTED_| |_|_|_MergeScanExec: REDACTED @@ -282,7 +280,7 @@ LIMIT 4; |_|_|_| | 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND (tier@2 = gold OR tier@2 = silver) metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")", "customer_id IS NOT NULL"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ @@ -380,20 +378,20 @@ WHERE c.tier IN ('gold', 'silver') | | Projection: o.id, o.customer_id, o.product_id, o.amount | | | MergeScan [is_placeholder=false, remote_input=[ | | | SubqueryAlias: o | -| | Filter: orders.product_id IS NOT NULL AND orders.customer_id IS NOT NULL | -| | TableScan: orders, partial_filters=[orders.product_id IS NOT NULL, orders.customer_id IS NOT NULL] | +| | Filter: orders.customer_id IS NOT NULL AND orders.product_id IS NOT NULL | +| | TableScan: orders, partial_filters=[orders.customer_id IS NOT NULL, orders.product_id IS NOT NULL] | | | ]] | | | Projection: c.customer_id, c.name, c.tier | | | MergeScan [is_placeholder=false, remote_input=[ | | | SubqueryAlias: c | | | Filter: customers.customer_id IS NOT NULL AND (customers.tier = Utf8("gold") OR customers.tier = Utf8("silver")) | -| | TableScan: customers, partial_filters=[customers.tier = Utf8("gold") OR customers.tier = Utf8("silver"), customers.customer_id IS NOT NULL] | +| | TableScan: customers, partial_filters=[customers.customer_id IS NOT NULL, customers.tier = Utf8("gold") OR customers.tier = Utf8("silver")] | | | ]] | | | Projection: p.product_id, p.name, p.category | | | MergeScan [is_placeholder=false, remote_input=[ | | | SubqueryAlias: p | | | Filter: products.product_id IS NOT NULL AND products.category = Utf8("electronics") | -| | TableScan: products, partial_filters=[products.category = Utf8("electronics"), products.product_id IS NOT NULL] | +| | TableScan: products, partial_filters=[products.product_id IS NOT NULL, products.category = Utf8("electronics")] | | | ]] | | physical_plan | ProjectionExec: expr=[id@0 as id, amount@1 as amount, name@2 as name, tier@3 as tier, name@4 as product_name, category@5 as category] | | | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(product_id@1, product_id@0)], projection=[id@0, amount@2, name@3, tier@4, name@6, category@7] | @@ -452,17 +450,17 @@ WHERE c.tier IN ('gold', 'silver') |_|_|_ProjectionExec: expr=[product_id@0 as product_id, name@1 as name, category@2 as category] metrics=REDACTED_| |_|_|_MergeScanExec: REDACTED |_|_|_| -| 1_| 0_|_FilterExec: product_id@2 IS NOT NULL AND customer_id@1 IS NOT NULL metrics=REDACTED_| +| 1_| 0_|_FilterExec: customer_id@1 IS NOT NULL AND product_id@2 IS NOT NULL metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "product_id", "amount", "ts"], "filters": ["product_id IS NOT NULL", "customer_id IS NOT NULL"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "product_id", "amount", "ts"], "filters": ["customer_id IS NOT NULL", "product_id IS NOT NULL"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| | 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND (tier@2 = gold OR tier@2 = silver) metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")", "customer_id IS NOT NULL"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| | 1_| 0_|_FilterExec: product_id@0 IS NOT NULL AND category@2 = electronics metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["product_id", "name", "category", "ts"], "filters": ["category = Utf8(\"electronics\")", "product_id IS NOT NULL"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["product_id", "name", "category", "ts"], "filters": ["product_id IS NOT NULL", "category = Utf8(\"electronics\")"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ diff --git a/tests/cases/standalone/common/flow/flow_advance_ttl.result b/tests/cases/standalone/common/flow/flow_advance_ttl.result index 6b085390aa6..d482d9cbac5 100644 --- a/tests/cases/standalone/common/flow/flow_advance_ttl.result +++ b/tests/cases/standalone/common/flow/flow_advance_ttl.result @@ -99,8 +99,10 @@ FROM SELECT number FROM distinct_basic; -++ -++ ++--------+ +| number | ++--------+ ++--------+ -- SQLNESS SLEEP 6s ADMIN FLUSH_TABLE('distinct_basic'); @@ -142,8 +144,10 @@ FROM SELECT number FROM distinct_basic; -++ -++ ++--------+ +| number | ++--------+ ++--------+ DROP FLOW test_distinct_basic; diff --git a/tests/cases/standalone/common/flow/flow_flush.result b/tests/cases/standalone/common/flow/flow_flush.result index c94963488e5..d6625767b3a 100644 --- a/tests/cases/standalone/common/flow/flow_flush.result +++ b/tests/cases/standalone/common/flow/flow_flush.result @@ -42,8 +42,10 @@ SELECT FROM out_num_cnt_basic; -++ -++ ++---------------------------------+-------------+ +| sum(numbers_input_basic.number) | time_window | ++---------------------------------+-------------+ ++---------------------------------+-------------+ DROP FLOW test_numbers_basic; diff --git a/tests/cases/standalone/common/flow/flow_incremental_sequence_range.result b/tests/cases/standalone/common/flow/flow_incremental_sequence_range.result new file mode 100644 index 00000000000..a193bcd8742 --- /dev/null +++ b/tests/cases/standalone/common/flow/flow_incremental_sequence_range.result @@ -0,0 +1,143 @@ +-- Validate incremental sequence_range reads for an append-only source whose rows +-- preserve insertion sequence across SST flushes. +CREATE TABLE flow_incr_seq_range_input ( + host_id INT, + n INT, + ts TIMESTAMP TIME INDEX, + PRIMARY KEY(host_id) +) WITH ( + append_mode = 'true', + preserve_row_sequence = 'true' +); + +Affected Rows: 0 + +CREATE FLOW flow_incr_seq_range SINK TO flow_incr_seq_range_sink +WITH (experimental_enable_incremental_read = 'true') +AS +SELECT + sum(n) AS total, + count(n) AS row_count, + min(n) AS min_n, + max(n) AS max_n, + date_bin(INTERVAL '1 minute', ts, '2024-01-01 00:00:00') AS time_window +FROM + flow_incr_seq_range_input +GROUP BY + time_window; + +Affected Rows: 0 + +-- ==== Phase 1: initial insert + checkpoint ==== +INSERT INTO flow_incr_seq_range_input VALUES + (1, 10, '2024-01-01 00:00:00'), + (2, 20, '2024-01-01 00:00:15'), + (3, 30, '2024-01-01 00:00:30'); + +Affected Rows: 3 + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_incr_seq_range'); + ++-----------------------------------------+ +| ADMIN FLUSH_FLOW('flow_incr_seq_range') | ++-----------------------------------------+ +| FLOW_FLUSHED | ++-----------------------------------------+ + +SELECT total, row_count, min_n, max_n, time_window +FROM flow_incr_seq_range_sink +ORDER BY time_window; + ++-------+-----------+-------+-------+---------------------+ +| total | row_count | min_n | max_n | time_window | ++-------+-----------+-------+-------+---------------------+ +| 60 | 3 | 10 | 30 | 2024-01-01T00:00:00 | ++-------+-----------+-------+-------+---------------------+ + +-- Move the checkpointed source and sink state into SST files. +ADMIN FLUSH_TABLE('flow_incr_seq_range_sink'); + ++-----------------------------------------------+ +| ADMIN FLUSH_TABLE('flow_incr_seq_range_sink') | ++-----------------------------------------------+ +| 0 | ++-----------------------------------------------+ + +ADMIN FLUSH_TABLE('flow_incr_seq_range_input'); + ++------------------------------------------------+ +| ADMIN FLUSH_TABLE('flow_incr_seq_range_input') | ++------------------------------------------------+ +| 0 | ++------------------------------------------------+ + +-- ==== Phase 2: flushed delta in the same window ==== +INSERT INTO flow_incr_seq_range_input VALUES + (4, 40, '2024-01-01 00:00:45'), + (5, 50, '2024-01-01 00:00:55'); + +Affected Rows: 2 + +-- Flush the delta to SST before the incremental flow run. The sequence range +-- scan must read only these new rows, not the already-checkpointed SST. +ADMIN FLUSH_TABLE('flow_incr_seq_range_input'); + ++------------------------------------------------+ +| ADMIN FLUSH_TABLE('flow_incr_seq_range_input') | ++------------------------------------------------+ +| 0 | ++------------------------------------------------+ + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_incr_seq_range'); + ++-----------------------------------------+ +| ADMIN FLUSH_FLOW('flow_incr_seq_range') | ++-----------------------------------------+ +| FLOW_FLUSHED | ++-----------------------------------------+ + +SELECT total, row_count, min_n, max_n, time_window +FROM flow_incr_seq_range_sink +ORDER BY time_window; + ++-------+-----------+-------+-------+---------------------+ +| total | row_count | min_n | max_n | time_window | ++-------+-----------+-------+-------+---------------------+ +| 150 | 5 | 10 | 50 | 2024-01-01T00:00:00 | ++-------+-----------+-------+-------+---------------------+ + +-- ==== Empty incremental run ==== +-- No new source rows must leave the aggregate unchanged. +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_incr_seq_range'); + ++-----------------------------------------+ +| ADMIN FLUSH_FLOW('flow_incr_seq_range') | ++-----------------------------------------+ +| FLOW_FLUSHED | ++-----------------------------------------+ + +SELECT total, row_count, min_n, max_n, time_window +FROM flow_incr_seq_range_sink +ORDER BY time_window; + ++-------+-----------+-------+-------+---------------------+ +| total | row_count | min_n | max_n | time_window | ++-------+-----------+-------+-------+---------------------+ +| 150 | 5 | 10 | 50 | 2024-01-01T00:00:00 | ++-------+-----------+-------+-------+---------------------+ + +DROP FLOW flow_incr_seq_range; + +Affected Rows: 0 + +DROP TABLE flow_incr_seq_range_input; + +Affected Rows: 0 + +DROP TABLE flow_incr_seq_range_sink; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/flow/flow_incremental_sequence_range.sql b/tests/cases/standalone/common/flow/flow_incremental_sequence_range.sql new file mode 100644 index 00000000000..ffe8aedf263 --- /dev/null +++ b/tests/cases/standalone/common/flow/flow_incremental_sequence_range.sql @@ -0,0 +1,71 @@ +-- Validate incremental sequence_range reads for an append-only source whose rows +-- preserve insertion sequence across SST flushes. +CREATE TABLE flow_incr_seq_range_input ( + host_id INT, + n INT, + ts TIMESTAMP TIME INDEX, + PRIMARY KEY(host_id) +) WITH ( + append_mode = 'true', + preserve_row_sequence = 'true' +); + +CREATE FLOW flow_incr_seq_range SINK TO flow_incr_seq_range_sink +WITH (experimental_enable_incremental_read = 'true') +AS +SELECT + sum(n) AS total, + count(n) AS row_count, + min(n) AS min_n, + max(n) AS max_n, + date_bin(INTERVAL '1 minute', ts, '2024-01-01 00:00:00') AS time_window +FROM + flow_incr_seq_range_input +GROUP BY + time_window; + +-- ==== Phase 1: initial insert + checkpoint ==== +INSERT INTO flow_incr_seq_range_input VALUES + (1, 10, '2024-01-01 00:00:00'), + (2, 20, '2024-01-01 00:00:15'), + (3, 30, '2024-01-01 00:00:30'); + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_incr_seq_range'); + +SELECT total, row_count, min_n, max_n, time_window +FROM flow_incr_seq_range_sink +ORDER BY time_window; + +-- Move the checkpointed source and sink state into SST files. +ADMIN FLUSH_TABLE('flow_incr_seq_range_sink'); +ADMIN FLUSH_TABLE('flow_incr_seq_range_input'); + +-- ==== Phase 2: flushed delta in the same window ==== +INSERT INTO flow_incr_seq_range_input VALUES + (4, 40, '2024-01-01 00:00:45'), + (5, 50, '2024-01-01 00:00:55'); + +-- Flush the delta to SST before the incremental flow run. The sequence range +-- scan must read only these new rows, not the already-checkpointed SST. +ADMIN FLUSH_TABLE('flow_incr_seq_range_input'); + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_incr_seq_range'); + +SELECT total, row_count, min_n, max_n, time_window +FROM flow_incr_seq_range_sink +ORDER BY time_window; + +-- ==== Empty incremental run ==== +-- No new source rows must leave the aggregate unchanged. +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_incr_seq_range'); + +SELECT total, row_count, min_n, max_n, time_window +FROM flow_incr_seq_range_sink +ORDER BY time_window; + +DROP FLOW flow_incr_seq_range; +DROP TABLE flow_incr_seq_range_input; +DROP TABLE flow_incr_seq_range_sink; diff --git a/tests/cases/standalone/common/flow/flow_pending.result b/tests/cases/standalone/common/flow/flow_pending.result index d6fe01b38a8..ca74021b5cb 100644 --- a/tests/cases/standalone/common/flow/flow_pending.result +++ b/tests/cases/standalone/common/flow/flow_pending.result @@ -47,6 +47,8 @@ Affected Rows: 0 SELECT flow_name FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name = 'pending_with_defer'; -++ -++ ++-----------+ +| flow_name | ++-----------+ ++-----------+ diff --git a/tests/cases/standalone/common/flow/flow_status.result b/tests/cases/standalone/common/flow/flow_status.result index 7b7ff8a1f2f..c58277ecef7 100644 --- a/tests/cases/standalone/common/flow/flow_status.result +++ b/tests/cases/standalone/common/flow/flow_status.result @@ -59,8 +59,10 @@ SELECT flow_id, flow_name FROM information_schema.flow_statistics WHERE flow_nam -- the like filter matches against flow_name; no matching flow returns empty. SHOW FLOW STATUS LIKE 'no_such_flow'; -++ -++ ++---------+-----------+------------+---------------------+----------------+------------+ +| flow_id | flow_name | start_time | last_execution_time | uptime_seconds | state_size | ++---------+-----------+------------+---------------------+----------------+------------+ ++---------+-----------+------------+---------------------+----------------+------------+ DROP FLOW test_flow_status; diff --git a/tests/cases/standalone/common/flow/flow_user_guide.result b/tests/cases/standalone/common/flow/flow_user_guide.result index 8bd9cab9263..a6be36e9b18 100644 --- a/tests/cases/standalone/common/flow/flow_user_guide.result +++ b/tests/cases/standalone/common/flow/flow_user_guide.result @@ -444,8 +444,10 @@ SELECT FROM temp_alerts; -++ -++ ++-----------+-----+----------+----------+ +| sensor_id | loc | max_temp | event_ts | ++-----------+-----+----------+----------+ ++-----------+-----+----------+----------+ INSERT INTO temp_sensor_data diff --git a/tests/cases/standalone/common/flow/show_create_flow.result b/tests/cases/standalone/common/flow/show_create_flow.result index 431d1dfbb55..9499618ec60 100644 --- a/tests/cases/standalone/common/flow/show_create_flow.result +++ b/tests/cases/standalone/common/flow/show_create_flow.result @@ -17,13 +17,17 @@ Affected Rows: 0 SELECT flow_name, table_catalog, flow_definition, source_table_names FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show'; -++ -++ ++-----------+---------------+-----------------+--------------------+ +| flow_name | table_catalog | flow_definition | source_table_names | ++-----------+---------------+-----------------+--------------------+ ++-----------+---------------+-----------------+--------------------+ SHOW FLOWS LIKE 'filter_numbers_show'; -++ -++ ++-------+ +| Flows | ++-------+ ++-------+ CREATE FLOW filter_numbers_show SINK TO out_num_cnt_show AS SELECT number FROM numbers_input_show where number > 10; @@ -63,13 +67,17 @@ Affected Rows: 0 SELECT flow_name, table_catalog, flow_definition, source_table_names FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show'; -++ -++ ++-----------+---------------+-----------------+--------------------+ +| flow_name | table_catalog | flow_definition | source_table_names | ++-----------+---------------+-----------------+--------------------+ ++-----------+---------------+-----------------+--------------------+ SHOW FLOWS LIKE 'filter_numbers_show'; -++ -++ ++-------+ +| Flows | ++-------+ ++-------+ -- also test `CREATE OR REPLACE` and `IF NOT EXISTS` -- (flow exists, replace, if not exists)=(false, false, false) @@ -235,8 +243,10 @@ Error: 1001(Unsupported), Unsupported operation Create flow with both `IF NOT EX SELECT flow_name, table_catalog, flow_definition, source_table_names FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show'; -++ -++ ++-----------+---------------+-----------------+--------------------+ +| flow_name | table_catalog | flow_definition | source_table_names | ++-----------+---------------+-----------------+--------------------+ ++-----------+---------------+-----------------+--------------------+ DROP FLOW filter_numbers_show; diff --git a/tests/cases/standalone/common/function/geo.result b/tests/cases/standalone/common/function/geo.result index 65ba42e6ced..31efbf401da 100644 --- a/tests/cases/standalone/common/function/geo.result +++ b/tests/cases/standalone/common/function/geo.result @@ -360,11 +360,11 @@ FROM( SELECT UNNEST(geo_path(37.76938, -122.3889, 1728083375::TimestampSecond)); -+-----------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ -| __unnest_placeholder(geo_path(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(1728083375),Utf8("Timestamp(s)")))).lat | __unnest_placeholder(geo_path(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(1728083375),Utf8("Timestamp(s)")))).lng | -+-----------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ -| [37.76938] | [-122.3889] | -+-----------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------+ ++-------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------+ +| geo_path(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(1728083375),Utf8("Timestamp(s)"))).lat | geo_path(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(1728083375),Utf8("Timestamp(s)"))).lng | ++-------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------+ +| [37.76938] | [-122.3889] | ++-------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------+ SELECT UNNEST(geo_path(lat, lon, ts)) FROM( @@ -377,11 +377,11 @@ FROM( SELECT 37.77001 AS lat, -122.3888 AS lon, 1728083372::TimestampSecond AS ts ); -+------------------------------------------------+------------------------------------------------+ -| __unnest_placeholder(geo_path(lat,lon,ts)).lat | __unnest_placeholder(geo_path(lat,lon,ts)).lng | -+------------------------------------------------+------------------------------------------------+ -| [37.77001, 37.76928, 37.76938, 37.7693] | [-122.3888, -122.3839, -122.3889, -122.382] | -+------------------------------------------------+------------------------------------------------+ ++-----------------------------------------+---------------------------------------------+ +| geo_path(lat,lon,ts).lat | geo_path(lat,lon,ts).lng | ++-----------------------------------------+---------------------------------------------+ +| [37.77001, 37.76928, 37.76938, 37.7693] | [-122.3888, -122.3839, -122.3889, -122.382] | ++-----------------------------------------+---------------------------------------------+ SELECT wkt_point_from_latlng(37.76938, -122.3889) AS point; diff --git a/tests/cases/standalone/common/function/json/json_get.result b/tests/cases/standalone/common/function/json/json_get.result index 830d4e73b9c..3c58ca91b93 100644 --- a/tests/cases/standalone/common/function/json/json_get.result +++ b/tests/cases/standalone/common/function/json/json_get.result @@ -450,8 +450,10 @@ SELECT json_to_string(j) FROM jsons WHERE json_to_string(json_get_object(j, 'a.b SELECT json_to_string(j) FROM jsons WHERE json_get_string(json_get_object(j, 'a.x'), 'c') == 'foo'; -++ -++ ++-------------------------+ +| json_to_string(jsons.j) | ++-------------------------+ ++-------------------------+ DROP TABLE jsons; diff --git a/tests/cases/standalone/common/function/matches_term.result b/tests/cases/standalone/common/function/matches_term.result index 37ecf5a55fa..d14bcf7ac22 100644 --- a/tests/cases/standalone/common/function/matches_term.result +++ b/tests/cases/standalone/common/function/matches_term.result @@ -488,13 +488,17 @@ SELECT * FROM zh_logs where `log_message` @@ 'ship_'; SELECT * FROM zh_logs where `log_message` @@ '登录_id'; -++ -++ ++----+-------------+ +| id | log_message | ++----+-------------+ ++----+-------------+ SELECT * FROM zh_logs where `log_message` @@ '手机号_trace'; -++ -++ ++----+-------------+ +| id | log_message | ++----+-------------+ ++----+-------------+ SELECT * FROM zh_logs where `log_message` @@ '手机'; diff --git a/tests/cases/standalone/common/function/string/replace.result b/tests/cases/standalone/common/function/string/replace.result index a4e1790d34e..d829016f869 100644 --- a/tests/cases/standalone/common/function/string/replace.result +++ b/tests/cases/standalone/common/function/string/replace.result @@ -38,7 +38,7 @@ SELECT REPLACE('hello world', '', 'xyz'); +---------------------------------------------------+ | replace(Utf8("hello world"),Utf8(""),Utf8("xyz")) | +---------------------------------------------------+ -| xyzhxyzexyzlxyzlxyzoxyz xyzwxyzoxyzrxyzlxyzdxyz | +| hello world | +---------------------------------------------------+ SELECT REPLACE('', 'xyz', 'abc'); diff --git a/tests/cases/standalone/common/insert/append_mode.result b/tests/cases/standalone/common/insert/append_mode.result index 1e715ec0044..cf8f4b3378e 100644 --- a/tests/cases/standalone/common/insert/append_mode.result +++ b/tests/cases/standalone/common/insert/append_mode.result @@ -12,8 +12,10 @@ Affected Rows: 0 SELECT host, ts from append_mode_on ORDER BY host, ts; -++ -++ ++------+----+ +| host | ts | ++------+----+ ++------+----+ INSERT INTO append_mode_on VALUES ('host1',0, 0), ('host2', 1, 1,); diff --git a/tests/cases/standalone/common/insert/logical_metric_table.result b/tests/cases/standalone/common/insert/logical_metric_table.result index fe35ce6aa82..4ea332c2f5c 100644 --- a/tests/cases/standalone/common/insert/logical_metric_table.result +++ b/tests/cases/standalone/common/insert/logical_metric_table.result @@ -25,8 +25,10 @@ Affected Rows: 0 SELECT * from t2; -++ -++ ++-----+----+-----+ +| job | ts | val | ++-----+----+-----+ ++-----+----+-----+ INSERT INTO t2 VALUES ('job1', 0, 0), ('job2', 1, 1); @@ -145,8 +147,10 @@ Affected Rows: 0 SELECT * from t2; -++ -++ ++-----+----+-----+ +| job | ts | val | ++-----+----+-----+ ++-----+----+-----+ INSERT INTO t2 VALUES ('job1', 0, 0), ('job2', 1, 1); diff --git a/tests/cases/standalone/common/insert/merge_mode.result b/tests/cases/standalone/common/insert/merge_mode.result index a5e03ae8b90..ff2d111a1d1 100644 --- a/tests/cases/standalone/common/insert/merge_mode.result +++ b/tests/cases/standalone/common/insert/merge_mode.result @@ -157,6 +157,7 @@ DROP TABLE `delete_between`; Affected Rows: 0 +-- SQLNESS REPLACE (\sat\sline\s\d+\scolumn\s\d+) create table if not exists invalid_merge_mode( host string, ts timestamp, @@ -168,7 +169,7 @@ create table if not exists invalid_merge_mode( engine=mito with('merge_mode'='first_row'); -Error: 1004(InvalidArguments), Invalid options: Matching variant not found at line 1 column 25 +Error: 1004(InvalidArguments), Invalid options: Matching variant not found create table if not exists invalid_merge_mode( host string, diff --git a/tests/cases/standalone/common/insert/merge_mode.sql b/tests/cases/standalone/common/insert/merge_mode.sql index 9d22cc13d65..c66354f0182 100644 --- a/tests/cases/standalone/common/insert/merge_mode.sql +++ b/tests/cases/standalone/common/insert/merge_mode.sql @@ -71,6 +71,7 @@ SELECT * FROM `delete_between`; DROP TABLE `delete_between`; +-- SQLNESS REPLACE (\sat\sline\s\d+\scolumn\s\d+) create table if not exists invalid_merge_mode( host string, ts timestamp, diff --git a/tests/cases/standalone/common/join/join_with_nulls.result b/tests/cases/standalone/common/join/join_with_nulls.result index 81fceba0ff6..df2f4290935 100644 --- a/tests/cases/standalone/common/join/join_with_nulls.result +++ b/tests/cases/standalone/common/join/join_with_nulls.result @@ -41,8 +41,10 @@ SELECT * FROM null_left l LEFT JOIN null_right r ON l."id" = r."id" ORDER BY l.t -- JOIN on string columns with NULLs SELECT * FROM null_left l INNER JOIN null_right r ON l.val = r.val ORDER BY l.ts; -++ -++ ++----+-----+----+----+-----+----+ +| id | val | ts | id | val | ts | ++----+-----+----+----+-----+----+ ++----+-----+----+----+-----+----+ -- JOIN with IS NOT DISTINCT FROM (treats NULL=NULL as true) SELECT * FROM null_left l INNER JOIN null_right r ON l."id" IS NOT DISTINCT FROM r."id" ORDER BY l.ts; diff --git a/tests/cases/standalone/common/join/self_join.result b/tests/cases/standalone/common/join/self_join.result index 4c6825a8365..47204026151 100644 --- a/tests/cases/standalone/common/join/self_join.result +++ b/tests/cases/standalone/common/join/self_join.result @@ -44,8 +44,10 @@ JOIN employees_self m ON e.manager_id = m."id" WHERE e.salary > m.salary ORDER BY e."name"; -++ -++ ++----------+--------+---------+----------------+ +| employee | salary | manager | manager_salary | ++----------+--------+---------+----------------+ ++----------+--------+---------+----------------+ -- Self join to find colleagues (same manager) SELECT e1."name" as employee1, e2."name" as employee2, m."name" as shared_manager diff --git a/tests/cases/standalone/common/order/limit.result b/tests/cases/standalone/common/order/limit.result index 54735fac2db..b69f44e0465 100644 --- a/tests/cases/standalone/common/order/limit.result +++ b/tests/cases/standalone/common/order/limit.result @@ -25,7 +25,7 @@ SELECT b FROM test ORDER BY b LIMIT 2 OFFSET 0; SELECT a FROM test LIMIT 1.25; -Error: 3001(EngineExecuteQuery), Error during planning: Expected LIMIT to be an integer or null, but got Float64 +Error: 3000(PlanQuery), Error during planning: Expected LIMIT to be an integer or null, but got Float64 SELECT a FROM test LIMIT 2-1; @@ -45,11 +45,11 @@ Error: 3000(PlanQuery), Failed to plan SQL: No field named a. SELECT a FROM test LIMIT SUM(42); -Error: 1001(Unsupported), This feature is not implemented: Unsupported LIMIT expression: Some(AggregateFunction(AggregateFunction { func: AggregateUDF { inner: Sum { signature: Signature { type_signature: OneOf([Coercible([Exact { desired_type: Decimal }]), Coercible([Implicit { desired_type: Native(LogicalType(Native(UInt64), UInt64)), implicit_coercion: ImplicitCoercion { allowed_source_types: [Native(LogicalType(Native(UInt8), UInt8)), Native(LogicalType(Native(UInt16), UInt16)), Native(LogicalType(Native(UInt32), UInt32))], default_casted_type: UInt64 } }]), Coercible([Implicit { desired_type: Native(LogicalType(Native(Int64), Int64)), implicit_coercion: ImplicitCoercion { allowed_source_types: [Native(LogicalType(Native(Int8), Int8)), Native(LogicalType(Native(Int16), Int16)), Native(LogicalType(Native(Int32), Int32))], default_casted_type: Int64 } }]), Coercible([Implicit { desired_type: Native(LogicalType(Native(Float64), Float64)), implicit_coercion: ImplicitCoercion { allowed_source_types: [Float], default_casted_type: Float64 } }]), Coercible([Exact { desired_type: Duration }])]), volatility: Immutable, parameter_names: None } } }, params: AggregateFunctionParams { args: [Literal(Int64(42), None)], distinct: false, filter: None, order_by: [], null_treatment: None } })) +Error: 1001(Unsupported), This feature is not implemented: Unsupported LIMIT expression: Some(AggregateFunction(AggregateFunction { func: AggregateUDF { inner: Sum { signature: Signature { type_signature: OneOf([Coercible([Exact { desired_type: Decimal, encoding_preservation: EncodingPreservation { preserve_dictionary: false } }]), Coercible([Implicit { desired_type: Native(LogicalType(Native(UInt64), UInt64)), implicit_coercion: ImplicitCoercion { allowed_source_types: [Native(LogicalType(Native(UInt8), UInt8)), Native(LogicalType(Native(UInt16), UInt16)), Native(LogicalType(Native(UInt32), UInt32))], default_casted_type: UInt64 }, encoding_preservation: EncodingPreservation { preserve_dictionary: false } }]), Coercible([Implicit { desired_type: Native(LogicalType(Native(Int64), Int64)), implicit_coercion: ImplicitCoercion { allowed_source_types: [Native(LogicalType(Native(Int8), Int8)), Native(LogicalType(Native(Int16), Int16)), Native(LogicalType(Native(Int32), Int32))], default_casted_type: Int64 }, encoding_preservation: EncodingPreservation { preserve_dictionary: false } }]), Coercible([Implicit { desired_type: Native(LogicalType(Native(Float64), Float64)), implicit_coercion: ImplicitCoercion { allowed_source_types: [Float], default_casted_type: Float64 }, encoding_preservation: EncodingPreservation { preserve_dictionary: false } }]), Coercible([Exact { desired_type: Duration, encoding_preservation: EncodingPreservation { preserve_dictionary: false } }]), Coercible([Exact { desired_type: Interval, encoding_preservation: EncodingPreservation { preserve_dictionary: false } }])]), volatility: Immutable, parameter_names: None } } }, params: AggregateFunctionParams { args: [Literal(Int64(42), None)], distinct: false, filter: None, order_by: [], null_treatment: None } })) SELECT a FROM test LIMIT row_number() OVER (); -Error: 3001(EngineExecuteQuery), This feature is not implemented: Unsupported LIMIT expression: Some(Cast(Cast { expr: WindowFunction(WindowFunction { fun: WindowUDF(WindowUDF { inner: RowNumber { signature: Signature { type_signature: Nullary, volatility: Immutable, parameter_names: None } } }), params: WindowFunctionParams { args: [], partition_by: [], order_by: [], window_frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }, filter: None, null_treatment: None, distinct: false } }), data_type: Int64 })) +Error: 1001(Unsupported), This feature is not implemented: Unsupported LIMIT expression: Some(Cast(Cast { expr: WindowFunction(WindowFunction { fun: WindowUDF(WindowUDF { inner: RowNumber { signature: Signature { type_signature: Nullary, volatility: Immutable, parameter_names: None } } }), params: WindowFunctionParams { args: [], partition_by: [], order_by: [], window_frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }, filter: None, null_treatment: None, distinct: false } }), field: Field { name: "", data_type: Int64, nullable: true } })) CREATE TABLE test2 (a STRING, ts TIMESTAMP TIME INDEX); @@ -69,7 +69,7 @@ SELECT * FROM test2 LIMIT 3; select 1 limit date '1992-01-01'; -Error: 3001(EngineExecuteQuery), Error during planning: Expected LIMIT to be an integer or null, but got Date32 +Error: 3000(PlanQuery), Error during planning: Expected LIMIT to be an integer or null, but got Date32 CREATE TABLE integers(i TIMESTAMP TIME INDEX); @@ -102,23 +102,23 @@ SELECT * FROM integers LIMIT 4; SELECT * FROM integers as int LIMIT (SELECT MIN(integers.i) FROM integers); -Error: 3001(EngineExecuteQuery), Error during planning: Expected LIMIT to be an integer or null, but got Timestamp(ms) +Error: 3000(PlanQuery), Error during planning: Expected LIMIT to be an integer or null, but got Timestamp(ms) SELECT * FROM integers as int OFFSET (SELECT MIN(integers.i) FROM integers); -Error: 3001(EngineExecuteQuery), Error during planning: Expected OFFSET to be an integer or null, but got Timestamp(ms) +Error: 3000(PlanQuery), Error during planning: Expected OFFSET to be an integer or null, but got Timestamp(ms) SELECT * FROM integers as int LIMIT (SELECT MAX(integers.i) FROM integers) OFFSET (SELECT MIN(integers.i) FROM integers); -Error: 3001(EngineExecuteQuery), Error during planning: Expected LIMIT to be an integer or null, but got Timestamp(ms) +Error: 3000(PlanQuery), Error during planning: Expected LIMIT to be an integer or null, but got Timestamp(ms) SELECT * FROM integers as int LIMIT (SELECT max(integers.i) FROM integers where i > 5); -Error: 3001(EngineExecuteQuery), Error during planning: Expected LIMIT to be an integer or null, but got Timestamp(ms) +Error: 3000(PlanQuery), Error during planning: Expected LIMIT to be an integer or null, but got Timestamp(ms) SELECT * FROM integers as int LIMIT (SELECT max(integers.i) FROM integers where i > 5); -Error: 3001(EngineExecuteQuery), Error during planning: Expected LIMIT to be an integer or null, but got Timestamp(ms) +Error: 3000(PlanQuery), Error during planning: Expected LIMIT to be an integer or null, but got Timestamp(ms) SELECT * FROM integers as int LIMIT (SELECT NULL); @@ -130,7 +130,7 @@ Error: 1001(Unsupported), This feature is not implemented: Unsupported LIMIT exp SELECT * FROM integers as int LIMIT (SELECT 'ab'); -Error: 3001(EngineExecuteQuery), Error during planning: Expected LIMIT to be an integer or null, but got Utf8 +Error: 3000(PlanQuery), Error during planning: Expected LIMIT to be an integer or null, but got Utf8 DROP TABLE integers; diff --git a/tests/cases/standalone/common/order/limit_zero.result b/tests/cases/standalone/common/order/limit_zero.result index 8460464b4f3..a520881614e 100644 --- a/tests/cases/standalone/common/order/limit_zero.result +++ b/tests/cases/standalone/common/order/limit_zero.result @@ -11,20 +11,26 @@ Affected Rows: 3 -- Test LIMIT 0 returns empty result SELECT * FROM test_data LIMIT 0; -++ -++ ++---+----+ +| i | ts | ++---+----+ ++---+----+ -- Test LIMIT 0 with aggregation SELECT SUM(i) FROM test_data LIMIT 0; -++ -++ ++------------------+ +| sum(test_data.i) | ++------------------+ ++------------------+ -- Test LIMIT 0 with WHERE clause SELECT * FROM test_data WHERE i > 0 LIMIT 0; -++ -++ ++---+----+ +| i | ts | ++---+----+ ++---+----+ -- Clean up DROP TABLE test_data; diff --git a/tests/cases/standalone/common/order/order_by.result b/tests/cases/standalone/common/order/order_by.result index aed1a491c27..da47fa663c6 100644 --- a/tests/cases/standalone/common/order/order_by.result +++ b/tests/cases/standalone/common/order/order_by.result @@ -196,7 +196,8 @@ SELECT a-10 AS k FROM test UNION SELECT a-10 AS l FROM test ORDER BY k; -- CONTROVERSIAL: SQLite allows both "k" and "l" to be referenced here, Postgres and MonetDB give an error. SELECT a-10 AS k FROM test UNION SELECT a-10 AS l FROM test ORDER BY l; -Error: 3000(PlanQuery), Failed to plan SQL: No field named l. Valid fields are k. +Error: 3000(PlanQuery), Failed to plan SQL: No field named l. +Valid fields are k. -- Not compatible with duckdb, work in gretimedb SELECT a-10 AS k FROM test UNION SELECT a-10 AS l FROM test ORDER BY 1-k; diff --git a/tests/cases/standalone/common/order/order_by_expressions.result b/tests/cases/standalone/common/order/order_by_expressions.result index f121fac188b..5c378671487 100644 --- a/tests/cases/standalone/common/order/order_by_expressions.result +++ b/tests/cases/standalone/common/order/order_by_expressions.result @@ -129,7 +129,14 @@ FROM test WHERE a IS NOT NULL ORDER BY a - (SELECT MIN(a) FROM test WHERE a IS NOT NULL); -Error: 1001(Unsupported), This feature is not implemented: Physical plan does not support logical expression ScalarSubquery() ++---+----+---------------+ +| a | b | diff_from_min | ++---+----+---------------+ +| 1 | 10 | 0 | +| 2 | 20 | 1 | +| 3 | 15 | 2 | +| 4 | 25 | 3 | ++---+----+---------------+ DROP TABLE test; diff --git a/tests/cases/standalone/common/partition.result b/tests/cases/standalone/common/partition.result index 9f512c33b98..fe6f99df591 100644 --- a/tests/cases/standalone/common/partition.result +++ b/tests/cases/standalone/common/partition.result @@ -104,8 +104,10 @@ Affected Rows: 5 SELECT * FROM my_table; -++ -++ ++---+---+----+ +| a | b | ts | ++---+---+----+ ++---+---+----+ DROP TABLE my_table; diff --git a/tests/cases/standalone/common/promql/absent.result b/tests/cases/standalone/common/promql/absent.result index 9c4fb4a2f72..468e173c775 100644 --- a/tests/cases/standalone/common/promql/absent.result +++ b/tests/cases/standalone/common/promql/absent.result @@ -23,14 +23,18 @@ Affected Rows: 8 -- SQLNESS SORT_RESULT 3 1 tql eval (0, 15, '5s') absent(t{job="job1"}); -++ -++ ++----+-----+-----+ +| ts | val | job | ++----+-----+-----+ ++----+-----+-----+ -- SQLNESS SORT_RESULT 3 1 tql eval (0, 15, '5s') absent(t{job="job2"}); -++ -++ ++----+-----+-----+ +| ts | val | job | ++----+-----+-----+ ++----+-----+-----+ -- SQLNESS SORT_RESULT 3 1 tql eval (0, 15, '5s') absent(t{job="job3"}); diff --git a/tests/cases/standalone/common/promql/encode_substrait.result b/tests/cases/standalone/common/promql/encode_substrait.result index bfc675b1eea..230eadb02f7 100644 --- a/tests/cases/standalone/common/promql/encode_substrait.result +++ b/tests/cases/standalone/common/promql/encode_substrait.result @@ -42,8 +42,10 @@ tql eval (0, 100, '1s') tag_a="ffa", }[1h])[12h:1h]; -++ -++ ++----+-----------------------------------------------+-------+-------+----------+ +| ts | prom_increase(ts_range,val,ts,Int64(3600000)) | tag_a | tag_b | ts_range | ++----+-----------------------------------------------+-------+-------+----------+ ++----+-----------------------------------------------+-------+-------+----------+ drop table count_total; diff --git a/tests/cases/standalone/common/promql/histogram_multi_partition.result b/tests/cases/standalone/common/promql/histogram_multi_partition.result index 3fba2ba8465..64e2f1e8551 100644 --- a/tests/cases/standalone/common/promql/histogram_multi_partition.result +++ b/tests/cases/standalone/common/promql/histogram_multi_partition.result @@ -44,9 +44,9 @@ tql analyze (0, 10, '10s') histogram_quantile(0.5, sum by (le) (histogram_gap_bu | 0_| 0_|_HistogramFoldExec: le=@0, field=@2, quantile=0.5 REDACTED |_|_|_SortExec: expr=[ts@1 ASC NULLS LAST, TRY_CAST(le@0 AS Float64) ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_RepartitionExec: partitioning=Hash([ts@1],REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[le@0 as le, ts@1 as ts], aggr=[sum(histogram_gap_bucket.val)] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[le@0 as le, ts@1 as ts], aggr=[__sum_merge(__sum_state(histogram_gap_bucket.val)) as sum(histogram_gap_bucket.val)] REDACTED |_|_|_RepartitionExec: partitioning=Hash([le@0, ts@1],REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[le@0 as le, ts@1 as ts], aggr=[sum(histogram_gap_bucket.val)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[le@0 as le, ts@1 as ts], aggr=[__sum_merge(__sum_state(histogram_gap_bucket.val)) as sum(histogram_gap_bucket.val)] REDACTED |_|_|_RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| diff --git a/tests/cases/standalone/common/promql/instant_last_row.result b/tests/cases/standalone/common/promql/instant_last_row.result index 5093a357ea8..16739197fb0 100644 --- a/tests/cases/standalone/common/promql/instant_last_row.result +++ b/tests/cases/standalone/common/promql/instant_last_row.result @@ -52,8 +52,10 @@ Affected Rows: 0 -- SQLNESS SORT_RESULT 3 1 TQL EVAL (1000, 1000, '1s') instant_last_empty; -++ -++ ++----+-----+--------+ +| ts | val | series | ++----+-----+--------+ ++----+-----+--------+ DROP TABLE instant_last_empty; @@ -444,8 +446,10 @@ Affected Rows: 1 -- SQLNESS SORT_RESULT 3 1 TQL EVAL (1, 1, '1s') instant_last_field_filter{__field__="val"} > 5; -++ -++ ++-----+------+----------+----+ +| val | host | instance | ts | ++-----+------+----------+----+ ++-----+------+----------+----+ -- A value matcher filters the scan before selecting a sample: the older 10 -- remains eligible, unlike the post-selection comparison above. diff --git a/tests/cases/standalone/common/promql/label.result b/tests/cases/standalone/common/promql/label.result index bd9228a324c..112b6261715 100644 --- a/tests/cases/standalone/common/promql/label.result +++ b/tests/cases/standalone/common/promql/label.result @@ -103,8 +103,10 @@ TQL EVAL (0, 15, '5s') label_join(test{host="host1"}, "new_host", "-", "idc", "h -- Should return empty result instead of error tql eval label_join(demo_num_cpus, "new_label", "-", "instance", "job"); -++ -++ ++------+-------+-----------+ +| time | value | new_label | ++------+-------+-----------+ ++------+-------+-----------+ -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 15, '5s') label_replace(test{host="host1"}, "new_idc", "$2", "idc", "(.*):(.*)"); @@ -202,14 +204,18 @@ TQL EVAL(0, 15, '5s') label_replace(vector(1), "host", "host1", "", ""); -- SQLNESS SORT_RESULT 3 1 TQL EVAL(0, 15, '5s') {__name__="test",host="host1"} * label_replace(vector(1), "host", "host1", "", ""); -++ -++ ++------+------+----------------------------+ +| host | time | test.val * .greptime_value | ++------+------+----------------------------+ ++------+------+----------------------------+ -- SQLNESS SORT_RESULT 3 1 TQL EVAL(0, 15, '5s') {__name__="test",host="host1"} + label_replace(vector(1), "host", "host1", "", ""); -++ -++ ++------+------+----------------------------+ +| host | time | test.val + .greptime_value | ++------+------+----------------------------+ ++------+------+----------------------------+ -- Empty regex with existing source label -- SQLNESS SORT_RESULT 3 1 @@ -272,15 +278,19 @@ TQL EVAL(0, 15, '5s') label_replace(test{host="host1"}, "host2", "", "instance", -- SQLNESS SORT_RESULT 3 1 TQL EVAL(0, 15, '5s') {__name__="test",host="host1"} * label_replace(vector(1), "host", "host2", "host", ""); -++ -++ ++------+------+----------------------------+ +| host | time | test.val * .greptime_value | ++------+------+----------------------------+ ++------+------+----------------------------+ -- Empty regex and not existing label in left expression -- SQLNESS SORT_RESULT 3 1 TQL EVAL(0, 15, '5s') {__name__="test",host="host1"} * label_replace(vector(1), "addr", "host1", "instance", ""); -++ -++ ++------+------+----------------------------+ +| addr | time | test.val * .greptime_value | ++------+------+----------------------------+ ++------+------+----------------------------+ TQL EVAL label_replace(demo_num_cpus, "~invalid", "", "src", "(.*)"); @@ -362,8 +372,10 @@ SELECT * FROM test; -- test the non-existent matchers -- TQL EVAL (0, 1, '5s') test{job=~"host1|host3"}; -++ -++ ++----+------+-----+ +| ts | host | val | ++----+------+-----+ ++----+------+-----+ -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 1, '5s') test{job=~".*"}; @@ -377,8 +389,10 @@ TQL EVAL (0, 1, '5s') test{job=~".*"}; TQL EVAL (0, 1, '5s') test{job=~".+"}; -++ -++ ++----+------+-----+ +| ts | host | val | ++----+------+-----+ ++----+------+-----+ -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 1, '5s') test{job=""}; @@ -392,8 +406,10 @@ TQL EVAL (0, 1, '5s') test{job=""}; TQL EVAL (0, 1, '5s') test{job!=""}; -++ -++ ++----+------+-----+ +| ts | host | val | ++----+------+-----+ ++----+------+-----+ DROP TABLE test; diff --git a/tests/cases/standalone/common/promql/native_time_selection.result b/tests/cases/standalone/common/promql/native_time_selection.result new file mode 100644 index 00000000000..2c0e20847cb --- /dev/null +++ b/tests/cases/standalone/common/promql/native_time_selection.result @@ -0,0 +1,619 @@ +-- Regression coverage for instant and range selection on native microsecond and +-- nanosecond time indexes. +CREATE TABLE native_time_us ( + ts TIMESTAMP(6) TIME INDEX, + series STRING PRIMARY KEY, + val DOUBLE, +); + +Affected Rows: 0 + +INSERT INTO native_time_us VALUES + (1000001, 'future', 101), + (1000000, 'exact', 201), + (1000001, 'exact', 202), + (-299000000, 'lowerbound', 301), + (-298999999, 'lowerplus', 302), + (1000000, 'positive_lowerbound', 701), + (1000001, 'positive_lowerplus', 702), + (1000001, 'multi', 401), + (-1000000, 'offset', 501), + (0, 'offset', 502), + (1000000, 'offset', 503), + (999999, 'past', 602), + (999001, 'past', 601), + (0, 'window', 1), + (1, 'window', 2), + (2, 'window', 5), + (1000000, 'window', 3), + (1000001, 'window', 4); + +Affected Rows: 18 + +-- The native projection and exact 1ms-lookback bounds must reach the scan; +-- the 1s+tick row must not displace 201. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE native_time_us.__table_id\s*=\s*UInt32\(\d+\) native_time_us.__table_id=UInt32(REDACTED) +TQL EXPLAIN (1, 1, '1s', '1ms') native_time_us{series="exact"}; + ++---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | PromInstantManipulate: range=[1000..1000], lookback=[1], interval=[1000], time index=[ts] | +| | PromSeriesDivide: tags=["series"] | +| | Sort: native_time_us.series ASC NULLS FIRST, native_time_us.ts ASC NULLS FIRST | +| | Projection: native_time_us.val, native_time_us.series, native_time_us.ts | +| | Filter: native_time_us.series = Utf8("exact") AND native_time_us.ts > TimestampMicrosecond(999000, None) AND native_time_us.ts <= TimestampMicrosecond(1000000, None) | +| | TableScan: native_time_us, partial_filters=[native_time_us.series = Utf8("exact"), native_time_us.ts > TimestampMicrosecond(999000, None), native_time_us.ts <= TimestampMicrosecond(1000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- The actual memtable scan must use LastRow { after_merge: true } with native +-- 1ms-lookback bounds; it must select exact 1s rather than the future tick. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +TQL ANALYZE VERBOSE (1, 1, '1s', '1ms') native_time_us{series="exact"}; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_CooperativeExec metrics=[]_| +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[1000..1000], lookback=[1], interval=[1000], time index=[ts] metrics=[output_rows: 1, REDACTED +|_|_|_PromSeriesDivideExec: tags=["series"] metrics=[output_rows: 1, REDACTED +|_|_|_ProjectionExec: expr=[val@2 as val, series@1 as series, ts@0 as ts] metrics=[output_rows: 1, REDACTED +|_|_|_CooperativeExec metrics=[]_| +|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "series", "val"], "filters": ["series = Dictionary(UInt32, Utf8(\"exact\"))", "ts > TimestampMicrosecond(999000, None)", "ts <= TimestampMicrosecond(1000000, None)"], "REDACTED +|_|_|_| +|_|_| Total rows: 1_| ++-+-+-+ + +-- The same-series future tick is in the memtable, while exact 1s remains selected. +TQL EVAL (1, 1, '1s', '1ms') native_time_us{series="exact"}; + ++-------+--------+---------------------+ +| val | series | ts | ++-------+--------+---------------------+ +| 201.0 | exact | 1970-01-01T00:00:01 | ++-------+--------+---------------------+ + +-- Future-only selection is empty before flushing, exercising the memtable path. +TQL EVAL (1, 1, '1s', '300s') native_time_us{series="future"}; + ++-----+--------+----+ +| val | series | ts | ++-----+--------+----+ ++-----+--------+----+ + +ADMIN FLUSH_TABLE('native_time_us'); + ++-------------------------------------+ +| ADMIN FLUSH_TABLE('native_time_us') | ++-------------------------------------+ +| 0 | ++-------------------------------------+ + +-- The exact native sample remains selected from the flushed SST. +TQL EVAL (1, 1, '1s', '1ms') native_time_us{series="exact"}; + ++-------+--------+---------------------+ +| val | series | ts | ++-------+--------+---------------------+ +| 201.0 | exact | 1970-01-01T00:00:01 | ++-------+--------+---------------------+ + +TQL EVAL (1, 1, '1s', '300s') timestamp(native_time_us{series="future"}); + ++----+-------+--------+ +| ts | value | series | ++----+-------+--------+ ++----+-------+--------+ + +TQL EVAL (1, 1, '1s', '300s') timestamp(native_time_us{series="exact"}); + ++---------------------+-------+--------+ +| ts | value | series | ++---------------------+-------+--------+ +| 1970-01-01T00:00:01 | 1.0 | exact | ++---------------------+-------+--------+ + +-- Instant lookback bounds are exclusive: these return only 302 and 702. +TQL EVAL (1, 1, '1s', '300s') native_time_us{series=~"lower.*"}; + ++-------+-----------+---------------------+ +| val | series | ts | ++-------+-----------+---------------------+ +| 302.0 | lowerplus | 1970-01-01T00:00:01 | ++-------+-----------+---------------------+ + +TQL EVAL (301, 301, '1s', '300s') native_time_us{series=~"positive_lower.*"}; + ++-------+--------------------+---------------------+ +| val | series | ts | ++-------+--------------------+---------------------+ +| 702.0 | positive_lowerplus | 1970-01-01T00:05:01 | ++-------+--------------------+---------------------+ + +-- The sub-millisecond point belongs only to the 2s evaluation step. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 2, '1s', '300s') native_time_us{series="multi"}; + ++-------+--------+---------------------+ +| val | series | ts | ++-------+--------+---------------------+ +| 401.0 | multi | 1970-01-01T00:00:02 | ++-------+--------+---------------------+ + +-- The latest native timestamp below 1s is retained even when inserts are unordered. +TQL EVAL (1, 1, '1s', '300s') native_time_us{series="past"}; + ++-------+--------+---------------------+ +| val | series | ts | ++-------+--------+---------------------+ +| 602.0 | past | 1970-01-01T00:00:01 | ++-------+--------+---------------------+ + +-- Offsets select native timestamps, including stored negative time. +TQL EVAL (0, 0, '1s', '300s') native_time_us{series="offset"}; + ++-------+--------+---------------------+ +| val | series | ts | ++-------+--------+---------------------+ +| 502.0 | offset | 1970-01-01T00:00:00 | ++-------+--------+---------------------+ + +TQL EVAL (0, 0, '1s', '300s') native_time_us{series="offset"} offset 1s; + ++-------+--------+---------------------+ +| val | series | ts | ++-------+--------+---------------------+ +| 501.0 | offset | 1970-01-01T00:00:00 | ++-------+--------+---------------------+ + +TQL EVAL (0, 0, '1s', '300s') native_time_us{series="offset"} offset -1s; + ++-------+--------+---------------------+ +| val | series | ts | ++-------+--------+---------------------+ +| 503.0 | offset | 1970-01-01T00:00:00 | ++-------+--------+---------------------+ + +-- [1s] at 1s excludes 0 and 1s+tick, retaining 0+tick, 0+2ticks, and 1s. +TQL EVAL (1, 1, '1s', '300s') count_over_time(native_time_us{series="window"}[1s]); + ++---------------------+------------------------------------+--------+ +| ts | prom_count_over_time(ts_range,val) | series | ++---------------------+------------------------------------+--------+ +| 1970-01-01T00:00:01 | 3.0 | window | ++---------------------+------------------------------------+--------+ + +TQL EVAL (1, 1, '1s', '300s') sum_over_time(native_time_us{series="window"}[1s]); + ++---------------------+----------------------------------+--------+ +| ts | prom_sum_over_time(ts_range,val) | series | ++---------------------+----------------------------------+--------+ +| 1970-01-01T00:00:01 | 10.0 | window | ++---------------------+----------------------------------+--------+ + +TQL EVAL (1, 1, '1s', '300s') last_over_time(native_time_us{series="window"}[1s]); + ++---------------------+-----------------------------------+--------+ +| ts | prom_last_over_time(ts_range,val) | series | ++---------------------+-----------------------------------+--------+ +| 1970-01-01T00:00:01 | 3.0 | window | ++---------------------+-----------------------------------+--------+ + +-- The inner selector consumes native time; the subquery consumes ms evaluations. +TQL EVAL (1, 1, '1s') last_over_time((native_time_us{series="exact"})[1s:1s]); + ++---------------------+-----------------------------------+--------+ +| ts | prom_last_over_time(ts_range,val) | series | ++---------------------+-----------------------------------+--------+ +| 1970-01-01T00:00:01 | 201.0 | exact | ++---------------------+-----------------------------------+--------+ + +-- Inner offsets are applied once when evaluating the subquery selector. +TQL EVAL (0, 0, '1s') last_over_time((native_time_us{series="offset"} offset 1s)[1s:1s]); + ++---------------------+-----------------------------------+--------+ +| ts | prom_last_over_time(ts_range,val) | series | ++---------------------+-----------------------------------+--------+ +| 1970-01-01T00:00:00 | 501.0 | offset | ++---------------------+-----------------------------------+--------+ + +TQL EVAL (0, 0, '1s') last_over_time((native_time_us{series="offset"} offset -1s)[1s:1s]); + ++---------------------+-----------------------------------+--------+ +| ts | prom_last_over_time(ts_range,val) | series | ++---------------------+-----------------------------------+--------+ +| 1970-01-01T00:00:00 | 503.0 | offset | ++---------------------+-----------------------------------+--------+ + +DROP TABLE native_time_us; + +Affected Rows: 0 + +CREATE TABLE native_time_ns ( + ts TIMESTAMP(9) TIME INDEX, + series STRING PRIMARY KEY, + val DOUBLE, +); + +Affected Rows: 0 + +INSERT INTO native_time_ns VALUES + (1000000001, 'future', 101), + (1000000000, 'exact', 201), + (1000000001, 'exact', 202), + (-299000000000, 'lowerbound', 301), + (-298999999999, 'lowerplus', 302), + (1000000000, 'positive_lowerbound', 701), + (1000000001, 'positive_lowerplus', 702), + (1000000001, 'multi', 401), + (-1000000000, 'offset', 501), + (0, 'offset', 502), + (1000000000, 'offset', 503), + (999999000, 'past', 602), + (999001000, 'past', 601), + (0, 'window', 1), + (1, 'window', 2), + (2, 'window', 5), + (1000000000, 'window', 3), + (1000000001, 'window', 4); + +Affected Rows: 18 + +-- The native projection and exact 1ms-lookback bounds must reach the scan; +-- the 1s+tick row must not displace 201. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE native_time_ns.__table_id\s*=\s*UInt32\(\d+\) native_time_ns.__table_id=UInt32(REDACTED) +TQL EXPLAIN (1, 1, '1s', '1ms') native_time_ns{series="exact"}; + ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | PromInstantManipulate: range=[1000..1000], lookback=[1], interval=[1000], time index=[ts] | +| | PromSeriesDivide: tags=["series"] | +| | Sort: native_time_ns.series ASC NULLS FIRST, native_time_ns.ts ASC NULLS FIRST | +| | Projection: native_time_ns.val, native_time_ns.series, native_time_ns.ts | +| | Filter: native_time_ns.series = Utf8("exact") AND native_time_ns.ts > TimestampNanosecond(999000000, None) AND native_time_ns.ts <= TimestampNanosecond(1000000000, None) | +| | TableScan: native_time_ns, partial_filters=[native_time_ns.series = Utf8("exact"), native_time_ns.ts > TimestampNanosecond(999000000, None), native_time_ns.ts <= TimestampNanosecond(1000000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- The actual memtable scan must use LastRow { after_merge: true } with native +-- 1ms-lookback bounds; it must select exact 1s rather than the future tick. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +TQL ANALYZE VERBOSE (1, 1, '1s', '1ms') native_time_ns{series="exact"}; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_CooperativeExec metrics=[]_| +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[1000..1000], lookback=[1], interval=[1000], time index=[ts] metrics=[output_rows: 1, REDACTED +|_|_|_PromSeriesDivideExec: tags=["series"] metrics=[output_rows: 1, REDACTED +|_|_|_ProjectionExec: expr=[val@2 as val, series@1 as series, ts@0 as ts] metrics=[output_rows: 1, REDACTED +|_|_|_CooperativeExec metrics=[]_| +|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "series", "val"], "filters": ["series = Dictionary(UInt32, Utf8(\"exact\"))", "ts > TimestampNanosecond(999000000, None)", "ts <= TimestampNanosecond(1000000000, None)"], "REDACTED +|_|_|_| +|_|_| Total rows: 1_| ++-+-+-+ + +-- The same-series future tick is in the memtable, while exact 1s remains selected. +TQL EVAL (1, 1, '1s', '1ms') native_time_ns{series="exact"}; + ++-------+--------+---------------------+ +| val | series | ts | ++-------+--------+---------------------+ +| 201.0 | exact | 1970-01-01T00:00:01 | ++-------+--------+---------------------+ + +-- Future-only selection is empty before flushing, exercising the memtable path. +TQL EVAL (1, 1, '1s', '300s') native_time_ns{series="future"}; + ++-----+--------+----+ +| val | series | ts | ++-----+--------+----+ ++-----+--------+----+ + +ADMIN FLUSH_TABLE('native_time_ns'); + ++-------------------------------------+ +| ADMIN FLUSH_TABLE('native_time_ns') | ++-------------------------------------+ +| 0 | ++-------------------------------------+ + +-- The exact native sample remains selected from the flushed SST. +TQL EVAL (1, 1, '1s', '1ms') native_time_ns{series="exact"}; + ++-------+--------+---------------------+ +| val | series | ts | ++-------+--------+---------------------+ +| 201.0 | exact | 1970-01-01T00:00:01 | ++-------+--------+---------------------+ + +TQL EVAL (1, 1, '1s', '300s') timestamp(native_time_ns{series="future"}); + ++----+-------+--------+ +| ts | value | series | ++----+-------+--------+ ++----+-------+--------+ + +TQL EVAL (1, 1, '1s', '300s') timestamp(native_time_ns{series="exact"}); + ++---------------------+-------+--------+ +| ts | value | series | ++---------------------+-------+--------+ +| 1970-01-01T00:00:01 | 1.0 | exact | ++---------------------+-------+--------+ + +-- Instant lookback bounds are exclusive: these return only 302 and 702. +TQL EVAL (1, 1, '1s', '300s') native_time_ns{series=~"lower.*"}; + ++-------+-----------+---------------------+ +| val | series | ts | ++-------+-----------+---------------------+ +| 302.0 | lowerplus | 1970-01-01T00:00:01 | ++-------+-----------+---------------------+ + +TQL EVAL (301, 301, '1s', '300s') native_time_ns{series=~"positive_lower.*"}; + ++-------+--------------------+---------------------+ +| val | series | ts | ++-------+--------------------+---------------------+ +| 702.0 | positive_lowerplus | 1970-01-01T00:05:01 | ++-------+--------------------+---------------------+ + +-- The sub-millisecond point belongs only to the 2s evaluation step. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 2, '1s', '300s') native_time_ns{series="multi"}; + ++-------+--------+---------------------+ +| val | series | ts | ++-------+--------+---------------------+ +| 401.0 | multi | 1970-01-01T00:00:02 | ++-------+--------+---------------------+ + +-- The latest native timestamp below 1s is retained even when inserts are unordered. +TQL EVAL (1, 1, '1s', '300s') native_time_ns{series="past"}; + ++-------+--------+---------------------+ +| val | series | ts | ++-------+--------+---------------------+ +| 602.0 | past | 1970-01-01T00:00:01 | ++-------+--------+---------------------+ + +-- Offsets select native timestamps, including stored negative time. +TQL EVAL (0, 0, '1s', '300s') native_time_ns{series="offset"}; + ++-------+--------+---------------------+ +| val | series | ts | ++-------+--------+---------------------+ +| 502.0 | offset | 1970-01-01T00:00:00 | ++-------+--------+---------------------+ + +TQL EVAL (0, 0, '1s', '300s') native_time_ns{series="offset"} offset 1s; + ++-------+--------+---------------------+ +| val | series | ts | ++-------+--------+---------------------+ +| 501.0 | offset | 1970-01-01T00:00:00 | ++-------+--------+---------------------+ + +TQL EVAL (0, 0, '1s', '300s') native_time_ns{series="offset"} offset -1s; + ++-------+--------+---------------------+ +| val | series | ts | ++-------+--------+---------------------+ +| 503.0 | offset | 1970-01-01T00:00:00 | ++-------+--------+---------------------+ + +-- [1s] at 1s excludes 0 and 1s+tick, retaining 0+tick, 0+2ticks, and 1s. +TQL EVAL (1, 1, '1s', '300s') count_over_time(native_time_ns{series="window"}[1s]); + ++---------------------+------------------------------------+--------+ +| ts | prom_count_over_time(ts_range,val) | series | ++---------------------+------------------------------------+--------+ +| 1970-01-01T00:00:01 | 3.0 | window | ++---------------------+------------------------------------+--------+ + +TQL EVAL (1, 1, '1s', '300s') sum_over_time(native_time_ns{series="window"}[1s]); + ++---------------------+----------------------------------+--------+ +| ts | prom_sum_over_time(ts_range,val) | series | ++---------------------+----------------------------------+--------+ +| 1970-01-01T00:00:01 | 10.0 | window | ++---------------------+----------------------------------+--------+ + +TQL EVAL (1, 1, '1s', '300s') last_over_time(native_time_ns{series="window"}[1s]); + ++---------------------+-----------------------------------+--------+ +| ts | prom_last_over_time(ts_range,val) | series | ++---------------------+-----------------------------------+--------+ +| 1970-01-01T00:00:01 | 3.0 | window | ++---------------------+-----------------------------------+--------+ + +-- The inner selector consumes native time; the subquery consumes ms evaluations. +TQL EVAL (1, 1, '1s') last_over_time((native_time_ns{series="exact"})[1s:1s]); + ++---------------------+-----------------------------------+--------+ +| ts | prom_last_over_time(ts_range,val) | series | ++---------------------+-----------------------------------+--------+ +| 1970-01-01T00:00:01 | 201.0 | exact | ++---------------------+-----------------------------------+--------+ + +-- Inner offsets are applied once when evaluating the subquery selector. +TQL EVAL (0, 0, '1s') last_over_time((native_time_ns{series="offset"} offset 1s)[1s:1s]); + ++---------------------+-----------------------------------+--------+ +| ts | prom_last_over_time(ts_range,val) | series | ++---------------------+-----------------------------------+--------+ +| 1970-01-01T00:00:00 | 501.0 | offset | ++---------------------+-----------------------------------+--------+ + +TQL EVAL (0, 0, '1s') last_over_time((native_time_ns{series="offset"} offset -1s)[1s:1s]); + ++---------------------+-----------------------------------+--------+ +| ts | prom_last_over_time(ts_range,val) | series | ++---------------------+-----------------------------------+--------+ +| 1970-01-01T00:00:00 | 503.0 | offset | ++---------------------+-----------------------------------+--------+ + +DROP TABLE native_time_ns; + +Affected Rows: 0 + +-- An unrepresentable native lower bound must not discard its representable upper bound. +-- The upper filter must reach LastRow so the 1ms-future row cannot hide the eligible row. +CREATE TABLE native_time_ns_lower_overflow ( + ts TIMESTAMP(9) TIME INDEX, + series STRING PRIMARY KEY, + val DOUBLE, +); + +Affected Rows: 0 + +INSERT INTO native_time_ns_lower_overflow VALUES + (-9223200000000000000, 'exact', 1), + (-9223199999999000000, 'exact', 2); + +Affected Rows: 2 + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE native_time_ns_lower_overflow.__table_id\s*=\s*UInt32\(\d+\) native_time_ns_lower_overflow.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 0, '1s', '2d') native_time_ns_lower_overflow{series="exact"} offset 106750d; + ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | PromInstantManipulate: range=[0..0], lookback=[172800000], interval=[1000], time index=[ts] | +| | PromSeriesNormalize: offset=[9223200000000], time index=[ts], filter NaN: [false] | +| | PromSeriesDivide: tags=["series"] | +| | Sort: native_time_ns_lower_overflow.series ASC NULLS FIRST, native_time_ns_lower_overflow.ts ASC NULLS FIRST | +| | Projection: native_time_ns_lower_overflow.val, native_time_ns_lower_overflow.series, native_time_ns_lower_overflow.ts | +| | Filter: native_time_ns_lower_overflow.series = Utf8("exact") AND native_time_ns_lower_overflow.ts <= TimestampNanosecond(-9223200000000000000, None) | +| | TableScan: native_time_ns_lower_overflow, partial_filters=[native_time_ns_lower_overflow.series = Utf8("exact"), native_time_ns_lower_overflow.ts <= TimestampNanosecond(-9223200000000000000, None)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +TQL ANALYZE VERBOSE (0, 0, '1s', '2d') native_time_ns_lower_overflow{series="exact"} offset 106750d; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_CooperativeExec metrics=[]_| +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..0], lookback=[172800000], interval=[1000], time index=[ts] metrics=[output_rows: 1, REDACTED +|_|_|_PromSeriesNormalizeExec: offset=[9223200000000], time index=[ts], filter NaN: [false] metrics=[output_rows: 1, REDACTED +|_|_|_PromSeriesDivideExec: tags=["series"] metrics=[output_rows: 1, REDACTED +|_|_|_ProjectionExec: expr=[val@2 as val, series@1 as series, ts@0 as ts] metrics=[output_rows: 1, REDACTED +|_|_|_CooperativeExec metrics=[]_| +|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "series", "val"], "filters": ["series = Dictionary(UInt32, Utf8(\"exact\"))", "ts <= TimestampNanosecond(-9223200000000000000, None)"], "REDACTED +|_|_|_| +|_|_| Total rows: 1_| ++-+-+-+ + +-- The representable upper bound selects only the exact row. +TQL EVAL (0, 0, '1s', '2d') native_time_ns_lower_overflow{series="exact"} offset 106750d; + ++-----+--------+---------------------+ +| val | series | ts | ++-----+--------+---------------------+ +| 1.0 | exact | 1970-01-01T00:00:00 | ++-----+--------+---------------------+ + +ADMIN FLUSH_TABLE('native_time_ns_lower_overflow'); + ++----------------------------------------------------+ +| ADMIN FLUSH_TABLE('native_time_ns_lower_overflow') | ++----------------------------------------------------+ +| 0 | ++----------------------------------------------------+ + +TQL EVAL (0, 0, '1s', '2d') native_time_ns_lower_overflow{series="exact"} offset 106750d; + ++-----+--------+---------------------+ +| val | series | ts | ++-----+--------+---------------------+ +| 1.0 | exact | 1970-01-01T00:00:00 | ++-----+--------+---------------------+ + +DROP TABLE native_time_ns_lower_overflow; + +Affected Rows: 0 + +-- Second precision is promoted before applying fractional-second offsets. +CREATE TABLE native_time_sec (ts TIMESTAMP(0) TIME INDEX, val DOUBLE); + +Affected Rows: 0 + +INSERT INTO native_time_sec VALUES (0, 10), (1, 11), (2, 12); + +Affected Rows: 3 + +TQL EVAL (1, 1, '1s', '1s') native_time_sec offset 500ms; + ++------+---------------------+ +| val | ts | ++------+---------------------+ +| 10.0 | 1970-01-01T00:00:01 | ++------+---------------------+ + +TQL EVAL (1, 1, '1s', '1s') native_time_sec offset -500ms; + ++------+---------------------+ +| val | ts | ++------+---------------------+ +| 11.0 | 1970-01-01T00:00:01 | ++------+---------------------+ + +DROP TABLE native_time_sec; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/promql/native_time_selection.sql b/tests/cases/standalone/common/promql/native_time_selection.sql new file mode 100644 index 00000000000..ca795b0b3ad --- /dev/null +++ b/tests/cases/standalone/common/promql/native_time_selection.sql @@ -0,0 +1,222 @@ +-- Regression coverage for instant and range selection on native microsecond and +-- nanosecond time indexes. + +CREATE TABLE native_time_us ( + ts TIMESTAMP(6) TIME INDEX, + series STRING PRIMARY KEY, + val DOUBLE, +); + +INSERT INTO native_time_us VALUES + (1000001, 'future', 101), + (1000000, 'exact', 201), + (1000001, 'exact', 202), + (-299000000, 'lowerbound', 301), + (-298999999, 'lowerplus', 302), + (1000000, 'positive_lowerbound', 701), + (1000001, 'positive_lowerplus', 702), + (1000001, 'multi', 401), + (-1000000, 'offset', 501), + (0, 'offset', 502), + (1000000, 'offset', 503), + (999999, 'past', 602), + (999001, 'past', 601), + (0, 'window', 1), + (1, 'window', 2), + (2, 'window', 5), + (1000000, 'window', 3), + (1000001, 'window', 4); + +-- The native projection and exact 1ms-lookback bounds must reach the scan; +-- the 1s+tick row must not displace 201. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE native_time_us.__table_id\s*=\s*UInt32\(\d+\) native_time_us.__table_id=UInt32(REDACTED) +TQL EXPLAIN (1, 1, '1s', '1ms') native_time_us{series="exact"}; + +-- The actual memtable scan must use LastRow { after_merge: true } with native +-- 1ms-lookback bounds; it must select exact 1s rather than the future tick. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +TQL ANALYZE VERBOSE (1, 1, '1s', '1ms') native_time_us{series="exact"}; + +-- The same-series future tick is in the memtable, while exact 1s remains selected. +TQL EVAL (1, 1, '1s', '1ms') native_time_us{series="exact"}; + +-- Future-only selection is empty before flushing, exercising the memtable path. +TQL EVAL (1, 1, '1s', '300s') native_time_us{series="future"}; + +ADMIN FLUSH_TABLE('native_time_us'); + +-- The exact native sample remains selected from the flushed SST. +TQL EVAL (1, 1, '1s', '1ms') native_time_us{series="exact"}; +TQL EVAL (1, 1, '1s', '300s') timestamp(native_time_us{series="future"}); +TQL EVAL (1, 1, '1s', '300s') timestamp(native_time_us{series="exact"}); + +-- Instant lookback bounds are exclusive: these return only 302 and 702. +TQL EVAL (1, 1, '1s', '300s') native_time_us{series=~"lower.*"}; +TQL EVAL (301, 301, '1s', '300s') native_time_us{series=~"positive_lower.*"}; + +-- The sub-millisecond point belongs only to the 2s evaluation step. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 2, '1s', '300s') native_time_us{series="multi"}; + +-- The latest native timestamp below 1s is retained even when inserts are unordered. +TQL EVAL (1, 1, '1s', '300s') native_time_us{series="past"}; + +-- Offsets select native timestamps, including stored negative time. +TQL EVAL (0, 0, '1s', '300s') native_time_us{series="offset"}; +TQL EVAL (0, 0, '1s', '300s') native_time_us{series="offset"} offset 1s; +TQL EVAL (0, 0, '1s', '300s') native_time_us{series="offset"} offset -1s; + +-- [1s] at 1s excludes 0 and 1s+tick, retaining 0+tick, 0+2ticks, and 1s. +TQL EVAL (1, 1, '1s', '300s') count_over_time(native_time_us{series="window"}[1s]); +TQL EVAL (1, 1, '1s', '300s') sum_over_time(native_time_us{series="window"}[1s]); +TQL EVAL (1, 1, '1s', '300s') last_over_time(native_time_us{series="window"}[1s]); + +-- The inner selector consumes native time; the subquery consumes ms evaluations. +TQL EVAL (1, 1, '1s') last_over_time((native_time_us{series="exact"})[1s:1s]); + +-- Inner offsets are applied once when evaluating the subquery selector. +TQL EVAL (0, 0, '1s') last_over_time((native_time_us{series="offset"} offset 1s)[1s:1s]); +TQL EVAL (0, 0, '1s') last_over_time((native_time_us{series="offset"} offset -1s)[1s:1s]); + +DROP TABLE native_time_us; + +CREATE TABLE native_time_ns ( + ts TIMESTAMP(9) TIME INDEX, + series STRING PRIMARY KEY, + val DOUBLE, +); + +INSERT INTO native_time_ns VALUES + (1000000001, 'future', 101), + (1000000000, 'exact', 201), + (1000000001, 'exact', 202), + (-299000000000, 'lowerbound', 301), + (-298999999999, 'lowerplus', 302), + (1000000000, 'positive_lowerbound', 701), + (1000000001, 'positive_lowerplus', 702), + (1000000001, 'multi', 401), + (-1000000000, 'offset', 501), + (0, 'offset', 502), + (1000000000, 'offset', 503), + (999999000, 'past', 602), + (999001000, 'past', 601), + (0, 'window', 1), + (1, 'window', 2), + (2, 'window', 5), + (1000000000, 'window', 3), + (1000000001, 'window', 4); + +-- The native projection and exact 1ms-lookback bounds must reach the scan; +-- the 1s+tick row must not displace 201. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +-- SQLNESS REPLACE native_time_ns.__table_id\s*=\s*UInt32\(\d+\) native_time_ns.__table_id=UInt32(REDACTED) +TQL EXPLAIN (1, 1, '1s', '1ms') native_time_ns{series="exact"}; + +-- The actual memtable scan must use LastRow { after_merge: true } with native +-- 1ms-lookback bounds; it must select exact 1s rather than the future tick. +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +TQL ANALYZE VERBOSE (1, 1, '1s', '1ms') native_time_ns{series="exact"}; + +-- The same-series future tick is in the memtable, while exact 1s remains selected. +TQL EVAL (1, 1, '1s', '1ms') native_time_ns{series="exact"}; + +-- Future-only selection is empty before flushing, exercising the memtable path. +TQL EVAL (1, 1, '1s', '300s') native_time_ns{series="future"}; + +ADMIN FLUSH_TABLE('native_time_ns'); + +-- The exact native sample remains selected from the flushed SST. +TQL EVAL (1, 1, '1s', '1ms') native_time_ns{series="exact"}; +TQL EVAL (1, 1, '1s', '300s') timestamp(native_time_ns{series="future"}); +TQL EVAL (1, 1, '1s', '300s') timestamp(native_time_ns{series="exact"}); + +-- Instant lookback bounds are exclusive: these return only 302 and 702. +TQL EVAL (1, 1, '1s', '300s') native_time_ns{series=~"lower.*"}; +TQL EVAL (301, 301, '1s', '300s') native_time_ns{series=~"positive_lower.*"}; + +-- The sub-millisecond point belongs only to the 2s evaluation step. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 2, '1s', '300s') native_time_ns{series="multi"}; + +-- The latest native timestamp below 1s is retained even when inserts are unordered. +TQL EVAL (1, 1, '1s', '300s') native_time_ns{series="past"}; + +-- Offsets select native timestamps, including stored negative time. +TQL EVAL (0, 0, '1s', '300s') native_time_ns{series="offset"}; +TQL EVAL (0, 0, '1s', '300s') native_time_ns{series="offset"} offset 1s; +TQL EVAL (0, 0, '1s', '300s') native_time_ns{series="offset"} offset -1s; + +-- [1s] at 1s excludes 0 and 1s+tick, retaining 0+tick, 0+2ticks, and 1s. +TQL EVAL (1, 1, '1s', '300s') count_over_time(native_time_ns{series="window"}[1s]); +TQL EVAL (1, 1, '1s', '300s') sum_over_time(native_time_ns{series="window"}[1s]); +TQL EVAL (1, 1, '1s', '300s') last_over_time(native_time_ns{series="window"}[1s]); + +-- The inner selector consumes native time; the subquery consumes ms evaluations. +TQL EVAL (1, 1, '1s') last_over_time((native_time_ns{series="exact"})[1s:1s]); + +-- Inner offsets are applied once when evaluating the subquery selector. +TQL EVAL (0, 0, '1s') last_over_time((native_time_ns{series="offset"} offset 1s)[1s:1s]); +TQL EVAL (0, 0, '1s') last_over_time((native_time_ns{series="offset"} offset -1s)[1s:1s]); + +DROP TABLE native_time_ns; + +-- An unrepresentable native lower bound must not discard its representable upper bound. +-- The upper filter must reach LastRow so the 1ms-future row cannot hide the eligible row. +CREATE TABLE native_time_ns_lower_overflow ( + ts TIMESTAMP(9) TIME INDEX, + series STRING PRIMARY KEY, + val DOUBLE, +); +INSERT INTO native_time_ns_lower_overflow VALUES + (-9223200000000000000, 'exact', 1), + (-9223199999999000000, 'exact', 2); + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE native_time_ns_lower_overflow.__table_id\s*=\s*UInt32\(\d+\) native_time_ns_lower_overflow.__table_id=UInt32(REDACTED) +TQL EXPLAIN (0, 0, '1s', '2d') native_time_ns_lower_overflow{series="exact"} offset 106750d; + +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +TQL ANALYZE VERBOSE (0, 0, '1s', '2d') native_time_ns_lower_overflow{series="exact"} offset 106750d; + +-- The representable upper bound selects only the exact row. +TQL EVAL (0, 0, '1s', '2d') native_time_ns_lower_overflow{series="exact"} offset 106750d; +ADMIN FLUSH_TABLE('native_time_ns_lower_overflow'); +TQL EVAL (0, 0, '1s', '2d') native_time_ns_lower_overflow{series="exact"} offset 106750d; +DROP TABLE native_time_ns_lower_overflow; + +-- Second precision is promoted before applying fractional-second offsets. +CREATE TABLE native_time_sec (ts TIMESTAMP(0) TIME INDEX, val DOUBLE); +INSERT INTO native_time_sec VALUES (0, 10), (1, 11), (2, 12); +TQL EVAL (1, 1, '1s', '1s') native_time_sec offset 500ms; +TQL EVAL (1, 1, '1s', '1s') native_time_sec offset -500ms; +DROP TABLE native_time_sec; diff --git a/tests/cases/standalone/common/promql/null_samples.result b/tests/cases/standalone/common/promql/null_samples.result new file mode 100644 index 00000000000..89270227a3d --- /dev/null +++ b/tests/cases/standalone/common/promql/null_samples.result @@ -0,0 +1,262 @@ +-- NULL fields are missing samples, not zero-valued samples. +CREATE TABLE null_samples ( + ts TIMESTAMP(3) TIME INDEX, + host STRING PRIMARY KEY, + val DOUBLE, +); + +Affected Rows: 0 + +INSERT INTO null_samples VALUES + (0, 'a', 1.0), + (1000, 'a', NULL), + (2000, 'a', NULL), + (3000, 'a', 4.0), + (0, 'b', NULL), + (1000, 'b', NULL), + (2000, 'b', NULL), + (3000, 'b', NULL); + +Affected Rows: 8 + +-- At t=2 the trailing NULLs must not hide 1; at t=3 count must be 2. +-- At t=7 the left-open window is empty. Valid results must disappear. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (2, 7, '1s') count_over_time(null_samples{host="a"}[4s]); + ++---------------------+------------------------------------+------+ +| ts | prom_count_over_time(ts_range,val) | host | ++---------------------+------------------------------------+------+ +| 1970-01-01T00:00:02 | 1.0 | a | +| 1970-01-01T00:00:03 | 2.0 | a | +| 1970-01-01T00:00:04 | 1.0 | a | +| 1970-01-01T00:00:05 | 1.0 | a | +| 1970-01-01T00:00:06 | 1.0 | a | ++---------------------+------------------------------------+------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (2, 7, '1s') last_over_time(null_samples{host="a"}[4s]); + ++---------------------+-----------------------------------+------+ +| ts | prom_last_over_time(ts_range,val) | host | ++---------------------+-----------------------------------+------+ +| 1970-01-01T00:00:02 | 1.0 | a | +| 1970-01-01T00:00:03 | 4.0 | a | +| 1970-01-01T00:00:04 | 4.0 | a | +| 1970-01-01T00:00:05 | 4.0 | a | +| 1970-01-01T00:00:06 | 4.0 | a | ++---------------------+-----------------------------------+------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (2, 7, '1s') present_over_time(null_samples{host="a"}[4s]); + ++---------------------+--------------------------------------+------+ +| ts | prom_present_over_time(ts_range,val) | host | ++---------------------+--------------------------------------+------+ +| 1970-01-01T00:00:02 | 1.0 | a | +| 1970-01-01T00:00:03 | 1.0 | a | +| 1970-01-01T00:00:04 | 1.0 | a | +| 1970-01-01T00:00:05 | 1.0 | a | +| 1970-01-01T00:00:06 | 1.0 | a | ++---------------------+--------------------------------------+------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (2, 7, '1s') absent_over_time(null_samples{host="a"}[4s]); + ++---------------------+-------------------------------------+------+ +| ts | prom_absent_over_time(ts_range,val) | host | ++---------------------+-------------------------------------+------+ +| 1970-01-01T00:00:07 | 1.0 | a | ++---------------------+-------------------------------------+------+ + +-- All-NULL windows have no samples: only absent_over_time returns 1. +TQL EVAL (3, 3, '1s') count_over_time(null_samples{host="b"}[4s]); + ++----+------------------------------------+------+ +| ts | prom_count_over_time(ts_range,val) | host | ++----+------------------------------------+------+ ++----+------------------------------------+------+ + +TQL EVAL (3, 3, '1s') last_over_time(null_samples{host="b"}[4s]); + ++----+-----------------------------------+------+ +| ts | prom_last_over_time(ts_range,val) | host | ++----+-----------------------------------+------+ ++----+-----------------------------------+------+ + +TQL EVAL (3, 3, '1s') present_over_time(null_samples{host="b"}[4s]); + ++----+--------------------------------------+------+ +| ts | prom_present_over_time(ts_range,val) | host | ++----+--------------------------------------+------+ ++----+--------------------------------------+------+ + +TQL EVAL (3, 3, '1s') absent_over_time(null_samples{host="b"}[4s]); + ++---------------------+-------------------------------------+------+ +| ts | prom_absent_over_time(ts_range,val) | host | ++---------------------+-------------------------------------+------+ +| 1970-01-01T00:00:03 | 1.0 | b | ++---------------------+-------------------------------------+------+ + +-- Every function below sees the same two samples, 1.0 at 0s and 4.0 at 3s. +TQL EVAL (3, 3, '1s') rate(null_samples{host="a"}[4s]); + ++---------------------+----------------------------------------+------+ +| ts | prom_rate(ts_range,val,ts,Int64(4000)) | host | ++---------------------+----------------------------------------+------+ +| 1970-01-01T00:00:03 | 1.0 | a | ++---------------------+----------------------------------------+------+ + +TQL EVAL (3, 3, '1s') increase(null_samples{host="a"}[4s]); + ++---------------------+--------------------------------------------+------+ +| ts | prom_increase(ts_range,val,ts,Int64(4000)) | host | ++---------------------+--------------------------------------------+------+ +| 1970-01-01T00:00:03 | 4.0 | a | ++---------------------+--------------------------------------------+------+ + +TQL EVAL (3, 3, '1s') delta(null_samples{host="a"}[4s]); + ++---------------------+-----------------------------------------+------+ +| ts | prom_delta(ts_range,val,ts,Int64(4000)) | host | ++---------------------+-----------------------------------------+------+ +| 1970-01-01T00:00:03 | 4.0 | a | ++---------------------+-----------------------------------------+------+ + +TQL EVAL (3, 3, '1s') idelta(null_samples{host="a"}[4s]); + ++---------------------+---------------------------+------+ +| ts | prom_idelta(ts_range,val) | host | ++---------------------+---------------------------+------+ +| 1970-01-01T00:00:03 | 3.0 | a | ++---------------------+---------------------------+------+ + +TQL EVAL (3, 3, '1s') irate(null_samples{host="a"}[4s]); + ++---------------------+--------------------------+------+ +| ts | prom_irate(ts_range,val) | host | ++---------------------+--------------------------+------+ +| 1970-01-01T00:00:03 | 1.0 | a | ++---------------------+--------------------------+------+ + +TQL EVAL (3, 3, '1s') changes(null_samples{host="a"}[4s]); + ++---------------------+----------------------------+------+ +| ts | prom_changes(ts_range,val) | host | ++---------------------+----------------------------+------+ +| 1970-01-01T00:00:03 | 1.0 | a | ++---------------------+----------------------------+------+ + +TQL EVAL (3, 3, '1s') resets(null_samples{host="a"}[4s]); + ++---------------------+---------------------------+------+ +| ts | prom_resets(ts_range,val) | host | ++---------------------+---------------------------+------+ +| 1970-01-01T00:00:03 | 0.0 | a | ++---------------------+---------------------------+------+ + +TQL EVAL (3, 3, '1s') avg_over_time(null_samples{host="a"}[4s]); + ++---------------------+----------------------------------+------+ +| ts | prom_avg_over_time(ts_range,val) | host | ++---------------------+----------------------------------+------+ +| 1970-01-01T00:00:03 | 2.5 | a | ++---------------------+----------------------------------+------+ + +TQL EVAL (3, 3, '1s') stddev_over_time(null_samples{host="a"}[4s]); + ++---------------------+-------------------------------------+------+ +| ts | prom_stddev_over_time(ts_range,val) | host | ++---------------------+-------------------------------------+------+ +| 1970-01-01T00:00:03 | 1.5 | a | ++---------------------+-------------------------------------+------+ + +TQL EVAL (3, 3, '1s') stdvar_over_time(null_samples{host="a"}[4s]); + ++---------------------+-------------------------------------+------+ +| ts | prom_stdvar_over_time(ts_range,val) | host | ++---------------------+-------------------------------------+------+ +| 1970-01-01T00:00:03 | 2.25 | a | ++---------------------+-------------------------------------+------+ + +TQL EVAL (3, 3, '1s') quantile_over_time(0.5, null_samples{host="a"}[4s]); + ++---------------------+----------------------------------------------------+------+ +| ts | prom_quantile_over_time(ts_range,val,Float64(0.5)) | host | ++---------------------+----------------------------------------------------+------+ +| 1970-01-01T00:00:03 | 2.5 | a | ++---------------------+----------------------------------------------------+------+ + +-- A window whose only slot is NULL holds no sample, like a window with no row. +TQL EVAL (1, 1, '1s') rate(null_samples{host="a"}[1s]); + ++----+----------------------------------------+------+ +| ts | prom_rate(ts_range,val,ts,Int64(1000)) | host | ++----+----------------------------------------+------+ ++----+----------------------------------------+------+ + +-- Prometheus returns an empty vector for a range without samples, not NaN. +TQL EVAL (1, 1, '1s') quantile_over_time(0.5, null_samples{host="a"}[1s]); + ++----+----------------------------------------------------+------+ +| ts | prom_quantile_over_time(ts_range,val,Float64(0.5)) | host | ++----+----------------------------------------------------+------+ ++----+----------------------------------------------------+------+ + +-- `b` never has a sample, so it must not reach the aggregation. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '1s') avg by (host) (rate(null_samples[4s])); + ++------+---------------------+---------------------------------------------+ +| host | ts | avg(prom_rate(ts_range,val,ts,Int64(4000))) | ++------+---------------------+---------------------------------------------+ +| a | 1970-01-01T00:00:03 | 1.0 | ++------+---------------------+---------------------------------------------+ + +DROP TABLE null_samples; + +Affected Rows: 0 + +CREATE TABLE multi_field ( + ts TIMESTAMP(3) TIME INDEX, + host STRING PRIMARY KEY, + f1 DOUBLE, + f2 DOUBLE +); + +Affected Rows: 0 + +INSERT INTO multi_field VALUES + (0, 'a', 1.0, 10.0), + (1000, 'a', 2.0, NULL), + (2000, 'a', 3.0, NULL), + (3000, 'a', 4.0, 40.0); + +Affected Rows: 4 + +-- f1 has two samples in this window and f2 only one, so f1 keeps its result while f2 is NULL. +-- Dropping the whole row would take f1's samples with it. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 1, '1s') rate(multi_field[4s]); + ++---------------------+---------------------------------------+---------------------------------------+------+ +| ts | prom_rate(ts_range,f1,ts,Int64(4000)) | prom_rate(ts_range,f2,ts,Int64(4000)) | host | ++---------------------+---------------------------------------+---------------------------------------+------+ +| 1970-01-01T00:00:01 | 0.375 | | a | ++---------------------+---------------------------------------+---------------------------------------+------+ + +-- A selector emits the same shape: the row stays, the field without a sample is NULL. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 1, '1s') multi_field; + ++---------------------+------+-----+----+ +| ts | host | f1 | f2 | ++---------------------+------+-----+----+ +| 1970-01-01T00:00:01 | a | 2.0 | | ++---------------------+------+-----+----+ + +DROP TABLE multi_field; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/promql/null_samples.sql b/tests/cases/standalone/common/promql/null_samples.sql new file mode 100644 index 00000000000..d0a02158a44 --- /dev/null +++ b/tests/cases/standalone/common/promql/null_samples.sql @@ -0,0 +1,92 @@ +-- NULL fields are missing samples, not zero-valued samples. +CREATE TABLE null_samples ( + ts TIMESTAMP(3) TIME INDEX, + host STRING PRIMARY KEY, + val DOUBLE, +); + +INSERT INTO null_samples VALUES + (0, 'a', 1.0), + (1000, 'a', NULL), + (2000, 'a', NULL), + (3000, 'a', 4.0), + (0, 'b', NULL), + (1000, 'b', NULL), + (2000, 'b', NULL), + (3000, 'b', NULL); + +-- At t=2 the trailing NULLs must not hide 1; at t=3 count must be 2. +-- At t=7 the left-open window is empty. Valid results must disappear. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (2, 7, '1s') count_over_time(null_samples{host="a"}[4s]); +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (2, 7, '1s') last_over_time(null_samples{host="a"}[4s]); +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (2, 7, '1s') present_over_time(null_samples{host="a"}[4s]); +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (2, 7, '1s') absent_over_time(null_samples{host="a"}[4s]); + +-- All-NULL windows have no samples: only absent_over_time returns 1. +TQL EVAL (3, 3, '1s') count_over_time(null_samples{host="b"}[4s]); +TQL EVAL (3, 3, '1s') last_over_time(null_samples{host="b"}[4s]); +TQL EVAL (3, 3, '1s') present_over_time(null_samples{host="b"}[4s]); +TQL EVAL (3, 3, '1s') absent_over_time(null_samples{host="b"}[4s]); + +-- Every function below sees the same two samples, 1.0 at 0s and 4.0 at 3s. +TQL EVAL (3, 3, '1s') rate(null_samples{host="a"}[4s]); + +TQL EVAL (3, 3, '1s') increase(null_samples{host="a"}[4s]); + +TQL EVAL (3, 3, '1s') delta(null_samples{host="a"}[4s]); + +TQL EVAL (3, 3, '1s') idelta(null_samples{host="a"}[4s]); + +TQL EVAL (3, 3, '1s') irate(null_samples{host="a"}[4s]); + +TQL EVAL (3, 3, '1s') changes(null_samples{host="a"}[4s]); + +TQL EVAL (3, 3, '1s') resets(null_samples{host="a"}[4s]); + +TQL EVAL (3, 3, '1s') avg_over_time(null_samples{host="a"}[4s]); + +TQL EVAL (3, 3, '1s') stddev_over_time(null_samples{host="a"}[4s]); + +TQL EVAL (3, 3, '1s') stdvar_over_time(null_samples{host="a"}[4s]); + +TQL EVAL (3, 3, '1s') quantile_over_time(0.5, null_samples{host="a"}[4s]); + +-- A window whose only slot is NULL holds no sample, like a window with no row. +TQL EVAL (1, 1, '1s') rate(null_samples{host="a"}[1s]); + +-- Prometheus returns an empty vector for a range without samples, not NaN. +TQL EVAL (1, 1, '1s') quantile_over_time(0.5, null_samples{host="a"}[1s]); + +-- `b` never has a sample, so it must not reach the aggregation. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 15, '1s') avg by (host) (rate(null_samples[4s])); + +DROP TABLE null_samples; + +CREATE TABLE multi_field ( + ts TIMESTAMP(3) TIME INDEX, + host STRING PRIMARY KEY, + f1 DOUBLE, + f2 DOUBLE +); + +INSERT INTO multi_field VALUES + (0, 'a', 1.0, 10.0), + (1000, 'a', 2.0, NULL), + (2000, 'a', 3.0, NULL), + (3000, 'a', 4.0, 40.0); + +-- f1 has two samples in this window and f2 only one, so f1 keeps its result while f2 is NULL. +-- Dropping the whole row would take f1's samples with it. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 1, '1s') rate(multi_field[4s]); + +-- A selector emits the same shape: the row stays, the field without a sample is NULL. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 1, '1s') multi_field; + +DROP TABLE multi_field; diff --git a/tests/cases/standalone/common/promql/offset.result b/tests/cases/standalone/common/promql/offset.result index 760e1ac333e..c62bdaee1b3 100644 --- a/tests/cases/standalone/common/promql/offset.result +++ b/tests/cases/standalone/common/promql/offset.result @@ -66,8 +66,10 @@ tql eval (1500, 1500, '1s') calculate_rate_offset_total offset -10m; -- SQLNESS SORT_RESULT 3 1 tql eval (0, 0, '1s') calculate_rate_offset_total offset 10m; -++ -++ ++----+-----+---+ +| ts | val | x | ++----+-----+---+ ++----+-----+---+ -- SQLNESS SORT_RESULT 3 1 tql eval (0, 0, '1s') calculate_rate_offset_total offset -10m; @@ -92,14 +94,18 @@ tql eval (3000, 3000, '1s') calculate_rate_offset_total offset 10m; -- SQLNESS SORT_RESULT 3 1 tql eval (3000, 3000, '1s') calculate_rate_offset_total offset -10m; -++ -++ ++----+-----+---+ +| ts | val | x | ++----+-----+---+ ++----+-----+---+ -- SQLNESS SORT_RESULT 3 1 tql eval (3000, 3000, '1s') rate(calculate_rate_window_total[10m]); -++ -++ ++------+------------------------------------------------+ +| time | prom_rate(time_range,value,time,Int64(600000)) | ++------+------------------------------------------------+ ++------+------------------------------------------------+ -- SQLNESS SORT_RESULT 3 1 tql eval (3000, 3000, '1s') rate(calculate_rate_offset_total[10m] offset 5m); diff --git a/tests/cases/standalone/common/promql/or_operation.result b/tests/cases/standalone/common/promql/or_operation.result index 29dbfd1f36d..d79df294e82 100644 --- a/tests/cases/standalone/common/promql/or_operation.result +++ b/tests/cases/standalone/common/promql/or_operation.result @@ -31,8 +31,10 @@ tql eval (3000, 3000, '1s') http_requests{job="api",instance="0",env="production tql eval (3000, 3000, '1s') http_requests{job="web",instance="1",env="production"} * 5; -++ -++ ++-----+----------+-----+----+-----------------------------+ +| job | instance | env | ts | greptime_value * Float64(5) | ++-----+----------+-----+----+-----------------------------+ ++-----+----------+-----+----+-----------------------------+ tql eval (3000, 3000, '1s') http_requests{job="web",instance="1",env="production"} * 5 or http_requests{job="api",instance="0",env="production"} + 5; diff --git a/tests/cases/standalone/common/promql/precisions.result b/tests/cases/standalone/common/promql/precisions.result index e57f5b04ee4..4026a358322 100644 --- a/tests/cases/standalone/common/promql/precisions.result +++ b/tests/cases/standalone/common/promql/precisions.result @@ -133,10 +133,8 @@ TQL EVAL (0, 15, '5s') avg_over_time(host_sec{host="host1"}[5s]) + avg_over_time -- Verify that PromQL time predicates on non-millisecond time indexes are -- pushed into the scan as native timestamp range filters. --- Original instant selector filter is built on the millisecond alias: --- host = "host1" AND ts_ms >= -299999ms AND ts_ms <= 10000ms --- After pushing through `CAST(raw_ts AS Timestamp(ms)) AS ts` and applying --- DataFusion cast preimage, it becomes a native half-open range on raw_ts. +-- Instant selection compares raw timestamps before millisecond output conversion: +-- host = "host1" AND ts_us > -300000000us AND ts_us <= 10000000us. -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED @@ -152,17 +150,17 @@ TQL EXPLAIN (0, 10, '5s') host_micro{host="host1"}; | | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts] | | | PromSeriesDivide: tags=["host"] | | | Sort: host_micro.host ASC NULLS FIRST, host_micro.ts ASC NULLS FIRST | -| | Projection: host_micro.val, host_micro.host, CAST(host_micro.ts AS Timestamp(ms)) AS ts | -| | Filter: host_micro.host = Utf8("host1") AND host_micro.ts >= TimestampMicrosecond(-299999999, None) AND host_micro.ts < TimestampMicrosecond(10001000, None) | -| | TableScan: host_micro, partial_filters=[host_micro.host = Utf8("host1"), host_micro.ts >= TimestampMicrosecond(-299999999, None), host_micro.ts < TimestampMicrosecond(10001000, None)] | +| | Projection: host_micro.val, host_micro.host, host_micro.ts | +| | Filter: host_micro.host = Utf8("host1") AND host_micro.ts > TimestampMicrosecond(-300000000, None) AND host_micro.ts <= TimestampMicrosecond(10000000, None) | +| | TableScan: host_micro, partial_filters=[host_micro.host = Utf8("host1"), host_micro.ts > TimestampMicrosecond(-300000000, None), host_micro.ts <= TimestampMicrosecond(10000000, None)] | | | ]] | | physical_plan | CooperativeExec | | | MergeScanExec: REDACTED | | | +---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ --- The same instant-selector cast-preimage path should work for nanosecond indexes. --- Expected native bounds: ts_ns >= -299999999999ns AND ts_ns < 10001000000ns. +-- The same exclusive-lower, inclusive-upper window applies to nanosecond indexes. +-- Expected native bounds: ts_ns > -300000000000ns AND ts_ns <= 10000000000ns. -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED @@ -177,9 +175,9 @@ TQL EXPLAIN (0, 10, '5s') host_nano{host="host1"}; | | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts] | | | PromSeriesDivide: tags=["host"] | | | Sort: host_nano.host ASC NULLS FIRST, host_nano.ts ASC NULLS FIRST | -| | Projection: host_nano.val, host_nano.host, CAST(host_nano.ts AS Timestamp(ms)) AS ts | -| | Filter: host_nano.host = Utf8("host1") AND host_nano.ts >= TimestampNanosecond(-299999999999, None) AND host_nano.ts < TimestampNanosecond(10001000000, None) | -| | TableScan: host_nano, partial_filters=[host_nano.host = Utf8("host1"), host_nano.ts >= TimestampNanosecond(-299999999999, None), host_nano.ts < TimestampNanosecond(10001000000, None)] | +| | Projection: host_nano.val, host_nano.host, host_nano.ts | +| | Filter: host_nano.host = Utf8("host1") AND host_nano.ts > TimestampNanosecond(-300000000000, None) AND host_nano.ts <= TimestampNanosecond(10000000000, None) | +| | TableScan: host_nano, partial_filters=[host_nano.host = Utf8("host1"), host_nano.ts > TimestampNanosecond(-300000000000, None), host_nano.ts <= TimestampNanosecond(10000000000, None)] | | | ]] | | physical_plan | CooperativeExec | | | MergeScanExec: REDACTED @@ -187,8 +185,8 @@ TQL EXPLAIN (0, 10, '5s') host_nano{host="host1"}; +---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -- Range selectors use their range window instead of the default lookback. --- Original range selector filter for [5s]: --- host = "host1" AND ts_ms >= -4999ms AND ts_ms <= 10000ms +-- Native range selector filter for [5s]: +-- host = "host1" AND ts_us > -5000000us AND ts_us <= 10000000us -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED @@ -206,17 +204,17 @@ TQL EXPLAIN (0, 10, '5s') avg_over_time(host_micro{host="host1"}[5s]); | | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] | | | PromSeriesDivide: tags=["host"] | | | Sort: host_micro.host ASC NULLS FIRST, host_micro.ts ASC NULLS FIRST | -| | Projection: host_micro.val, host_micro.host, CAST(host_micro.ts AS Timestamp(ms)) AS ts | -| | Filter: host_micro.host = Utf8("host1") AND host_micro.ts >= TimestampMicrosecond(-4999999, None) AND host_micro.ts < TimestampMicrosecond(10001000, None) | -| | TableScan: host_micro, partial_filters=[host_micro.host = Utf8("host1"), host_micro.ts >= TimestampMicrosecond(-4999999, None), host_micro.ts < TimestampMicrosecond(10001000, None)] | +| | Projection: host_micro.val, host_micro.host, host_micro.ts | +| | Filter: host_micro.host = Utf8("host1") AND host_micro.ts > TimestampMicrosecond(-5000000, None) AND host_micro.ts <= TimestampMicrosecond(10000000, None) | +| | TableScan: host_micro, partial_filters=[host_micro.host = Utf8("host1"), host_micro.ts > TimestampMicrosecond(-5000000, None), host_micro.ts <= TimestampMicrosecond(10000000, None)] | | | ]] | | physical_plan | CooperativeExec | | | MergeScanExec: REDACTED | | | +---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ --- The same range-selector cast-preimage path should work for nanosecond indexes. --- Expected native bounds: ts_ns >= -4999999999ns AND ts_ns < 10001000000ns. +-- Range membership also retains nanosecond precision. +-- Expected native bounds: ts_ns > -5000000000ns AND ts_ns <= 10000000000ns. -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED @@ -234,9 +232,9 @@ TQL EXPLAIN (0, 10, '5s') avg_over_time(host_nano{host="host1"}[5s]); | | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] | | | PromSeriesDivide: tags=["host"] | | | Sort: host_nano.host ASC NULLS FIRST, host_nano.ts ASC NULLS FIRST | -| | Projection: host_nano.val, host_nano.host, CAST(host_nano.ts AS Timestamp(ms)) AS ts | -| | Filter: host_nano.host = Utf8("host1") AND host_nano.ts >= TimestampNanosecond(-4999999999, None) AND host_nano.ts < TimestampNanosecond(10001000000, None) | -| | TableScan: host_nano, partial_filters=[host_nano.host = Utf8("host1"), host_nano.ts >= TimestampNanosecond(-4999999999, None), host_nano.ts < TimestampNanosecond(10001000000, None)] | +| | Projection: host_nano.val, host_nano.host, host_nano.ts | +| | Filter: host_nano.host = Utf8("host1") AND host_nano.ts > TimestampNanosecond(-5000000000, None) AND host_nano.ts <= TimestampNanosecond(10000000000, None) | +| | TableScan: host_nano, partial_filters=[host_nano.host = Utf8("host1"), host_nano.ts > TimestampNanosecond(-5000000000, None), host_nano.ts <= TimestampNanosecond(10000000000, None)] | | | ]] | | physical_plan | CooperativeExec | | | MergeScanExec: REDACTED diff --git a/tests/cases/standalone/common/promql/precisions.sql b/tests/cases/standalone/common/promql/precisions.sql index 01e6cae9fec..8f7fec37a86 100644 --- a/tests/cases/standalone/common/promql/precisions.sql +++ b/tests/cases/standalone/common/promql/precisions.sql @@ -68,10 +68,8 @@ TQL EVAL (0, 15, '5s') avg_over_time(host_sec{host="host1"}[5s]) + avg_over_time -- Verify that PromQL time predicates on non-millisecond time indexes are -- pushed into the scan as native timestamp range filters. --- Original instant selector filter is built on the millisecond alias: --- host = "host1" AND ts_ms >= -299999ms AND ts_ms <= 10000ms --- After pushing through `CAST(raw_ts AS Timestamp(ms)) AS ts` and applying --- DataFusion cast preimage, it becomes a native half-open range on raw_ts. +-- Instant selection compares raw timestamps before millisecond output conversion: +-- host = "host1" AND ts_us > -300000000us AND ts_us <= 10000000us. -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED @@ -80,8 +78,8 @@ TQL EVAL (0, 15, '5s') avg_over_time(host_sec{host="host1"}[5s]) + avg_over_time -- SQLNESS REPLACE host_nano.__table_id\s*=\s*UInt32\(\d+\) host_nano.__table_id=UInt32(REDACTED) TQL EXPLAIN (0, 10, '5s') host_micro{host="host1"}; --- The same instant-selector cast-preimage path should work for nanosecond indexes. --- Expected native bounds: ts_ns >= -299999999999ns AND ts_ns < 10001000000ns. +-- The same exclusive-lower, inclusive-upper window applies to nanosecond indexes. +-- Expected native bounds: ts_ns > -300000000000ns AND ts_ns <= 10000000000ns. -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED @@ -90,8 +88,8 @@ TQL EXPLAIN (0, 10, '5s') host_micro{host="host1"}; TQL EXPLAIN (0, 10, '5s') host_nano{host="host1"}; -- Range selectors use their range window instead of the default lookback. --- Original range selector filter for [5s]: --- host = "host1" AND ts_ms >= -4999ms AND ts_ms <= 10000ms +-- Native range selector filter for [5s]: +-- host = "host1" AND ts_us > -5000000us AND ts_us <= 10000000us -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED @@ -99,8 +97,8 @@ TQL EXPLAIN (0, 10, '5s') host_nano{host="host1"}; -- SQLNESS REPLACE host_micro.__table_id\s*=\s*UInt32\(\d+\) host_micro.__table_id=UInt32(REDACTED) TQL EXPLAIN (0, 10, '5s') avg_over_time(host_micro{host="host1"}[5s]); --- The same range-selector cast-preimage path should work for nanosecond indexes. --- Expected native bounds: ts_ns >= -4999999999ns AND ts_ns < 10001000000ns. +-- Range membership also retains nanosecond precision. +-- Expected native bounds: ts_ns > -5000000000ns AND ts_ns <= 10000000000ns. -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED diff --git a/tests/cases/standalone/common/promql/range_presence_null.result b/tests/cases/standalone/common/promql/range_presence_null.result deleted file mode 100644 index d0f28ad9fb4..00000000000 --- a/tests/cases/standalone/common/promql/range_presence_null.result +++ /dev/null @@ -1,99 +0,0 @@ --- NULL fields are missing samples, not zero-valued samples. -CREATE TABLE range_presence_null ( - ts TIMESTAMP(3) TIME INDEX, - host STRING PRIMARY KEY, - val DOUBLE, -); - -Affected Rows: 0 - -INSERT INTO range_presence_null VALUES - (0, 'a', 1.0), - (1000, 'a', NULL), - (2000, 'a', NULL), - (3000, 'a', 4.0), - (0, 'b', NULL), - (1000, 'b', NULL), - (2000, 'b', NULL), - (3000, 'b', NULL); - -Affected Rows: 8 - --- At t=2 the trailing NULLs must not hide 1; at t=3 count must be 2. --- At t=7 the left-open window is empty. Valid results must disappear. --- SQLNESS SORT_RESULT 3 1 -TQL EVAL (2, 7, '1s') count_over_time(range_presence_null{host="a"}[4s]); - -+---------------------+------------------------------------+------+ -| ts | prom_count_over_time(ts_range,val) | host | -+---------------------+------------------------------------+------+ -| 1970-01-01T00:00:02 | 1.0 | a | -| 1970-01-01T00:00:03 | 2.0 | a | -| 1970-01-01T00:00:04 | 1.0 | a | -| 1970-01-01T00:00:05 | 1.0 | a | -| 1970-01-01T00:00:06 | 1.0 | a | -+---------------------+------------------------------------+------+ - --- SQLNESS SORT_RESULT 3 1 -TQL EVAL (2, 7, '1s') last_over_time(range_presence_null{host="a"}[4s]); - -+---------------------+-----------------------------------+------+ -| ts | prom_last_over_time(ts_range,val) | host | -+---------------------+-----------------------------------+------+ -| 1970-01-01T00:00:02 | 1.0 | a | -| 1970-01-01T00:00:03 | 4.0 | a | -| 1970-01-01T00:00:04 | 4.0 | a | -| 1970-01-01T00:00:05 | 4.0 | a | -| 1970-01-01T00:00:06 | 4.0 | a | -+---------------------+-----------------------------------+------+ - --- SQLNESS SORT_RESULT 3 1 -TQL EVAL (2, 7, '1s') present_over_time(range_presence_null{host="a"}[4s]); - -+---------------------+--------------------------------------+------+ -| ts | prom_present_over_time(ts_range,val) | host | -+---------------------+--------------------------------------+------+ -| 1970-01-01T00:00:02 | 1.0 | a | -| 1970-01-01T00:00:03 | 1.0 | a | -| 1970-01-01T00:00:04 | 1.0 | a | -| 1970-01-01T00:00:05 | 1.0 | a | -| 1970-01-01T00:00:06 | 1.0 | a | -+---------------------+--------------------------------------+------+ - --- SQLNESS SORT_RESULT 3 1 -TQL EVAL (2, 7, '1s') absent_over_time(range_presence_null{host="a"}[4s]); - -+---------------------+-------------------------------------+------+ -| ts | prom_absent_over_time(ts_range,val) | host | -+---------------------+-------------------------------------+------+ -| 1970-01-01T00:00:07 | 1.0 | a | -+---------------------+-------------------------------------+------+ - --- All-NULL windows have no samples: only absent_over_time returns 1. -TQL EVAL (3, 3, '1s') count_over_time(range_presence_null{host="b"}[4s]); - -++ -++ - -TQL EVAL (3, 3, '1s') last_over_time(range_presence_null{host="b"}[4s]); - -++ -++ - -TQL EVAL (3, 3, '1s') present_over_time(range_presence_null{host="b"}[4s]); - -++ -++ - -TQL EVAL (3, 3, '1s') absent_over_time(range_presence_null{host="b"}[4s]); - -+---------------------+-------------------------------------+------+ -| ts | prom_absent_over_time(ts_range,val) | host | -+---------------------+-------------------------------------+------+ -| 1970-01-01T00:00:03 | 1.0 | b | -+---------------------+-------------------------------------+------+ - -DROP TABLE range_presence_null; - -Affected Rows: 0 - diff --git a/tests/cases/standalone/common/promql/range_presence_null.sql b/tests/cases/standalone/common/promql/range_presence_null.sql deleted file mode 100644 index e442679b2b3..00000000000 --- a/tests/cases/standalone/common/promql/range_presence_null.sql +++ /dev/null @@ -1,35 +0,0 @@ --- NULL fields are missing samples, not zero-valued samples. -CREATE TABLE range_presence_null ( - ts TIMESTAMP(3) TIME INDEX, - host STRING PRIMARY KEY, - val DOUBLE, -); - -INSERT INTO range_presence_null VALUES - (0, 'a', 1.0), - (1000, 'a', NULL), - (2000, 'a', NULL), - (3000, 'a', 4.0), - (0, 'b', NULL), - (1000, 'b', NULL), - (2000, 'b', NULL), - (3000, 'b', NULL); - --- At t=2 the trailing NULLs must not hide 1; at t=3 count must be 2. --- At t=7 the left-open window is empty. Valid results must disappear. --- SQLNESS SORT_RESULT 3 1 -TQL EVAL (2, 7, '1s') count_over_time(range_presence_null{host="a"}[4s]); --- SQLNESS SORT_RESULT 3 1 -TQL EVAL (2, 7, '1s') last_over_time(range_presence_null{host="a"}[4s]); --- SQLNESS SORT_RESULT 3 1 -TQL EVAL (2, 7, '1s') present_over_time(range_presence_null{host="a"}[4s]); --- SQLNESS SORT_RESULT 3 1 -TQL EVAL (2, 7, '1s') absent_over_time(range_presence_null{host="a"}[4s]); - --- All-NULL windows have no samples: only absent_over_time returns 1. -TQL EVAL (3, 3, '1s') count_over_time(range_presence_null{host="b"}[4s]); -TQL EVAL (3, 3, '1s') last_over_time(range_presence_null{host="b"}[4s]); -TQL EVAL (3, 3, '1s') present_over_time(range_presence_null{host="b"}[4s]); -TQL EVAL (3, 3, '1s') absent_over_time(range_presence_null{host="b"}[4s]); - -DROP TABLE range_presence_null; diff --git a/tests/cases/standalone/common/promql/regex.result b/tests/cases/standalone/common/promql/regex.result index bccbd0f3402..7e0132c2f40 100644 --- a/tests/cases/standalone/common/promql/regex.result +++ b/tests/cases/standalone/common/promql/regex.result @@ -38,8 +38,10 @@ TQL EVAL (0, 100, '15s') test{host=~"(10.0.160.237:8080|10.0.160.237:9090)"}; TQL EVAL (0, 100, '15s') test{host=~"10\\.0\\.160\\.237:808|nonexistence"}; -++ -++ ++----+------+-----+ +| ts | host | val | ++----+------+-----+ ++----+------+-----+ TQL EVAL (0, 100, '15s') test{host=~"(10\\.0\\.160\\.237:8080|10\\.0\\.160\\.237:9090)"}; diff --git a/tests/cases/standalone/common/promql/set_operation.result b/tests/cases/standalone/common/promql/set_operation.result index 50a0b6cf3b6..2f0041f1a36 100644 --- a/tests/cases/standalone/common/promql/set_operation.result +++ b/tests/cases/standalone/common/promql/set_operation.result @@ -225,8 +225,10 @@ tql eval (3000, 3000, '1s') http_requests{g="canary"} unless http_requests{insta -- eval instant at 50m http_requests{group="canary"} unless on(job) http_requests{instance="0"} tql eval (3000, 3000, '1s') http_requests{g="canary"} unless on(job) http_requests{instance="0"}; -++ -++ ++----+-----+----------+---+----------------+ +| ts | job | instance | g | greptime_value | ++----+-----+----------+---+----------------+ ++----+-----+----------+---+----------------+ -- eval instant at 50m http_requests{group="canary"} unless on(job, instance) http_requests{instance="0"} -- http_requests{group="canary", instance="1", job="api-server"} 400 @@ -244,8 +246,10 @@ tql eval (3000, 3000, '1s') http_requests{g="canary"} unless on(job, instance) h -- eval instant at 50m http_requests{group="canary"} unless ignoring(group, instance) http_requests{instance="0"} tql eval (3000, 3000, '1s') http_requests{g="canary"} unless ignoring(g, instance) http_requests{instance="0"}; -++ -++ ++----+-----+----------+---+----------------+ +| ts | job | instance | g | greptime_value | ++----+-----+----------+---+----------------+ ++----+-----+----------+---+----------------+ -- eval instant at 50m http_requests{group="canary"} unless ignoring(group) http_requests{instance="0"} -- http_requests{group="canary", instance="1", job="api-server"} 400 @@ -816,12 +820,12 @@ tql eval(1000, 2000, '300s') sum by (src, src_pod, src_namespace, src_node, dest +---------------------+-----------+---------+----------+---------+----------+---------------+----------+---------+----------------------------------------------------------------------------------------------+ | greptime_timestamp | az | cloud | dest | region | src | src_namespace | src_node | src_pod | sum(prom_increase(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(900000))) | +---------------------+-----------+---------+----------+---------+----------+---------------+----------+---------+----------------------------------------------------------------------------------------------+ -| 1970-01-01T00:21:40 | us-west-6 | cloud-1 | 10.0.0.2 | us-west | 10.0.0.1 | namespace-1 | node-1 | pod-1 | 2500.0 | +| 1970-01-01T00:21:40 | us-west-6 | cloud-1 | 10.0.0.2 | us-west | 10.0.0.1 | namespace-1 | node-1 | pod-1 | 2000.0 | | 1970-01-01T00:21:40 | us-west-6 | cloud-1 | 10.0.0.3 | us-west | 10.0.0.1 | namespace-1 | node-2 | pod-2 | 2000.0 | -| 1970-01-01T00:21:40 | us-west-6 | cloud-2 | 10.0.0.5 | us-west | 10.0.0.4 | namespace-2 | node-3 | pod-3 | 2300.0 | -| 1970-01-01T00:26:40 | us-west-6 | cloud-1 | 10.0.0.2 | us-west | 10.0.0.1 | namespace-1 | node-1 | pod-1 | 2500.0 | +| 1970-01-01T00:21:40 | us-west-6 | cloud-2 | 10.0.0.5 | us-west | 10.0.0.4 | namespace-2 | node-3 | pod-3 | 2000.0 | +| 1970-01-01T00:26:40 | us-west-6 | cloud-1 | 10.0.0.2 | us-west | 10.0.0.1 | namespace-1 | node-1 | pod-1 | 2000.0 | | 1970-01-01T00:26:40 | us-west-6 | cloud-1 | 10.0.0.3 | us-west | 10.0.0.1 | namespace-1 | node-2 | pod-2 | 2000.0 | -| 1970-01-01T00:26:40 | us-west-6 | cloud-2 | 10.0.0.5 | us-west | 10.0.0.4 | namespace-2 | node-3 | pod-3 | 2300.0 | +| 1970-01-01T00:26:40 | us-west-6 | cloud-2 | 10.0.0.5 | us-west | 10.0.0.4 | namespace-2 | node-3 | pod-3 | 2000.0 | +---------------------+-----------+---------+----------+---------+----------+---------------+----------+---------+----------------------------------------------------------------------------------------------+ DROP TABLE node_network_transmit_bytes_total; diff --git a/tests/cases/standalone/common/promql/simple_histogram.result b/tests/cases/standalone/common/promql/simple_histogram.result index e02e138f80e..a794a30e8b9 100644 --- a/tests/cases/standalone/common/promql/simple_histogram.result +++ b/tests/cases/standalone/common/promql/simple_histogram.result @@ -134,8 +134,10 @@ tql eval (3000, 3000, '1s') label_replace(histogram_quantile(0.8, histogram_buck -- SQLNESS SORT_RESULT 3 1 tql eval (3000, 3000, '1s') histogram_quantile(0.2, rate(histogram_bucket[10m])); -++ -++ ++----+------------------------------------------+---+ +| ts | prom_rate(ts_range,val,ts,Int64(600000)) | s | ++----+------------------------------------------+---+ ++----+------------------------------------------+---+ drop table histogram_bucket; @@ -347,8 +349,10 @@ Affected Rows: 0 -- SQLNESS SORT_RESULT 3 1 tql eval(0, 10, '10s') histogram_quantile(0.99, sum by(pod,instance, fff) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"xxx"}[1m]))); -++ -++ ++------+----------------------------------------------------+ +| time | sum(prom_rate(time_range,value,time,Int64(60000))) | ++------+----------------------------------------------------+ ++------+----------------------------------------------------+ -- test case where table exists but doesn't have 'le' column should raise error CREATE TABLE greptime_servers_postgres_query_elapsed_no_le ( @@ -365,14 +369,18 @@ Affected Rows: 0 -- SQLNESS SORT_RESULT 3 1 tql eval(0, 10, '10s') histogram_quantile(0.99, sum by(pod,instance, le) (rate(greptime_servers_postgres_query_elapsed_no_le{instance=~"xxx"}[1m]))); -++ -++ ++-----+----------+---+------------------------------------------+ +| pod | instance | t | sum(prom_rate(t_range,v,t,Int64(60000))) | ++-----+----------+---+------------------------------------------+ ++-----+----------+---+------------------------------------------+ -- SQLNESS SORT_RESULT 3 1 tql eval(0, 10, '10s') histogram_quantile(0.99, sum by(pod,instance, fbf) (rate(greptime_servers_postgres_query_elapsed_no_le{instance=~"xxx"}[1m]))); -++ -++ ++-----+----------+---+------------------------------------------+ +| pod | instance | t | sum(prom_rate(t_range,v,t,Int64(60000))) | ++-----+----------+---+------------------------------------------+ ++-----+----------+---+------------------------------------------+ drop table greptime_servers_postgres_query_elapsed_no_le; diff --git a/tests/cases/standalone/common/promql/time_fn.result b/tests/cases/standalone/common/promql/time_fn.result index 92ee594150c..25e0cd11030 100644 --- a/tests/cases/standalone/common/promql/time_fn.result +++ b/tests/cases/standalone/common/promql/time_fn.result @@ -206,8 +206,10 @@ tql eval (1701413023, 1701413023, '1s') hour(); tql eval (1701413023, 1701413023, '1s') hour(metrics); -++ -++ ++----+----------------------------+ +| ts | date_part(Utf8("hour"),ts) | ++----+----------------------------+ ++----+----------------------------+ tql eval (1701413023, 1701413023, '1s') minute(); diff --git a/tests/cases/standalone/common/promql/timestamp_fn.result b/tests/cases/standalone/common/promql/timestamp_fn.result index 7cc2b24484e..1f5682649a4 100644 --- a/tests/cases/standalone/common/promql/timestamp_fn.result +++ b/tests/cases/standalone/common/promql/timestamp_fn.result @@ -105,19 +105,25 @@ tql eval (0, 60, '30s') timestamp(timestamp_test) - time(); -- Test timestamp() with other functions tql eval (0, 60, '30s') abs(timestamp(timestamp_test) - avg(timestamp(timestamp_test))) > 20; -++ -++ ++----+---------------------------------+ +| ts | abs(lhs.value - rhs.avg(value)) | ++----+---------------------------------+ ++----+---------------------------------+ -- Test Issue 6707 tql eval timestamp(demo_memory_usage_bytes * 1); -++ -++ ++------+--------------------+ +| time | value * Float64(1) | ++------+--------------------+ ++------+--------------------+ tql eval timestamp(-demo_memory_usage_bytes); -++ -++ ++------+-----------+ +| time | (- value) | ++------+-----------+ ++------+-----------+ tql eval (0, 60, '30s') timestamp(timestamp_test) == 60; diff --git a/tests/cases/standalone/common/promql/tsid_binary_join_regression.result b/tests/cases/standalone/common/promql/tsid_binary_join_regression.result index a3983260e33..d80d29ce000 100644 --- a/tests/cases/standalone/common/promql/tsid_binary_join_regression.result +++ b/tests/cases/standalone/common/promql/tsid_binary_join_regression.result @@ -115,8 +115,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / tsid_binary_join_right; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[host@2 as host, job@3 as job, ts@5 as ts, __tsid@4 as __tsid, greptime_value@0 / greptime_value@1 as tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value] REDACTED -|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@2, ts@4)], projection=[greptime_value@0, greptime_value@3, host@4, job@5, __tsid@6, ts@7], NullsEqual: true REDACTED +| 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, greptime_value@4 / greptime_value@5 as tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@2, ts@4)], projection=[host@4, job@5, ts@7, __tsid@6, greptime_value@0, greptime_value@3], NullsEqual: true REDACTED |_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED |_|_|_CooperativeExec REDACTED |_|_|_MergeScanExec: REDACTED @@ -155,8 +155,8 @@ TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left + tsid_binary_join_right) / tsid +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[host@1 as host, job@2 as job, ts@4 as ts, __tsid@3 as __tsid, (greptime_value@0 + greptime_value@5) / greptime_value@0 as tsid_binary_join_left.greptime_value + tsid_binary_join_right.greptime_value / tsid_binary_join_left.greptime_value] REDACTED -|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@3, __tsid@1), (ts@4, ts@2)], projection=[greptime_value@0, host@1, job@2, __tsid@3, ts@4, greptime_value@5], NullsEqual: true REDACTED +| 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, (greptime_value@4 + greptime_value@5) / greptime_value@4 as tsid_binary_join_left.greptime_value + tsid_binary_join_right.greptime_value / tsid_binary_join_left.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@3, __tsid@1), (ts@4, ts@2)], projection=[host@1, job@2, ts@4, __tsid@3, greptime_value@0, greptime_value@5], NullsEqual: true REDACTED |_|_|_RepartitionExec: partitioning=Hash([__tsid@3, ts@4],REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_RepartitionExec: partitioning=Hash([__tsid@1, ts@2],REDACTED @@ -194,8 +194,8 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left + tsid_binary_join_right) * (ts +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[host@1 as host, job@2 as job, ts@4 as ts, __tsid@3 as __tsid, (greptime_value@0 + greptime_value@5) * (greptime_value@0 - greptime_value@6) / (greptime_value@0 + 2) as tsid_binary_join_left.greptime_value + tsid_binary_join_right.greptime_value * tsid_binary_join_left.greptime_value - tsid_binary_join_third.greptime_value / tsid_binary_join_left.greptime_value + Float64(2)] REDACTED -|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@3, __tsid@1), (ts@4, ts@2)], projection=[greptime_value@0, host@1, job@2, __tsid@3, ts@4, greptime_value@5, greptime_value@6], NullsEqual: true REDACTED +| 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, (greptime_value@4 + greptime_value@5) * (greptime_value@4 - greptime_value@6) / (greptime_value@4 + 2) as tsid_binary_join_left.greptime_value + tsid_binary_join_right.greptime_value * tsid_binary_join_left.greptime_value - tsid_binary_join_third.greptime_value / tsid_binary_join_left.greptime_value + Float64(2)] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@3, __tsid@1), (ts@4, ts@2)], projection=[host@1, job@2, ts@4, __tsid@3, greptime_value@0, greptime_value@5, greptime_value@6], NullsEqual: true REDACTED |_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@3, __tsid@1), (ts@4, ts@2)], projection=[greptime_value@0, host@1, job@2, __tsid@3, ts@4, greptime_value@5], NullsEqual: true REDACTED |_|_|_RepartitionExec: partitioning=Hash([REDACTED |_|_|_MergeScanExec: REDACTED @@ -243,8 +243,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / ignoring(host) tsid_binary_join +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[host@2 as host, job@3 as job, ts@5 as ts, __tsid@4 as __tsid, greptime_value@0 / greptime_value@1 as tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value] REDACTED -|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(job@1, job@2), (ts@2, ts@4)], projection=[greptime_value@0, greptime_value@3, host@4, job@5, __tsid@6, ts@7], NullsEqual: true REDACTED +| 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, greptime_value@4 / greptime_value@5 as tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(job@1, job@2), (ts@2, ts@4)], projection=[host@4, job@5, ts@7, __tsid@6, greptime_value@0, greptime_value@3], NullsEqual: true REDACTED |_|_|_RepartitionExec: partitioning=Hash([job@1, ts@2],REDACTED |_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, job@2 as job, ts@4 as ts] REDACTED |_|_|_MergeScanExec: REDACTED @@ -282,8 +282,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / on(job) tsid_binary_join_right_ +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[job@2 as job, ts@4 as ts, __tsid@3 as __tsid, greptime_value@0 / greptime_value@1 as tsid_binary_join_left.greptime_value / tsid_binary_join_right_by_job.greptime_value] REDACTED -|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(job@1, job@1), (ts@2, ts@3)], projection=[greptime_value@0, greptime_value@3, job@4, __tsid@5, ts@6], NullsEqual: true REDACTED +| 0_| 0_|_ProjectionExec: expr=[job@0 as job, ts@1 as ts, __tsid@2 as __tsid, greptime_value@3 / greptime_value@4 as tsid_binary_join_left.greptime_value / tsid_binary_join_right_by_job.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(job@1, job@1), (ts@2, ts@3)], projection=[job@4, ts@6, __tsid@5, greptime_value@0, greptime_value@3], NullsEqual: true REDACTED |_|_|_RepartitionExec: partitioning=Hash([job@1, ts@2],REDACTED |_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, job@2 as job, ts@4 as ts] REDACTED |_|_|_MergeScanExec: REDACTED @@ -322,8 +322,7 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > tsid_binary_join_right; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[ts@4 as ts, greptime_value@0 as greptime_value, host@1 as host, job@2 as job, __tsid@3 as __tsid] REDACTED -|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@3, __tsid@1), (ts@4, ts@2)], filter=greptime_value@1 < greptime_value@0, projection=[greptime_value@0, host@1, job@2, __tsid@3, ts@4], NullsEqual: true REDACTED +| 0_| 0_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@3, __tsid@1), (ts@4, ts@2)], filter=greptime_value@1 < greptime_value@0, projection=[ts@4, greptime_value@0, host@1, job@2, __tsid@3], NullsEqual: true REDACTED |_|_|_RepartitionExec: partitioning=Hash([__tsid@3, ts@4],REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_RepartitionExec: partitioning=Hash([__tsid@1, ts@2],REDACTED @@ -361,8 +360,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > bool tsid_binary_join_right; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[host@2 as host, job@3 as job, ts@5 as ts, __tsid@4 as __tsid, CAST(greptime_value@1 < greptime_value@0 AS Float64) as tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value] REDACTED -|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@2, ts@4)], projection=[greptime_value@0, greptime_value@3, host@4, job@5, __tsid@6, ts@7], NullsEqual: true REDACTED +| 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, CAST(greptime_value@4 < greptime_value@5 AS Float64) as tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@2, ts@4)], projection=[host@4, job@5, ts@7, __tsid@6, greptime_value@3, greptime_value@0], NullsEqual: true REDACTED |_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED |_|_|_CooperativeExec REDACTED |_|_|_MergeScanExec: REDACTED @@ -400,11 +399,10 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > tsid_binary_join_right) / tsi +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[host@2 as host, job@3 as job, ts@5 as ts, __tsid@4 as __tsid, greptime_value@0 / greptime_value@1 * 100 as .greptime_value / tsid_binary_join_left.greptime_value * Float64(100)] REDACTED -|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@2, __tsid@3), (ts@0, ts@4)], projection=[greptime_value@1, greptime_value@3, host@4, job@5, __tsid@6, ts@7], NullsEqual: true REDACTED +| 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, greptime_value@4 / greptime_value@5 * 100 as .greptime_value / tsid_binary_join_left.greptime_value * Float64(100)] REDACTED +|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@2, __tsid@3), (ts@0, ts@4)], projection=[host@4, job@5, ts@7, __tsid@6, greptime_value@1, greptime_value@3], NullsEqual: true REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_ProjectionExec: expr=[ts@2 as ts, greptime_value@0 as greptime_value, __tsid@1 as __tsid] REDACTED -|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@2, ts@2)], filter=greptime_value@1 < greptime_value@0, projection=[greptime_value@0, __tsid@1, ts@2], NullsEqual: true REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@2, ts@2)], filter=greptime_value@1 < greptime_value@0, projection=[ts@2, greptime_value@0, __tsid@1], NullsEqual: true REDACTED |_|_|_RepartitionExec: partitioning=Hash([REDACTED |_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED |_|_|_MergeScanExec: REDACTED @@ -450,13 +448,13 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > bool tsid_binary_join_right) +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[host@2 as host, job@3 as job, ts@5 as ts, __tsid@4 as __tsid, tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value@0 / greptime_value@1 as lhs.tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value / rhs.greptime_value] REDACTED -|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@0, ts@4)], projection=[tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value@2, greptime_value@3, host@4, job@5, __tsid@6, ts@7], NullsEqual: true REDACTED +| 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value@4 / greptime_value@5 as lhs.tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value / rhs.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@0, ts@4)], projection=[host@4, job@5, ts@7, __tsid@6, tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value@2, greptime_value@3], NullsEqual: true REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_ProjectionExec: expr=[ts@3 as ts, __tsid@2 as __tsid, tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value@0 + greptime_value@1 as tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value] REDACTED -|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@0, ts@2)], projection=[tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value@2, greptime_value@3, __tsid@4, ts@5], NullsEqual: true REDACTED -|_|_|_ProjectionExec: expr=[ts@3 as ts, __tsid@2 as __tsid, CAST(greptime_value@1 < greptime_value@0 AS Float64) as tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value] REDACTED -|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@2, ts@2)], projection=[greptime_value@0, greptime_value@3, __tsid@4, ts@5], NullsEqual: true REDACTED +|_|_|_ProjectionExec: expr=[ts@0 as ts, __tsid@1 as __tsid, tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value@2 + greptime_value@3 as tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@0, ts@2)], projection=[ts@5, __tsid@4, tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value@2, greptime_value@3], NullsEqual: true REDACTED +|_|_|_ProjectionExec: expr=[ts@0 as ts, __tsid@1 as __tsid, CAST(greptime_value@2 < greptime_value@3 AS Float64) as tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@2, ts@2)], projection=[ts@5, __tsid@4, greptime_value@3, greptime_value@0], NullsEqual: true REDACTED |_|_|_RepartitionExec: partitioning=Hash([REDACTED |_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED |_|_|_MergeScanExec: REDACTED @@ -512,8 +510,8 @@ TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left or tsid_binary_join_right) / tsi +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[host@2 as host, job@3 as job, ts@5 as ts, __tsid@4 as __tsid, greptime_value@0 / greptime_value@1 as lhs.greptime_value / rhs.greptime_value] REDACTED -|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@0, ts@4)], projection=[greptime_value@2, greptime_value@3, host@4, job@5, __tsid@6, ts@7], NullsEqual: true REDACTED +| 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, greptime_value@4 / greptime_value@5 as lhs.greptime_value / rhs.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@0, ts@4)], projection=[host@4, job@5, ts@7, __tsid@6, greptime_value@2, greptime_value@3], NullsEqual: true REDACTED |_|_|_ProjectionExec: expr=[ts@0 as ts, __tsid@1 as __tsid, greptime_value@2 as greptime_value] REDACTED |_|_|_UnionDistinctOnExec: on col=[5, 6], ts_col=0 REDACTED |_|_|_CooperativeExec REDACTED @@ -564,11 +562,11 @@ TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left / ignoring(host) group_left tsid +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[host@2 as host, job@3 as job, ts@5 as ts, __tsid@4 as __tsid, tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value@0 / greptime_value@1 as tsid_binary_join_right.tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value / tsid_binary_join_left.greptime_value] REDACTED -|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@0, ts@4)], projection=[tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value@2, greptime_value@3, host@4, job@5, __tsid@6, ts@7], NullsEqual: true REDACTED +| 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value@4 / greptime_value@5 as tsid_binary_join_right.tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value / tsid_binary_join_left.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@0, ts@4)], projection=[host@4, job@5, ts@7, __tsid@6, tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value@2, greptime_value@3], NullsEqual: true REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_ProjectionExec: expr=[ts@3 as ts, __tsid@2 as __tsid, greptime_value@0 / greptime_value@1 as tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value] REDACTED -|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(job@1, job@1), (ts@2, ts@3)], projection=[greptime_value@0, greptime_value@3, __tsid@5, ts@6], NullsEqual: true REDACTED +|_|_|_ProjectionExec: expr=[ts@0 as ts, __tsid@1 as __tsid, greptime_value@2 / greptime_value@3 as tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(job@1, job@1), (ts@2, ts@3)], projection=[ts@6, __tsid@5, greptime_value@0, greptime_value@3], NullsEqual: true REDACTED |_|_|_RepartitionExec: partitioning=Hash([REDACTED |_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, job@2 as job, ts@4 as ts] REDACTED |_|_|_MergeScanExec: REDACTED diff --git a/tests/cases/standalone/common/range/by.result b/tests/cases/standalone/common/range/by.result index fd2feb1da46..ef842f30787 100644 --- a/tests/cases/standalone/common/range/by.result +++ b/tests/cases/standalone/common/range/by.result @@ -63,7 +63,8 @@ SELECT ts, length(host)::INT64 + 2, max(val) RANGE '5s' FROM host ALIGN '20s' BY -- project non-aggregation key SELECT ts, host, max(val) RANGE '5s' FROM host ALIGN '20s' BY () ORDER BY ts; -Error: 3001(EngineExecuteQuery), No field named host.host. Did you mean 'host.ts'?. +Error: 3001(EngineExecuteQuery), No field named host.host. Did you mean 'host.ts'? +Valid fields are "max(host.val) RANGE 5s", host.ts, "Int64(1)". DROP TABLE host; diff --git a/tests/cases/standalone/common/range/calculate.result b/tests/cases/standalone/common/range/calculate.result index f27cbef398e..7037854f9e3 100644 --- a/tests/cases/standalone/common/range/calculate.result +++ b/tests/cases/standalone/common/range/calculate.result @@ -126,16 +126,16 @@ SELECT ts, host, approx_percentile_cont(0.5) WITHIN GROUP (ORDER BY val) RANGE ' +---------------------+-------+--------------------------------------------------------------------------------------+ | ts | host | approx_percentile_cont(Float64(0.5)) WITHIN GROUP [host.val ASC NULLS LAST] RANGE 5s | +---------------------+-------+--------------------------------------------------------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 0 | +| 1970-01-01T00:00:00 | host1 | 0.0 | | 1970-01-01T00:00:05 | host1 | | -| 1970-01-01T00:00:10 | host1 | 1 | +| 1970-01-01T00:00:10 | host1 | 1.0 | | 1970-01-01T00:00:15 | host1 | | -| 1970-01-01T00:00:20 | host1 | 2 | -| 1970-01-01T00:00:00 | host2 | 3 | +| 1970-01-01T00:00:20 | host1 | 2.0 | +| 1970-01-01T00:00:00 | host2 | 3.0 | | 1970-01-01T00:00:05 | host2 | | -| 1970-01-01T00:00:10 | host2 | 4 | +| 1970-01-01T00:00:10 | host2 | 4.0 | | 1970-01-01T00:00:15 | host2 | | -| 1970-01-01T00:00:20 | host2 | 5 | +| 1970-01-01T00:00:20 | host2 | 5.0 | +---------------------+-------+--------------------------------------------------------------------------------------+ -- Test complex range expr calculate diff --git a/tests/cases/standalone/common/range/error.result b/tests/cases/standalone/common/range/error.result index ff0795f4505..feaee54d1a0 100644 --- a/tests/cases/standalone/common/range/error.result +++ b/tests/cases/standalone/common/range/error.result @@ -33,23 +33,24 @@ Error: 2000(InvalidSyntax), Invalid SQL syntax: sql parser error: not a valid du -- 2.1 no range param SELECT min(val) FROM host ALIGN '5s'; -Error: 2000(InvalidSyntax), Invalid SQL syntax: sql parser error: Illegal Range select, no RANGE keyword found in any SelectItem +Error: 2000(InvalidSyntax), Invalid SQL syntax: sql parser error: Illegal range select: no RANGE keyword found in any SELECT item SELECT 1 FROM host ALIGN '5s'; -Error: 2000(InvalidSyntax), Invalid SQL syntax: sql parser error: Illegal Range select, no RANGE keyword found in any SelectItem +Error: 2000(InvalidSyntax), Invalid SQL syntax: sql parser error: Illegal range select: no RANGE keyword found in any SELECT item SELECT min(val) RANGE '10s', max(val) FROM host ALIGN '5s'; -Error: 3001(EngineExecuteQuery), No field named "max(host.val)". Valid fields are "min(host.val) RANGE 10s", host.ts, host.host. +Error: 3001(EngineExecuteQuery), No field named "max(host.val)". +Valid fields are "min(host.val) RANGE 10s", host.ts, host.host. SELECT min(val) * 2 RANGE '10s' FROM host ALIGN '5s'; -Error: 2000(InvalidSyntax), Invalid SQL syntax: sql parser error: Can't use the RANGE keyword in Expr 2 without function +Error: 2000(InvalidSyntax), Invalid SQL syntax: sql parser error: Can't use RANGE in expression 2 without a function SELECT 1 RANGE '10s' FILL NULL FROM host ALIGN '1h' FILL NULL; -Error: 2000(InvalidSyntax), Invalid SQL syntax: sql parser error: Can't use the RANGE keyword in Expr 1 without function +Error: 2000(InvalidSyntax), Invalid SQL syntax: sql parser error: Can't use RANGE in expression 1 without a function -- 2.2 no align param SELECT min(val) RANGE '5s' FROM host; @@ -85,7 +86,7 @@ SELECT covar(ceil(val), floor(val)) RANGE '20s' FROM host ALIGN '10s'; -- 2.4 nest query SELECT min(max(val) RANGE '20s') RANGE '20s' FROM host ALIGN '10s'; -Error: 2000(InvalidSyntax), Range Query: Nest Range Query is not allowed +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Aggregate function calls cannot be nested: 'max(host.val)' is nested inside 'min(range_fn(max(host.val), Utf8("20s"), Utf8("")))' -- 2.5 wrong Aggregate SELECT rank() OVER (PARTITION BY host ORDER BY ts DESC) RANGE '10s' FROM host ALIGN '5s'; diff --git a/tests/cases/standalone/common/range/nest.result b/tests/cases/standalone/common/range/nest.result index 184b27545d0..e283c457526 100644 --- a/tests/cases/standalone/common/range/nest.result +++ b/tests/cases/standalone/common/range/nest.result @@ -132,7 +132,6 @@ EXPLAIN SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s'; | plan_type_| plan_| +-+-+ | logical_plan_| RangeSelect: range_exprs=[min(host.val) RANGE 5s], align=5000ms, align_to=0ms, align_by=[host.host], time_index=ts | -|_|_Projection: host.ts, host.host, host.val_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Projection: host.ts, host.host, host.val_| |_|_TableScan: host_| diff --git a/tests/cases/standalone/common/range/special_aggr.result b/tests/cases/standalone/common/range/special_aggr.result index f6659eaa807..a4071349c9c 100644 --- a/tests/cases/standalone/common/range/special_aggr.result +++ b/tests/cases/standalone/common/range/special_aggr.result @@ -231,7 +231,7 @@ SELECT ts, host, count(distinct *) RANGE '5s' FROM host ALIGN '5s' ORDER BY host -- Test error first_value/last_value SELECT ts, host, first_value(val, val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: The function 'first_value' expected 1 arguments but received 2 No function matches the given name and argument types 'first_value(Int64, Int64)'. You might need to add explicit type casts. +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: The function 'first_value' expected 1 arguments but received 2. No function matches the given name and argument types 'first_value(Int64, Int64)'. You might need to add explicit type casts. Candidate functions: first_value(Any) diff --git a/tests/cases/standalone/common/select/matches.result b/tests/cases/standalone/common/select/matches.result index 084cec8cd82..6a7db78ae32 100644 --- a/tests/cases/standalone/common/select/matches.result +++ b/tests/cases/standalone/common/select/matches.result @@ -80,13 +80,17 @@ select fox from fox where matches(fox, 'fox AND lazy') order by ts; select fox from fox where matches(fox, '-over -lazy') order by ts; -++ -++ ++-----+ +| fox | ++-----+ ++-----+ select fox from fox where matches(fox, '-over AND -lazy') order by ts; -++ -++ ++-----+ +| fox | ++-----+ ++-----+ select fox from fox where matches(fox, 'fox AND jumps OR over') order by ts; @@ -228,8 +232,10 @@ select fox from fox where matches(fox, '+(fox jumps) AND over') order by ts; select fox from fox where matches(fox, 'over -(fox jumps)') order by ts; -++ -++ ++-----+ +| fox | ++-----+ ++-----+ select fox from fox where matches(fox, 'over -(fox AND jumps)') order by ts; diff --git a/tests/cases/standalone/common/select/prune.result b/tests/cases/standalone/common/select/prune.result index d5e2b5c3de4..5e219a195db 100644 --- a/tests/cases/standalone/common/select/prune.result +++ b/tests/cases/standalone/common/select/prune.result @@ -48,8 +48,10 @@ select * from demo where host='test2' and idc='idc1' and collector='disk'; select * from demo where host='test2' and idc='idc2'; -++ -++ ++----+-------+------+-----+-----------+ +| ts | value | host | idc | collector | ++----+-------+------+-----+-----------+ ++----+-------+------+-----+-----------+ select * from demo where host='test3' and idc>'idc1'; diff --git a/tests/cases/standalone/common/select/schema_reference.result b/tests/cases/standalone/common/select/schema_reference.result index 37d2de4c8c9..9ae3ebcac58 100644 --- a/tests/cases/standalone/common/select/schema_reference.result +++ b/tests/cases/standalone/common/select/schema_reference.result @@ -25,7 +25,8 @@ SELECT s1.tbl.i FROM s1.tbl ORDER BY i; -- Test schema mismatch error - should fail SELECT s2.tbl.i FROM s1.tbl; -Error: 3000(PlanQuery), Failed to plan SQL: No field named s2.tbl.i. Valid fields are s1.tbl.i, s1.tbl.ts. +Error: 3000(PlanQuery), Failed to plan SQL: No field named s2.tbl.i. Did you mean 's1.tbl.i'? +Valid fields are s1.tbl.i, s1.tbl.ts. -- Clean up DROP TABLE s1.tbl; diff --git a/tests/cases/standalone/common/select/tql_filter.result b/tests/cases/standalone/common/select/tql_filter.result index b4fbfdb5862..45f85ead00d 100644 --- a/tests/cases/standalone/common/select/tql_filter.result +++ b/tests/cases/standalone/common/select/tql_filter.result @@ -95,8 +95,10 @@ Affected Rows: 5 -- SQLNESS SORT_RESULT 3 1 tql eval (0, 0, '1s') t2{a=~"10"}; -++ -++ ++---+---+---+ +| a | b | c | ++---+---+---+ ++---+---+---+ -- SQLNESS SORT_RESULT 3 1 tql eval (0, 0, '1s') t2{a=~"10.*"}; diff --git a/tests/cases/standalone/common/select/unnest.result b/tests/cases/standalone/common/select/unnest.result index e082a8a6c15..8555a49506b 100644 --- a/tests/cases/standalone/common/select/unnest.result +++ b/tests/cases/standalone/common/select/unnest.result @@ -29,11 +29,11 @@ SELECT unnest([1,2,3]); SELECT unnest(struct(1,2,3)); -+-------------------------------------------------------------+-------------------------------------------------------------+-------------------------------------------------------------+ -| __unnest_placeholder(struct(Int64(1),Int64(2),Int64(3))).c0 | __unnest_placeholder(struct(Int64(1),Int64(2),Int64(3))).c1 | __unnest_placeholder(struct(Int64(1),Int64(2),Int64(3))).c2 | -+-------------------------------------------------------------+-------------------------------------------------------------+-------------------------------------------------------------+ -| 1 | 2 | 3 | -+-------------------------------------------------------------+-------------------------------------------------------------+-------------------------------------------------------------+ ++---------------------------------------+---------------------------------------+---------------------------------------+ +| struct(Int64(1),Int64(2),Int64(3)).c0 | struct(Int64(1),Int64(2),Int64(3)).c1 | struct(Int64(1),Int64(2),Int64(3)).c2 | ++---------------------------------------+---------------------------------------+---------------------------------------+ +| 1 | 2 | 3 | ++---------------------------------------+---------------------------------------+---------------------------------------+ -- Table function is not supported for now -- SELECT * FROM unnest([1,2,3]); diff --git a/tests/cases/standalone/common/setops/basic_setops.result b/tests/cases/standalone/common/setops/basic_setops.result index 2bee6caa0f1..c1b6dd14eca 100644 --- a/tests/cases/standalone/common/setops/basic_setops.result +++ b/tests/cases/standalone/common/setops/basic_setops.result @@ -49,8 +49,10 @@ SELECT NULL UNION SELECT NULL; SELECT NULL EXCEPT SELECT NULL; -++ -++ ++------+ +| NULL | ++------+ ++------+ SELECT NULL INTERSECT SELECT NULL; @@ -120,8 +122,10 @@ SELECT 1 EXCEPT SELECT 2; SELECT 1 EXCEPT SELECT 1; -++ -++ ++----------+ +| Int64(1) | ++----------+ ++----------+ SELECT 1 INTERSECT SELECT 1; @@ -133,8 +137,10 @@ SELECT 1 INTERSECT SELECT 1; SELECT 1 INTERSECT SELECT 2; -++ -++ ++----------+ +| Int64(1) | ++----------+ ++----------+ DROP TABLE test; diff --git a/tests/cases/standalone/common/show/show_charset.result b/tests/cases/standalone/common/show/show_charset.result index 10b25864d3a..abd9924e68a 100644 --- a/tests/cases/standalone/common/show/show_charset.result +++ b/tests/cases/standalone/common/show/show_charset.result @@ -24,8 +24,10 @@ SHOW CHARACTER SET LIKE 'utf8'; SHOW CHARACTER SET LIKE 'latin1'; -++ -++ ++---------+-------------+-------------------+--------+ +| Charset | Description | Default collation | Maxlen | ++---------+-------------+-------------------+--------+ ++---------+-------------+-------------------+--------+ SHOW CHARSET LIKE 'utf8'; @@ -53,6 +55,8 @@ SHOW CHARSET WHERE Charset = 'utf8'; SHOW CHARSET WHERE Charset = 'latin1'; -++ -++ ++---------+-------------+-------------------+--------+ +| Charset | Description | Default collation | Maxlen | ++---------+-------------+-------------------+--------+ ++---------+-------------+-------------------+--------+ diff --git a/tests/cases/standalone/common/show/show_collation.result b/tests/cases/standalone/common/show/show_collation.result index 4dcf9530595..99bd07346cf 100644 --- a/tests/cases/standalone/common/show/show_collation.result +++ b/tests/cases/standalone/common/show/show_collation.result @@ -8,8 +8,10 @@ SHOW COLLATION; SHOW COLLATION LIKE 'utf8'; -++ -++ ++-----------+---------+----+---------+----------+---------+ +| Collation | Charset | Id | Default | Compiled | Sortlen | ++-----------+---------+----+---------+----------+---------+ ++-----------+---------+----+---------+----------+---------+ SHOW COLLATION WHERE Charset = 'utf8'; @@ -21,11 +23,15 @@ SHOW COLLATION WHERE Charset = 'utf8'; SHOW COLLATION WHERE Charset = 'latin1'; -++ -++ ++-----------+---------+----+---------+----------+---------+ +| Collation | Charset | Id | Default | Compiled | Sortlen | ++-----------+---------+----+---------+----------+---------+ ++-----------+---------+----+---------+----------+---------+ SHOW COLLATION LIKE 'latin1'; -++ -++ ++-----------+---------+----+---------+----------+---------+ +| Collation | Charset | Id | Default | Compiled | Sortlen | ++-----------+---------+----+---------+----------+---------+ ++-----------+---------+----+---------+----------+---------+ diff --git a/tests/cases/standalone/common/show/show_databases_tables.result b/tests/cases/standalone/common/show/show_databases_tables.result index bdfd3b79ee3..1c9e3a32460 100644 --- a/tests/cases/standalone/common/show/show_databases_tables.result +++ b/tests/cases/standalone/common/show/show_databases_tables.result @@ -46,8 +46,10 @@ SHOW TABLES; | optimizer_trace | | parameters | | partitions | +| plugins | | procedure_info | | process_list | +| processlist | | profiling | | referential_constraints | | region_info | @@ -65,6 +67,7 @@ SHOW TABLES; | table_privileges | | table_semantics | | tables | +| user_privileges | | views | +---------------------------------------+ @@ -100,8 +103,10 @@ SHOW FULL TABLES; | optimizer_trace | LOCAL TEMPORARY | | parameters | LOCAL TEMPORARY | | partitions | LOCAL TEMPORARY | +| plugins | LOCAL TEMPORARY | | procedure_info | LOCAL TEMPORARY | | process_list | LOCAL TEMPORARY | +| processlist | LOCAL TEMPORARY | | profiling | LOCAL TEMPORARY | | referential_constraints | LOCAL TEMPORARY | | region_info | LOCAL TEMPORARY | @@ -119,6 +124,7 @@ SHOW FULL TABLES; | table_privileges | LOCAL TEMPORARY | | table_semantics | LOCAL TEMPORARY | | tables | LOCAL TEMPORARY | +| user_privileges | LOCAL TEMPORARY | | views | LOCAL TEMPORARY | +---------------------------------------+-----------------+ @@ -148,8 +154,10 @@ SHOW TABLE STATUS; |optimizer_trace||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| |parameters||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| |partitions||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| +|plugins||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| |procedure_info||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| |process_list||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| +|processlist||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| |profiling||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| |referential_constraints||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| |region_info||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| @@ -167,6 +175,7 @@ SHOW TABLE STATUS; |table_privileges||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| |table_semantics||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| |tables||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| +|user_privileges||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| |views||11|Fixed|0|0|0|0|0|0|0|DATETIME|DATETIME||utf8_bin|0||| +++++++++++++++++++ diff --git a/tests/cases/standalone/common/show/show_region.result b/tests/cases/standalone/common/show/show_region.result index 17d36c7438f..914c480f7a5 100644 --- a/tests/cases/standalone/common/show/show_region.result +++ b/tests/cases/standalone/common/show/show_region.result @@ -45,8 +45,10 @@ SHOW REGION FROM another_table in public; -- SQLNESS REPLACE (\d{1}) PEER_ID SHOW REGION FROM another_table WHERE Leader = 'No'; -++ -++ ++-------+--------+------+--------+ +| Table | Region | Peer | Leader | ++-------+--------+------+--------+ ++-------+--------+------+--------+ DROP TABLE my_table; diff --git a/tests/cases/standalone/common/skip_wal.result b/tests/cases/standalone/common/skip_wal.result index 5f3ae29f52b..25c18fed20e 100644 --- a/tests/cases/standalone/common/skip_wal.result +++ b/tests/cases/standalone/common/skip_wal.result @@ -22,8 +22,10 @@ Affected Rows: 3 -- SQLNESS ARG restart=true SELECT * FROM system_metrics; -++ -++ ++------+-----+----------+-------------+-----------+----+ +| host | idc | cpu_util | memory_util | disk_util | ts | ++------+-----+----------+-------------+-----------+----+ ++------+-----+----------+-------------+-----------+----+ INSERT INTO system_metrics VALUES @@ -52,6 +54,11 @@ SELECT * FROM system_metrics; | host2 | idc_a | 80.0 | 70.3 | 90.0 | 2022-11-03T03:39:57.450 | +-------+-------+----------+-------------+-----------+-------------------------+ +-- A table created without a real WAL provider cannot enable WAL later. +ALTER TABLE system_metrics SET 'skip_wal' = 'false'; + +Error: 1001(Unsupported), Unsupported operation setting skip_wal = 'false' without an existing WAL provider + DROP TABLE system_metrics; Affected Rows: 0 @@ -93,15 +100,45 @@ SHOW CREATE TABLE alter_skip_wal; | | ) | +----------------+-----------------------------------------------+ +-- This row is intentionally not written to WAL. +INSERT INTO alter_skip_wal VALUES ('host2', 2, 2000); + +Affected Rows: 1 + ALTER TABLE alter_skip_wal SET 'skip_wal' = 'false'; -Error: 1004(InvalidArguments), Invalid set table option request: Invalid set region option request, key: skip_wal, value: false +Affected Rows: 0 + +SHOW CREATE TABLE alter_skip_wal; + ++----------------+-----------------------------------------------+ +| Table | Create Table | ++----------------+-----------------------------------------------+ +| alter_skip_wal | CREATE TABLE IF NOT EXISTS "alter_skip_wal" ( | +| | "host" STRING NULL, | +| | "val" DOUBLE NULL, | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | skip_wal = 'false' | +| | ) | ++----------------+-----------------------------------------------+ + +-- Repeating the transition is idempotent. +ALTER TABLE alter_skip_wal SET 'skip_wal' = 'false'; + +Affected Rows: 0 ALTER TABLE alter_skip_wal UNSET 'skip_wal'; Error: 1004(InvalidArguments), Invalid unset table option request: Invalid set region option request, key: skip_wal -INSERT INTO alter_skip_wal VALUES ('host2', 2, 2000); +-- This row uses the restored WAL provider. +INSERT INTO alter_skip_wal VALUES ('host3', 3, 3000); Affected Rows: 1 @@ -112,9 +149,10 @@ SELECT * FROM alter_skip_wal ORDER BY ts; +-------+-----+---------------------+ | host1 | 1.0 | 1970-01-01T00:00:01 | | host2 | 2.0 | 1970-01-01T00:00:02 | +| host3 | 3.0 | 1970-01-01T00:00:03 | +-------+-----+---------------------+ --- The post-ALTER row can be lost because restart does not flush skip-WAL memtables. +-- Re-enabling WAL does not retroactively write the skipped row to WAL. -- SQLNESS ARG restart=true SHOW CREATE TABLE alter_skip_wal; @@ -131,7 +169,7 @@ SHOW CREATE TABLE alter_skip_wal; | | | | | ENGINE=mito | | | WITH( | -| | skip_wal = 'true' | +| | skip_wal = 'false' | | | ) | +----------------+-----------------------------------------------+ @@ -141,6 +179,7 @@ SELECT * FROM alter_skip_wal ORDER BY ts; | host | val | ts | +-------+-----+---------------------+ | host1 | 1.0 | 1970-01-01T00:00:01 | +| host3 | 3.0 | 1970-01-01T00:00:03 | +-------+-----+---------------------+ DROP TABLE alter_skip_wal; diff --git a/tests/cases/standalone/common/skip_wal.sql b/tests/cases/standalone/common/skip_wal.sql index cdf0d551ab6..2e9cac1f272 100644 --- a/tests/cases/standalone/common/skip_wal.sql +++ b/tests/cases/standalone/common/skip_wal.sql @@ -29,6 +29,9 @@ ADMIN flush_table('system_metrics'); -- SQLNESS ARG restart=true SELECT * FROM system_metrics; +-- A table created without a real WAL provider cannot enable WAL later. +ALTER TABLE system_metrics SET 'skip_wal' = 'false'; + DROP TABLE system_metrics; CREATE TABLE alter_skip_wal ( @@ -45,15 +48,24 @@ ALTER TABLE alter_skip_wal SET 'skip_wal' = 'true'; SHOW CREATE TABLE alter_skip_wal; +-- This row is intentionally not written to WAL. +INSERT INTO alter_skip_wal VALUES ('host2', 2, 2000); + +ALTER TABLE alter_skip_wal SET 'skip_wal' = 'false'; + +SHOW CREATE TABLE alter_skip_wal; + +-- Repeating the transition is idempotent. ALTER TABLE alter_skip_wal SET 'skip_wal' = 'false'; ALTER TABLE alter_skip_wal UNSET 'skip_wal'; -INSERT INTO alter_skip_wal VALUES ('host2', 2, 2000); +-- This row uses the restored WAL provider. +INSERT INTO alter_skip_wal VALUES ('host3', 3, 3000); SELECT * FROM alter_skip_wal ORDER BY ts; --- The post-ALTER row can be lost because restart does not flush skip-WAL memtables. +-- Re-enabling WAL does not retroactively write the skipped row to WAL. -- SQLNESS ARG restart=true SHOW CREATE TABLE alter_skip_wal; diff --git a/tests/cases/standalone/common/subquery/neumann.result b/tests/cases/standalone/common/subquery/neumann.result index e575b1694c1..66d167bfbcf 100644 --- a/tests/cases/standalone/common/subquery/neumann.result +++ b/tests/cases/standalone/common/subquery/neumann.result @@ -55,7 +55,7 @@ WHERE s."id"=e.sid AND e.grade <= (SELECT AVG(e2.grade) - 1 FROM exams e2 WHERE s."id"=e2.sid OR (e2.curriculum=s.major AND s."year">=e2."year")) ORDER BY "name", course; -Error: 3001(EngineExecuteQuery), Error during planning: Correlated scalar subquery can only be used in Projection, Filter, Aggregate plan nodes +Error: 1001(Unsupported), This feature is not implemented: Physical plan does not support logical expression ScalarSubquery() -- Test 3: EXISTS subquery SELECT "name", major diff --git a/tests/cases/standalone/common/subquery/offset.result b/tests/cases/standalone/common/subquery/offset.result index aa5398b38ed..7c9d67651f5 100644 --- a/tests/cases/standalone/common/subquery/offset.result +++ b/tests/cases/standalone/common/subquery/offset.result @@ -11,8 +11,11 @@ Affected Rows: 1 SELECT (SELECT c0 FROM temp_values OFFSET 1) as result; -++ -++ ++--------+ +| result | ++--------+ +| | ++--------+ -- Test with actual data SELECT (SELECT c0 FROM temp_values OFFSET 0) as result; diff --git a/tests/cases/standalone/common/system/database.result b/tests/cases/standalone/common/system/database.result index 4a85ffaddae..d49a511faa9 100644 --- a/tests/cases/standalone/common/system/database.result +++ b/tests/cases/standalone/common/system/database.result @@ -10,6 +10,22 @@ select database(); | public | +------------+ +select schema(); + ++----------+ +| schema() | ++----------+ +| public | ++----------+ + +select user(), current_user(), system_user(); + ++----------+----------------+---------------+ +| user() | current_user() | system_user() | ++----------+----------------+---------------+ +| greptime | greptime | greptime | ++----------+----------------+---------------+ + use information_schema; Affected Rows: 0 @@ -22,6 +38,14 @@ select database(); | information_schema | +--------------------+ +select schema(); + ++--------------------+ +| schema() | ++--------------------+ +| information_schema | ++--------------------+ + use public; Affected Rows: 0 diff --git a/tests/cases/standalone/common/system/database.sql b/tests/cases/standalone/common/system/database.sql index fcad11ce889..6ea3447076f 100644 --- a/tests/cases/standalone/common/system/database.sql +++ b/tests/cases/standalone/common/system/database.sql @@ -2,8 +2,14 @@ use public; select database(); +select schema(); + +select user(), current_user(), system_user(); + use information_schema; select database(); +select schema(); + use public; diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result index 2546bb34d5a..7fdfed0defc 100644 --- a/tests/cases/standalone/common/system/information_schema.result +++ b/tests/cases/standalone/common/system/information_schema.result @@ -34,8 +34,10 @@ order by table_schema, table_name; |greptime|information_schema|optimizer_trace|LOCALTEMPORARY|17|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| |greptime|information_schema|parameters|LOCALTEMPORARY|18|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| |greptime|information_schema|partitions|LOCALTEMPORARY|28|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| +|greptime|information_schema|plugins|LOCALTEMPORARY|47|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| |greptime|information_schema|procedure_info|LOCALTEMPORARY|34|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| |greptime|information_schema|process_list|LOCALTEMPORARY|36|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| +|greptime|information_schema|processlist|LOCALTEMPORARY|49|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| |greptime|information_schema|profiling|LOCALTEMPORARY|19|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| |greptime|information_schema|referential_constraints|LOCALTEMPORARY|20|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| |greptime|information_schema|region_info|LOCALTEMPORARY|41|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| @@ -53,6 +55,7 @@ order by table_schema, table_name; |greptime|information_schema|table_privileges|LOCALTEMPORARY|23|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| |greptime|information_schema|table_semantics|LOCALTEMPORARY|42|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| |greptime|information_schema|tables|LOCALTEMPORARY|3|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| +|greptime|information_schema|user_privileges|LOCALTEMPORARY|48|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| |greptime|information_schema|views|LOCALTEMPORARY|32|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| |greptime|public|numbers|LOCALTEMPORARY|2|0|0|0|0|0|test_engine|11|Fixed|0|0|0|DATETIME|DATETIME||utf8_bin|0|||Y| +++++++++++++++++++++++++ @@ -287,6 +290,17 @@ order by table_schema, table_name, column_name; | greptime | information_schema | partitions | table_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | | greptime | information_schema | partitions | tablespace_name | 25 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | YES | string | | | | greptime | information_schema | partitions | update_time | 20 | | | | | 0 | | | | | select,insert | | TimestampSecond | timestamp(0) | FIELD | | YES | timestamp(0) | | | +| greptime | information_schema | plugins | load_option | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | plugins | plugin_author | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | plugins | plugin_description | 9 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | plugins | plugin_library | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | plugins | plugin_library_version | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | plugins | plugin_license | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | plugins | plugin_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | plugins | plugin_status | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | plugins | plugin_type | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | plugins | plugin_type_version | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | plugins | plugin_version | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | | greptime | information_schema | procedure_info | end_time | 4 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | YES | timestamp(3) | | | | greptime | information_schema | procedure_info | lock_keys | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | YES | string | | | | greptime | information_schema | procedure_info | procedure_id | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | @@ -301,6 +315,14 @@ order by table_schema, table_name, column_name; | greptime | information_schema | process_list | query | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | | greptime | information_schema | process_list | schemas | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | | greptime | information_schema | process_list | start_timestamp | 7 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | NO | timestamp(3) | | | +| greptime | information_schema | processlist | command | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | processlist | db | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | processlist | host | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | processlist | id | 1 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | NO | bigint | | | +| greptime | information_schema | processlist | info | 8 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | processlist | state | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | processlist | time | 6 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | NO | bigint | | | +| greptime | information_schema | processlist | user | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | | greptime | information_schema | profiling | block_ops_in | 9 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | NO | bigint | | | | greptime | information_schema | profiling | block_ops_out | 10 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | NO | bigint | | | | greptime | information_schema | profiling | context_involuntary | 8 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | NO | bigint | | | @@ -526,6 +548,10 @@ order by table_schema, table_name, column_name; | greptime | information_schema | tables | temporary | 24 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | YES | string | | | | greptime | information_schema | tables | update_time | 18 | | | | | 0 | | | | | select,insert | | TimestampSecond | timestamp(0) | FIELD | | YES | timestamp(0) | | | | greptime | information_schema | tables | version | 12 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | YES | bigint unsigned | | | +| greptime | information_schema | user_privileges | grantee | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | user_privileges | is_grantable | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | user_privileges | privilege_type | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | +| greptime | information_schema | user_privileges | table_catalog | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | NO | string | | | | greptime | information_schema | views | character_set_client | 9 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | YES | string | | | | greptime | information_schema | views | check_option | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | YES | string | | | | greptime | information_schema | views | collation_connection | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | YES | string | | | @@ -700,8 +726,10 @@ Affected Rows: 0 -- test query filter for key_column_usage -- select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME = 'TIME INDEX' and TABLE_SCHEMA != 'greptime_private'; -++ -++ ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+---------------------+ +| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | greptime_index_type | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+---------------------+ ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+---------------------+ select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME != 'TIME INDEX' and TABLE_SCHEMA != 'greptime_private'; @@ -713,8 +741,10 @@ select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME != 'TIME INDEX' and TABLE_S select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME LIKE '%INDEX' and TABLE_SCHEMA != 'greptime_private'; -++ -++ ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+---------------------+ +| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | greptime_index_type | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+---------------------+ ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+---------------------+ select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME NOT LIKE '%INDEX' and TABLE_SCHEMA != 'greptime_private'; @@ -726,8 +756,10 @@ select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME NOT LIKE '%INDEX' and TABLE select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME == 'TIME INDEX' AND CONSTRAINT_SCHEMA != 'my_db' and TABLE_SCHEMA != 'greptime_private'; -++ -++ ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+---------------------+ +| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | greptime_index_type | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+---------------------+ ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+---------------------+ -- schemata -- desc table schemata; @@ -828,8 +860,10 @@ DESC TABLE COLUMN_PRIVILEGES; SELECT * FROM COLUMN_PRIVILEGES; -++ -++ ++---------+---------------+--------------+------------+-------------+----------------+--------------+ +| grantee | table_catalog | table_schema | table_name | column_name | privilege_type | is_grantable | ++---------+---------------+--------------+------------+-------------+----------------+--------------+ ++---------+---------------+--------------+------------+-------------+----------------+--------------+ DESC TABLE COLUMN_STATISTICS; @@ -844,8 +878,10 @@ DESC TABLE COLUMN_STATISTICS; SELECT * FROM COLUMN_STATISTICS; -++ -++ ++-------------+------------+-------------+-----------+ +| schema_name | table_name | column_name | histogram | ++-------------+------------+-------------+-----------+ ++-------------+------------+-------------+-----------+ SELECT * FROM CHARACTER_SETS; @@ -884,8 +920,10 @@ DESC TABLE CHECK_CONSTRAINTS; SELECT * FROM CHECK_CONSTRAINTS; -++ -++ ++--------------------+-------------------+-----------------+--------------+ +| constraint_catalog | constraint_schema | constraint_name | check_clause | ++--------------------+-------------------+-----------------+--------------+ ++--------------------+-------------------+-----------------+--------------+ DESC TABLE REGION_PEERS; diff --git a/tests/cases/standalone/common/system/pg_catalog.result b/tests/cases/standalone/common/system/pg_catalog.result index a36f0fe606b..5a582fe7fe3 100644 --- a/tests/cases/standalone/common/system/pg_catalog.result +++ b/tests/cases/standalone/common/system/pg_catalog.result @@ -1112,21 +1112,70 @@ CREATE table foo Affected Rows: 0 -- SQLNESS PROTOCOL POSTGRES -SELECT attname, atttypid FROM pg_catalog.pg_class AS cls INNER JOIN +SELECT attr.attnum, attname, atttypid FROM pg_catalog.pg_class AS cls INNER JOIN pg_catalog.pg_attribute AS attr ON cls.oid = attr.attrelid INNER JOIN pg_catalog.pg_type AS typ ON attr.atttypid = typ.oid WHERE attr.attnum >= 0 AND cls.oid = 'foo'::regclass::oid ORDER BY attr.attnum; -+-----------+----------+ -| attname | atttypid | -+-----------+----------+ -| ts | 1114 | -| log_data | 25 | -| count_num | 20 | -+-----------+----------+ ++--------+-----------+----------+ +| attnum | attname | atttypid | ++--------+-----------+----------+ +| 1 | ts | 1114 | +| 2 | log_data | 25 | +| 3 | count_num | 20 | ++--------+-----------+----------+ -- SQLNESS PROTOCOL POSTGRES DROP TABLE foo; Affected Rows: 0 +-- array_upper / array_lower UDFs (DataFusion ships array_length but not these) +SELECT array_upper(ARRAY[1,2,3], 1), array_lower(ARRAY[5,6,7], 1); + ++--------------------------------------------------------------+--------------------------------------------------------------+ +| array_upper(make_array(Int64(1),Int64(2),Int64(3)),Int64(1)) | array_lower(make_array(Int64(5),Int64(6),Int64(7)),Int64(1)) | ++--------------------------------------------------------------+--------------------------------------------------------------+ +| 3 | 1 | ++--------------------------------------------------------------+--------------------------------------------------------------+ + +-- NULL semantics: out-of-range dim (dim < 1) yields NULL +SELECT array_upper(ARRAY[1,2], 0), array_lower(ARRAY[1,2], 0); + ++-----------------------------------------------------+-----------------------------------------------------+ +| array_upper(make_array(Int64(1),Int64(2)),Int64(0)) | array_lower(make_array(Int64(1),Int64(2)),Int64(0)) | ++-----------------------------------------------------+-----------------------------------------------------+ +| | | ++-----------------------------------------------------+-----------------------------------------------------+ + +-- generate_series with int4 bounds from array_upper must execute (widened to int8) +SELECT n FROM generate_series(1, array_upper(ARRAY[10,20,30], 1)) AS t(n); + ++---+ +| n | ++---+ +| 1 | +| 2 | +| 3 | ++---+ + +-- ADBC type-info predicate: retain bool, excluding zero receivers and arrays. +-- SQLNESS PROTOCOL POSTGRES +WITH type_info AS ( + SELECT oid, typname, typreceive, typbasetype, typrelid, typarray + FROM pg_catalog.pg_type + WHERE (typreceive != 0 OR typsend != 0) + AND typtype != 'r' + AND typreceive::TEXT != 'array_recv' +) +SELECT oid, typname, typreceive +FROM type_info +WHERE oid IN (16, 269, 1000) +ORDER BY oid; + ++-----+---------+------------+ +| oid | typname | typreceive | ++-----+---------+------------+ +| 16 | bool | boolrecv | ++-----+---------+------------+ + diff --git a/tests/cases/standalone/common/system/pg_catalog.sql b/tests/cases/standalone/common/system/pg_catalog.sql index 1dd97c9dc38..89f45d24bd1 100644 --- a/tests/cases/standalone/common/system/pg_catalog.sql +++ b/tests/cases/standalone/common/system/pg_catalog.sql @@ -274,10 +274,33 @@ CREATE table foo ); -- SQLNESS PROTOCOL POSTGRES -SELECT attname, atttypid FROM pg_catalog.pg_class AS cls INNER JOIN +SELECT attr.attnum, attname, atttypid FROM pg_catalog.pg_class AS cls INNER JOIN pg_catalog.pg_attribute AS attr ON cls.oid = attr.attrelid INNER JOIN pg_catalog.pg_type AS typ ON attr.atttypid = typ.oid WHERE attr.attnum >= 0 AND cls.oid = 'foo'::regclass::oid ORDER BY attr.attnum; -- SQLNESS PROTOCOL POSTGRES DROP TABLE foo; + +-- array_upper / array_lower UDFs (DataFusion ships array_length but not these) +SELECT array_upper(ARRAY[1,2,3], 1), array_lower(ARRAY[5,6,7], 1); + +-- NULL semantics: out-of-range dim (dim < 1) yields NULL +SELECT array_upper(ARRAY[1,2], 0), array_lower(ARRAY[1,2], 0); + +-- generate_series with int4 bounds from array_upper must execute (widened to int8) +SELECT n FROM generate_series(1, array_upper(ARRAY[10,20,30], 1)) AS t(n); + +-- ADBC type-info predicate: retain bool, excluding zero receivers and arrays. +-- SQLNESS PROTOCOL POSTGRES +WITH type_info AS ( + SELECT oid, typname, typreceive, typbasetype, typrelid, typarray + FROM pg_catalog.pg_type + WHERE (typreceive != 0 OR typsend != 0) + AND typtype != 'r' + AND typreceive::TEXT != 'array_recv' +) +SELECT oid, typname, typreceive +FROM type_info +WHERE oid IN (16, 269, 1000) +ORDER BY oid; diff --git a/tests/cases/standalone/common/system/semantic_graph.result b/tests/cases/standalone/common/system/semantic_graph.result index e1728c4ae03..9b23c4a506b 100644 --- a/tests/cases/standalone/common/system/semantic_graph.result +++ b/tests/cases/standalone/common/system/semantic_graph.result @@ -2,13 +2,17 @@ -- virtual tables: readable, but rejecting every DDL/DML path. select observed_at, entity_type, entity_id, scope from greptime_private.semantic_entities; -++ -++ ++-------------+-------------+-----------+-------+ +| observed_at | entity_type | entity_id | scope | ++-------------+-------------+-----------+-------+ ++-------------+-------------+-----------+-------+ select observed_at, src_id, dst_id, rel_type from greptime_private.semantic_relationships; -++ -++ ++-------------+--------+--------+----------+ +| observed_at | src_id | dst_id | rel_type | ++-------------+--------+--------+----------+ ++-------------+--------+--------+----------+ insert into greptime_private.semantic_entities (observed_at, entity_type, entity_id) values (now(), 'service', 'svc-a'); @@ -137,8 +141,10 @@ Affected Rows: 1 select entity_type, entity_id from greptime_private.semantic_entities order by entity_type, entity_id; -++ -++ ++-------------+-----------+ +| entity_type | entity_id | ++-------------+-----------+ ++-------------+-----------+ alter table graph_late_metrics set 'greptime.semantic.entity.service.id' = 'svc', 'greptime.semantic.entity.service.scope' = 'env'; @@ -167,8 +173,10 @@ Affected Rows: 0 select entity_type, entity_id from greptime_private.semantic_entities order by entity_type, entity_id; -++ -++ ++-------------+-----------+ +| entity_type | entity_id | ++-------------+-----------+ ++-------------+-----------+ drop table graph_late_metrics; @@ -362,8 +370,10 @@ Affected Rows: 4 select src_id from greptime_private.semantic_relationships order by src_id; -++ -++ ++--------+ +| src_id | ++--------+ ++--------+ -- DROP is allowed (nothing structural is lost: the next INSERT recreates the -- canonical table) and cleans up after this test. diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.result b/tests/cases/standalone/common/tql-explain-analyze/explain.result index 2d6eab49da3..fed75f0448e 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/explain.result +++ b/tests/cases/standalone/common/tql-explain-analyze/explain.result @@ -105,6 +105,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; | analyzed_logical_plan_| SAME TEXT AS ABOVE_| | logical_plan after rewrite_set_comparison_| SAME TEXT AS ABOVE_| | logical_plan after optimize_unions_| SAME TEXT AS ABOVE_| +| logical_plan after unions_to_filter_| SAME TEXT AS ABOVE_| | logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| | logical_plan after replace_distinct_aggregate_| SAME TEXT AS ABOVE_| | logical_plan after eliminate_join_| SAME TEXT AS ABOVE_| @@ -137,6 +138,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; | logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_| | logical_plan after rewrite_set_comparison_| SAME TEXT AS ABOVE_| | logical_plan after optimize_unions_| SAME TEXT AS ABOVE_| +| logical_plan after unions_to_filter_| SAME TEXT AS ABOVE_| | logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| | logical_plan after replace_distinct_aggregate_| SAME TEXT AS ABOVE_| | logical_plan after eliminate_join_| SAME TEXT AS ABOVE_| @@ -185,16 +187,18 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; | physical_plan after PassDistributionRule_| SAME TEXT AS ABOVE_| | physical_plan after PromqlTsidNarrowJoin_| SAME TEXT AS ABOVE_| | physical_plan after EnforceSorting_| SAME TEXT AS ABOVE_| -| physical_plan after EnforceDistribution_| SAME TEXT AS ABOVE_| +| physical_plan after WindowTopN_| SAME TEXT AS ABOVE_| +| physical_plan after EnsureRequirements_| SAME TEXT AS ABOVE_| | physical_plan after CombinePartialFinalAggregate_| SAME TEXT AS ABOVE_| -| physical_plan after EnforceSorting_| SAME TEXT AS ABOVE_| | physical_plan after OptimizeAggregateOrder_| SAME TEXT AS ABOVE_| | physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_| | physical_plan after OutputRequirements_| MergeScanExec: REDACTED |_|_| | physical_plan after LimitAggregation_| SAME TEXT AS ABOVE_| | physical_plan after LimitPushPastWindows_| SAME TEXT AS ABOVE_| +| physical_plan after HashJoinBuffering_| SAME TEXT AS ABOVE_| | physical_plan after LimitPushdown_| SAME TEXT AS ABOVE_| +| physical_plan after TopKRepartition_| SAME TEXT AS ABOVE_| | physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_| | physical_plan after PushdownSort_| SAME TEXT AS ABOVE_| | physical_plan after EnsureCooperative_| CooperativeExec_| @@ -256,6 +260,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test AS series; | analyzed_logical_plan_| SAME TEXT AS ABOVE_| | logical_plan after rewrite_set_comparison_| SAME TEXT AS ABOVE_| | logical_plan after optimize_unions_| SAME TEXT AS ABOVE_| +| logical_plan after unions_to_filter_| SAME TEXT AS ABOVE_| | logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| | logical_plan after replace_distinct_aggregate_| SAME TEXT AS ABOVE_| | logical_plan after eliminate_join_| SAME TEXT AS ABOVE_| @@ -289,6 +294,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test AS series; | logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_| | logical_plan after rewrite_set_comparison_| SAME TEXT AS ABOVE_| | logical_plan after optimize_unions_| SAME TEXT AS ABOVE_| +| logical_plan after unions_to_filter_| SAME TEXT AS ABOVE_| | logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| | logical_plan after replace_distinct_aggregate_| SAME TEXT AS ABOVE_| | logical_plan after eliminate_join_| SAME TEXT AS ABOVE_| @@ -338,16 +344,18 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test AS series; | physical_plan after PassDistributionRule_| SAME TEXT AS ABOVE_| | physical_plan after PromqlTsidNarrowJoin_| SAME TEXT AS ABOVE_| | physical_plan after EnforceSorting_| SAME TEXT AS ABOVE_| -| physical_plan after EnforceDistribution_| SAME TEXT AS ABOVE_| +| physical_plan after WindowTopN_| SAME TEXT AS ABOVE_| +| physical_plan after EnsureRequirements_| SAME TEXT AS ABOVE_| | physical_plan after CombinePartialFinalAggregate_| SAME TEXT AS ABOVE_| -| physical_plan after EnforceSorting_| SAME TEXT AS ABOVE_| | physical_plan after OptimizeAggregateOrder_| SAME TEXT AS ABOVE_| | physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_| | physical_plan after OutputRequirements_| MergeScanExec: REDACTED |_|_| | physical_plan after LimitAggregation_| SAME TEXT AS ABOVE_| | physical_plan after LimitPushPastWindows_| SAME TEXT AS ABOVE_| +| physical_plan after HashJoinBuffering_| SAME TEXT AS ABOVE_| | physical_plan after LimitPushdown_| SAME TEXT AS ABOVE_| +| physical_plan after TopKRepartition_| SAME TEXT AS ABOVE_| | physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_| | physical_plan after PushdownSort_| SAME TEXT AS ABOVE_| | physical_plan after EnsureCooperative_| CooperativeExec_| @@ -391,9 +399,9 @@ TQL EXPLAIN (0, 10, '5s') test_nano; | | PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] | | | PromSeriesDivide: tags=["k"] | | | Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST | -| | Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j | -| | Filter: test_nano.j >= TimestampNanosecond(-299999999999, None) AND test_nano.j < TimestampNanosecond(10001000000, None) | -| | TableScan: test_nano, partial_filters=[test_nano.j >= TimestampNanosecond(-299999999999, None), test_nano.j < TimestampNanosecond(10001000000, None)] | +| | Projection: test_nano.i, test_nano.k, test_nano.j | +| | Filter: test_nano.j > TimestampNanosecond(-300000000000, None) AND test_nano.j <= TimestampNanosecond(10000000000, None) | +| | TableScan: test_nano, partial_filters=[test_nano.j > TimestampNanosecond(-300000000000, None), test_nano.j <= TimestampNanosecond(10000000000, None)] | | | ]] | | physical_plan | CooperativeExec | | | MergeScanExec: REDACTED @@ -415,8 +423,8 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; | initial_logical_plan_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| |_|_PromSeriesDivide: tags=["k"]_| |_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| -|_|_Filter: test_nano.j >= TimestampMillisecond(-299999, None) AND test_nano.j <= TimestampMillisecond(10000, None)_| -|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| +|_|_Filter: test_nano.j > TimestampNanosecond(-300000000000, None) AND test_nano.j <= TimestampNanosecond(10000000000, None)_| +|_|_Projection: test_nano.i, test_nano.k, test_nano.j_| |_|_TableScan: test_nano_| | logical_plan after apply_function_rewrites_| SAME TEXT AS ABOVE_| | logical_plan after count_wildcard_to_time_index_rule_| SAME TEXT AS ABOVE_| @@ -430,15 +438,16 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; |_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| |_|_PromSeriesDivide: tags=["k"]_| |_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| -|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| -|_|_Filter: test_nano.j >= TimestampNanosecond(-299999999999, None) AND test_nano.j < TimestampNanosecond(10001000000, None)_| -|_|_TableScan: test_nano, partial_filters=[test_nano.j >= TimestampNanosecond(-299999999999, None), test_nano.j < TimestampNanosecond(10001000000, None)] | +|_|_Projection: test_nano.i, test_nano.k, test_nano.j_| +|_|_Filter: test_nano.j > TimestampNanosecond(-300000000000, None) AND test_nano.j <= TimestampNanosecond(10000000000, None)_| +|_|_TableScan: test_nano, partial_filters=[test_nano.j > TimestampNanosecond(-300000000000, None), test_nano.j <= TimestampNanosecond(10000000000, None)] | |_| ]]_| | logical_plan after JsonSchemaConcretizeRule_| SAME TEXT AS ABOVE_| | logical_plan after FixStateUdafOrderingAnalyzer_| SAME TEXT AS ABOVE_| | analyzed_logical_plan_| SAME TEXT AS ABOVE_| | logical_plan after rewrite_set_comparison_| SAME TEXT AS ABOVE_| | logical_plan after optimize_unions_| SAME TEXT AS ABOVE_| +| logical_plan after unions_to_filter_| SAME TEXT AS ABOVE_| | logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| | logical_plan after replace_distinct_aggregate_| SAME TEXT AS ABOVE_| | logical_plan after eliminate_join_| SAME TEXT AS ABOVE_| @@ -464,14 +473,15 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; |_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| |_|_PromSeriesDivide: tags=["k"]_| |_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| -|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| -|_|_Filter: test_nano.j >= TimestampNanosecond(-299999999999, None) AND test_nano.j < TimestampNanosecond(10001000000, None)_| -|_|_TableScan: test_nano, partial_filters=[test_nano.j >= TimestampNanosecond(-299999999999, None), test_nano.j < TimestampNanosecond(10001000000, None)] | +|_|_Projection: test_nano.i, test_nano.k, test_nano.j_| +|_|_Filter: test_nano.j > TimestampNanosecond(-300000000000, None) AND test_nano.j <= TimestampNanosecond(10000000000, None)_| +|_|_TableScan: test_nano, partial_filters=[test_nano.j > TimestampNanosecond(-300000000000, None), test_nano.j <= TimestampNanosecond(10000000000, None)] | |_| ]]_| | logical_plan after ScanHintRule_| SAME TEXT AS ABOVE_| | logical_plan after JsonTypeConcretizeRule_| SAME TEXT AS ABOVE_| | logical_plan after rewrite_set_comparison_| SAME TEXT AS ABOVE_| | logical_plan after optimize_unions_| SAME TEXT AS ABOVE_| +| logical_plan after unions_to_filter_| SAME TEXT AS ABOVE_| | logical_plan after simplify_expressions_| SAME TEXT AS ABOVE_| | logical_plan after replace_distinct_aggregate_| SAME TEXT AS ABOVE_| | logical_plan after eliminate_join_| SAME TEXT AS ABOVE_| @@ -500,9 +510,9 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; |_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[j]_| |_|_PromSeriesDivide: tags=["k"]_| |_|_Sort: test_nano.k ASC NULLS FIRST, test_nano.j ASC NULLS FIRST_| -|_|_Projection: test_nano.i, test_nano.k, CAST(test_nano.j AS Timestamp(ms)) AS j_| -|_|_Filter: test_nano.j >= TimestampNanosecond(-299999999999, None) AND test_nano.j < TimestampNanosecond(10001000000, None)_| -|_|_TableScan: test_nano, partial_filters=[test_nano.j >= TimestampNanosecond(-299999999999, None), test_nano.j < TimestampNanosecond(10001000000, None)] | +|_|_Projection: test_nano.i, test_nano.k, test_nano.j_| +|_|_Filter: test_nano.j > TimestampNanosecond(-300000000000, None) AND test_nano.j <= TimestampNanosecond(10000000000, None)_| +|_|_TableScan: test_nano, partial_filters=[test_nano.j > TimestampNanosecond(-300000000000, None), test_nano.j <= TimestampNanosecond(10000000000, None)] | |_| ]]_| | initial_physical_plan_| MergeScanExec: REDACTED |_|_| @@ -521,16 +531,18 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test_nano; | physical_plan after PassDistributionRule_| SAME TEXT AS ABOVE_| | physical_plan after PromqlTsidNarrowJoin_| SAME TEXT AS ABOVE_| | physical_plan after EnforceSorting_| SAME TEXT AS ABOVE_| -| physical_plan after EnforceDistribution_| SAME TEXT AS ABOVE_| +| physical_plan after WindowTopN_| SAME TEXT AS ABOVE_| +| physical_plan after EnsureRequirements_| SAME TEXT AS ABOVE_| | physical_plan after CombinePartialFinalAggregate_| SAME TEXT AS ABOVE_| -| physical_plan after EnforceSorting_| SAME TEXT AS ABOVE_| | physical_plan after OptimizeAggregateOrder_| SAME TEXT AS ABOVE_| | physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_| | physical_plan after OutputRequirements_| MergeScanExec: REDACTED |_|_| | physical_plan after LimitAggregation_| SAME TEXT AS ABOVE_| | physical_plan after LimitPushPastWindows_| SAME TEXT AS ABOVE_| +| physical_plan after HashJoinBuffering_| SAME TEXT AS ABOVE_| | physical_plan after LimitPushdown_| SAME TEXT AS ABOVE_| +| physical_plan after TopKRepartition_| SAME TEXT AS ABOVE_| | physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_| | physical_plan after PushdownSort_| SAME TEXT AS ABOVE_| | physical_plan after EnsureCooperative_| CooperativeExec_| diff --git a/tests/cases/standalone/common/tql/binary_operator.result b/tests/cases/standalone/common/tql/binary_operator.result index 47d524c952e..9952615e6bf 100644 --- a/tests/cases/standalone/common/tql/binary_operator.result +++ b/tests/cases/standalone/common/tql/binary_operator.result @@ -8,8 +8,10 @@ Affected Rows: 3 tql eval (0, 30, '10s'), data < 1; -++ -++ ++----+-----+ +| ts | val | ++----+-----+ ++----+-----+ tql eval (0, 30, '10s'), data + (1 < bool 2); diff --git a/tests/cases/standalone/common/tql/case_sensitive.result b/tests/cases/standalone/common/tql/case_sensitive.result index 2da24736901..de3ef60a40c 100644 --- a/tests/cases/standalone/common/tql/case_sensitive.result +++ b/tests/cases/standalone/common/tql/case_sensitive.result @@ -60,14 +60,18 @@ Affected Rows: 0 tql eval (0,10,'5s') sum(MemAvailable / 4) + sum(MemTotal / 4); -++ -++ ++------+---------------------------------------------------------------+ +| time | MemAvailable.sum(val / Float64(4)) + .sum(value / Float64(4)) | ++------+---------------------------------------------------------------+ ++------+---------------------------------------------------------------+ -- Cross schema is not supported tql eval (0,10,'5s') sum(MemAvailable / 4) + sum({__name__="AnotherSchema.MemTotal"} / 4); -++ -++ ++------+---------------------------------------------------------------+ +| time | MemAvailable.sum(val / Float64(4)) + .sum(value / Float64(4)) | ++------+---------------------------------------------------------------+ ++------+---------------------------------------------------------------+ drop table "MemAvailable"; diff --git a/tests/cases/standalone/common/tql/partition.result b/tests/cases/standalone/common/tql/partition.result index e32d13043bd..ac3072342d7 100644 --- a/tests/cases/standalone/common/tql/partition.result +++ b/tests/cases/standalone/common/tql/partition.result @@ -68,9 +68,9 @@ tql analyze (0, 10, '1s') 100 - (avg by (k) (irate(t[1m])) * 100); |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_SortPreservingMergeExec: [k@0 ASC NULLS LAST, j@1 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[k@0 ASC NULLS LAST, j@1 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[k@0 as k, j@1 as j], aggr=[avg(prom_irate(j_range,i))] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[k@0 as k, j@1 as j], aggr=[__avg_merge(__avg_state(prom_irate(j_range,i))) as avg(prom_irate(j_range,i))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[k@0 as k, j@1 as j], aggr=[avg(prom_irate(j_range,i))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[k@0 as k, j@1 as j], aggr=[__avg_merge(__avg_state(prom_irate(j_range,i))) as avg(prom_irate(j_range,i))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| diff --git a/tests/cases/standalone/common/tql/range.result b/tests/cases/standalone/common/tql/range.result index c650222448d..4eab8d6dc5c 100644 --- a/tests/cases/standalone/common/tql/range.result +++ b/tests/cases/standalone/common/tql/range.result @@ -177,44 +177,58 @@ TQL EVAL (60, 180, '60s') sum by(host) (rate(metrics[1m])) * 60; -- Test querying non-existent table TQL EVAL (60, 180, '60s') sum(rate(non_existent_table[1m])); -++ -++ ++------+----------------------------------------------------+ +| time | sum(prom_rate(time_range,value,time,Int64(60000))) | ++------+----------------------------------------------------+ ++------+----------------------------------------------------+ -- Test querying non-existent label TQL EVAL (60, 180, '60s') sum(rate(metrics{non_existent_label="value"}[1m])); -++ -++ ++----+----------------------------------------------+ +| ts | sum(prom_rate(ts_range,val,ts,Int64(60000))) | ++----+----------------------------------------------+ ++----+----------------------------------------------+ -- Test querying non-existent label value TQL EVAL (60, 180, '60s') sum(rate(metrics{host="non_existent_host"}[1m])); -++ -++ ++----+----------------------------------------------+ +| ts | sum(prom_rate(ts_range,val,ts,Int64(60000))) | ++----+----------------------------------------------+ ++----+----------------------------------------------+ -- Test querying multiple non-existent labels TQL EVAL (60, 180, '60s') sum(rate(metrics{non_existent_label1="value1", non_existent_label2="value2"}[1m])); -++ -++ ++----+----------------------------------------------+ +| ts | sum(prom_rate(ts_range,val,ts,Int64(60000))) | ++----+----------------------------------------------+ ++----+----------------------------------------------+ -- Test querying mix of existing and non-existent labels TQL EVAL (60, 180, '60s') sum(rate(metrics{host="host1", non_existent_label="value"}[1m])); -++ -++ ++----+----------------------------------------------+ +| ts | sum(prom_rate(ts_range,val,ts,Int64(60000))) | ++----+----------------------------------------------+ ++----+----------------------------------------------+ -- Test querying non-existent table with non-existent labels TQL EVAL (60, 180, '60s') sum(rate(non_existent_table{non_existent_label="value"}[1m])); -++ -++ ++------+----------------------------------------------------+ +| time | sum(prom_rate(time_range,value,time,Int64(60000))) | ++------+----------------------------------------------------+ ++------+----------------------------------------------------+ -- Test querying non-existent table with multiple non-existent labels TQL EVAL (60, 180, '60s') sum(rate(non_existent_table{label1="value1", label2="value2"}[1m])); -++ -++ ++------+----------------------------------------------------+ +| time | sum(prom_rate(time_range,value,time,Int64(60000))) | ++------+----------------------------------------------------+ ++------+----------------------------------------------------+ DROP TABLE metrics; diff --git a/tests/cases/standalone/common/tql/tql-cte.result b/tests/cases/standalone/common/tql/tql-cte.result index 8efe983763b..7fbc621319c 100644 --- a/tests/cases/standalone/common/tql/tql-cte.result +++ b/tests/cases/standalone/common/tql/tql-cte.result @@ -201,7 +201,7 @@ SELECT sum(val) FROM filtered; | | Projection: tql_data.ts, tql_data.val | | | SubqueryAlias: tql_data | | | Projection: metric.ts AS ts, prom_rate(ts_range,val,ts,Int64(20000)) AS val | -| | Filter: prom_rate(ts_range,val,ts,Int64(20000)) > Float64(0) AND prom_rate(ts_range,val,ts,Int64(20000)) IS NOT NULL | +| | Filter: prom_rate(ts_range,val,ts,Int64(20000)) IS NOT NULL AND prom_rate(ts_range,val,ts,Int64(20000)) > Float64(0) | | | Projection: metric.ts, prom_rate(ts_range, val, metric.ts, Int64(20000)) AS prom_rate(ts_range,val,ts,Int64(20000)) | | | PromRangeManipulate: req range=[0..40000], interval=[10000], eval range=[20000], time index=[ts], values=["val"] | | | PromSeriesNormalize: offset=[0], time index=[ts], filter NaN: [true] | @@ -244,8 +244,8 @@ SELECT round(avg(summary)) as avg_sum FROM tql_agg; | | Aggregate: groupBy=[[labels.ts]], aggr=[[sum(labels.cpu)]] | | | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] | | | PromSeriesDivide: tags=["host"] | -| | Filter: labels.host ~ Utf8("^(?:host.*)$") AND labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) | -| | TableScan: labels, partial_filters=[labels.host ~ Utf8("^(?:host.*)$"), labels.ts >= TimestampMillisecond(-299999, None), labels.ts <= TimestampMillisecond(40000, None)] | +| | Filter: labels.ts >= TimestampMillisecond(-299999, None) AND labels.ts <= TimestampMillisecond(40000, None) AND labels.host ~ Utf8("^(?:host.*)$") | +| | TableScan: labels, partial_filters=[labels.ts >= TimestampMillisecond(-299999, None), labels.ts <= TimestampMillisecond(40000, None), labels.host ~ Utf8("^(?:host.*)$")] | | | ]] | | physical_plan | CooperativeExec | | | MergeScanExec: REDACTED @@ -389,8 +389,8 @@ LIMIT 3; | physical_plan | ProjectionExec: expr=[metric_val@0 as metric_val, label_val@1 as label_val] | | | SortPreservingMergeExec: [ts@2 ASC NULLS LAST], fetch=3 | | | SortExec: TopK(fetch=3), expr=[ts@2 ASC NULLS LAST], preserve_REDACTED -| | ProjectionExec: expr=[val@1 as metric_val, cpu@2 as label_val, ts@0 as ts] | -| | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ts@0, ts@0)], projection=[ts@0, val@1, cpu@3] | +| | ProjectionExec: expr=[val@0 as metric_val, cpu@1 as label_val, ts@2 as ts] | +| | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ts@0, ts@0)], projection=[val@1, cpu@3, ts@0] | | | RepartitionExec: REDACTED | | MergeScanExec: REDACTED | | RepartitionExec: REDACTED @@ -651,7 +651,7 @@ SELECT count(*) as high_values FROM final; | | Projection: base_tql.ts, base_tql.val * Float64(100) AS percent | | | SubqueryAlias: base_tql | | | Projection: metric.ts AS ts, metric.val AS val | -| | Filter: metric.val * Float64(100) > Float64(200) AND metric.val > Float64(0) | +| | Filter: metric.val > Float64(0) AND metric.val * Float64(100) > Float64(200) | | | PromInstantManipulate: range=[0..40000], lookback=[300000], interval=[10000], time index=[ts] | | | PromSeriesDivide: tags=[] | | | Filter: metric.ts >= TimestampMillisecond(-299999, None) AND metric.ts <= TimestampMillisecond(40000, None) | @@ -668,8 +668,10 @@ WITH time_shifted AS ( ) SELECT * FROM time_shifted; -++ -++ ++----+-----+ +| ts | val | ++----+-----+ ++----+-----+ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED @@ -844,8 +846,8 @@ LIMIT 5; | | ]] | | physical_plan | SortPreservingMergeExec: [ts@0 ASC NULLS LAST, host@2 ASC NULLS LAST, avg_value@1 ASC NULLS LAST], fetch=5 | | | SortExec: TopK(fetch=5), expr=[ts@0 ASC NULLS LAST, host@2 ASC NULLS LAST, avg_value@1 ASC NULLS LAST], preserve_REDACTED -| | ProjectionExec: expr=[ts@1 as ts, cpu@0 as avg_value, host@2 as host] | -| | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(date_trunc(Utf8("second"),t.ts)@2, date_trunc(Utf8("second"),l.ts)@2)], projection=[cpu@0, ts@1, host@4] | +| | ProjectionExec: expr=[ts@0 as ts, cpu@1 as avg_value, host@2 as host] | +| | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(date_trunc(Utf8("second"),t.ts)@2, date_trunc(Utf8("second"),l.ts)@2)], projection=[ts@1, cpu@0, host@4] | | | RepartitionExec: REDACTED | | ProjectionExec: expr=[cpu@0 as cpu, ts@1 as ts, date_trunc(second, ts@1) as date_trunc(Utf8("second"),t.ts)] | | | RepartitionExec: REDACTED diff --git a/tests/cases/standalone/common/truncate/truncate.result b/tests/cases/standalone/common/truncate/truncate.result index 6cd490e26d9..27fcbd90265 100644 --- a/tests/cases/standalone/common/truncate/truncate.result +++ b/tests/cases/standalone/common/truncate/truncate.result @@ -41,8 +41,10 @@ Affected Rows: 0 SELECT ts, host, cpu, memory FROM monitor ORDER BY ts; -++ -++ ++----+------+-----+--------+ +| ts | host | cpu | memory | ++----+------+-----+--------+ ++----+------+-----+--------+ -- truncate with time range INSERT INTO monitor(ts, host, cpu, memory) VALUES @@ -102,8 +104,10 @@ Affected Rows: 0 SELECT ts, host, cpu, memory FROM monitor ORDER BY ts; -++ -++ ++----+------+-----+--------+ +| ts | host | cpu | memory | ++----+------+-----+--------+ ++----+------+-----+--------+ INSERT INTO monitor(ts, host, cpu, memory) VALUES (1695217660000, 'host1', 88.8, 4096), @@ -128,8 +132,10 @@ Affected Rows: 0 SELECT ts, host, cpu, memory FROM monitor ORDER BY ts; -++ -++ ++----+------+-----+--------+ +| ts | host | cpu | memory | ++----+------+-----+--------+ ++----+------+-----+--------+ DROP TABLE monitor; diff --git a/tests/cases/standalone/common/ttl/alter_table_ttl.result b/tests/cases/standalone/common/ttl/alter_table_ttl.result index c93610907df..ac7c6710640 100644 --- a/tests/cases/standalone/common/ttl/alter_table_ttl.result +++ b/tests/cases/standalone/common/ttl/alter_table_ttl.result @@ -61,8 +61,10 @@ ADMIN compact_table('test_ttl'); SELECT val from test_ttl; -++ -++ ++-----+ +| val | ++-----+ ++-----+ ALTER TABLE test_ttl SET ttl = '1 minute'; diff --git a/tests/cases/standalone/common/ttl/basic.result b/tests/cases/standalone/common/ttl/basic.result index b381b970753..1b24cf16a95 100644 --- a/tests/cases/standalone/common/ttl/basic.result +++ b/tests/cases/standalone/common/ttl/basic.result @@ -34,8 +34,10 @@ ADMIN compact_table('test_ttl'); SELECT val from test_ttl; -++ -++ ++-----+ +| val | ++-----+ ++-----+ DROP TABLE test_ttl; diff --git a/tests/cases/standalone/common/ttl/database_ttl.result b/tests/cases/standalone/common/ttl/database_ttl.result index 81d1edef78b..cce95a9f153 100644 --- a/tests/cases/standalone/common/ttl/database_ttl.result +++ b/tests/cases/standalone/common/ttl/database_ttl.result @@ -44,8 +44,10 @@ ADMIN compact_table('test_ttl'); -- Must be expired -- SELECT val from test_ttl; -++ -++ ++-----+ +| val | ++-----+ ++-----+ ALTER DATABASE test_ttl_db SET ttl = '1 day'; diff --git a/tests/cases/standalone/common/ttl/database_ttl_with_metric_engine.result b/tests/cases/standalone/common/ttl/database_ttl_with_metric_engine.result index ad59b7bd1af..0eacfccf469 100644 --- a/tests/cases/standalone/common/ttl/database_ttl_with_metric_engine.result +++ b/tests/cases/standalone/common/ttl/database_ttl_with_metric_engine.result @@ -52,8 +52,10 @@ ADMIN compact_table('phy'); --- should be expired -- SELECT val, host FROM test_ttl; -++ -++ ++-----+------+ +| val | host | ++-----+------+ ++-----+------+ ALTER DATABASE test_ttl_db SET ttl = '1 day'; diff --git a/tests/cases/standalone/common/ttl/metric_engine_ttl.result b/tests/cases/standalone/common/ttl/metric_engine_ttl.result index 6152c0cd584..3f0ac57d871 100644 --- a/tests/cases/standalone/common/ttl/metric_engine_ttl.result +++ b/tests/cases/standalone/common/ttl/metric_engine_ttl.result @@ -43,8 +43,10 @@ ADMIN compact_table('phy'); --- should be expired -- SELECT val, host FROM test_ttl; -++ -++ ++-----+------+ +| val | host | ++-----+------+ ++-----+------+ ALTER TABLE phy SET ttl = '1 day'; diff --git a/tests/cases/standalone/common/ttl/ttl_instant.result b/tests/cases/standalone/common/ttl/ttl_instant.result index 49913fd90ec..110fd92d92b 100644 --- a/tests/cases/standalone/common/ttl/ttl_instant.result +++ b/tests/cases/standalone/common/ttl/ttl_instant.result @@ -40,8 +40,10 @@ from ORDER BY val; -++ -++ ++-----+ +| val | ++-----+ ++-----+ -- SQLNESS SLEEP 2s ADMIN flush_table('test_ttl'); @@ -67,8 +69,10 @@ from ORDER BY val; -++ -++ ++-----+ +| val | ++-----+ ++-----+ ALTER TABLE test_ttl UNSET 'ttl'; @@ -215,8 +219,10 @@ from ORDER BY val; -++ -++ ++-----+ +| val | ++-----+ ++-----+ ALTER TABLE test_ttl @@ -279,8 +285,10 @@ from ORDER BY val; -++ -++ ++-----+ +| val | ++-----+ ++-----+ -- to make sure alter back and forth from duration to/from instant wouldn't break anything ALTER TABLE @@ -345,8 +353,10 @@ from ORDER BY val; -++ -++ ++-----+ +| val | ++-----+ ++-----+ DROP TABLE test_ttl; diff --git a/tests/cases/standalone/common/types/float/ieee_floating_points.result b/tests/cases/standalone/common/types/float/ieee_floating_points.result index 69198d490ec..0432048dd53 100644 --- a/tests/cases/standalone/common/types/float/ieee_floating_points.result +++ b/tests/cases/standalone/common/types/float/ieee_floating_points.result @@ -58,8 +58,10 @@ SELECT d, d < -1000000 FROM float_special ORDER BY ts; -- NaN != NaN SELECT f, f = f FROM float_special WHERE f != f ORDER BY ts; -++ -++ ++---+-----------------------------------+ +| f | float_special.f = float_special.f | ++---+-----------------------------------+ ++---+-----------------------------------+ SELECT d, d IS NULL FROM float_special ORDER BY ts; diff --git a/tests/cases/standalone/common/types/float/infinity.result b/tests/cases/standalone/common/types/float/infinity.result index c35287f1cd4..c27c9a53ee9 100644 --- a/tests/cases/standalone/common/types/float/infinity.result +++ b/tests/cases/standalone/common/types/float/infinity.result @@ -127,8 +127,10 @@ SELECT f FROM floats WHERE f>'-inf'::FLOAT ORDER BY 1; SELECT f FROM floats WHERE f>'inf'::FLOAT; -++ -++ ++---+ +| f | ++---+ ++---+ -- >= SELECT f FROM floats WHERE f>=1 ORDER BY f; @@ -178,8 +180,10 @@ SELECT f FROM floats WHERE f<'inf'::FLOAT ORDER BY f; SELECT f FROM floats WHERE f<'-inf'::FLOAT; -++ -++ ++---+ +| f | ++---+ ++---+ -- <= SELECT f FROM floats WHERE f<=1 ORDER BY f; @@ -341,8 +345,10 @@ SELECT d FROM doubles WHERE d>'-inf'::DOUBLE ORDER BY 1; SELECT d FROM doubles WHERE d>'inf'::DOUBLE; -++ -++ ++---+ +| d | ++---+ ++---+ -- >= SELECT d FROM doubles WHERE d>=1 ORDER BY d; @@ -392,8 +398,10 @@ SELECT d FROM doubles WHERE d<'inf'::DOUBLE ORDER BY d; SELECT d FROM doubles WHERE d<'-inf'::DOUBLE; -++ -++ ++---+ +| d | ++---+ ++---+ -- <= SELECT d FROM doubles WHERE d<=1 ORDER BY d; diff --git a/tests/cases/standalone/common/types/float/nan.result b/tests/cases/standalone/common/types/float/nan.result index 9ab48fe88ea..e7472e7da65 100644 --- a/tests/cases/standalone/common/types/float/nan.result +++ b/tests/cases/standalone/common/types/float/nan.result @@ -100,8 +100,10 @@ SELECT f FROM floats WHERE f>0; SELECT f FROM floats WHERE f>'nan'::FLOAT; -++ -++ ++---+ +| f | ++---+ ++---+ -- >= SELECT f FROM floats WHERE f>=1; @@ -124,8 +126,10 @@ SELECT f FROM floats WHERE f>='nan'::FLOAT; -- < SELECT f FROM floats WHERE f<1; -++ -++ ++---+ +| f | ++---+ ++---+ SELECT f FROM floats WHERE f<'nan'::FLOAT; @@ -257,8 +261,10 @@ SELECT d FROM doubles WHERE d>0; SELECT d FROM doubles WHERE d>'nan'::DOUBLE; -++ -++ ++---+ +| d | ++---+ ++---+ -- >= SELECT d FROM doubles WHERE d>=1; @@ -281,8 +287,10 @@ SELECT d FROM doubles WHERE d>='nan'::DOUBLE; -- < SELECT d FROM doubles WHERE d<1; -++ -++ ++---+ +| d | ++---+ ++---+ SELECT d FROM doubles WHERE d<'nan'::DOUBLE; diff --git a/tests/cases/standalone/common/types/json/json.result b/tests/cases/standalone/common/types/json/json.result index 8fad9632b12..541606a3ed6 100644 --- a/tests/cases/standalone/common/types/json/json.result +++ b/tests/cases/standalone/common/types/json/json.result @@ -131,8 +131,10 @@ Error: 3001(EngineExecuteQuery), Execution error: cannot parse 'Morning my frien SELECT json_to_string(j), t FROM jsons; -++ -++ ++-------------------------+---+ +| json_to_string(jsons.j) | t | ++-------------------------+---+ ++-------------------------+---+ CREATE TABLE json_empty (j JSON, t timestamp time index); diff --git a/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.result b/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.result new file mode 100644 index 00000000000..77d48225039 --- /dev/null +++ b/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.result @@ -0,0 +1,142 @@ +-- Regression test: min-max pruning must look up row group statistics by +-- parquet leaf column index. On flat-format tables with a JSON2 column, the +-- JSON2 struct expands to multiple leaf columns, so columns after it were +-- previously misaligned with leaves of the JSON2 struct when reading row +-- group statistics. Here `event_time` is the 8th logical column (2 tags + 5 +-- fields), so its statistics were read from the 8th parquet leaf, which is +-- the small-integer `ns_edge` path of the payload. SWCS compaction reads +-- inputs with a time window predicate, so min-max pruning compared the window +-- against the tiny `ns_edge` statistics, dropped every row group and silently +-- lost all rows. The same misaligned statistics also affected plain queries +-- with time-range predicates. +create table json2_swcs_minmax_stats ( + workspace_id string not null, + session_id string not null, + seq bigint not null, + event_time timestamp(9) not null time index, + entry_kind string not null, + payload json2 not null, + schema_version int not null, + is_error boolean not null, + primary key (workspace_id, session_id) +) with ( + 'append_mode' = 'true', + 'sst_format' = 'flat', + 'ttl' = 'forever' +); + +Affected Rows: 0 + +-- 9 rows in the 2026-09-10 window and 2 rows in the 2026-09-11 window. +insert into json2_swcs_minmax_stats values + ('ws', 's1', 1, 1788998400000000001, 'synthetic', '{"case":"swcs-repro","nested":[1,true],"ns_edge":0,"ordinal":1,"synthetic":true}', 1, false), + ('ws', 's2', 2, 1788998400000000002, 'synthetic', '{"case":"swcs-repro","nested":[2,false],"ns_edge":1,"ordinal":2,"synthetic":true}', 1, false), + ('ws', 's3', 3, 1788998400000000003, 'synthetic', '{"case":"swcs-repro","nested":[3,null],"ns_edge":2,"ordinal":3,"synthetic":true}', 1, false), + ('ws', 's1', 4, 1788998400000000004, 'synthetic', '{"case":"swcs-repro","nested":[4,true],"ns_edge":3,"ordinal":4,"synthetic":true}', 1, false), + ('ws', 's2', 5, 1788998400000000005, 'synthetic', '{"case":"swcs-repro","nested":[5,false],"ns_edge":4,"ordinal":5,"synthetic":true}', 1, false), + ('ws', 's3', 6, 1788998400000000006, 'synthetic', '{"case":"swcs-repro","nested":[6,null],"ns_edge":5,"ordinal":6,"synthetic":true}', 1, false), + ('ws', 's1', 7, 1788998400000000007, 'synthetic', '{"case":"swcs-repro","nested":[7,true],"ns_edge":6,"ordinal":7,"synthetic":true}', 1, false), + ('ws', 's2', 8, 1788998400000000008, 'synthetic', '{"case":"swcs-repro","nested":[8,false],"ns_edge":7,"ordinal":8,"synthetic":true}', 1, false), + ('ws', 's3', 9, 1788998400000000009, 'synthetic', '{"case":"swcs-repro","nested":[9,null],"ns_edge":8,"ordinal":9,"synthetic":true}', 1, false), + ('ws', 's1', 10, 1789084800000000001, 'synthetic', '{"case":"swcs-repro","nested":[10,true],"ns_edge":9,"ordinal":10,"synthetic":true}', 1, false), + ('ws', 's2', 11, 1789084800000000002, 'synthetic', '{"case":"swcs-repro","nested":[11,false],"ns_edge":10,"ordinal":11,"synthetic":true}', 1, false); + +Affected Rows: 11 + +select count(*) from json2_swcs_minmax_stats; + ++----------+ +| count(*) | ++----------+ +| 11 | ++----------+ + +admin flush_table('json2_swcs_minmax_stats'); + ++----------------------------------------------+ +| ADMIN flush_table('json2_swcs_minmax_stats') | ++----------------------------------------------+ +| 0 | ++----------------------------------------------+ + +-- Check pruning on flush-written SSTs before any manual compaction. +select count(*) from json2_swcs_minmax_stats +where event_time >= 1788998400000000000 and event_time < 1789084800000000000; + ++----------+ +| count(*) | ++----------+ +| 9 | ++----------+ + +-- SWCS compaction reads the SST with a time window predicate. Before the +-- fix, min-max pruning compared the window against the small `ns_edge` +-- statistics and dropped every row group. +admin compact_table('json2_swcs_minmax_stats', 'swcs', '86400'); + ++-----------------------------------------------------------------+ +| ADMIN compact_table('json2_swcs_minmax_stats', 'swcs', '86400') | ++-----------------------------------------------------------------+ +| 0 | ++-----------------------------------------------------------------+ + +select count(*) from json2_swcs_minmax_stats; + ++----------+ +| count(*) | ++----------+ +| 11 | ++----------+ + +-- Compact once more to also exercise reading compaction-written SSTs. +admin compact_table('json2_swcs_minmax_stats', 'swcs', '86400'); + ++-----------------------------------------------------------------+ +| ADMIN compact_table('json2_swcs_minmax_stats', 'swcs', '86400') | ++-----------------------------------------------------------------+ +| 0 | ++-----------------------------------------------------------------+ + +select count(*) from json2_swcs_minmax_stats; + ++----------+ +| count(*) | ++----------+ +| 11 | ++----------+ + +-- Time-range predicates use the same statistics and must not prune the rows +-- either. +select count(*) from json2_swcs_minmax_stats +where event_time >= 1788998400000000000 and event_time < 1789084800000000000; + ++----------+ +| count(*) | ++----------+ +| 9 | ++----------+ + +select workspace_id, session_id, seq, event_time, entry_kind, payload, schema_version, is_error +from json2_swcs_minmax_stats +order by seq; + ++--------------+------------+-----+-------------------------------+------------+--------------------------------------------------------------------------------------+----------------+----------+ +| workspace_id | session_id | seq | event_time | entry_kind | payload | schema_version | is_error | ++--------------+------------+-----+-------------------------------+------------+--------------------------------------------------------------------------------------+----------------+----------+ +| ws | s1 | 1 | 2026-09-10T00:00:00.000000001 | synthetic | {"case":"swcs-repro","nested":[1,true],"ns_edge":0,"ordinal":1,"synthetic":true} | 1 | false | +| ws | s2 | 2 | 2026-09-10T00:00:00.000000002 | synthetic | {"case":"swcs-repro","nested":[2,false],"ns_edge":1,"ordinal":2,"synthetic":true} | 1 | false | +| ws | s3 | 3 | 2026-09-10T00:00:00.000000003 | synthetic | {"case":"swcs-repro","nested":[3,null],"ns_edge":2,"ordinal":3,"synthetic":true} | 1 | false | +| ws | s1 | 4 | 2026-09-10T00:00:00.000000004 | synthetic | {"case":"swcs-repro","nested":[4,true],"ns_edge":3,"ordinal":4,"synthetic":true} | 1 | false | +| ws | s2 | 5 | 2026-09-10T00:00:00.000000005 | synthetic | {"case":"swcs-repro","nested":[5,false],"ns_edge":4,"ordinal":5,"synthetic":true} | 1 | false | +| ws | s3 | 6 | 2026-09-10T00:00:00.000000006 | synthetic | {"case":"swcs-repro","nested":[6,null],"ns_edge":5,"ordinal":6,"synthetic":true} | 1 | false | +| ws | s1 | 7 | 2026-09-10T00:00:00.000000007 | synthetic | {"case":"swcs-repro","nested":[7,true],"ns_edge":6,"ordinal":7,"synthetic":true} | 1 | false | +| ws | s2 | 8 | 2026-09-10T00:00:00.000000008 | synthetic | {"case":"swcs-repro","nested":[8,false],"ns_edge":7,"ordinal":8,"synthetic":true} | 1 | false | +| ws | s3 | 9 | 2026-09-10T00:00:00.000000009 | synthetic | {"case":"swcs-repro","nested":[9,null],"ns_edge":8,"ordinal":9,"synthetic":true} | 1 | false | +| ws | s1 | 10 | 2026-09-11T00:00:00.000000001 | synthetic | {"case":"swcs-repro","nested":[10,true],"ns_edge":9,"ordinal":10,"synthetic":true} | 1 | false | +| ws | s2 | 11 | 2026-09-11T00:00:00.000000002 | synthetic | {"case":"swcs-repro","nested":[11,false],"ns_edge":10,"ordinal":11,"synthetic":true} | 1 | false | ++--------------+------------+-----+-------------------------------+------------+--------------------------------------------------------------------------------------+----------------+----------+ + +drop table json2_swcs_minmax_stats; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.sql b/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.sql new file mode 100644 index 00000000000..7bfea334420 --- /dev/null +++ b/tests/cases/standalone/common/types/json/json2_swcs_minmax_stats.sql @@ -0,0 +1,72 @@ +-- Regression test: min-max pruning must look up row group statistics by +-- parquet leaf column index. On flat-format tables with a JSON2 column, the +-- JSON2 struct expands to multiple leaf columns, so columns after it were +-- previously misaligned with leaves of the JSON2 struct when reading row +-- group statistics. Here `event_time` is the 8th logical column (2 tags + 5 +-- fields), so its statistics were read from the 8th parquet leaf, which is +-- the small-integer `ns_edge` path of the payload. SWCS compaction reads +-- inputs with a time window predicate, so min-max pruning compared the window +-- against the tiny `ns_edge` statistics, dropped every row group and silently +-- lost all rows. The same misaligned statistics also affected plain queries +-- with time-range predicates. + +create table json2_swcs_minmax_stats ( + workspace_id string not null, + session_id string not null, + seq bigint not null, + event_time timestamp(9) not null time index, + entry_kind string not null, + payload json2 not null, + schema_version int not null, + is_error boolean not null, + primary key (workspace_id, session_id) +) with ( + 'append_mode' = 'true', + 'sst_format' = 'flat', + 'ttl' = 'forever' +); + +-- 9 rows in the 2026-09-10 window and 2 rows in the 2026-09-11 window. +insert into json2_swcs_minmax_stats values + ('ws', 's1', 1, 1788998400000000001, 'synthetic', '{"case":"swcs-repro","nested":[1,true],"ns_edge":0,"ordinal":1,"synthetic":true}', 1, false), + ('ws', 's2', 2, 1788998400000000002, 'synthetic', '{"case":"swcs-repro","nested":[2,false],"ns_edge":1,"ordinal":2,"synthetic":true}', 1, false), + ('ws', 's3', 3, 1788998400000000003, 'synthetic', '{"case":"swcs-repro","nested":[3,null],"ns_edge":2,"ordinal":3,"synthetic":true}', 1, false), + ('ws', 's1', 4, 1788998400000000004, 'synthetic', '{"case":"swcs-repro","nested":[4,true],"ns_edge":3,"ordinal":4,"synthetic":true}', 1, false), + ('ws', 's2', 5, 1788998400000000005, 'synthetic', '{"case":"swcs-repro","nested":[5,false],"ns_edge":4,"ordinal":5,"synthetic":true}', 1, false), + ('ws', 's3', 6, 1788998400000000006, 'synthetic', '{"case":"swcs-repro","nested":[6,null],"ns_edge":5,"ordinal":6,"synthetic":true}', 1, false), + ('ws', 's1', 7, 1788998400000000007, 'synthetic', '{"case":"swcs-repro","nested":[7,true],"ns_edge":6,"ordinal":7,"synthetic":true}', 1, false), + ('ws', 's2', 8, 1788998400000000008, 'synthetic', '{"case":"swcs-repro","nested":[8,false],"ns_edge":7,"ordinal":8,"synthetic":true}', 1, false), + ('ws', 's3', 9, 1788998400000000009, 'synthetic', '{"case":"swcs-repro","nested":[9,null],"ns_edge":8,"ordinal":9,"synthetic":true}', 1, false), + ('ws', 's1', 10, 1789084800000000001, 'synthetic', '{"case":"swcs-repro","nested":[10,true],"ns_edge":9,"ordinal":10,"synthetic":true}', 1, false), + ('ws', 's2', 11, 1789084800000000002, 'synthetic', '{"case":"swcs-repro","nested":[11,false],"ns_edge":10,"ordinal":11,"synthetic":true}', 1, false); + +select count(*) from json2_swcs_minmax_stats; + +admin flush_table('json2_swcs_minmax_stats'); + +-- Check pruning on flush-written SSTs before any manual compaction. +select count(*) from json2_swcs_minmax_stats +where event_time >= 1788998400000000000 and event_time < 1789084800000000000; + +-- SWCS compaction reads the SST with a time window predicate. Before the +-- fix, min-max pruning compared the window against the small `ns_edge` +-- statistics and dropped every row group. +admin compact_table('json2_swcs_minmax_stats', 'swcs', '86400'); + +select count(*) from json2_swcs_minmax_stats; + +-- Compact once more to also exercise reading compaction-written SSTs. +admin compact_table('json2_swcs_minmax_stats', 'swcs', '86400'); + +select count(*) from json2_swcs_minmax_stats; + +-- Time-range predicates use the same statistics and must not prune the rows +-- either. +select count(*) from json2_swcs_minmax_stats +where event_time >= 1788998400000000000 and event_time < 1789084800000000000; + +select workspace_id, session_id, seq, event_time, entry_kind, payload, schema_version, is_error +from json2_swcs_minmax_stats +order by seq; + +drop table json2_swcs_minmax_stats; diff --git a/tests/cases/standalone/common/types/timestamp/timestamp.result b/tests/cases/standalone/common/types/timestamp/timestamp.result index 50cba0eac4a..9ce5c3d8539 100644 --- a/tests/cases/standalone/common/types/timestamp/timestamp.result +++ b/tests/cases/standalone/common/types/timestamp/timestamp.result @@ -75,21 +75,22 @@ SELECT MAX(t) FROM timestamp; SELECT SUM(t) FROM timestamp; -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Execution error: Function 'sum' failed to match any signature, errors: Error during planning: Function 'sum' requires TypeSignatureClass::Decimal, but received Timestamp(ms) (DataType: Timestamp(ms)).,Error during planning: Function 'sum' requires TypeSignatureClass::Native(LogicalType(Native(UInt64), UInt64)), but received Timestamp(ms) (DataType: Timestamp(ms)).,Error during planning: Function 'sum' requires TypeSignatureClass::Native(LogicalType(Native(Int64), Int64)), but received Timestamp(ms) (DataType: Timestamp(ms)).,Error during planning: Function 'sum' requires TypeSignatureClass::Native(LogicalType(Native(Float64), Float64)), but received Timestamp(ms) (DataType: Timestamp(ms)).,Error during planning: Function 'sum' requires TypeSignatureClass::Duration, but received Timestamp(ms) (DataType: Timestamp(ms)). No function matches the given name and argument types 'sum(Timestamp(ms))'. You might need to add explicit type casts. +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Execution error: Function 'sum' failed to match any signature, errors: Error during planning: Function 'sum' requires Decimal, but received Timestamp(ms) (DataType: Timestamp(ms)).,Error during planning: Function 'sum' requires UInt64, but received Timestamp(ms) (DataType: Timestamp(ms)).,Error during planning: Function 'sum' requires Int64, but received Timestamp(ms) (DataType: Timestamp(ms)).,Error during planning: Function 'sum' requires Float64, but received Timestamp(ms) (DataType: Timestamp(ms)).,Error during planning: Function 'sum' requires Duration, but received Timestamp(ms) (DataType: Timestamp(ms)).,Error during planning: Function 'sum' requires Interval, but received Timestamp(ms) (DataType: Timestamp(ms)).. No function matches the given name and argument types 'sum(Timestamp(ms))'. You might need to add explicit type casts. Candidate functions: - sum(Coercion(TypeSignatureClass::Decimal)) - sum(Coercion(TypeSignatureClass::Native(LogicalType(Native(UInt64), UInt64)), implicit_coercion=ImplicitCoercion([Native(LogicalType(Native(UInt8), UInt8)), Native(LogicalType(Native(UInt16), UInt16)), Native(LogicalType(Native(UInt32), UInt32))], default_type=UInt64)) - sum(Coercion(TypeSignatureClass::Native(LogicalType(Native(Int64), Int64)), implicit_coercion=ImplicitCoercion([Native(LogicalType(Native(Int8), Int8)), Native(LogicalType(Native(Int16), Int16)), Native(LogicalType(Native(Int32), Int32))], default_type=Int64)) - sum(Coercion(TypeSignatureClass::Native(LogicalType(Native(Float64), Float64)), implicit_coercion=ImplicitCoercion([Float], default_type=Float64)) - sum(Coercion(TypeSignatureClass::Duration)) + sum(Decimal) + sum(UInt64) + sum(Int64) + sum(Float64) + sum(Duration) + sum(Interval) SELECT AVG(t) FROM timestamp; -Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Execution error: Function 'avg' failed to match any signature, errors: Error during planning: Function 'avg' requires TypeSignatureClass::Decimal, but received Timestamp(ms) (DataType: Timestamp(ms)).,Error during planning: Function 'avg' requires TypeSignatureClass::Duration, but received Timestamp(ms) (DataType: Timestamp(ms)).,Error during planning: Function 'avg' requires TypeSignatureClass::Native(LogicalType(Native(Float64), Float64)), but received Timestamp(ms) (DataType: Timestamp(ms)). No function matches the given name and argument types 'avg(Timestamp(ms))'. You might need to add explicit type casts. +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Execution error: Function 'avg' failed to match any signature, errors: Error during planning: Function 'avg' requires Decimal, but received Timestamp(ms) (DataType: Timestamp(ms)).,Error during planning: Function 'avg' requires Duration, but received Timestamp(ms) (DataType: Timestamp(ms)).,Error during planning: Function 'avg' requires Float64, but received Timestamp(ms) (DataType: Timestamp(ms)).. No function matches the given name and argument types 'avg(Timestamp(ms))'. You might need to add explicit type casts. Candidate functions: - avg(Coercion(TypeSignatureClass::Decimal)) - avg(Coercion(TypeSignatureClass::Duration)) - avg(Coercion(TypeSignatureClass::Native(LogicalType(Native(Float64), Float64)), implicit_coercion=ImplicitCoercion([Integer, Float], default_type=Float64)) + avg(Decimal) + avg(Duration) + avg(Float64) SELECT t+t FROM timestamp; diff --git a/tests/cases/standalone/common/view/columns.result b/tests/cases/standalone/common/view/columns.result index 7184cd3da17..b9e9f819b3f 100644 --- a/tests/cases/standalone/common/view/columns.result +++ b/tests/cases/standalone/common/view/columns.result @@ -58,7 +58,8 @@ SELECT a FROM v1; SELECT n FROM v1; -Error: 3000(PlanQuery), Failed to plan SQL: No field named n. Valid fields are v1.a. +Error: 3000(PlanQuery), Failed to plan SQL: No field named n. +Valid fields are v1.a. CREATE OR REPLACE VIEW v1 (a, b) AS SELECT n, n+1 FROM t1; @@ -167,11 +168,13 @@ SELECT a,b FROM v1; SELECT n FROM v1; -Error: 3000(PlanQuery), Failed to plan SQL: No field named n. Valid fields are v1.a, v1.b. +Error: 3000(PlanQuery), Failed to plan SQL: No field named n. +Valid fields are v1.a, v1.b. SELECT * FROM v1 WHERE n > 5; -Error: 3000(PlanQuery), Failed to plan SQL: No field named n. Valid fields are v1.a, v1.b. +Error: 3000(PlanQuery), Failed to plan SQL: No field named n. +Valid fields are v1.a, v1.b. -- test view after altering table t1 -- CREATE OR REPLACE VIEW v1 AS SELECT n, ts FROM t1 LIMIT 5; @@ -212,7 +215,8 @@ Affected Rows: 0 SELECT * FROM v1; -Error: 1002(Unexpected), Failed to decode DataFusion plan: No field named n. Valid fields are greptime.public.t1.ts, greptime.public.t1.s. +Error: 1002(Unexpected), Failed to decode DataFusion plan: No field named n. +Valid fields are greptime.public.t1.ts, greptime.public.t1.s. DROP VIEW v1; diff --git a/tests/cases/standalone/common/view/create.result b/tests/cases/standalone/common/view/create.result index 76b5ed78935..577583515e9 100644 --- a/tests/cases/standalone/common/view/create.result +++ b/tests/cases/standalone/common/view/create.result @@ -116,8 +116,10 @@ ORDER BY TABLE_NAME, TABLE_TYPE; |greptime|information_schema|optimizer_trace|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||Y| |greptime|information_schema|parameters|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||Y| |greptime|information_schema|partitions|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||Y| +|greptime|information_schema|plugins|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||Y| |greptime|information_schema|procedure_info|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||Y| |greptime|information_schema|process_list|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||Y| +|greptime|information_schema|processlist|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||Y| |greptime|information_schema|profiling|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||Y| |greptime|information_schema|referential_constraints|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||Y| |greptime|information_schema|region_info|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||Y| @@ -138,6 +140,7 @@ ORDER BY TABLE_NAME, TABLE_TYPE; |greptime|public|test_table|BASETABLE|ID|ID|ID|ID|ID|ID|mito|ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||N| |greptime|public|test_view|VIEW|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||N| |greptime|public|test_view2|VIEW|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||N| +|greptime|information_schema|user_privileges|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||Y| |greptime|information_schema|views|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|DATETIME||utf8_bin|ID|||Y| +++++++++++++++++++++++++ @@ -154,18 +157,24 @@ SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'VIEW' ORDER BY TABLE SHOW COLUMNS FROM test_view; -++ -++ ++-------+------+------+-----+---------+-------+---------------+ +| Field | Type | Null | Key | Default | Extra | Greptime_type | ++-------+------+------+-----+---------+-------+---------------+ ++-------+------+------+-----+---------+-------+---------------+ SHOW FULL COLUMNS FROM test_view; -++ -++ ++-------+------+-----------+------+-----+---------+---------+------------+-------+---------------+ +| Field | Type | Collation | Null | Key | Default | Comment | Privileges | Extra | Greptime_type | ++-------+------+-----------+------+-----+---------+---------+------------+-------+---------------+ ++-------+------+-----------+------+-----+---------+---------+------------+-------+---------------+ SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 'test_view'; -++ -++ ++---------------+--------------+------------+-------------+------------------+--------------------------+------------------------+-------------------+---------------+--------------------+--------------------+----------------+------------+-------+------------+-----------------------+--------------------+-----------+---------------+----------------+-------------+-------------+----------------+--------+ +| table_catalog | table_schema | table_name | column_name | ordinal_position | character_maximum_length | character_octet_length | numeric_precision | numeric_scale | datetime_precision | character_set_name | collation_name | column_key | extra | privileges | generation_expression | greptime_data_type | data_type | semantic_type | column_default | is_nullable | column_type | column_comment | srs_id | ++---------------+--------------+------------+-------------+------------------+--------------------------+------------------------+-------------------+---------------+--------------------+--------------------+----------------+------------+-------+------------+-----------------------+--------------------+-----------+---------------+----------------+-------------+-------------+----------------+--------+ ++---------------+--------------+------------+-------------+------------------+--------------------------+------------------------+-------------------+---------------+--------------------+--------------------+----------------+------------+-------+------------+-----------------------+--------------------+-----------+---------------+----------------+-------------+-------------+----------------+--------+ SELECT * FROM test_view LIMIT 10; diff --git a/tests/cases/standalone/common/view/view.result b/tests/cases/standalone/common/view/view.result index 21d674e54e1..701a5d0abdc 100644 --- a/tests/cases/standalone/common/view/view.result +++ b/tests/cases/standalone/common/view/view.result @@ -58,8 +58,10 @@ Error: 4001(TableNotFound), Failed to plan SQL: Table not found: greptime.public SHOW VIEWS; -++ -++ ++-------+ +| Views | ++-------+ ++-------+ DROP VIEW v1; @@ -92,6 +94,8 @@ SHOW TABLES; SHOW VIEWS; -++ -++ ++-------+ +| Views | ++-------+ ++-------+ diff --git a/tests/cases/standalone/copy/copy_database_from_fs_parquet.result b/tests/cases/standalone/copy/copy_database_from_fs_parquet.result index 3ec38aa7ca7..662355099cc 100644 --- a/tests/cases/standalone/copy/copy_database_from_fs_parquet.result +++ b/tests/cases/standalone/copy/copy_database_from_fs_parquet.result @@ -20,8 +20,10 @@ Affected Rows: 2 SELECT * FROM demo ORDER BY ts; -++ -++ ++------+-----+--------+----+ +| host | cpu | memory | ts | ++------+-----+--------+----+ ++------+-----+--------+----+ COPY DATABASE public FROM '${SQLNESS_HOME}/demo/export/parquet/'; diff --git a/tests/cases/standalone/flow-tql/flow_tql.result b/tests/cases/standalone/flow-tql/flow_tql.result index 50e1d000bc2..de93fb7c518 100644 --- a/tests/cases/standalone/flow-tql/flow_tql.result +++ b/tests/cases/standalone/flow-tql/flow_tql.result @@ -43,8 +43,10 @@ SHOW CREATE TABLE cnt_reqs; -- test if sink table is tql queryable TQL EVAL (now() - '1m'::interval, now(), '5s') count_values("status_code", cnt_reqs); -++ -++ ++------------------------------------------+----+-------------+ +| count(cnt_reqs.count(http_requests.val)) | ts | status_code | ++------------------------------------------+----+-------------+ ++------------------------------------------+----+-------------+ INSERT INTO TABLE http_requests VALUES (now() - '17s'::interval, 'host1', 'idc1', 200), @@ -187,8 +189,10 @@ SHOW CREATE TABLE cnt_reqs; -- test if sink table is tql queryable TQL EVAL (now() - '1m'::interval, now(), '5s') count_values("status_code", cnt_reqs); -++ -++ ++------------------------------------------+----+-------------+ +| count(cnt_reqs.count(http_requests.val)) | ts | status_code | ++------------------------------------------+----+-------------+ ++------------------------------------------+----+-------------+ INSERT INTO TABLE http_requests VALUES (0::Timestamp, 'host1', 'idc1', 200), @@ -278,8 +282,10 @@ SHOW CREATE TABLE rate_reqs; -- test if sink table is tql queryable TQL EVAL (now() - '1m'::interval, now(), '5s') count_values("status_code", rate_reqs); -++ -++ ++-----------------------------------------------------------+----+-------------+ +| count(rate_reqs.prom_rate(ts_range,val,ts,Int64(300000))) | ts | status_code | ++-----------------------------------------------------------+----+-------------+ ++-----------------------------------------------------------+----+-------------+ INSERT INTO TABLE http_requests VALUES (now() - '1m'::interval, 0), @@ -359,8 +365,10 @@ SHOW CREATE TABLE rate_reqs; -- test if sink table is tql queryable TQL EVAL (now() - '1m'::interval, now(), '5s') count_values("status_code", rate_reqs); -++ -++ ++-----------------------------------------------------------+----+-------------+ +| count(rate_reqs.prom_rate(ts_range,byte,ts,Int64(60000))) | ts | status_code | ++-----------------------------------------------------------+----+-------------+ ++-----------------------------------------------------------+----+-------------+ INSERT INTO TABLE http_requests_total VALUES ('localhost', 'my_service', 'instance1', 100, now() - '1min'::interval), diff --git a/tests/cases/standalone/information_schema/cluster_info.result b/tests/cases/standalone/information_schema/cluster_info.result index 04567ff721b..9fe338fbb00 100644 --- a/tests/cases/standalone/information_schema/cluster_info.result +++ b/tests/cases/standalone/information_schema/cluster_info.result @@ -45,8 +45,10 @@ SELECT peer_id, peer_type, peer_addr, version, git_commit, start_time, uptime, a SELECT peer_id, peer_type, peer_addr, version, git_commit, start_time, uptime, active_time FROM CLUSTER_INFO WHERE PEER_TYPE != 'STANDALONE'; -++ -++ ++---------+-----------+-----------+---------+------------+------------+--------+-------------+ +| peer_id | peer_type | peer_addr | version | git_commit | start_time | uptime | active_time | ++---------+-----------+-----------+---------+------------+------------+--------+-------------+ ++---------+-----------+-----------+---------+------------+------------+--------+-------------+ -- SQLNESS REPLACE version node_version -- SQLNESS REPLACE (\s[\-0-9T:\.]{15,}) Start_time @@ -60,8 +62,10 @@ SELECT peer_id, peer_type, peer_addr, version, git_commit, start_time, uptime, a SELECT peer_id, peer_type, peer_addr, version, git_commit, start_time, uptime, active_time FROM CLUSTER_INFO WHERE PEER_ID > 0; -++ -++ ++---------+-----------+-----------+---------+------------+------------+--------+-------------+ +| peer_id | peer_type | peer_addr | version | git_commit | start_time | uptime | active_time | ++---------+-----------+-----------+---------+------------+------------+--------+-------------+ ++---------+-----------+-----------+---------+------------+------------+--------+-------------+ SELECT peer_type, total_cpu_millicores!=0, total_memory_bytes!=0 FROM CLUSTER_INFO ORDER BY peer_type; diff --git a/tests/cases/standalone/limit/limit.result b/tests/cases/standalone/limit/limit.result index 41738535a2d..860637de2e4 100644 --- a/tests/cases/standalone/limit/limit.result +++ b/tests/cases/standalone/limit/limit.result @@ -1,7 +1,9 @@ SELECT * FROM (SELECT SUM(number) FROM numbers LIMIT 100000000000) LIMIT 0; -++ -++ ++---------------------+ +| sum(numbers.number) | ++---------------------+ ++---------------------+ EXPLAIN SELECT * FROM (SELECT SUM(number) FROM numbers LIMIT 100000000000) LIMIT 0; diff --git a/tests/cases/standalone/optimizer/count.result b/tests/cases/standalone/optimizer/count.result index 8a128a92000..925e89eeb23 100644 --- a/tests/cases/standalone/optimizer/count.result +++ b/tests/cases/standalone/optimizer/count.result @@ -135,9 +135,9 @@ select count(1) from count_where_bug where `tag` = 'b'; | 0_| 0_|_CooperativeExec REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| -| 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +| 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(count_where_bug.ts) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(count_where_bug.ts) as count(Int64(1))] REDACTED |_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED |_|_|_| |_|_| Total rows: 1_| @@ -176,9 +176,9 @@ select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; | 0_| 0_|_CooperativeExec REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| -| 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +| 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(count_where_bug.ts) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(count_where_bug.ts) as count(Int64(1))] REDACTED |_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED |_|_|_| |_|_| Total rows: 1_| @@ -215,9 +215,9 @@ select count(1) from count_where_bug where num != 3; | 0_| 0_|_CooperativeExec REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| -| 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +| 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(count_where_bug.ts) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(count_where_bug.ts) as count(Int64(1))] REDACTED |_|_|_FilterExec: num@1 != 3, projection=[ts@0] REDACTED |_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED |_|_|_| @@ -275,9 +275,9 @@ select count(1) from count_where_bug; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -311,9 +311,9 @@ select count(1) from count_where_bug where `tag` = 'b'; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -338,9 +338,9 @@ select count(1) from count_where_bug where `tag` = 'b'; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -375,9 +375,9 @@ select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -415,9 +415,9 @@ select count(1) from count_where_bug where num != 3; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(count_where_bug.ts)) as count(Int64(1))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| diff --git a/tests/cases/standalone/optimizer/filter_column_pruning.result b/tests/cases/standalone/optimizer/filter_column_pruning.result index 9cff5fb6951..3a16bef50df 100644 --- a/tests/cases/standalone/optimizer/filter_column_pruning.result +++ b/tests/cases/standalone/optimizer/filter_column_pruning.result @@ -54,9 +54,9 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'ho |_|_|_| | 1_| 0_|_ProjectionExec: expr=[cpu_usage@0 as cpu_usage] REDACTED |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED -|_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED +|_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED +|_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))"], "flat_format": REDACTED, "REDACTED |_|_|_| |_|_| Total rows: 3_| @@ -90,9 +90,9 @@ EXPLAIN ANALYZE VERBOSE SELECT mem_usage, cpu_usage FROM filter_prune_test WHERE |_|_|_| | 1_| 0_|_ProjectionExec: expr=[mem_usage@0 as mem_usage, cpu_usage@1 as cpu_usage] REDACTED |_|_|_SortPreservingMergeExec: [ts@2 ASC NULLS LAST] REDACTED -|_|_|_WindowedSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_PartSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[mem_usage@2 as mem_usage, cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED +|_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED +|_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage", "mem_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], "flat_format": REDACTED, "REDACTED |_|_|_| |_|_| Total rows: 2_| @@ -130,8 +130,7 @@ EXPLAIN ANALYZE VERBOSE SELECT host, `region` FROM filter_prune_test WHERE cpu_u |_|_|_SortPreservingMergeExec: [ts@2 ASC NULLS LAST] REDACTED |_|_|_WindowedSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_ProjectionExec: expr=[host@1 as host, region@2 as region, ts@0 as ts] REDACTED -|_|_|_FilterExec: cpu_usage@3 > 20, projection=[ts@0, host@1, region@2] REDACTED +|_|_|_FilterExec: cpu_usage@3 > 20, projection=[host@1, region@2, ts@0] REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["cpu_usage > Float64(20)"], "flat_format": REDACTED, "REDACTED |_|_|_| |_|_| Total rows: 4_| @@ -167,9 +166,9 @@ EXPLAIN ANALYZE VERBOSE SELECT host, cpu_usage FROM filter_prune_test WHERE ts > |_|_|_| | 1_| 0_|_ProjectionExec: expr=[host@0 as host, cpu_usage@1 as cpu_usage] REDACTED |_|_|_SortPreservingMergeExec: [ts@2 ASC NULLS LAST] REDACTED -|_|_|_WindowedSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_PartSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[host@1 as host, cpu_usage@2 as cpu_usage, ts@0 as ts] REDACTED +|_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED +|_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "cpu_usage"], "filters": ["ts > TimestampMillisecond(2000, None)"], "flat_format": REDACTED, "REDACTED |_|_|_| |_|_| Total rows: 4_| @@ -205,8 +204,7 @@ EXPLAIN ANALYZE VERBOSE SELECT mem_usage FROM filter_prune_test WHERE host = 'ho |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_ProjectionExec: expr=[mem_usage@1 as mem_usage, ts@0 as ts] REDACTED -|_|_|_FilterExec: cpu_usage@1 > 10, projection=[ts@0, mem_usage@2] REDACTED +|_|_|_FilterExec: cpu_usage@1 > 10, projection=[mem_usage@2, ts@0] REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage", "mem_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "cpu_usage > Float64(10)", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], "flat_format": REDACTED, "REDACTED |_|_|_| |_|_| Total rows: 2_| @@ -274,8 +272,8 @@ EXPLAIN ANALYZE VERBOSE SELECT `region`, AVG(cpu_usage) as avg_cpu FROM filter_p |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [region@0 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[region@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_ProjectionExec: expr=[region@0 as region, avg(filter_prune_test.cpu_usage)@1 as avg_cpu] REDACTED +|_|_|_SortExec: expr=[region@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_AggregateExec: mode=FinalPartitioned, gby=[region@0 as region], aggr=[avg(filter_prune_test.cpu_usage)] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[region@0 as region], aggr=[avg(filter_prune_test.cpu_usage)] REDACTED @@ -316,9 +314,9 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host IN (' |_|_|_| | 1_| 0_|_ProjectionExec: expr=[cpu_usage@0 as cpu_usage] REDACTED |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED -|_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED +|_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED +|_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\")) OR host = Dictionary(UInt32, Utf8(\"host2\"))"], "flat_format": REDACTED, "REDACTED |_|_|_| |_|_| Total rows: 5_| @@ -355,8 +353,7 @@ EXPLAIN ANALYZE VERBOSE SELECT host FROM filter_prune_test WHERE cpu_usage BETWE |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_ProjectionExec: expr=[host@1 as host, ts@0 as ts] REDACTED -|_|_|_FilterExec: cpu_usage@2 >= 15 AND cpu_usage@2 <= 30, projection=[ts@0, host@1] REDACTED +|_|_|_FilterExec: cpu_usage@2 >= 15 AND cpu_usage@2 <= 30, projection=[host@1, ts@0] REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "cpu_usage"], "filters": ["cpu_usage >= Float64(15)", "cpu_usage <= Float64(30)"], "flat_format": REDACTED, "REDACTED |_|_|_| |_|_| Total rows: 3_| @@ -395,8 +392,7 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host LIKE |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED -|_|_|_FilterExec: host@1 LIKE host% AND region@2 LIKE us-%, projection=[ts@0, cpu_usage@3] REDACTED +|_|_|_FilterExec: host@1 LIKE host% AND region@2 LIKE us-%, projection=[cpu_usage@3, ts@0] REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["host LIKE Dictionary(UInt32, Utf8(\"host%\"))", "region LIKE Dictionary(UInt32, Utf8(\"us-%\"))"], "flat_format": REDACTED, "REDACTED |_|_|_| |_|_| Total rows: 5_| @@ -434,8 +430,7 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'ho |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED -|_|_|_FilterExec: host@1 = host1 OR region@2 = eu-west, projection=[ts@0, cpu_usage@3] REDACTED +|_|_|_FilterExec: host@1 = host1 OR region@2 = eu-west, projection=[cpu_usage@3, ts@0] REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\")) OR region = Dictionary(UInt32, Utf8(\"eu-west\"))"], "flat_format": REDACTED, "REDACTED |_|_|_| |_|_| Total rows: 4_| @@ -469,10 +464,10 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM (SELECT * FROM filter_prune_test W |_|_|_| | 1_| 0_|_ProjectionExec: expr=[cpu_usage@0 as cpu_usage] REDACTED |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED -|_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage"], "filters": ["region = Dictionary(UInt32, Utf8(\"us-east\"))", "host = Dictionary(UInt32, Utf8(\"host1\"))"], "flat_format": REDACTED, "REDACTED +|_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED +|_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], "flat_format": REDACTED, "REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -516,9 +511,9 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'ho |_|_|_| | 1_| 0_|_ProjectionExec: expr=[cpu_usage@0 as cpu_usage] REDACTED |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED -|_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED +|_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED +|_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))"], \"file\":REDACTED |_|_|_| |_|_| Total rows: 3_| @@ -553,9 +548,9 @@ EXPLAIN ANALYZE VERBOSE SELECT mem_usage, cpu_usage FROM filter_prune_test WHERE |_|_|_| | 1_| 0_|_ProjectionExec: expr=[mem_usage@0 as mem_usage, cpu_usage@1 as cpu_usage] REDACTED |_|_|_SortPreservingMergeExec: [ts@2 ASC NULLS LAST] REDACTED -|_|_|_WindowedSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_PartSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[mem_usage@2 as mem_usage, cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED +|_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED +|_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage", "mem_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], \"file\":REDACTED |_|_|_| |_|_| Total rows: 2_| @@ -594,8 +589,7 @@ EXPLAIN ANALYZE VERBOSE SELECT host, `region` FROM filter_prune_test WHERE cpu_u |_|_|_SortPreservingMergeExec: [ts@2 ASC NULLS LAST] REDACTED |_|_|_WindowedSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_ProjectionExec: expr=[host@1 as host, region@2 as region, ts@0 as ts] REDACTED -|_|_|_FilterExec: cpu_usage@3 > 20, projection=[ts@0, host@1, region@2] REDACTED +|_|_|_FilterExec: cpu_usage@3 > 20, projection=[host@1, region@2, ts@0] REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["cpu_usage > Float64(20)"], \"file\":REDACTED |_|_|_| |_|_| Total rows: 4_| @@ -632,9 +626,9 @@ EXPLAIN ANALYZE VERBOSE SELECT host, cpu_usage FROM filter_prune_test WHERE ts > |_|_|_| | 1_| 0_|_ProjectionExec: expr=[host@0 as host, cpu_usage@1 as cpu_usage] REDACTED |_|_|_SortPreservingMergeExec: [ts@2 ASC NULLS LAST] REDACTED -|_|_|_WindowedSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_PartSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[host@1 as host, cpu_usage@2 as cpu_usage, ts@0 as ts] REDACTED +|_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED +|_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "cpu_usage"], "filters": ["ts > TimestampMillisecond(2000, None)"], \"file\":REDACTED |_|_|_| |_|_| Total rows: 4_| @@ -671,8 +665,7 @@ EXPLAIN ANALYZE VERBOSE SELECT mem_usage FROM filter_prune_test WHERE host = 'ho |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_ProjectionExec: expr=[mem_usage@1 as mem_usage, ts@0 as ts] REDACTED -|_|_|_FilterExec: cpu_usage@1 > 10, projection=[ts@0, mem_usage@2] REDACTED +|_|_|_FilterExec: cpu_usage@1 > 10, projection=[mem_usage@2, ts@0] REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage", "mem_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "cpu_usage > Float64(10)", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], \"file\":REDACTED |_|_|_| |_|_| Total rows: 2_| @@ -742,8 +735,8 @@ EXPLAIN ANALYZE VERBOSE SELECT `region`, AVG(cpu_usage) as avg_cpu FROM filter_p |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [region@0 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[region@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_ProjectionExec: expr=[region@0 as region, avg(filter_prune_test.cpu_usage)@1 as avg_cpu] REDACTED +|_|_|_SortExec: expr=[region@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_AggregateExec: mode=FinalPartitioned, gby=[region@0 as region], aggr=[avg(filter_prune_test.cpu_usage)] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[region@0 as region], aggr=[avg(filter_prune_test.cpu_usage)] REDACTED @@ -785,9 +778,9 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host IN (' |_|_|_| | 1_| 0_|_ProjectionExec: expr=[cpu_usage@0 as cpu_usage] REDACTED |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED -|_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED +|_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED +|_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\")) OR host = Dictionary(UInt32, Utf8(\"host2\"))"], \"file\":REDACTED |_|_|_| |_|_| Total rows: 5_| @@ -825,8 +818,7 @@ EXPLAIN ANALYZE VERBOSE SELECT host FROM filter_prune_test WHERE cpu_usage BETWE |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_ProjectionExec: expr=[host@1 as host, ts@0 as ts] REDACTED -|_|_|_FilterExec: cpu_usage@2 >= 15 AND cpu_usage@2 <= 30, projection=[ts@0, host@1] REDACTED +|_|_|_FilterExec: cpu_usage@2 >= 15 AND cpu_usage@2 <= 30, projection=[host@1, ts@0] REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "cpu_usage"], "filters": ["cpu_usage >= Float64(15)", "cpu_usage <= Float64(30)"], \"file\":REDACTED |_|_|_| |_|_| Total rows: 3_| @@ -866,8 +858,7 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host LIKE |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED -|_|_|_FilterExec: host@1 LIKE host% AND region@2 LIKE us-%, projection=[ts@0, cpu_usage@3] REDACTED +|_|_|_FilterExec: host@1 LIKE host% AND region@2 LIKE us-%, projection=[cpu_usage@3, ts@0] REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["host LIKE Dictionary(UInt32, Utf8(\"host%\"))", "region LIKE Dictionary(UInt32, Utf8(\"us-%\"))"], \"file\":REDACTED |_|_|_| |_|_| Total rows: 5_| @@ -906,8 +897,7 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'ho |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED -|_|_|_FilterExec: host@1 = host1 OR region@2 = eu-west, projection=[ts@0, cpu_usage@3] REDACTED +|_|_|_FilterExec: host@1 = host1 OR region@2 = eu-west, projection=[cpu_usage@3, ts@0] REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\")) OR region = Dictionary(UInt32, Utf8(\"eu-west\"))"], \"file\":REDACTED |_|_|_| |_|_| Total rows: 4_| @@ -942,10 +932,10 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM (SELECT * FROM filter_prune_test W |_|_|_| | 1_| 0_|_ProjectionExec: expr=[cpu_usage@0 as cpu_usage] REDACTED |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED -|_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage"], "filters": ["region = Dictionary(UInt32, Utf8(\"us-east\"))", "host = Dictionary(UInt32, Utf8(\"host1\"))"], \"file\":REDACTED +|_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED +|_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], \"file\":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -1031,9 +1021,9 @@ EXPLAIN ANALYZE VERBOSE SELECT field1 FROM filter_prune_files WHERE tag_key = 'a |_|_|_| | 1_| 0_|_ProjectionExec: expr=[field1@0 as field1] REDACTED |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED -|_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[field1@1 as field1, ts@0 as ts] REDACTED +|_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED +|_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":3, "mem_ranges":1, "files":2, "file_ranges":2}, "projection": ["ts", "field1"], "filters": ["tag_key = Dictionary(UInt32, Utf8(\"a\"))"], \"file\":REDACTED |_|_|_| |_|_| Total rows: 2_| @@ -1073,8 +1063,7 @@ EXPLAIN ANALYZE VERBOSE SELECT field1 FROM filter_prune_files WHERE field2 > 5.0 |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_ProjectionExec: expr=[field1@1 as field1, ts@0 as ts] REDACTED -|_|_|_FilterExec: field2@2 > 5, projection=[ts@0, field1@1] REDACTED +|_|_|_FilterExec: field2@2 > 5, projection=[field1@1, ts@0] REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":3, "mem_ranges":1, "files":2, "file_ranges":2}, "projection": ["ts", "field1", "field2"], "filters": ["field2 > Float64(5)"], \"file\":REDACTED |_|_|_| |_|_| Total rows: 5_| @@ -1111,8 +1100,7 @@ EXPLAIN ANALYZE VERBOSE SELECT field3 FROM filter_prune_files WHERE tag_key = 'b |_|_|_SortPreservingMergeExec: [ts@1 ASC NULLS LAST] REDACTED |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_ProjectionExec: expr=[field3@1 as field3, ts@0 as ts] REDACTED -|_|_|_FilterExec: field2@1 > 7, projection=[ts@0, field3@2] REDACTED +|_|_|_FilterExec: field2@1 > 7, projection=[field3@2, ts@0] REDACTED |_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":3, "mem_ranges":1, "files":2, "file_ranges":2}, "projection": ["ts", "field2", "field3"], "filters": ["tag_key = Dictionary(UInt32, Utf8(\"b\"))", "field2 > Float64(7)"], \"file\":REDACTED |_|_|_| |_|_| Total rows: 2_| diff --git a/tests/cases/standalone/optimizer/filter_push_down.result b/tests/cases/standalone/optimizer/filter_push_down.result index b959e063f0c..242cebd1167 100644 --- a/tests/cases/standalone/optimizer/filter_push_down.result +++ b/tests/cases/standalone/optimizer/filter_push_down.result @@ -139,7 +139,7 @@ SELECT * FROM integers i1 WHERE NOT EXISTS(SELECT i FROM integers WHERE i=i1.i) SELECT i1.i,i2.i FROM integers i1, integers i2 WHERE i1.i=(SELECT i FROM integers WHERE i1.i=i) AND i1.i=i2.i ORDER BY i1.i; -Error: 3001(EngineExecuteQuery), Error during planning: Correlated scalar subquery must be aggregated to return at most one row +Error: 3000(PlanQuery), Error during planning: Correlated scalar subquery must be aggregated to return at most one row SELECT * FROM (SELECT i1.i AS a, i2.i AS b FROM integers i1, integers i2) a1 WHERE a=b ORDER BY 1; @@ -190,14 +190,18 @@ SELECT * FROM (SELECT i1.i AS a, i2.i AS b, row_number() OVER (ORDER BY i1.i, i2 -- Align the result to PostgreSQL: empty. SELECT * FROM (SELECT 0=1 AS cond FROM integers i1, integers i2) a1 WHERE cond ORDER BY 1; -++ -++ ++------+ +| cond | ++------+ ++------+ -- Align the result to PostgreSQL: empty. SELECT * FROM (SELECT 0=1 AS cond FROM integers i1, integers i2 GROUP BY 1) a1 WHERE cond ORDER BY 1; -++ -++ ++------+ +| cond | ++------+ ++------+ DROP TABLE integers; diff --git a/tests/cases/standalone/optimizer/first_value_advance.result b/tests/cases/standalone/optimizer/first_value_advance.result index c543a8df2a2..f3b24db6d45 100644 --- a/tests/cases/standalone/optimizer/first_value_advance.result +++ b/tests/cases/standalone/optimizer/first_value_advance.result @@ -314,14 +314,14 @@ explain select first_value(ts order by ts) from t; +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]] | +| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[]], aggr=[[__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_TableScan: t_| |_| ]]_| -| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[__first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -343,9 +343,9 @@ explain analyze +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -400,19 +400,19 @@ order by time_window, ordered_host; +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Sort: time_window ASC NULLS LAST, ordered_host ASC NULLS LAST_| +| logical_plan_| Sort: time_window ASC NULLS LAST_| |_|_Projection: first_value(t.host) ORDER BY [t.ts ASC NULLS LAST] AS ordered_host, first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t.ts) AS time_window_| -|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t.ts)]], aggr=[[__first_value_merge(__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]] | +|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t.ts)]], aggr=[[__first_value_merge(__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }"), t.ts) AS date_bin(Utf8("5 milliseconds"),t.ts)]], aggr=[[__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_TableScan: t_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST]_| -|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST]_| |_|_ProjectionExec: expr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@1 as ordered_host, first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]@3 as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t.ts)@0 as time_window]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_merge(__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED -|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_merge(__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -441,12 +441,12 @@ order by time_window, ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST] REDACTED |_|_|_ProjectionExec: expr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@1 as ordered_host, first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]@3 as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t.ts)@0 as time_window] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_merge(__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_merge(__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -518,14 +518,14 @@ explain +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]] | +| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[]], aggr=[[__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_TableScan: t1_| |_| ]]_| -| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[__first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -547,9 +547,9 @@ explain analyze +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -698,19 +698,19 @@ order by time_window, ordered_host; +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Sort: time_window ASC NULLS LAST, ordered_host ASC NULLS LAST_| +| logical_plan_| Sort: time_window ASC NULLS LAST_| |_|_Projection: first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST] AS ordered_host, first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t1.ts) AS time_window_| -|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t1.ts)]], aggr=[[__first_value_merge(__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]] | +|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t1.ts)]], aggr=[[__first_value_merge(__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }"), t1.ts) AS date_bin(Utf8("5 milliseconds"),t1.ts)]], aggr=[[__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_TableScan: t1_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST]_| -|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST]_| |_|_ProjectionExec: expr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@1 as ordered_host, first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@2 as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]@3 as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t1.ts)@0 as time_window]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_merge(__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED -|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_merge(__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -739,12 +739,12 @@ order by time_window, ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST] REDACTED |_|_|_ProjectionExec: expr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@1 as ordered_host, first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@2 as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]@3 as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t1.ts)@0 as time_window] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +|_|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_merge(__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_merge(__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_merge(__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as first_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| diff --git a/tests/cases/standalone/optimizer/join_filter_pushdown_edge.result b/tests/cases/standalone/optimizer/join_filter_pushdown_edge.result index e12ea22f4a0..c9b1f37c9ad 100644 --- a/tests/cases/standalone/optimizer/join_filter_pushdown_edge.result +++ b/tests/cases/standalone/optimizer/join_filter_pushdown_edge.result @@ -100,7 +100,7 @@ EXPLAIN SELECT f.k, f.val FROM fact f | | MergeScan [is_placeholder=false, remote_input=[ | | | SubqueryAlias: f | | | Filter: CAST(fact.val AS Int32) IS NOT NULL AND fact.ts >= TimestampMillisecond(1706572800000, None) AND fact.val > Float64(1) | -| | TableScan: fact, partial_filters=[fact.ts >= TimestampMillisecond(1706572800000, None), fact.val > Float64(1), CAST(fact.val AS Int32) IS NOT NULL] | +| | TableScan: fact, partial_filters=[CAST(fact.val AS Int32) IS NOT NULL, fact.ts >= TimestampMillisecond(1706572800000, None), fact.val > Float64(1)] | | | ]] | | | Projection: d.k | | | MergeScan [is_placeholder=false, remote_input=[ | diff --git a/tests/cases/standalone/optimizer/last_value_advance.result b/tests/cases/standalone/optimizer/last_value_advance.result index 15e5ca80cbf..6b4960be6a1 100644 --- a/tests/cases/standalone/optimizer/last_value_advance.result +++ b/tests/cases/standalone/optimizer/last_value_advance.result @@ -314,14 +314,14 @@ explain select last_value(ts order by ts) from t; +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]] | +| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[]], aggr=[[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_TableScan: t_| |_| ]]_| -| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[__last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -343,9 +343,9 @@ explain analyze +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -400,19 +400,19 @@ order by time_window, ordered_host; +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Sort: time_window ASC NULLS LAST, ordered_host ASC NULLS LAST_| +| logical_plan_| Sort: time_window ASC NULLS LAST_| |_|_Projection: last_value(t.host) ORDER BY [t.ts ASC NULLS LAST] AS ordered_host, last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t.ts) AS time_window_| -|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t.ts)]], aggr=[[__last_value_merge(__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]] | +|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t.ts)]], aggr=[[__last_value_merge(__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) AS last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }"), t.ts) AS date_bin(Utf8("5 milliseconds"),t.ts)]], aggr=[[__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]]]_| |_|_TableScan: t_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST]_| -|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST]_| |_|_ProjectionExec: expr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@1 as ordered_host, last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]@3 as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t.ts)@0 as time_window]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_merge(__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED -|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_merge(__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]]_| |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -441,12 +441,12 @@ order by time_window, ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST] REDACTED |_|_|_ProjectionExec: expr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@1 as ordered_host, last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]@3 as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t.ts)@0 as time_window] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_merge(__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_merge(__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]) as last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -518,14 +518,14 @@ explain +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]] | +| logical_plan_| Aggregate: groupBy=[[]], aggr=[[__last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[]], aggr=[[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_TableScan: t1_| |_| ]]_| -| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +| physical_plan | AggregateExec: mode=Final, gby=[], aggr=[__last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| |_|_CoalescePartitionsExec_| -|_|_AggregateExec: mode=Partial, gby=[], aggr=[last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -547,9 +547,9 @@ explain analyze +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +| 0_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| @@ -698,19 +698,19 @@ order by time_window, ordered_host; +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| Sort: time_window ASC NULLS LAST, ordered_host ASC NULLS LAST_| +| logical_plan_| Sort: time_window ASC NULLS LAST_| |_|_Projection: last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST] AS ordered_host, last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t1.ts) AS time_window_| -|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t1.ts)]], aggr=[[__last_value_merge(__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]] | +|_|_Aggregate: groupBy=[[date_bin(Utf8("5 milliseconds"),t1.ts)]], aggr=[[__last_value_merge(__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) AS last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| Aggregate: groupBy=[[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }"), t1.ts) AS date_bin(Utf8("5 milliseconds"),t1.ts)]], aggr=[[__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]]_| |_|_TableScan: t1_| |_| ]]_| -| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST]_| -|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| physical_plan | SortPreservingMergeExec: [time_window@3 ASC NULLS LAST]_| |_|_ProjectionExec: expr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@1 as ordered_host, last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@2 as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]@3 as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t1.ts)@0 as time_window]_| -|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_merge(__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] | |_|_RepartitionExec: REDACTED -|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| +|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_merge(__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]]_| |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| @@ -739,12 +739,12 @@ order by time_window, ordered_host; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[time_window@3 ASC NULLS LAST, ordered_host@0 ASC NULLS LAST], preserve_REDACTED +| 0_| 0_|_SortPreservingMergeExec: [time_window@3 ASC NULLS LAST] REDACTED |_|_|_ProjectionExec: expr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@1 as ordered_host, last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@2 as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]@3 as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST], date_bin(Utf8("5 milliseconds"),t1.ts)@0 as time_window] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +|_|_|_SortExec: expr=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 ASC NULLS LAST], preserve_REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_merge(__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_merge(__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_merge(__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]) as last_value(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| diff --git a/tests/cases/standalone/optimizer/lateral_join_guard.result b/tests/cases/standalone/optimizer/lateral_join_guard.result index e14ab5c9a76..d2a565089e3 100644 --- a/tests/cases/standalone/optimizer/lateral_join_guard.result +++ b/tests/cases/standalone/optimizer/lateral_join_guard.result @@ -1,9 +1,7 @@ --- Document the current aliased SQL LATERAL limitation and guard the remote --- scan boundary. DataFusion's DecorrelateLateralJoin does not currently match --- the SubqueryAlias(Subquery) shape produced by `LATERAL (...) d`, so this query --- is still expected to fail physical planning with an outer_ref expression. The --- important regression assertion is that the remaining outer_ref predicate must --- NOT be advertised as a remote TableScan.partial_filters predicate. +-- Guard the remote scan boundary for an aliased SQL LATERAL query. +-- DataFusion 55 decorrelates this shape into an inner join. The resulting +-- scan-local IS NOT NULL predicates may be pushed down, but no outer_ref +-- expression may be advertised as a remote TableScan.partial_filters predicate. CREATE TABLE lateral_fact ( ts TIMESTAMP(3) TIME INDEX, k STRING, @@ -52,6 +50,9 @@ ADMIN FLUSH_TABLE('lateral_dim'); -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE partitioning=Hash\(\[k@(\d+)\],\s*\d+\) partitioning=Hash([k@$1], REDACTED) +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE (input_partitions=REDACTED)(\s+)\| $1| EXPLAIN SELECT f.k, d.threshold FROM lateral_fact f, LATERAL ( @@ -60,28 +61,35 @@ LATERAL ( WHERE f.val > d.threshold ORDER BY f.k; -+---------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| plan_type | plan | -+---------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| logical_plan | Sort: f.k ASC NULLS LAST | -| | Projection: f.k, d.threshold | -| | Inner Join: Filter: f.val > d.threshold | -| | Projection: f.k, f.val | -| | MergeScan [is_placeholder=false, remote_input=[ | -| | SubqueryAlias: f | -| | TableScan: lateral_fact | -| | ]] | -| | SubqueryAlias: d | -| | Subquery: | -| | SubqueryAlias: d | -| | Projection: lateral_dim.threshold | -| | Filter: lateral_dim.k = outer_ref(f.k) | -| | Projection: lateral_dim.k, lateral_dim.threshold | -| | MergeScan [is_placeholder=false, remote_input=[ | -| | TableScan: lateral_dim | -| | ]] | -| physical_plan_error | This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(Field { name: "k", data_type: Dictionary(UInt32, Utf8), nullable: true }, Column { relation: Some(Bare { table: "f" }), name: "k" }) | -+---------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++---------------+---------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | Sort: f.k ASC NULLS LAST | +| | Projection: f.k, d.threshold | +| | Inner Join: f.k = d.k Filter: f.val > d.threshold | +| | Projection: f.k, f.val | +| | MergeScan [is_placeholder=false, remote_input=[ | +| | SubqueryAlias: f | +| | Filter: lateral_fact.k IS NOT NULL | +| | TableScan: lateral_fact, partial_filters=[lateral_fact.k IS NOT NULL] | +| | ]] | +| | MergeScan [is_placeholder=false, remote_input=[ | +| | SubqueryAlias: d | +| | Projection: d.threshold, d.k | +| | SubqueryAlias: d | +| | Filter: lateral_dim.k IS NOT NULL | +| | TableScan: lateral_dim, partial_filters=[lateral_dim.k IS NOT NULL] | +| | ]] | +| physical_plan | SortPreservingMergeExec: [k@0 ASC NULLS LAST] | +| | SortExec: expr=[k@0 ASC NULLS LAST], preserve_partitioning=[true] | +| | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(k@0, k@1)], filter=val@0 > threshold@1, projection=[k@0, threshold@2] | +| | RepartitionExec: partitioning=Hash([k@0], REDACTED), input_partitions=REDACTED| +| | ProjectionExec: expr=[k@1 as k, val@2 as val] | +| | MergeScanExec: REDACTED +| | RepartitionExec: partitioning=Hash([k@1], REDACTED), input_partitions=REDACTED| +| | MergeScanExec: REDACTED +| | | ++---------------+---------------------------------------------------------------------------------------------------------------------------------+ DROP TABLE lateral_fact; diff --git a/tests/cases/standalone/optimizer/lateral_join_guard.sql b/tests/cases/standalone/optimizer/lateral_join_guard.sql index 20ed6b5cef5..95004756617 100644 --- a/tests/cases/standalone/optimizer/lateral_join_guard.sql +++ b/tests/cases/standalone/optimizer/lateral_join_guard.sql @@ -1,9 +1,7 @@ --- Document the current aliased SQL LATERAL limitation and guard the remote --- scan boundary. DataFusion's DecorrelateLateralJoin does not currently match --- the SubqueryAlias(Subquery) shape produced by `LATERAL (...) d`, so this query --- is still expected to fail physical planning with an outer_ref expression. The --- important regression assertion is that the remaining outer_ref predicate must --- NOT be advertised as a remote TableScan.partial_filters predicate. +-- Guard the remote scan boundary for an aliased SQL LATERAL query. +-- DataFusion 55 decorrelates this shape into an inner join. The resulting +-- scan-local IS NOT NULL predicates may be pushed down, but no outer_ref +-- expression may be advertised as a remote TableScan.partial_filters predicate. CREATE TABLE lateral_fact ( ts TIMESTAMP(3) TIME INDEX, @@ -32,6 +30,9 @@ ADMIN FLUSH_TABLE('lateral_dim'); -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE partitioning=Hash\(\[k@(\d+)\],\s*\d+\) partitioning=Hash([k@$1], REDACTED) +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE (input_partitions=REDACTED)(\s+)\| $1| EXPLAIN SELECT f.k, d.threshold FROM lateral_fact f, LATERAL ( diff --git a/tests/cases/standalone/optimizer/order_by.result b/tests/cases/standalone/optimizer/order_by.result index 06b06ae4421..020e1dc28ac 100644 --- a/tests/cases/standalone/optimizer/order_by.result +++ b/tests/cases/standalone/optimizer/order_by.result @@ -140,10 +140,9 @@ EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY t DESC LIMIT 5; | 0_| 0_|_CooperativeExec REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| -| 1_| 0_|_ProjectionExec: expr=[i@0 as i, alias_ts@1 as alias_ts] REDACTED -|_|_|_SortPreservingMergeExec: [t@2 DESC], fetch=5 REDACTED -|_|_|_SortExec: TopK(fetch=5), expr=[alias_ts@1 DESC], preserve_partitioning=[true], filter=[alias_ts@1 IS NULL OR alias_ts@1 > 2] REDACTED -|_|_|_ProjectionExec: expr=[i@0 as i, t@1 as alias_ts, t@1 as t] REDACTED +| 1_| 0_|_SortPreservingMergeExec: [alias_ts@1 DESC], fetch=5 REDACTED +|_|_|_ProjectionExec: expr=[i@0 as i, t@1 as alias_ts] REDACTED +|_|_|_SortExec: TopK(fetch=5), expr=[t@1 DESC], preserve_partitioning=[true], filter=[t@1 IS NULL OR t@1 > 2] REDACTED |_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED |_|_|_| |_|_| Total rows: 5_| @@ -164,8 +163,8 @@ EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY alias_ts DESC LIMI |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [alias_ts@1 DESC], fetch=5 REDACTED -|_|_|_SortExec: TopK(fetch=5), expr=[alias_ts@1 DESC], preserve_partitioning=[true], filter=[alias_ts@1 IS NULL OR alias_ts@1 > 2] REDACTED |_|_|_ProjectionExec: expr=[i@0 as i, t@1 as alias_ts] REDACTED +|_|_|_SortExec: TopK(fetch=5), expr=[t@1 DESC], preserve_partitioning=[true], filter=[t@1 IS NULL OR t@1 > 2] REDACTED |_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED |_|_|_| |_|_| Total rows: 5_| diff --git a/tests/cases/standalone/optimizer/rewrite_set_comparison.result b/tests/cases/standalone/optimizer/rewrite_set_comparison.result index 6d9e04458c7..f7463099de2 100644 --- a/tests/cases/standalone/optimizer/rewrite_set_comparison.result +++ b/tests/cases/standalone/optimizer/rewrite_set_comparison.result @@ -147,8 +147,10 @@ EXPLAIN SELECT v FROM sc_t WHERE v != ALL(SELECT v FROM sc_s) ORDER BY v; SELECT v FROM sc_t WHERE v != ALL(SELECT v FROM sc_s) ORDER BY v; -++ -++ ++---+ +| v | ++---+ ++---+ DROP TABLE sc_t; diff --git a/tests/cases/standalone/optimizer/windowed_sort.result b/tests/cases/standalone/optimizer/windowed_sort.result index 3be0dd1c3e4..9bffb3d58d3 100644 --- a/tests/cases/standalone/optimizer/windowed_sort.result +++ b/tests/cases/standalone/optimizer/windowed_sort.result @@ -71,8 +71,8 @@ ORDER BY |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [collect_time@0 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[collect_time@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_ProjectionExec: expr=[collect_time_utc@0 as collect_time, peak_current@1 as peak_current] REDACTED +|_|_|_SortExec: expr=[collect_time_utc@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED |_|_|_| |_|_| Total rows: 8_| @@ -120,8 +120,8 @@ ORDER BY |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [collect_time_0@0 ASC NULLS LAST] REDACTED -|_|_|_SortExec: expr=[collect_time_0@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_ProjectionExec: expr=[collect_time_utc@0 as collect_time_0, peak_current@1 as peak_current] REDACTED +|_|_|_SortExec: expr=[collect_time_utc@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED |_|_|_| |_|_| Total rows: 8_| @@ -172,9 +172,9 @@ ORDER BY |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [true_collect_time@0 DESC] REDACTED -|_|_|_WindowedSortExec: expr=true_collect_time@0 DESC num_ranges=REDACTED REDACTED -|_|_|_PartSortExec: expr=true_collect_time@0 DESC num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[collect_time@0 as true_collect_time, collect_time_utc@1 as collect_time, peak_current@2 as peak_current] REDACTED +|_|_|_WindowedSortExec: expr=collect_time@0 DESC num_ranges=REDACTED REDACTED +|_|_|_PartSortExec: expr=collect_time@0 DESC num_ranges=REDACTED REDACTED |_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED |_|_|_| |_|_| Total rows: 8_| @@ -225,9 +225,9 @@ ORDER BY |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [true_collect_time@1 DESC] REDACTED -|_|_|_WindowedSortExec: expr=true_collect_time@1 DESC num_ranges=REDACTED REDACTED -|_|_|_PartSortExec: expr=true_collect_time@1 DESC num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[collect_time_utc@1 as collect_time, collect_time@0 as true_collect_time, peak_current@2 as peak_current] REDACTED +|_|_|_WindowedSortExec: expr=collect_time@0 DESC num_ranges=REDACTED REDACTED +|_|_|_PartSortExec: expr=collect_time@0 DESC num_ranges=REDACTED REDACTED |_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED |_|_|_| |_|_| Total rows: 8_| diff --git a/tests/cases/standalone/optimizer/windowed_sort_advance.result b/tests/cases/standalone/optimizer/windowed_sort_advance.result index c0be1bece92..46ff90a69f1 100644 --- a/tests/cases/standalone/optimizer/windowed_sort_advance.result +++ b/tests/cases/standalone/optimizer/windowed_sort_advance.result @@ -65,8 +65,8 @@ EXPLAIN ANALYZE select ts as ts, status, value from `a` where ts >= '2026-03-12T |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [ts@0 ASC NULLS LAST] REDACTED -|_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_ProjectionExec: expr=[ts@2 as ts, status@1 as status, value@0 as value] REDACTED +|_|_|_WindowedSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED |_|_|_| |_|_| Total rows: 10_| diff --git a/tests/cases/standalone/tql-explain-analyze/analyze.result b/tests/cases/standalone/tql-explain-analyze/analyze.result index cbbce8eaf1b..7c0b9b16e6b 100644 --- a/tests/cases/standalone/tql-explain-analyze/analyze.result +++ b/tests/cases/standalone/tql-explain-analyze/analyze.result @@ -268,8 +268,10 @@ Affected Rows: 0 TQL EVAL sum(test2); -++ -++ ++--------------------+---------------------------+ +| greptime_timestamp | sum(test2.greptime_value) | ++--------------------+---------------------------+ ++--------------------+---------------------------+ -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED @@ -285,9 +287,9 @@ TQL ANALYZE sum(test2); +-+-+-+ | 0_| 0_|_SortPreservingMergeExec: [greptime_timestamp@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[greptime_timestamp@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[sum(test2.greptime_value)] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[__sum_merge(__sum_state(test2.greptime_value)) as sum(test2.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[sum(test2.greptime_value)] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[__sum_merge(__sum_state(test2.greptime_value)) as sum(test2.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| diff --git a/tests/cases/standalone/tql-explain-analyze/tsid_column.result b/tests/cases/standalone/tql-explain-analyze/tsid_column.result index cf2deb76647..c761267bd5a 100644 --- a/tests/cases/standalone/tql-explain-analyze/tsid_column.result +++ b/tests/cases/standalone/tql-explain-analyze/tsid_column.result @@ -76,9 +76,9 @@ TQL ANALYZE (0, 10, '5s') sum by (job, instance) (tsid_metric); |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [job@0 ASC NULLS LAST, instance@1 ASC NULLS LAST, ts@2 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[job@0 ASC NULLS LAST, instance@1 ASC NULLS LAST, ts@2 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[job@0 as job, instance@1 as instance, ts@2 as ts], aggr=[sum(tsid_metric.val), __tsid] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[job@0 as job, instance@1 as instance, ts@2 as ts], aggr=[sum(tsid_metric.val), first_value(tsid_metric.__tsid) as __tsid] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[job@2 as job, instance@1 as instance, ts@4 as ts], aggr=[sum(tsid_metric.val), __tsid] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[job@2 as job, instance@1 as instance, ts@4 as ts], aggr=[sum(tsid_metric.val), first_value(tsid_metric.__tsid) as __tsid] REDACTED |_|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[val@1 as val, instance@3 as instance, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED @@ -99,7 +99,7 @@ TQL ANALYZE (0, 10, '5s') sum(irate(tsid_metric[1h])) / scalar(count(count(tsid +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[ts@1 as ts, sum(prom_irate(ts_range,val))@2 / scalar(count(count(tsid_metric.val)))@0 as lhs.sum(prom_irate(ts_range,val)) / rhs.scalar(count(count(tsid_metric.val)))] REDACTED +| 0_| 0_|_ProjectionExec: expr=[ts@0 as ts, sum(prom_irate(ts_range,val))@1 / scalar(count(count(tsid_metric.val)))@2 as lhs.sum(prom_irate(ts_range,val)) / rhs.scalar(count(count(tsid_metric.val)))] REDACTED |_|_|_REDACTED |_|_|_ScalarCalculateExec: tags=[] REDACTED |_|_|_CooperativeExec REDACTED @@ -109,9 +109,9 @@ TQL ANALYZE (0, 10, '5s') sum(irate(tsid_metric[1h])) / scalar(count(count(tsid |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [ts@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[count(count(tsid_metric.val))] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[count(tsid_metric.ts) as count(count(tsid_metric.val))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[count(count(tsid_metric.val))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[count(tsid_metric.ts) as count(count(tsid_metric.val))] REDACTED |_|_|_ProjectionExec: expr=[ts@0 as ts] REDACTED |_|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts, job@1 as job], aggr=[] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED @@ -150,7 +150,7 @@ TQL ANALYZE (0, 10, '5s') sum(irate(tsid_metric[1h])) / scalar(count(sum(tsid_m +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[ts@1 as ts, sum(prom_irate(ts_range,val))@2 / scalar(count(sum(tsid_metric.val)))@0 as lhs.sum(prom_irate(ts_range,val)) / rhs.scalar(count(sum(tsid_metric.val)))] REDACTED +| 0_| 0_|_ProjectionExec: expr=[ts@0 as ts, sum(prom_irate(ts_range,val))@1 / scalar(count(sum(tsid_metric.val)))@2 as lhs.sum(prom_irate(ts_range,val)) / rhs.scalar(count(sum(tsid_metric.val)))] REDACTED |_|_|_REDACTED |_|_|_ScalarCalculateExec: tags=[] REDACTED |_|_|_CooperativeExec REDACTED @@ -160,15 +160,14 @@ TQL ANALYZE (0, 10, '5s') sum(irate(tsid_metric[1h])) / scalar(count(sum(tsid_m |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [ts@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[count(sum(tsid_metric.val))] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[count(tsid_metric.ts) as count(sum(tsid_metric.val))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[count(sum(tsid_metric.val))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts], aggr=[count(tsid_metric.ts) as count(sum(tsid_metric.val))] REDACTED |_|_|_ProjectionExec: expr=[ts@0 as ts] REDACTED |_|_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts, job@1 as job], aggr=[] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[ts@0 as ts, job@1 as job], aggr=[] REDACTED -|_|_|_ProjectionExec: expr=[ts@1 as ts, job@0 as job] REDACTED -|_|_|_FilterExec: val@0 IS NOT NULL, projection=[job@1, ts@2] REDACTED +|_|_|_FilterExec: val@0 IS NOT NULL, projection=[ts@2, job@1] REDACTED |_|_|_ProjectionExec: expr=[val@0 as val, job@1 as job, ts@3 as ts] REDACTED |_|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED diff --git a/tests/cases/standalone/tql/general_table.result b/tests/cases/standalone/tql/general_table.result new file mode 100644 index 00000000000..e4add60c60b --- /dev/null +++ b/tests/cases/standalone/tql/general_table.result @@ -0,0 +1,53 @@ +-- run PromQL on non-typical prometheus table schema +CREATE TABLE IF NOT EXISTS `cpu_usage` ( + `job` STRING NULL, + `value` DOUBLE NULL, + `ts` TIMESTAMP(9) NOT NULL, + TIME INDEX (`ts`), + PRIMARY KEY (`job`) +) +ENGINE=mito +WITH( + merge_mode = 'last_non_null' +); + +Affected Rows: 0 + +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (cpu_usage\.ts_range) ts_range +-- SQLNESS REPLACE (cpu_usage\.ts) ts +-- SQLNESS REPLACE (SeriesScan:.*|SeqScan:.*) ScanExec: REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +TQL analyze (0, 10, '1s') sum by(job) (irate(cpu_usage{job="fire"}[5s])) / 1e9; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_CooperativeExec REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_ProjectionExec: expr=[job@0 as job, ts@1 as ts, sum(prom_irate(ts_range,value))@2 / 1000000000 as sum(prom_irate(ts_range,value)) / Float64(1000000000)] REDACTED +|_|_|_RepartitionExec: partitioning=REDACTED +|_|_|_SortPreservingMergeExec: [job@0 ASC NULLS LAST, ts@1 ASC NULLS LAST] REDACTED +|_|_|_SortExec: expr=[job@0 ASC NULLS LAST, ts@1 ASC NULLS LAST], preserve_partitioning=[true] REDACTED +|_|_|_AggregateExec: mode=SinglePartitioned, gby=[job@2 as job, ts@0 as ts], aggr=[sum(prom_irate(ts_range,value))] REDACTED +|_|_|_FilterExec: prom_irate(ts_range,value)@1 IS NOT NULL REDACTED +|_|_|_ProjectionExec: expr=[ts@2 as ts, prom_irate(ts_range@3, value@0) as prom_irate(ts_range,value), job@1 as job] REDACTED +|_|_|_PromRangeManipulateExec: req range=[0..10000], interval=[1000], eval range=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesNormalizeExec: offset=[0], time index=[ts], filter NaN: [true] REDACTED +|_|_|_PromSeriesDivideExec: tags=["job"] REDACTED +|_|_|_ProjectionExec: expr=[value@1 as value, job@0 as job, ts@2 as ts] REDACTED +|_|_|_ScanExec: REDACTED +|_|_|_| +|_|_| Total rows: 0_| ++-+-+-+ + +drop table `cpu_usage`; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/tql/general_table.sql b/tests/cases/standalone/tql/general_table.sql new file mode 100644 index 00000000000..53dd5b4e7f8 --- /dev/null +++ b/tests/cases/standalone/tql/general_table.sql @@ -0,0 +1,26 @@ +-- run PromQL on non-typical prometheus table schema +CREATE TABLE IF NOT EXISTS `cpu_usage` ( + `job` STRING NULL, + `value` DOUBLE NULL, + `ts` TIMESTAMP(9) NOT NULL, + TIME INDEX (`ts`), + PRIMARY KEY (`job`) +) +ENGINE=mito +WITH( + merge_mode = 'last_non_null' +); + +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (cpu_usage\.ts_range) ts_range +-- SQLNESS REPLACE (cpu_usage\.ts) ts +-- SQLNESS REPLACE (SeriesScan:.*|SeqScan:.*) ScanExec: REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +TQL analyze (0, 10, '1s') sum by(job) (irate(cpu_usage{job="fire"}[5s])) / 1e9; + +drop table `cpu_usage`; diff --git a/tests/compatibility/cases/analyze_verbose_remote_metrics_extension/verify.result b/tests/compatibility/cases/analyze_verbose_remote_metrics_extension/verify.result index 45aac8d9c39..7394ca8faee 100644 --- a/tests/compatibility/cases/analyze_verbose_remote_metrics_extension/verify.result +++ b/tests/compatibility/cases/analyze_verbose_remote_metrics_extension/verify.result @@ -14,9 +14,9 @@ EXPLAIN ANALYZE VERBOSE SELECT count(*) FROM t_analyze_verbose_remote_metrics_ex | stage | node | plan_| +-+-+-+ | 0_| 0_|_ProjectionExec: expr=[count(Int64(1))@0 as count(*)] REDACTED -|_|_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Final, gby=[], aggr=[__count_merge(__count_state(t_analyze_verbose_remote_REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_merge(__count_state(t_analyze_verbose_remote_REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| diff --git a/tests/compatibility/cases/datafusion_substrait_view/case.toml b/tests/compatibility/cases/datafusion_substrait_view/case.toml new file mode 100644 index 00000000000..3794292b673 --- /dev/null +++ b/tests/compatibility/cases/datafusion_substrait_view/case.toml @@ -0,0 +1,8 @@ +name = "datafusion_substrait_view" +reason = "Verify persisted CREATE VIEW Substrait plans remain executable across DataFusion upgrades." +introduced_by = "DataFusion 54 upgrade" +topologies = ["distributed"] +from_range = ["*"] +to_range = ["*"] +features = ["view", "substrait", "table", "query"] +owner = "query" diff --git a/tests/compatibility/cases/datafusion_substrait_view/setup.sql b/tests/compatibility/cases/datafusion_substrait_view/setup.sql new file mode 100644 index 00000000000..43f9aa43f45 --- /dev/null +++ b/tests/compatibility/cases/datafusion_substrait_view/setup.sql @@ -0,0 +1,80 @@ +CREATE TABLE view_metrics ( + host STRING, + "region" STRING, + "value" INT, + event_ts TIMESTAMP TIME INDEX, + PRIMARY KEY (host) +); + +INSERT INTO view_metrics VALUES + ('alpha', 'east', 10, '2024-01-01 00:00:00'), + ('beta', 'east', NULL, '2024-01-01 00:01:00'), + ('gamma', 'west', 30, '2024-01-01 00:02:00'), + ('delta', 'west', 5, '2024-01-01 00:03:00'); + +CREATE TABLE view_labels ( + host STRING, + label STRING, + enabled BOOLEAN, + label_ts TIMESTAMP TIME INDEX, + PRIMARY KEY (host) +); + +INSERT INTO view_labels VALUES + ('alpha', 'production', TRUE, '2024-01-01 00:00:00'), + ('beta', 'staging', NULL, '2024-01-01 00:01:00'), + ('gamma', NULL, FALSE, '2024-01-01 00:02:00'), + ('orphan', 'unused', TRUE, '2024-01-01 00:03:00'); + +CREATE VIEW persisted_scan_filter AS +SELECT + host AS host_name, + value AS metric_value, + CAST(event_ts AS TIMESTAMP(3)) AS event_time +FROM view_metrics +WHERE value IS NOT NULL AND value >= 10; + +CREATE VIEW persisted_grouped_aggregate AS +SELECT + "region" AS region_name, + COUNT(*) AS row_count, + SUM(value) AS value_sum, + AVG(value) AS value_avg +FROM view_metrics +GROUP BY "region"; + +CREATE VIEW persisted_union AS +SELECT host AS host_name, "region" AS region_name +FROM view_metrics +WHERE "region" = 'east' +UNION +SELECT host AS host_name, "region" AS region_name +FROM view_metrics +WHERE host = 'beta' +UNION ALL +SELECT host AS host_name, "region" AS region_name +FROM view_metrics +WHERE host = 'beta'; + +CREATE VIEW persisted_inner_join AS +SELECT + metrics.host AS host_name, + metrics.value AS metric_value, + labels.label AS label_name +FROM view_metrics AS metrics +INNER JOIN view_labels AS labels ON metrics.host = labels.host +WHERE metrics.value IS NOT NULL OR labels.enabled IS NULL; + +CREATE VIEW persisted_window AS +SELECT + "region" AS region_name, + host AS host_name, + event_ts AS event_time, + ROW_NUMBER() OVER (PARTITION BY "region" ORDER BY event_ts) AS row_num +FROM view_metrics; + +CREATE VIEW persisted_date_format AS +SELECT + host AS host_name, + date_format(event_ts, '%Y-%m-%d') AS event_day +FROM view_metrics; diff --git a/tests/compatibility/cases/datafusion_substrait_view/verify.result b/tests/compatibility/cases/datafusion_substrait_view/verify.result new file mode 100644 index 00000000000..28dabc3fd2c --- /dev/null +++ b/tests/compatibility/cases/datafusion_substrait_view/verify.result @@ -0,0 +1,71 @@ +SELECT host_name, metric_value, event_time +FROM persisted_scan_filter +ORDER BY host_name; + ++-----------+--------------+---------------------+ +| host_name | metric_value | event_time | ++-----------+--------------+---------------------+ +| alpha | 10 | 2024-01-01T00:00:00 | +| gamma | 30 | 2024-01-01T00:02:00 | ++-----------+--------------+---------------------+ + +SELECT region_name, row_count, value_sum, value_avg +FROM persisted_grouped_aggregate +ORDER BY region_name; + ++-------------+-----------+-----------+-----------+ +| region_name | row_count | value_sum | value_avg | ++-------------+-----------+-----------+-----------+ +| east | 2 | 10 | 10.0 | +| west | 2 | 35 | 17.5 | ++-------------+-----------+-----------+-----------+ + +SELECT host_name, region_name +FROM persisted_union +ORDER BY host_name, region_name; + ++-----------+-------------+ +| host_name | region_name | ++-----------+-------------+ +| alpha | east | +| beta | east | +| beta | east | ++-----------+-------------+ + +SELECT host_name, metric_value, label_name +FROM persisted_inner_join +ORDER BY host_name; + ++-----------+--------------+------------+ +| host_name | metric_value | label_name | ++-----------+--------------+------------+ +| alpha | 10 | production | +| beta | | staging | +| gamma | 30 | | ++-----------+--------------+------------+ + +SELECT region_name, host_name, event_time, row_num +FROM persisted_window +ORDER BY region_name, row_num; + ++-------------+-----------+---------------------+---------+ +| region_name | host_name | event_time | row_num | ++-------------+-----------+---------------------+---------+ +| east | alpha | 2024-01-01T00:00:00 | 1 | +| east | beta | 2024-01-01T00:01:00 | 2 | +| west | gamma | 2024-01-01T00:02:00 | 1 | +| west | delta | 2024-01-01T00:03:00 | 2 | ++-------------+-----------+---------------------+---------+ + +SELECT host_name, event_day +FROM persisted_date_format +ORDER BY host_name; + ++-----------+------------+ +| host_name | event_day | ++-----------+------------+ +| alpha | 2024-01-01 | +| beta | 2024-01-01 | +| delta | 2024-01-01 | +| gamma | 2024-01-01 | ++-----------+------------+ diff --git a/tests/compatibility/cases/datafusion_substrait_view/verify.sql b/tests/compatibility/cases/datafusion_substrait_view/verify.sql new file mode 100644 index 00000000000..0f2f39de480 --- /dev/null +++ b/tests/compatibility/cases/datafusion_substrait_view/verify.sql @@ -0,0 +1,23 @@ +SELECT host_name, metric_value, event_time +FROM persisted_scan_filter +ORDER BY host_name; + +SELECT region_name, row_count, value_sum, value_avg +FROM persisted_grouped_aggregate +ORDER BY region_name; + +SELECT host_name, region_name +FROM persisted_union +ORDER BY host_name, region_name; + +SELECT host_name, metric_value, label_name +FROM persisted_inner_join +ORDER BY host_name; + +SELECT region_name, host_name, event_time, row_num +FROM persisted_window +ORDER BY region_name, row_num; + +SELECT host_name, event_day +FROM persisted_date_format +ORDER BY host_name; diff --git a/tests/compatibility/cases/downgrade_compatibility/case.toml b/tests/compatibility/cases/downgrade_compatibility/case.toml index ba5ee5065ed..ab0496e523d 100644 --- a/tests/compatibility/cases/downgrade_compatibility/case.toml +++ b/tests/compatibility/cases/downgrade_compatibility/case.toml @@ -1,8 +1,8 @@ name = "downgrade_compatibility" -reason = "Verify v1.1.4 can reopen a table whose region WAL options were written by the current binary, and can read all rows of an append-only table flushed and compacted with preserve_row_sequence enabled." +reason = "Verify v1.1.4 can reopen tables whose region WAL options or byte-stream-split (BSS) float SSTs were written by the current binary, and can read all rows of an append-only table flushed and compacted with preserve_row_sequence enabled." introduced_by = "fix: preserve legacy region WAL options format; preserve_row_sequence" topologies = ["distributed", "standalone"] from_range = [">=v1.2.0"] to_range = ["=v1.1.4"] -features = ["table", "wal", "downgrade", "append", "preserve_row_sequence"] +features = ["table", "wal", "downgrade", "append", "preserve_row_sequence", "sst", "float", "byte_stream_split"] owner = "metasrv" diff --git a/tests/compatibility/cases/downgrade_compatibility/setup.sql b/tests/compatibility/cases/downgrade_compatibility/setup.sql index a9415570d31..267466f9dfe 100644 --- a/tests/compatibility/cases/downgrade_compatibility/setup.sql +++ b/tests/compatibility/cases/downgrade_compatibility/setup.sql @@ -32,3 +32,18 @@ INSERT INTO t_preserve_sequence_downgrade VALUES ADMIN FLUSH_TABLE('t_preserve_sequence_downgrade'); ADMIN COMPACT_TABLE('t_preserve_sequence_downgrade'); + +CREATE TABLE t_sst_float_field_encoding_downgrade( + ts TIMESTAMP TIME INDEX, + host STRING PRIMARY KEY, + f FLOAT, + d DOUBLE +) ENGINE=mito +WITH('experimental_sst_float_field_encoding'='byte_stream_split'); + +INSERT INTO t_sst_float_field_encoding_downgrade VALUES +('2024-02-10 00:00:00+0000', 'host_a', 1.25, 10.5), +('2024-02-10 00:01:00+0000', 'host_b', -2.5, 20.25), +('2024-02-10 00:02:00+0000', 'host_c', NULL, NULL); + +ADMIN FLUSH_TABLE('t_sst_float_field_encoding_downgrade'); diff --git a/tests/compatibility/cases/downgrade_compatibility/verify.result b/tests/compatibility/cases/downgrade_compatibility/verify.result index 3c052392927..c350288d19d 100644 --- a/tests/compatibility/cases/downgrade_compatibility/verify.result +++ b/tests/compatibility/cases/downgrade_compatibility/verify.result @@ -17,3 +17,15 @@ SELECT ts, host, val FROM t_preserve_sequence_downgrade ORDER BY ts, host; | 2024-02-09T00:02:00 | host_a | 3 | | 2024-02-09T00:03:00 | host_c | 4 | +---------------------+--------+-----+ + +SELECT ts, host, f, d +FROM t_sst_float_field_encoding_downgrade +ORDER BY ts, host; + ++---------------------+--------+------+-------+ +| ts | host | f | d | ++---------------------+--------+------+-------+ +| 2024-02-10T00:00:00 | host_a | 1.25 | 10.5 | +| 2024-02-10T00:01:00 | host_b | -2.5 | 20.25 | +| 2024-02-10T00:02:00 | host_c | | | ++---------------------+--------+------+-------+ diff --git a/tests/compatibility/cases/downgrade_compatibility/verify.sql b/tests/compatibility/cases/downgrade_compatibility/verify.sql index 764968bf1b8..6b73c886117 100644 --- a/tests/compatibility/cases/downgrade_compatibility/verify.sql +++ b/tests/compatibility/cases/downgrade_compatibility/verify.sql @@ -1,3 +1,7 @@ SELECT ts, host, val FROM t_downgrade_compatibility ORDER BY ts, host; SELECT ts, host, val FROM t_preserve_sequence_downgrade ORDER BY ts, host; + +SELECT ts, host, f, d +FROM t_sst_float_field_encoding_downgrade +ORDER BY ts, host; diff --git a/tests/compatibility/cases/json2_unknown_type/case.toml b/tests/compatibility/cases/json2_unknown_type/case.toml new file mode 100644 index 00000000000..d28a3b4228d --- /dev/null +++ b/tests/compatibility/cases/json2_unknown_type/case.toml @@ -0,0 +1,8 @@ +name = "json2_unknown_type" +reason = "Reopen unknown JSON2 schemas across versions using the historical absent extension or the recursive JsonNativeType marker, then write and reopen JSON2 values. Run in both upgrade and downgrade directions." +introduced_by = "PR #9145" +topologies = ["distributed", "standalone"] +from_range = [">=v1.2.0"] +to_range = [">=v1.2.0"] +features = ["json2", "table", "write", "read", "downgrade"] +owner = "query" diff --git a/tests/compatibility/cases/json2_unknown_type/setup.sql b/tests/compatibility/cases/json2_unknown_type/setup.sql new file mode 100644 index 00000000000..8be5bef5d9f --- /dev/null +++ b/tests/compatibility/cases/json2_unknown_type/setup.sql @@ -0,0 +1,7 @@ +-- Keep the native JSON2 type unknown so schema encoding uses the Null form. +CREATE TABLE unknown_json2 (ts TIMESTAMP TIME INDEX, j JSON2) +WITH ('append_mode' = 'true', 'memtable.type' = 'bulk'); + +INSERT INTO unknown_json2 VALUES (0, NULL); + +ADMIN FLUSH_TABLE('unknown_json2'); diff --git a/tests/compatibility/cases/json2_unknown_type/verify.result b/tests/compatibility/cases/json2_unknown_type/verify.result new file mode 100644 index 00000000000..7c18af8e9ee --- /dev/null +++ b/tests/compatibility/cases/json2_unknown_type/verify.result @@ -0,0 +1,72 @@ +SELECT ts, j FROM unknown_json2 ORDER BY ts; + ++---------------------+---+ +| ts | j | ++---------------------+---+ +| 1970-01-01T00:00:00 | | ++---------------------+---+ + +-- Decode the previous version's unknown type and evolve it on a new write. +INSERT INTO unknown_json2 VALUES + (1, '{"items":[1,"two",null],"ok":true}'), + (2, NULL); + +Affected Rows: 2 + +-- Exercise schema creation with this version's encoding as well. +CREATE TABLE new_unknown_json2 (ts TIMESTAMP TIME INDEX, j JSON2) +WITH ('append_mode' = 'true', 'memtable.type' = 'bulk'); + +Affected Rows: 0 + +INSERT INTO new_unknown_json2 VALUES (0, NULL); + +Affected Rows: 1 + +ADMIN FLUSH_TABLE('unknown_json2'); + ++------------------------------------+ +| ADMIN FLUSH_TABLE('unknown_json2') | ++------------------------------------+ +| 0 | ++------------------------------------+ + +ADMIN FLUSH_TABLE('new_unknown_json2'); + ++----------------------------------------+ +| ADMIN FLUSH_TABLE('new_unknown_json2') | ++----------------------------------------+ +| 0 | ++----------------------------------------+ + +-- SQLNESS ARG restart=true +SELECT ts, j FROM new_unknown_json2 ORDER BY ts; + ++---------------------+---+ +| ts | j | ++---------------------+---+ +| 1970-01-01T00:00:00 | | ++---------------------+---+ + +INSERT INTO new_unknown_json2 VALUES (1, '{"items":[1,"two",null],"ok":true}'); + +Affected Rows: 1 + +SELECT ts, j FROM unknown_json2 ORDER BY ts; + ++-------------------------+------------------------------------+ +| ts | j | ++-------------------------+------------------------------------+ +| 1970-01-01T00:00:00 | | +| 1970-01-01T00:00:00.001 | {"items":[1,"two",null],"ok":true} | +| 1970-01-01T00:00:00.002 | | ++-------------------------+------------------------------------+ + +SELECT ts, j FROM new_unknown_json2 ORDER BY ts; + ++-------------------------+------------------------------------+ +| ts | j | ++-------------------------+------------------------------------+ +| 1970-01-01T00:00:00 | | +| 1970-01-01T00:00:00.001 | {"items":[1,"two",null],"ok":true} | ++-------------------------+------------------------------------+ diff --git a/tests/compatibility/cases/json2_unknown_type/verify.sql b/tests/compatibility/cases/json2_unknown_type/verify.sql new file mode 100644 index 00000000000..69a3ce69bad --- /dev/null +++ b/tests/compatibility/cases/json2_unknown_type/verify.sql @@ -0,0 +1,23 @@ +SELECT ts, j FROM unknown_json2 ORDER BY ts; + +-- Decode the previous version's unknown type and evolve it on a new write. +INSERT INTO unknown_json2 VALUES + (1, '{"items":[1,"two",null],"ok":true}'), + (2, NULL); + +-- Exercise schema creation with this version's encoding as well. +CREATE TABLE new_unknown_json2 (ts TIMESTAMP TIME INDEX, j JSON2) +WITH ('append_mode' = 'true', 'memtable.type' = 'bulk'); + +INSERT INTO new_unknown_json2 VALUES (0, NULL); + +ADMIN FLUSH_TABLE('unknown_json2'); +ADMIN FLUSH_TABLE('new_unknown_json2'); + +-- SQLNESS ARG restart=true +SELECT ts, j FROM new_unknown_json2 ORDER BY ts; + +INSERT INTO new_unknown_json2 VALUES (1, '{"items":[1,"two",null],"ok":true}'); + +SELECT ts, j FROM unknown_json2 ORDER BY ts; +SELECT ts, j FROM new_unknown_json2 ORDER BY ts; diff --git a/tests/compatibility/cases/sst_float_field_encoding_upgrade/case.toml b/tests/compatibility/cases/sst_float_field_encoding_upgrade/case.toml new file mode 100644 index 00000000000..4f008775bb2 --- /dev/null +++ b/tests/compatibility/cases/sst_float_field_encoding_upgrade/case.toml @@ -0,0 +1,8 @@ +name = "sst_float_field_encoding_upgrade" +reason = "Verify newer binaries can read default-encoded Mito SSTs containing FLOAT and DOUBLE fields written by older binaries." +introduced_by = "experimental_sst_float_field_encoding" +topologies = ["distributed", "standalone"] +from_range = ["*"] +to_range = ["*"] +features = ["table", "sst", "float", "upgrade"] +owner = "storage" diff --git a/tests/compatibility/cases/sst_float_field_encoding_upgrade/setup.sql b/tests/compatibility/cases/sst_float_field_encoding_upgrade/setup.sql new file mode 100644 index 00000000000..a84b4339dfc --- /dev/null +++ b/tests/compatibility/cases/sst_float_field_encoding_upgrade/setup.sql @@ -0,0 +1,13 @@ +CREATE TABLE t_sst_float_field_encoding_upgrade( + ts TIMESTAMP TIME INDEX, + host STRING PRIMARY KEY, + f FLOAT, + d DOUBLE +) ENGINE=mito; + +INSERT INTO t_sst_float_field_encoding_upgrade VALUES +('2024-02-10 00:00:00+0000', 'host_a', 1.25, 10.5), +('2024-02-10 00:01:00+0000', 'host_b', -2.5, 20.25), +('2024-02-10 00:02:00+0000', 'host_c', NULL, NULL); + +ADMIN FLUSH_TABLE('t_sst_float_field_encoding_upgrade'); diff --git a/tests/compatibility/cases/sst_float_field_encoding_upgrade/verify.result b/tests/compatibility/cases/sst_float_field_encoding_upgrade/verify.result new file mode 100644 index 00000000000..d6ae25d9747 --- /dev/null +++ b/tests/compatibility/cases/sst_float_field_encoding_upgrade/verify.result @@ -0,0 +1,11 @@ +SELECT ts, host, f, d +FROM t_sst_float_field_encoding_upgrade +ORDER BY ts, host; + ++---------------------+--------+------+-------+ +| ts | host | f | d | ++---------------------+--------+------+-------+ +| 2024-02-10T00:00:00 | host_a | 1.25 | 10.5 | +| 2024-02-10T00:01:00 | host_b | -2.5 | 20.25 | +| 2024-02-10T00:02:00 | host_c | | | ++---------------------+--------+------+-------+ diff --git a/tests/compatibility/cases/sst_float_field_encoding_upgrade/verify.sql b/tests/compatibility/cases/sst_float_field_encoding_upgrade/verify.sql new file mode 100644 index 00000000000..81822cb0ac8 --- /dev/null +++ b/tests/compatibility/cases/sst_float_field_encoding_upgrade/verify.sql @@ -0,0 +1,3 @@ +SELECT ts, host, f, d +FROM t_sst_float_field_encoding_upgrade +ORDER BY ts, host; diff --git a/tests/perf/README.md b/tests/perf/README.md index 6f7893ae668..0361f598a3b 100644 --- a/tests/perf/README.md +++ b/tests/perf/README.md @@ -86,15 +86,19 @@ sample values without changing label cardinality or the runner lifecycle: [scenario.remote_write.value] pattern = "quantized_signal" # linear, constant, modulo, unique, seeded_random, # run_length, quantized_signal, - # signal_with_sporadic_stalls, mixed_signal_repeated + # signal_with_sporadic_stalls, mixed_signal_repeated, + # bounded_mixed base = 0.0 step = 0.125 -cardinality = 4096 # buckets for modulo/seeded_random/quantized_signal/run_length -seed = 12345 # deterministic seeded_random input -run_length = 8 # adjacent samples per bucket for run_length/quantized_signal +cardinality = 4096 # buckets for modulo/seeded_random/quantized_signal/run_length; + # baseline buckets for bounded_mixed +seed = 12345 # deterministic seeded_random input and bounded_mixed series hash +run_length = 8 # adjacent samples per bucket for run_length/quantized_signal; + # interpolation interval for bounded_mixed stall_every = 100 # interval for signal_with_sporadic_stalls stall_length = 16 # held samples inside each stall interval -mixed_every = 5 # every Nth sample becomes the repeated base value +mixed_every = 5 # every Nth sample becomes the repeated base value; + # one fractional bounded_mixed series per N series ``` The default `linear` pattern preserves the helper's historical formula. Use @@ -103,12 +107,29 @@ data shapes, `run_length` for run-heavy low-cardinality series, `quantized_signa for signal-like values collapsed into a finite bucket set, `signal_with_sporadic_stalls` for mostly continuous signals with periodic flat spots, and `mixed_signal_repeated` for signal-plus-periodic-default mixtures. `unique` or high-cardinality buckets -still work for broad sample-value distributions. This is a generic sample-value -control for query/ingestion cases; it does not inspect or assert storage -encoding, Parquet footers, or storage policy choices. For chunked remote-write -ingestion, the runner passes the sample offset and total sample count to the -helper so non-linear value patterns use a stable global/per-series ordinal across -chunks. +still work for broad sample-value distributions. + +`bounded_mixed` is synthetic rather than an empirical workload. For series `s`, +`h = splitmix64(s ^ seed)` selects a span from `[10, 1_000, 100_000, 10_000_000]` +using its top two bits, multiplied by `1 + ((h >> 32) % 10)`. Its baseline is +`(h % max(cardinality, 1)) * span / 10`. Ranges can overlap. At local sample +`l = sample_offset + sample_idx`, it linearly interpolates hash-derived anchors +in `[-span + 1, span - 1]`, changing anchors every `max(run_length, 1)` samples. +It applies `base + step * signal`, then rounds to an integer. Every +`max(mixed_every, 1)`-th series adds a hash-derived fraction from `1/1000` to +`999/1000`. The 95/5 finite, nonintegral guarantee applies to the case's bounded +parameters, not arbitrary huge or nonfinite base/step inputs. With `base = 0` +and `step = 1`, output lies within `baseline ± span`, allowing one unit for +rounding and the fractional addition. The explicit case uses 1,000 equal-length +series and `mixed_every = 20`, giving exactly 95% integer-valued and 5% +fractional Float64 samples. Its ranges and temporal shape are synthetic design +parameters, not measured properties from the survey. + +This is a generic sample-value control for query/ingestion cases; it does not +inspect or assert storage encoding, Parquet footers, or storage policy choices. +For chunked remote-write ingestion, the runner passes the sample offset and total +sample count to the helper so non-linear value patterns use a stable global/per-series +ordinal across chunks. Case schema, value distribution defaults, storage defaults, and read-bench defaults are owned by Rust. The outer CI driver calls @@ -162,12 +183,15 @@ measures region scan cost, and query measurements still exercise the SQL/TQL frontend path. Treat all performance conclusions as release-only; debug builds are suitable only for command wiring and correctness checks. -`prepare-remote` creates the configured database if needed. The outer driver writes a per-target -frontend config enabling `[prom_store]` with metric engine storage and a non-zero -`pending_rows_flush_interval`, and validates that the logical metric table reaches -`series_count * samples_per_series` rows before trusting the query measurements. -Use `--fixture-generator /path/to/query_perf_fixture` to provide the Rust helper -to the outer driver. +`prepare-remote` creates the configured database if needed. A remote-write case +can provide `base_setup_sql` and `candidate_setup_sql` lists; each target runs its +own complete statements in order after database creation and before ingestion. +The outer driver writes a per-target frontend config enabling `[prom_store]` with +metric engine storage and a non-zero `pending_rows_flush_interval`, and validates +that the logical metric table reaches `series_count * samples_per_series` rows +before trusting the query measurements. Use +`--fixture-generator /path/to/query_perf_fixture` to provide the Rust helper to +the outer driver. Large manual remote-write cases can set `sample_chunk_size` to split ingestion by time. For each chunk, `prepare-remote` invokes `query_perf_fixture prom-remote-write` with the @@ -204,7 +228,9 @@ case for issue #7913. It writes 8192 series × 20160 samples through remote-writ in 1440-sample daily time chunks, flushing after each chunk before running 1d/7d/14d TQL selectors. It is not included in the default `all` case set because ingestion cost dominates routine CI validation. Commenting `/query-regression heavy` runs -only this case; `/query-regression` runs the eight routine default cases. Manual +only this case; `/query-regression` runs the nine routine default cases, including +`sst_float_bss`, `promql_instant_last_row_9034`, and +`mito_prefilter_all_match`. Manual workflow dispatch accepts the `heavy` token to select this case. The routine default set also includes @@ -371,6 +397,89 @@ uv run --no-project python .github/scripts/query-regression-run.py \ --work-dir /tmp/query-regression-work ``` +### SST float BSS comparison + +`tests/perf/query_cases/sst_float_bss/case.toml` is included in the routine +`all` default case group and compares a default empty physical metric table with +a candidate byte-stream-split (BSS) physical table. It writes 1,000 series × 4,320 samples +(4,320,000 rows) using synthetic `bounded_mixed` values: 95% integral series and +5% nonintegral series, with bounded per-series fluctuation rather than globally +unique values. This deliberately matches a 95/5 design; it is not an empirical +claim about any production population. A separate survey sample found 94.79% +integral values, but that observation does not make its values globally unique. +Current mixed-data evidence, including SST inspection, exact-bit row verification, +warmed endpoint SQL, and both warm-reader projections, is recorded in +[`query_cases/sst_float_bss/RESULTS.md`](query_cases/sst_float_bss/RESULTS.md). +Timing observations were collected while other builds saturated the shared host; +they are not performance acceptance evidence. Before rerunning timings, ensure +the host is idle (not merely this agent), record load throughout, and avoid +concurrent builds. CPU affinity alone does not isolate memory or I/O contention. +The historical unique-integer workload is preserved only in Git history and does +not apply to this case. +Both targets must use the exact same release `greptime` binary; only the +per-target table setup SQL differs. Run the existing driver from a checkout +containing that binary, with absolute paths and fresh data directories for every +run: + +```bash +REPO="$(pwd -P)" +RELEASE_GREPTIME="/absolute/path/to/release/greptime" +FIXTURE_GENERATOR="/absolute/path/to/release/query_perf_fixture" +RUNNER="/absolute/path/to/release/query_regression_runner" +WORK_DIR="/absolute/path/to/fresh/sst-float-bss-run-1" +cd "$REPO" +uv run --no-project python "$REPO/.github/scripts/query-regression-run.py" \ + --cases "$REPO/tests/perf/query_cases/sst_float_bss/case.toml" \ + --base-src "$REPO" \ + --candidate-src . \ + --base-bin "$RELEASE_GREPTIME" \ + --candidate-bin "$RELEASE_GREPTIME" \ + --fixture-generator "$FIXTURE_GENERATOR" \ + --runner "$RUNNER" \ + --work-dir "$WORK_DIR" \ + --summary-script "$REPO/.github/scripts/query-regression-summary.py" +``` + +Repeat the command three times with a different fresh absolute `WORK_DIR` each +run. In each `query-regression-report.json`, compare base and candidate +`targets[].storage_inspection.summary.summary.total_file_size`, and each query's +`targets[].measurements[].latency_ms_median`. Storage percentage is +`(candidate_total_file_size - base_total_file_size) / base_total_file_size * 100`; +query latency percentage is +`(candidate_latency_ms_median - base_latency_ms_median) / base_latency_ms_median * 100`. +The configured three warmups occur after the initial query validation and before +that query's 15 measured endpoint requests, so query latency is a warmed +frontend/cache measurement. The case restricts storage inspection to +`data/greptime/public`, excluding `greptime_private` SSTs. After the datanodes +stop, it also runs seven iterations of value-only `parquetbench` for all +inspected SST files and sequential `scanbench` over the corresponding regions, +with parallelism one. The three explicit flushes yielded six SSTs in the observed +mixed runs—three 1,105,920-row files and three 334,080-row files—because storage split each +flush; this is an observation, not a guaranteed file layout. Their per-run +output and aggregate `parquetbench_median_average_ms` and +`scanbench_median_average_ms` are under +`targets[].read_bench`; they are quiescent local-file read/scan diagnostics, not +warmed frontend-query latency measurements. Bench averages include iteration one; +no OS cache is dropped, and the driver runs base before candidate. + +Historical [PR #8548](https://github.com/GreptimeTeam/greptimedb/pull/8548) +reported storage savings alongside warm-read slowdowns for a different mixed +counter/gauge study. It used value-only and all-column projections, discarded the +first iteration, and alternated target order. This case reuses that reader +measurement approach, not its dataset or results. For the supplementary warm +comparison, reuse recorded commands against stopped data directories with eight +iterations, discard iteration one, and alternate target order for eight rounds. +For parquetbench, sum all per-file warm medians before taking the outer median; +for scanbench, take the outer median of whole-region warm medians. Keep +post-flush and post-compaction measurements separate. + +The case's `-5.0` storage target and `25` query-latency guardrail are experimental +acceptance targets, not observed-benefit claims; do not relax them if a run +fails. Its three periodic flushes and shared high TWCS trigger avoid the normal +four-file compaction trigger from confounding the layout. Check footer encodings +(BSS on candidate and no BSS on base) and data equality from the artifacts rather +than through a new harness framework. + For a focused manual reproduction of the Mito prefilter all-match optimization, use the existing lifecycle command above with `--cases tests/perf/query_cases/mito_prefilter_all_match/case.toml`. Its four count probes require explicit result inspection rather than automatic validation: expect `262144`, `131072`, `16896`, and `16896` in query order. The default lifecycle retains caches, so the optimization remains exercised but its timing includes warm-cache effects; use an explicitly configured cold environment when a cold comparison is required. The Rust runner subcommands are also useful for focused diagnostics: @@ -400,8 +509,8 @@ parquetbench/scanbench` as the read-bench tool against each target's data direct The workflow runs when an allowlisted repository admin comments `/query-regression` on a non-draft PR. It does not rerun on pushes, -ready-for-review, or reopen events. `/query-regression` runs the eight routine -default cases, including `promql_instant_last_row_9034` and +ready-for-review, or reopen events. `/query-regression` runs the nine routine +default cases, including `sst_float_bss`, `promql_instant_last_row_9034`, and `mito_prefilter_all_match`; `/query-regression heavy` runs only the high-cardinality remote-write #7913 case. PR runs build base/candidate once and use `--allow-large-fixture`. Manual `workflow_dispatch` runs can pass `all`, diff --git a/tests/perf/query_cases/sst_float_bss/RESULTS.md b/tests/perf/query_cases/sst_float_bss/RESULTS.md new file mode 100644 index 00000000000..ba932d0e42a --- /dev/null +++ b/tests/perf/query_cases/sst_float_bss/RESULTS.md @@ -0,0 +1,145 @@ +# SST float BSS: mixed-data post-flush evidence + +This report replaces the earlier unique-integer experiment as the primary evidence +for `sst_float_bss`. The detailed historical numbers remain in Git history; they +apply only to that superseded workload and must not be used for the current mixed +case. + +> **Timing measurements are confounded.** After these runs, the user reported +> that other agents' builds saturated all 20 logical CPUs during measurement. +> No host-wide idle/load gate was enforced. The latency tables below are retained +> as raw observations only: neither improvements nor threshold failures establish +> BSS performance. SST byte counts and cross-target stored-row checks remain +> valid for the inspected dataset. Stable-host timing verification is pending. + +## Case and data shape + +- Generator/case revision: `149b49de5f3`. +- Base and candidate used the same release `greptime` binary. The only physical + table difference was `experimental_sst_float_field_encoding`: `default` versus + `byte_stream_split`; base used dictionary encoding and candidate used BSS + without dictionaries. This is not a dictionary-policy-held-constant comparison. +- Each of three fresh post-flush runs wrote 1,000 series × 4,320 samples at + 60-second intervals: 4,320,000 rows per target. Three explicit flushes produced + six SSTs per target: three × 1,105,920 rows and three × 334,080 rows, with 45 + `greptime_value` row groups total. This is observed layout, not a layout + guarantee. +- `bounded_mixed` is a synthetic, uncalibrated design: a series hash selects its + baseline and span band, and 60-sample anchor interpolation produces bounded + fluctuation. It does not represent a measured production distribution. +- Exactly 950 equal-length series are integer-only and 50 are fractional-only, + yielding 4,104,000 integer rows and 216,000 fractional rows per target. The + equal-length 1,000-series design deliberately makes both the series and row + shares 95%/5%. +- The 95% design is not an empirical claim. A [local preview survey project](http://192.168.50.85:8765/view/prometheus-metrics-186-project-preliminary) + observed 94.79% integral **samples**; it does not establish a global rate, + global uniqueness, or a 95% series distribution. + +## Initial SST inspection and value verification + +Inspection is limited to `data/greptime/public`; bytes exclude WAL, indexes, +metadata, and internal/private-table storage. Each of the three runs had the +same totals: + +| Measure | Default | BSS | BSS change | +| --- | ---: | ---: | ---: | +| Selected SST bytes | 20,443,131 | 8,884,743 | -56.5392% | +| `greptime_value` compressed bytes | 20,201,188 | 8,643,964 | -57.2106% | +| SST files / rows / value row groups | 6 / 4,320,000 / 45 | 6 / 4,320,000 / 45 | — | + +Footer inspection found `PLAIN`/`RLE`/`RLE_DICTIONARY` for the default value +column and `RLE`/`BYTE_STREAM_SPLIT` for BSS (both ZSTD level 1). The configured +storage threshold remains a 5% reduction; this repeated SST-size result passes +that threshold. It is post-flush evidence only, not a compaction result or a +general float-compression claim. + +For every base/candidate group in all three runs, rows were compared by the same +primary key and timestamp and their Float64 bits were exact matches: 950 +integer-only and 50 fractional-only series, 4,104,000 and 216,000 rows +respectively. The six canonical sorted-row SHA-256 results are identical: + +``` +48f87d7d01bc8a0ce9a9cdb3033f322e787784fcae61f1417ac25b05c4a40375 +``` + +This verifies exact cross-target stored rows, not independent reconstruction of +the generator output. All 1,000 series had both rises and falls, and every one +of the 36 inspected SSTs contained both integer-valued and fractional values; +these were not stored as two disjoint file sets. + +## Warmed endpoint SQL + +Each query used three warmups followed by 15 measured endpoint requests. Values +are median/p95 milliseconds; delta is `(BSS median - default median) / default +median × 100%`. The guardrail remains a maximum 25% candidate median regression and +was not changed for these runs. + +| Run | Query | Default median / p95 | BSS median / p95 | Delta | Guardrail | +| --- | --- | ---: | ---: | ---: | --- | +| mixed-1 | `sum_all_values` | 27.13 / 37.38 | 25.45 / 38.99 | -6.21% | pass | +| mixed-1 | `hourly_sum_values` | 118.78 / 136.63 | 130.93 / 185.26 | +10.23% | pass | +| mixed-2 | `sum_all_values` | 24.02 / 35.02 | 36.34 / 56.01 | +51.31% | **fail** | +| mixed-2 | `hourly_sum_values` | 95.58 / 120.40 | 85.45 / 148.03 | -10.60% | pass | +| mixed-3 | `sum_all_values` | 38.25 / 55.41 | 19.68 / 29.49 | -48.55% | pass | +| mixed-3 | `hourly_sum_values` | 56.77 / 148.29 | 121.87 / 143.16 | +114.66% | **fail** | + +The two recorded threshold failures remain in the artifacts, but the saturated +shared host prevents attributing them to BSS. Passing entries likewise do not +establish performance acceptance. No CPU isolation or cache flush was used. + +## Stopped-directory warm readers + +For each of the three datasets, eight outer rounds alternate base/candidate order. +Each command runs eight iterations, discards the first, and takes the median of +iterations 2–8. Parquetbench reads every SST: sum the six per-file medians, then +take the median of the eight round totals. Scanbench scans the whole region and +reports the outer median of its eight warm medians. These are distinct metrics, +not interchangeable whole-query latencies. Deltas are ratios of separately +aggregated medians. Value projection selects `greptime_value`; all-column +parquetbench returns five physical columns, while scanbench uses `{}` (all +region columns). Projection order is fixed; target order alternates. + +| Dataset | Projection | Reader | Base ms | BSS ms | Change | +| --- | --- | --- | ---: | ---: | ---: | +| mixed-1 | value | parquetbench | 95.957 | 68.442 | -28.67% | +| mixed-1 | value | scanbench | 144.293 | 115.421 | -20.01% | +| mixed-1 | allcolumns | parquetbench | 176.620 | 103.761 | -41.25% | +| mixed-1 | allcolumns | scanbench | 200.881 | 202.660 | +0.89% | +| mixed-2 | value | parquetbench | 112.151 | 74.275 | -33.77% | +| mixed-2 | value | scanbench | 109.824 | 78.189 | -28.80% | +| mixed-2 | allcolumns | parquetbench | 141.944 | 102.681 | -27.66% | +| mixed-2 | allcolumns | scanbench | 211.414 | 163.261 | -22.78% | +| mixed-3 | value | parquetbench | 121.827 | 67.565 | -44.54% | +| mixed-3 | value | scanbench | 100.968 | 79.878 | -20.89% | +| mixed-3 | allcolumns | parquetbench | 113.984 | 94.361 | -17.22% | +| mixed-3 | allcolumns | scanbench | 193.101 | 164.175 | -14.98% | + +All 672 commands / 5,376 raw iterations returned the expected rows; timing, +projection configurations, parquet schemas and order metadata were checked. +Run 1 all-column scanbench differed by **+0.89%**, negligible relative to the +observed variation, not evidence of a regression. Its round medians ranged from +167.197–663.465 ms (base) and 133.339–557.352 ms (BSS). There is substantial +variability; these measurements do not establish a stable gain on every read +path. The reported concurrent CPU saturation affects apparent improvements as +well as slowdowns; no causal performance conclusion is drawn from these runs. + +## Scope and recorded checks + +This case compares default and BSS post-flush data only. It adds no benchmark +framework and no public artifact upload. Historical unique-integer results and +other mixed counter/gauge studies have different data and methods; they are +context only and do not apply to this workload. + +Recorded checks for this change set: 37 targeted Rust tests, 20 Python tooling +tests, and 15 plans passed. Full-workspace tests were not run. + +Local artifacts are under +`/mnt/nvme_rust/rust-targets/metric-bss-perf/experiments/mixed-{1,2,3}/`, +including `value-verification.json`, `verification.txt`, and each run's +`sst_float_bss/query-regression-report.json`. + +Warm artifacts are in sibling `warm-mixed-{1,2,3}/` directories; +`mixed-warm-comparison.json` contains the table and per-round ranges. +`mixed-file-composition.json` records integer/fractional counts for each SST. +The shared release `greptime` SHA-256 is +`2ce0ab1670cd4bdb0f60c87af3e1f494ee66c3c6ab83bce943a073638514eb2b`. diff --git a/tests/perf/query_cases/sst_float_bss/case.toml b/tests/perf/query_cases/sst_float_bss/case.toml new file mode 100644 index 00000000000..272e0eb4bc4 --- /dev/null +++ b/tests/perf/query_cases/sst_float_bss/case.toml @@ -0,0 +1,83 @@ +# Default all-group default-versus-BSS storage and warmed-query comparison. + +[case] +name = "sst_float_bss" +description = "Compare default and byte-stream-split floating-point SST bytes and warmed value-scanning query latency with bounded 95/5 per-series DOUBLE values" + +[scenario] +kind = "prom_remote_write_then_query" + +[scenario.remote_write] +database = "public" +metric = "sst_float_bss" +physical_table = "sst_float_bss_physical" +series_count = 1000 +samples_per_series = 4320 +sample_chunk_size = 1440 +flush_every_sample_chunks = 1 +start_unix_millis = 1_704_067_200_000 # 2024-01-01T00:00:00Z +step_millis = 60000 +chunk_series_count = 128 +timeout_seconds = 600 +visibility_timeout_seconds = 300 + +# Omit the default encoding from base setup because older binaries do not recognize it. +# The high shared TWCS trigger avoids compaction changing the three-flush layout. +base_setup_sql = [ + "CREATE TABLE sst_float_bss_physical (greptime_timestamp TIMESTAMP TIME INDEX, greptime_value DOUBLE) ENGINE=metric WITH ('physical_metric_table'='', 'compaction.twcs.trigger_file_num'='100')", +] +candidate_setup_sql = [ + "CREATE TABLE sst_float_bss_physical (greptime_timestamp TIMESTAMP TIME INDEX, greptime_value DOUBLE) ENGINE=metric WITH ('physical_metric_table'='', 'experimental_sst_float_field_encoding'='byte_stream_split', 'compaction.twcs.trigger_file_num'='100')", +] + +[scenario.remote_write.value] +# Synthetic, not empirical: 95% integral and 5% nonintegral series vary +# smoothly within deterministic per-series ranges; series are not a global monotonic counter. +pattern = "bounded_mixed" +base = 0 +step = 1 +cardinality = 1000 +seed = 42 +mixed_every = 20 +run_length = 60 + +[scenario.remote_write.prom_store] +pending_rows_flush_interval = "1s" +max_batch_rows = 100000 + +[scenario.remote_write.storage] +root_suffix = "data/greptime/public" +column = "greptime_value" +min_files = 1 +min_files_with_column = 1 +max_candidate_total_file_size_regression_pct = -5.0 + +[scenario.remote_write.read_bench] +enabled = true +parquetbench = true +scanbench = true +iterations = 7 +projection = ["greptime_value"] +parquet_reader = "direct" +scan_scanner = "seq" +parallelism = 1 + +[[scenario.queries]] +name = "sum_all_values" +kind = "sql" +query = "SELECT sum(greptime_value) FROM sst_float_bss" +warmup = 3 +iterations = 15 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 25 + +[[scenario.queries]] +name = "hourly_sum_values" +kind = "sql" +query = "SELECT date_bin(INTERVAL '1 hour', greptime_timestamp) AS time_window, sum(greptime_value) FROM sst_float_bss WHERE greptime_timestamp >= TIMESTAMP '2024-01-01 00:00:00' AND greptime_timestamp < TIMESTAMP '2024-01-04 00:00:00' GROUP BY time_window ORDER BY time_window" +warmup = 3 +iterations = 15 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 25 diff --git a/tests/perf/test_query_regression_case_selection.py b/tests/perf/test_query_regression_case_selection.py index 6ccf18a4423..04a542e23ad 100644 --- a/tests/perf/test_query_regression_case_selection.py +++ b/tests/perf/test_query_regression_case_selection.py @@ -16,6 +16,7 @@ """Coverage for query-regression case group selection.""" import importlib.util +import re import sys import unittest from pathlib import Path @@ -35,6 +36,7 @@ class QueryRegressionCaseSelectionTest(unittest.TestCase): runner.split_cases(["all"]), [ "tests/perf/query_cases/smoke_direct_sst/case.toml", + "tests/perf/query_cases/sst_float_bss/case.toml", "tests/perf/query_cases/prom_remote_write_seeded_random/case.toml", "tests/perf/query_cases/prom_remote_write_run_heavy/case.toml", "tests/perf/query_cases/prom_remote_write_mixed_every/case.toml", @@ -45,8 +47,8 @@ class QueryRegressionCaseSelectionTest(unittest.TestCase): ], ) - def test_implicit_selection_has_eight_routine_cases(self) -> None: - self.assertEqual(len(runner.DEFAULT_CASES), 8) + def test_implicit_selection_has_nine_routine_cases(self) -> None: + self.assertEqual(len(runner.DEFAULT_CASES), 9) self.assertEqual(runner.split_cases([]), runner.DEFAULT_CASES) def test_heavy_selects_only_remote_write_7913(self) -> None: @@ -59,6 +61,23 @@ class QueryRegressionCaseSelectionTest(unittest.TestCase): case = "tests/perf/query_cases/sql_topk_order_by/case.toml" self.assertEqual(runner.split_cases([case]), [case]) + def test_sst_float_bss_omits_base_encoding_for_older_binaries(self) -> None: + case_path = Path(__file__).parent / "query_cases/sst_float_bss/case.toml" + setup_sql = dict(re.findall( + r'(base|candidate)_setup_sql = \[\s*"([^"\n]+)"', + case_path.read_text(), + )) + base_setup = setup_sql["base"] + candidate_setup = setup_sql["candidate"] + candidate_option = "'experimental_sst_float_field_encoding'='byte_stream_split'" + + self.assertNotIn("experimental_sst_float_field_encoding", base_setup) + self.assertIn(candidate_option, candidate_setup) + self.assertEqual( + base_setup, + candidate_setup.replace(f", {candidate_option}", ""), + ) + if __name__ == "__main__": unittest.main() diff --git a/typos.toml b/typos.toml index bb14c40bab4..44dcda61687 100644 --- a/typos.toml +++ b/typos.toml @@ -12,6 +12,9 @@ consts = "consts" # We need to extend the word list to avoid typos in our codebase. Exceded = "Exceded" +[default.extend-identifiers] +allocatedp = "allocatedp" + [files] extend-exclude = [ "corrupted",