diff --git a/.github/actions/build-linux-artifacts/action.yml b/.github/actions/build-linux-artifacts/action.yml index c5212c28998..26028485061 100644 --- a/.github/actions/build-linux-artifacts/action.yml +++ b/.github/actions/build-linux-artifacts/action.yml @@ -57,6 +57,7 @@ runs: # Without GT_S3_*, every remote object-store case is skipped and the # release only ever exercises the local file backend. env: + RUST_MIN_STACK: 8388608 # 8 MiB, aligned with Rust CI tests. GT_S3_BUCKET: ${{ inputs.aws-ci-test-bucket }} GT_S3_REGION: ${{ inputs.aws-region }} GT_S3_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} diff --git a/.github/actions/build-macos-artifacts/action.yml b/.github/actions/build-macos-artifacts/action.yml index 2caefe1c382..2affb5b94f7 100644 --- a/.github/actions/build-macos-artifacts/action.yml +++ b/.github/actions/build-macos-artifacts/action.yml @@ -64,6 +64,7 @@ runs: shell: bash env: CARGO_BUILD_RUSTFLAGS: "-Clink-arg=-Wl,-ld_classic" + RUST_MIN_STACK: 8388608 # 8 MiB, aligned with Rust CI and release tests. SQLNESS_OPTS: "--preserve-state" run: | make test sqlness-test diff --git a/.github/actions/build-windows-artifacts/action.yml b/.github/actions/build-windows-artifacts/action.yml index 8fe077996d2..d25dd27e117 100644 --- a/.github/actions/build-windows-artifacts/action.yml +++ b/.github/actions/build-windows-artifacts/action.yml @@ -50,6 +50,7 @@ runs: run: make test sqlness-test env: RUST_BACKTRACE: 1 + RUST_MIN_STACK: 8388608 # 8 MiB, aligned with Rust CI tests. SQLNESS_OPTS: "--preserve-state" - name: Upload sqlness logs diff --git a/.github/scripts/previous-release-tag-test.py b/.github/scripts/previous-release-tag-test.py new file mode 100644 index 00000000000..5b1465eef93 --- /dev/null +++ b/.github/scripts/previous-release-tag-test.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +import subprocess +import sys +import unittest +from pathlib import Path + + +def load_module(): + path = Path(__file__).resolve().parent / "previous-release-tag.py" + spec = importlib.util.spec_from_file_location("previous_release_tag", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +TAGS = [ + "v1.1.0", + "v1.2.0", + "v1.2.1", + "v1.2.1^{}", + "v1.3.0-alpha.1", + "v1.3.0-beta.1", + "v1.3.0-beta.2", + "v1.2.0-nightly-20260706", +] + + +class PreviousReleaseTagTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.mod = load_module() + + def test_beta_uses_latest_older_stable_not_an_ancestor(self) -> None: + self.assertEqual(self.mod.previous_stable_tag("v1.3.0-beta.2", TAGS), "v1.2.1") + + def test_stable_release_skips_itself(self) -> None: + self.assertEqual(self.mod.previous_stable_tag("v1.2.1", TAGS), "v1.2.0") + + def test_same_line_stable_is_not_baseline_for_its_prerelease(self) -> None: + tags = TAGS + ["v1.3.0"] + self.assertEqual(self.mod.previous_stable_tag("v1.3.0-beta.2", tags), "v1.2.1") + self.assertEqual(self.mod.previous_stable_tag("v1.3.0", tags), "v1.2.1") + + def test_nightly_of_unreleased_version_uses_previous_stable(self) -> None: + self.assertEqual( + self.mod.previous_stable_tag("v1.3.0-nightly-20261013", TAGS), + "v1.2.1", + ) + + def test_no_older_stable_tag(self) -> None: + self.assertIsNone(self.mod.previous_stable_tag("v0.1.0", ["v0.1.0", "v1.2.1"])) + + def test_invalid_version(self) -> None: + with self.assertRaises(ValueError): + self.mod.previous_stable_tag("nightly-20261013", TAGS) + + def test_unordered_tags_use_numeric_order(self) -> None: + forward = ["v1.2.1", "v1.9.0", "v1.10.0", "v0.10.0", "v1.9.9"] + backward = list(reversed(forward)) + for tags in (forward, backward): + self.assertEqual(self.mod.previous_stable_tag("v1.10.0", tags), "v1.9.9") + self.assertEqual(self.mod.previous_stable_tag("v1.10.0-rc.1", tags), "v1.9.9") + + def test_multi_digit_components_compare_as_integers(self) -> None: + tags = ["v1.9.0", "v1.10.0", "v0.9.10", "v0.10.0", "v2.9.15", "v2.10.0", "v10.2.0"] + self.assertEqual(self.mod.previous_stable_tag("v1.10.0", tags), "v1.9.0") + self.assertEqual(self.mod.previous_stable_tag("v0.10.0", tags), "v0.9.10") + self.assertEqual(self.mod.previous_stable_tag("v2.10.0", tags), "v2.9.15") + self.assertEqual(self.mod.previous_stable_tag("v10.2.0", ["v10.0.9", "v9.20.0"]), "v10.0.9") + + +class PreviousReleaseTagCliTest(unittest.TestCase): + SCRIPT = Path(__file__).resolve().parent / "previous-release-tag.py" + + def run_cli( + self, version: str, tags: list[str] | None = None, stdin: str | None = None + ) -> subprocess.CompletedProcess[str]: + command = [sys.executable, str(self.SCRIPT), "--version", version] + if tags is not None: + command.extend(tags) + return subprocess.run( + command, + input=stdin, + text=True, + capture_output=True, + check=False, + ) + + def test_stdin_selects_baseline_and_exits_zero(self) -> None: + completed = self.run_cli( + "v1.10.0", + stdin="v1.2.1\nv1.9.0\n\nv1.10.0\nv1.9.0^{}\n", + ) + self.assertEqual(completed.returncode, 0) + self.assertEqual(completed.stdout, "v1.9.0\n") + self.assertEqual(completed.stderr, "") + + def test_positional_tags_ignore_stdin(self) -> None: + completed = self.run_cli("v1.10.0", tags=["v1.9.0"], stdin="v1.9.1\n") + self.assertEqual(completed.returncode, 0) + self.assertEqual(completed.stdout, "v1.9.0\n") + + def test_no_older_tag_exits_zero_with_empty_stdout(self) -> None: + completed = self.run_cli("v0.1.0", stdin="v0.1.0\nv1.2.1\n") + self.assertEqual(completed.returncode, 0) + self.assertEqual(completed.stdout, "") + self.assertEqual(completed.stderr, "") + + def test_invalid_version_exits_one(self) -> None: + completed = self.run_cli("nightly", stdin="v1.2.1\n") + self.assertEqual(completed.returncode, 1) + self.assertEqual(completed.stdout, "") + self.assertIn("not a release version: nightly", completed.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/previous-release-tag.py b/.github/scripts/previous-release-tag.py new file mode 100644 index 00000000000..b8a8f42ef72 --- /dev/null +++ b/.github/scripts/previous-release-tag.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pick the query-regression baseline tag for a release. + +Compatibility tests do not use this tag. They read ``tests/compatibility/ci.toml``. + +Release branches diverge, so a tag on ``release/v1.3`` is not a descendant of +``v1.2.1``. Ancestry (``git tag --merged``) therefore skips the real previous +release. Compare version numbers instead: the baseline is the greatest +``vX.Y.Z`` tag whose version is strictly older than the release being built. +Prerelease and nightly tags are not baselines. +""" + +from __future__ import annotations + +import argparse +import re +import sys + +STABLE_TAG = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") +RELEASE_VERSION = re.compile(r"^v(\d+)\.(\d+)\.(\d+)(?:-.+)?$") + + +def _triple(match: re.Match[str]) -> tuple[int, int, int]: + return int(match.group(1)), int(match.group(2)), int(match.group(3)) + + +def release_version(version: str) -> tuple[int, int, int]: + match = RELEASE_VERSION.fullmatch(version.strip()) + if match is None: + raise ValueError(f"not a release version: {version}") + return _triple(match) + + +def previous_stable_tag(version: str, tags: list[str]) -> str | None: + """Return the newest stable tag older than ``version``, or None.""" + current = release_version(version) + best: tuple[int, int, int] | None = None + best_tag: str | None = None + for raw in tags: + tag = raw.strip() + if tag.endswith("^{}"): + tag = tag[: -len("^{}")] + match = STABLE_TAG.fullmatch(tag) + if match is None: + continue + triple = _triple(match) + if triple < current and (best is None or triple > best): + best = triple + best_tag = tag + return best_tag + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True, help="Release version, e.g. v1.3.0-beta.2") + parser.add_argument( + "tags", + nargs="*", + help="Candidate tags. Read stdin when omitted.", + ) + args = parser.parse_args(argv) + tags = list(args.tags) + if not tags and not sys.stdin.isatty(): + tags = sys.stdin.read().splitlines() + try: + tag = previous_stable_tag(args.version, tags) + except ValueError as exc: + print(exc, file=sys.stderr) + return 1 + if tag is None: + return 0 + print(tag) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ada296aeaaf..dff9032b6d5 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -82,6 +82,8 @@ jobs: version: "latest" - name: Run check-version script tests run: uv run --no-project python .github/scripts/check-version-test.py + - name: Run previous-release-tag script tests + run: python3 .github/scripts/previous-release-tag-test.py - name: Run enterprise license check tests run: python3 scripts/check-enterprise-license-test.py - name: Run fuzz orchestration script tests diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index ca705b4178b..f6158a8d7ad 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -157,6 +157,7 @@ jobs: env: CARGO_BUILD_RUSTFLAGS: "-C linker=lld-link" RUST_BACKTRACE: 1 + RUST_MIN_STACK: 8388608 # 8 MiB, aligned with Rust CI and release tests. CARGO_INCREMENTAL: 0 GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }} GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e2e634084e8..948391b2aa3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -415,11 +415,15 @@ jobs: REQUESTED_VALIDATION_POLICY: ${{ inputs.release_validation }} run: | set -euo pipefail - base_commit="${GITHUB_SHA}" - if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then - base_commit="${GITHUB_SHA}^" - fi - previous_release_tag="$(git tag --merged "${base_commit}" --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -n 1 || true)" + # Stable tags on older release branches are not ancestors of this + # commit. Pick the newest vX.Y.Z older than the version being released. + release_version="${{ needs.allocate-runners.outputs.version }}" + previous_release_tag="$( + git ls-remote --tags origin 'refs/tags/v*' \ + | awk '{print $2}' \ + | sed -e 's#^refs/tags/##' \ + | python3 .github/scripts/previous-release-tag.py --version "${release_version}" + )" if [[ -z "${previous_release_tag}" ]]; then previous_release_tag="main" fi diff --git a/Cargo.lock b/Cargo.lock index 632fdc9b017..eb993be2ef5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -230,7 +230,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" [[package]] name = "api" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "arrow-schema 59.2.0", "common-base", @@ -959,7 +959,7 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "auth" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "async-trait", @@ -1002,9 +1002,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" -version = "1.17.0" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -1013,14 +1013,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.41.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -1624,7 +1625,7 @@ dependencies = [ [[package]] name = "cache" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "catalog", "common-error", @@ -1690,7 +1691,7 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "catalog" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "arrow 59.2.0", @@ -2046,7 +2047,7 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "cli" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "async-stream", "async-trait", @@ -2101,7 +2102,7 @@ dependencies = [ [[package]] name = "client" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "arc-swap", @@ -2133,7 +2134,7 @@ dependencies = [ "serde_json", "snafu 0.8.6", "store-api", - "substrait 1.3.0-beta.2", + "substrait 1.3.0", "tokio", "tokio-stream", "tonic 0.14.2", @@ -2182,7 +2183,7 @@ dependencies = [ [[package]] name = "cmd" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "async-trait", @@ -2318,7 +2319,7 @@ dependencies = [ [[package]] name = "common-base" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "ahash 0.8.12", "anymap2", @@ -2340,7 +2341,7 @@ dependencies = [ [[package]] name = "common-batcher" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "dashmap", "futures", @@ -2349,14 +2350,14 @@ dependencies = [ [[package]] name = "common-catalog" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "const_format", ] [[package]] name = "common-config" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "common-base", "common-error", @@ -2379,7 +2380,7 @@ dependencies = [ [[package]] name = "common-datasource" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -2416,7 +2417,7 @@ dependencies = [ [[package]] name = "common-decimal" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "bigdecimal 0.4.8", "common-error", @@ -2429,7 +2430,7 @@ dependencies = [ [[package]] name = "common-error" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "common-macro", "http 1.5.0", @@ -2441,7 +2442,7 @@ dependencies = [ [[package]] name = "common-event-recorder" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "async-trait", @@ -2466,7 +2467,7 @@ dependencies = [ [[package]] name = "common-frontend" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "async-trait", @@ -2485,7 +2486,7 @@ dependencies = [ [[package]] name = "common-function" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "ahash 0.8.12", "api", @@ -2546,7 +2547,7 @@ dependencies = [ [[package]] name = "common-greptimedb-telemetry" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "async-trait", "common-runtime", @@ -2563,7 +2564,7 @@ dependencies = [ [[package]] name = "common-grpc" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "arrow-flight", @@ -2598,7 +2599,7 @@ dependencies = [ [[package]] name = "common-grpc-expr" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "common-catalog", @@ -2616,7 +2617,7 @@ dependencies = [ [[package]] name = "common-macro" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "greptime-proto", "once_cell", @@ -2627,7 +2628,7 @@ dependencies = [ [[package]] name = "common-mem-prof" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "anyhow", "common-error", @@ -2643,7 +2644,7 @@ dependencies = [ [[package]] name = "common-memory-manager" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "common-error", "common-macro", @@ -2655,7 +2656,7 @@ dependencies = [ [[package]] name = "common-meta" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "anymap2", "api", @@ -2683,6 +2684,7 @@ dependencies = [ "common-wal", "common-workload", "datafusion-common", + "datafusion-expr", "datatypes", "deadpool", "deadpool-postgres", @@ -2726,7 +2728,7 @@ dependencies = [ [[package]] name = "common-options" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "common-base", "common-grpc", @@ -2737,11 +2739,11 @@ dependencies = [ [[package]] name = "common-plugins" -version = "1.3.0-beta.2" +version = "1.3.0" [[package]] name = "common-pprof" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "common-error", "common-macro", @@ -2752,7 +2754,7 @@ dependencies = [ [[package]] name = "common-procedure" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "async-stream", @@ -2781,7 +2783,7 @@ dependencies = [ [[package]] name = "common-procedure-test" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "async-trait", "common-procedure", @@ -2790,7 +2792,7 @@ dependencies = [ [[package]] name = "common-query" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "async-trait", @@ -2820,7 +2822,7 @@ dependencies = [ [[package]] name = "common-recordbatch" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "arc-swap", "common-base", @@ -2845,7 +2847,7 @@ dependencies = [ [[package]] name = "common-runtime" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "async-trait", "catio", @@ -2873,7 +2875,7 @@ dependencies = [ [[package]] name = "common-session" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "serde", "strum 0.27.1", @@ -2881,7 +2883,7 @@ dependencies = [ [[package]] name = "common-sql" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "arrow-schema 59.2.0", "common-base", @@ -2900,7 +2902,7 @@ dependencies = [ [[package]] name = "common-stat" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "common-base", "common-runtime", @@ -2914,7 +2916,7 @@ dependencies = [ [[package]] name = "common-telemetry" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "backtrace", "common-base", @@ -2944,7 +2946,7 @@ dependencies = [ [[package]] name = "common-test-util" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "common-grpc", "common-query", @@ -2956,7 +2958,7 @@ dependencies = [ [[package]] name = "common-time" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "arrow 59.2.0", "chrono", @@ -2974,7 +2976,7 @@ dependencies = [ [[package]] name = "common-version" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "cargo-manifest", "const_format", @@ -2984,7 +2986,7 @@ dependencies = [ [[package]] name = "common-wal" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "common-base", "common-error", @@ -3007,7 +3009,7 @@ dependencies = [ [[package]] name = "common-workload" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "common-telemetry", "serde", @@ -3760,7 +3762,7 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "datafusion" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -3812,7 +3814,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3836,7 +3838,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3859,7 +3861,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "arrow-ipc 59.2.0", @@ -3885,7 +3887,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "futures", "log", @@ -3895,7 +3897,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "async-compression", @@ -3931,7 +3933,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "arrow-ipc 59.2.0", @@ -3955,7 +3957,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3978,7 +3980,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "async-trait", @@ -4001,7 +4003,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -4033,12 +4035,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" [[package]] name = "datafusion-execution" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "arrow-buffer 59.2.0", @@ -4063,7 +4065,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -4087,7 +4089,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4098,7 +4100,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "arrow-buffer 59.2.0", @@ -4129,7 +4131,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4149,7 +4151,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4160,7 +4162,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "arrow-ord 59.2.0", @@ -4184,7 +4186,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "async-trait", @@ -4199,7 +4201,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4215,7 +4217,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -4224,7 +4226,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "datafusion-doc", "quote", @@ -4234,7 +4236,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "chrono", @@ -4282,7 +4284,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -4305,7 +4307,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4319,7 +4321,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "chrono", @@ -4336,7 +4338,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4355,7 +4357,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "arrow-data 59.2.0", @@ -4392,7 +4394,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "datafusion-catalog", @@ -4418,7 +4420,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4428,7 +4430,7 @@ dependencies = [ [[package]] name = "datafusion-proto-models" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "datafusion-common", "datafusion-proto-common", @@ -4438,7 +4440,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4453,7 +4455,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow-schema 59.2.0", "async-trait", @@ -4467,7 +4469,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "arrow 59.2.0", "bigdecimal 0.4.8", @@ -4486,7 +4488,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=cc5534492d5408831395609cdafb6520c7d514b7#cc5534492d5408831395609cdafb6520c7d514b7" dependencies = [ "async-recursion", "async-trait", @@ -4504,7 +4506,7 @@ dependencies = [ [[package]] name = "datanode" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "arrow-flight", @@ -4577,7 +4579,7 @@ checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" [[package]] name = "datatypes" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "arrow 59.2.0", "arrow-array 59.2.0", @@ -5330,7 +5332,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file-engine" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "async-trait", @@ -5471,7 +5473,7 @@ checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8" [[package]] name = "flow" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "arrow 59.2.0", @@ -5523,7 +5525,7 @@ dependencies = [ "snafu 0.8.6", "sql", "store-api", - "substrait 1.3.0-beta.2", + "substrait 1.3.0", "table", "tokio", "tokio-stream", @@ -5631,7 +5633,7 @@ dependencies = [ [[package]] name = "frontend" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "arrow-flight", @@ -7026,7 +7028,7 @@ dependencies = [ [[package]] name = "index" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "ahash 0.8.12", "async-trait", @@ -8175,7 +8177,7 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "log-query" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "chrono", "common-error", @@ -8187,7 +8189,7 @@ dependencies = [ [[package]] name = "log-store" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "async-stream", "async-trait", @@ -8539,7 +8541,7 @@ dependencies = [ [[package]] name = "meta-client" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "async-trait", @@ -8568,7 +8570,7 @@ dependencies = [ [[package]] name = "meta-srv" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "async-trait", @@ -8659,7 +8661,7 @@ dependencies = [ [[package]] name = "metric-engine" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "aquamarine", @@ -8759,7 +8761,7 @@ dependencies = [ [[package]] name = "mito-codec" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "bytes", @@ -8784,7 +8786,7 @@ dependencies = [ [[package]] name = "mito2" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "aquamarine", @@ -9627,7 +9629,7 @@ dependencies = [ [[package]] name = "object-store" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "anyhow", "bytes", @@ -10172,7 +10174,7 @@ dependencies = [ [[package]] name = "operator" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "ahash 0.8.12", "api", @@ -10238,7 +10240,7 @@ dependencies = [ "sql", "sqlparser", "store-api", - "substrait 1.3.0-beta.2", + "substrait 1.3.0", "table", "tokio", "tokio-stream", @@ -10573,7 +10575,7 @@ checksum = "e3c406c9e2aa74554e662d2c2ee11cd3e73756988800be7e6f5eddb16fed4699" [[package]] name = "partition" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "common-base", @@ -10978,7 +10980,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pipeline" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "ahash 0.8.12", "api", @@ -11155,7 +11157,7 @@ dependencies = [ [[package]] name = "plugins" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "auth", "catalog", @@ -11253,8 +11255,7 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "pprof" version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187" +source = "git+https://github.com/tikv/pprof-rs?rev=01cff82dbe6fe110a707bf2b38d8ebb1d14a18f8#01cff82dbe6fe110a707bf2b38d8ebb1d14a18f8" dependencies = [ "aligned-vec", "backtrace", @@ -11498,7 +11499,7 @@ dependencies = [ [[package]] name = "promql" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "ahash 0.8.12", "async-trait", @@ -11915,7 +11916,7 @@ dependencies = [ [[package]] name = "puffin" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "async-compression", "async-trait", @@ -11967,7 +11968,7 @@ dependencies = [ [[package]] name = "query" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "ahash 0.8.12", "api", @@ -12034,7 +12035,7 @@ dependencies = [ "sql", "sqlparser", "store-api", - "substrait 1.3.0-beta.2", + "substrait 1.3.0", "table", "tokio", "toml 0.8.23", @@ -13136,9 +13137,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.28" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", @@ -13223,9 +13224,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -13677,7 +13678,7 @@ dependencies = [ [[package]] name = "servers" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "ahash 0.8.12", "api", @@ -13820,7 +13821,7 @@ dependencies = [ [[package]] name = "session" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "ahash 0.8.12", "api", @@ -14200,7 +14201,7 @@ dependencies = [ [[package]] name = "sql" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "arrow-buffer 59.2.0", @@ -14255,7 +14256,7 @@ dependencies = [ [[package]] name = "sqlness-runner" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "async-trait", "clap", @@ -14709,7 +14710,7 @@ dependencies = [ [[package]] name = "standalone" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "async-trait", @@ -14760,7 +14761,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "store-api" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "aquamarine", @@ -14953,7 +14954,7 @@ dependencies = [ [[package]] name = "substrait" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "async-trait", "bytes", @@ -15089,7 +15090,7 @@ dependencies = [ [[package]] name = "table" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "arc-swap", @@ -15369,7 +15370,7 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "tests-fuzz" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "arbitrary", "async-trait", @@ -15417,7 +15418,7 @@ dependencies = [ [[package]] name = "tests-integration" -version = "1.3.0-beta.2" +version = "1.3.0" dependencies = [ "api", "arrow-flight", @@ -15496,7 +15497,7 @@ dependencies = [ "sqlx 0.9.0", "standalone", "store-api", - "substrait 1.3.0-beta.2", + "substrait 1.3.0", "table", "tempfile", "time", diff --git a/Cargo.toml b/Cargo.toml index c6249bdabb8..3afea04c390 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.3.0-beta.2" +version = "1.3.0" edition = "2024" license = "Apache-2.0" @@ -237,7 +237,7 @@ rskafka = { git = "https://github.com/influxdata/rskafka.git", rev = "80f42eb15e rstest = "0.25" rstest_reuse = "0.7" rust_decimal = "1.33" -rustls = { version = "0.23.25", default-features = false } +rustls = { version = "0.23.46", default-features = false } ryu = "1.0" sea-query = "0.32" serde = { version = "1.0", features = ["derive"] } @@ -365,26 +365,30 @@ rev = "f743c3a5fe8c1f57363eac5866c356b7d91f9b39" [patch.crates-io] substrait = { git = "https://github.com/GreptimeTeam/substrait-rs.git", rev = "91ec978b0649417ad3da8390e7a515baec723b1b" } -datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-execution = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } -datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-execution = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } +datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "cc5534492d5408831395609cdafb6520c7d514b7" } datafusion-pg-catalog = { git = "https://github.com/datafusion-contrib/datafusion-postgres.git", rev = "ffb14a52b3d7c7489559812e1ec38096f225ce93" } arrow-pg = { git = "https://github.com/datafusion-contrib/datafusion-postgres.git", rev = "ffb14a52b3d7c7489559812e1ec38096f225ce93" } sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "9e9019bb1c7040ed956f654e39378dd42ab17884" } +# pprof 0.15.0 initializes framehop inside SIGPROF; pin the unreleased fix +# that initializes it before sampling. Remove once tikv/pprof-rs#282 is released. +pprof = { git = "https://github.com/tikv/pprof-rs", rev = "01cff82dbe6fe110a707bf2b38d8ebb1d14a18f8" } + # Temporary: use the GreptimeTeam fork of tikv-jemalloc-sys embedding # jemalloc 5.3.1 + backport of 54f22c83 ("Initialize TSD tcache before # enabling it"), fixing the TSD/tcache enable-before-init reentrancy diff --git a/Makefile b/Makefile index 7f92098262f..21c87cc3707 100644 --- a/Makefile +++ b/Makefile @@ -262,6 +262,7 @@ run-it-in-container: start-etcd ## Run integration tests in dev-builder. docker run --network=host \ -v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry -v ${CARGO_GIT_CACHE}:/root/.cargo/git -v /tmp:/tmp \ -e GT_S3_BUCKET -e GT_S3_ACCESS_KEY_ID -e GT_S3_ACCESS_KEY -e GT_S3_REGION \ + -e RUST_MIN_STACK \ -w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:${DEV_BUILDER_IMAGE_TAG} \ make test sqlness-test BUILD_JOBS=${BUILD_JOBS} diff --git a/src/catalog/src/information_extension.rs b/src/catalog/src/information_extension.rs index 9d55f9f7a1e..f22c169df13 100644 --- a/src/catalog/src/information_extension.rs +++ b/src/catalog/src/information_extension.rs @@ -310,3 +310,122 @@ impl InformationExtension for DistributedInformationExtension { )))) } } + +#[cfg(test)] +mod tests { + use common_meta::node_manager::{DatanodeManager, DatanodeRef}; + use common_meta::peer::Peer; + use datafusion::common::tree_node::TreeNode; + use datafusion::logical_expr::LogicalPlan; + use datafusion::physical_plan::sorts::sort::SortExec; + use meta_client::client::MetaClientBuilder; + use store_api::sst_entry::ManifestSstEntry; + use store_api::storage::ScanRequest; + + use super::*; + use crate::information_schema::DatanodeInspectKind; + + struct UnusedDatanodeManager; + + #[async_trait::async_trait] + impl DatanodeManager for UnusedDatanodeManager { + async fn datanode(&self, _node: &Peer) -> DatanodeRef { + panic!("plan tests must not contact datanodes") + } + } + + #[derive(Debug)] + struct ManifestScanProvider; + + #[async_trait::async_trait] + impl datafusion::catalog::TableProvider for ManifestScanProvider { + fn schema(&self) -> ArrowSchemaRef { + ManifestSstEntry::schema().arrow_schema().clone() + } + + fn table_type(&self) -> datafusion::logical_expr::TableType { + datafusion::logical_expr::TableType::View + } + + fn supports_filters_pushdown( + &self, + filters: &[&datafusion::logical_expr::Expr], + ) -> Result> { + Ok(vec![datafusion::logical_expr::TableProviderFilterPushDown::Inexact; filters.len()]) + } + + async fn scan( + &self, + _state: &dyn datafusion::catalog::Session, + projection: Option<&Vec>, + filters: &[datafusion::logical_expr::Expr], + limit: Option, + ) -> Result> { + let schema = ManifestSstEntry::schema(); + let schema = match projection { + Some(projection) => Arc::new(schema.try_project(projection).unwrap()), + None => schema, + }; + Ok(Arc::new(DistributedInspectExec::new( + DatanodeInspectRequest { + kind: DatanodeInspectKind::SstManifest, + scan: ScanRequest { + projection: projection.cloned(), + filters: filters.to_vec(), + limit, + ..Default::default() + }, + }, + schema, + Arc::new(MetaClientBuilder::new(0, api::v1::meta::Role::Frontend).build()), + Arc::new(UnusedDatanodeManager), + ))) + } + } + + #[tokio::test] + async fn test_sql_manifest_sort_stays_on_frontend() { + let ctx = datafusion::prelude::SessionContext::new(); + ctx.register_table("ssts_manifest", Arc::new(ManifestScanProvider)) + .unwrap(); + for sql in [ + "SELECT region_id, file_id, min_ts, max_ts, file_size FROM ssts_manifest WHERE region_id = 42 ORDER BY min_ts LIMIT 100", + "SELECT file_id FROM ssts_manifest WHERE region_id = 42 ORDER BY max_ts DESC LIMIT 10 OFFSET 3", + "SELECT min_ts AS ts, file_id FROM ssts_manifest WHERE region_id = 42 ORDER BY ts DESC NULLS FIRST LIMIT 10", + "SELECT file_id FROM ssts_manifest WHERE region_id = 42 ORDER BY min_ts, max_ts DESC LIMIT 10", + ] { + let plan = ctx + .sql(sql) + .await + .unwrap() + .create_physical_plan() + .await + .unwrap(); + let mut found_scan = false; + let mut found_sort = false; + plan.apply(|plan| { + found_sort |= plan.is::(); + if let Some(scan) = plan.downcast_ref::() { + found_scan = true; + // All matching rows must reach the frontend global sort. + assert_eq!(scan.request.scan.limit, None); + let remote = scan.request.clone().build_plan().unwrap(); + remote.apply(|node| { + assert!(!matches!( + node, + LogicalPlan::Sort(_) | LogicalPlan::Limit(_) + )); + Ok(TreeNodeRecursion::Continue) + })?; + } + Ok(TreeNodeRecursion::Continue) + }) + .unwrap(); + assert!( + found_scan && found_sort, + "{}", + datafusion::physical_plan::displayable(plan.as_ref()).indent(true) + ); + } + } +} diff --git a/src/catalog/src/system_schema/information_schema/ssts.rs b/src/catalog/src/system_schema/information_schema/ssts.rs index 9006ab6fed9..ace6a87bce5 100644 --- a/src/catalog/src/system_schema/information_schema/ssts.rs +++ b/src/catalog/src/system_schema/information_schema/ssts.rs @@ -349,6 +349,13 @@ mod tests { .unwrap() .unwrap(); + let filter = + datafusion::logical_expr::col("region_id").eq(datafusion::logical_expr::lit(42_u64)); + assert_eq!( + table.supports_filters_pushdown(&[&filter]).unwrap(), + vec![table::metadata::FilterPushDownType::Inexact], + ); + let request = ScanRequest { projection: Some(vec![0]), ..Default::default() diff --git a/src/common/meta/Cargo.toml b/src/common/meta/Cargo.toml index 02db875760e..d235f6e9ad1 100644 --- a/src/common/meta/Cargo.toml +++ b/src/common/meta/Cargo.toml @@ -46,6 +46,7 @@ common-time.workspace = true common-wal.workspace = true common-workload.workspace = true datafusion-common.workspace = true +datafusion-expr.workspace = true datatypes.workspace = true deadpool = { workspace = true, optional = true } deadpool-postgres = { workspace = true, optional = true } diff --git a/src/common/meta/src/ddl/alter_table.rs b/src/common/meta/src/ddl/alter_table.rs index 2b4ac1c3655..232191def88 100644 --- a/src/common/meta/src/ddl/alter_table.rs +++ b/src/common/meta/src/ddl/alter_table.rs @@ -14,8 +14,11 @@ mod executor; mod metadata; +mod partition_compat; mod region_request; +pub use partition_compat::validate_region_partition_bounds; + use std::collections::HashSet; use std::vec; @@ -147,6 +150,10 @@ impl AlterTableProcedure { )?; self.new_table_info = Some(new_table_info); + // Reject schema changes that the persisted partition expressions of some + // region cannot survive, before any region is altered. + self.validate_region_partition_exprs().await?; + // Safety: Checked in `AlterTableProcedure::new`. let alter_kind = self.data.task.alter_table.kind.as_ref().unwrap(); if sets_skip_wal(alter_kind) { @@ -241,6 +248,44 @@ impl AlterTableProcedure { Ok(Status::executing(true)) } + /// Validates the persisted partition expressions of every region against the + /// schema proposed by this ALTER TABLE, so an ALTER that would break a region's + /// partition predicate is rejected before any region is modified. + async fn validate_region_partition_exprs(&self) -> Result<()> { + // Safety: filled in `fill_table_info`. + let old_schema = &self + .data + .table_info_value + .as_ref() + .unwrap() + .table_info + .meta + .schema; + // Safety: set right after the new table info is built in `on_prepare`. + let new_schema = &self.new_table_info.as_ref().unwrap().meta.schema; + let changed_columns = partition_compat::changed_column_types(old_schema, new_schema); + if changed_columns.is_empty() { + return Ok(()); + } + + let (physical_table_id, physical_table_route) = self + .context + .table_metadata_manager + .table_route_manager() + .get_physical_table_route(self.data.table_id()) + .await?; + if physical_table_id != self.data.table_id() { + // Partition expressions are persisted on the regions of the physical + // table; the altered schema here belongs to a logical table. + return Ok(()); + } + + partition_compat::validate_region_partition_bounds( + &changed_columns, + &physical_table_route.region_routes, + ) + } + fn table_poison_key(&self) -> PoisonKey { table_poison_key(self.data.table_id()) } diff --git a/src/common/meta/src/ddl/alter_table/partition_compat.rs b/src/common/meta/src/ddl/alter_table/partition_compat.rs new file mode 100644 index 00000000000..9fc94004d5c --- /dev/null +++ b/src/common/meta/src/ddl/alter_table/partition_compat.rs @@ -0,0 +1,565 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Preflight validation of persisted partition expressions against an ALTER TABLE schema change. +//! +//! Persisted partition bounds keep the types they were written with. A column whose +//! type changes while a partition expression references it breaks the region on both +//! paths that consume the bounds: +//! +//! * On scan, the region coerces the bound literal and the new column type to a +//! common comparison type while building the partition predicate. If no common type +//! exists, or the bound cannot be cast to it (a classic example is a millisecond +//! timestamp bound that overflows when the time index column is widened to +//! nanoseconds), every scan of that region fails at that point. +//! * On write, the region routes rows through `MultiDimPartitionRule::find_region`, +//! which compares the persisted bound against row values directly (`Value::cmp`): a +//! bound whose variant no longer matches the column's panics on ordered comparisons +//! and misroutes silently on equality. Only a timestamp unit widening keeps the two +//! comparable — `Timestamp` comparisons convert units before comparing — so every +//! other type change of a referenced column is rejected outright, and the widening +//! keeps the scan check below. +//! +//! Such an ALTER must be rejected before it is submitted to any region, otherwise +//! already-altered regions keep failing while the rest of the table is untouched. +//! +//! The `partition` crate depends on `common-meta`, so this crate cannot depend on +//! `partition` back and cannot reuse [`partition::expr::PartitionExpr`]. The `Persisted*` +//! types below mirror the JSON representation of `partition::partition::PartitionBound` and +//! `partition::expr::PartitionExpr` and must be kept in sync with `partition/src/partition.rs` +//! and `partition/src/expr.rs`; round-trip tests in `partition` run real serialized +//! expressions through this validator so drift fails in CI. The compatibility check itself +//! deliberately repeats the same conversion (value -> [`ScalarValue`]) and the same +//! comparison coercion the region performs when it builds the partition predicate +//! (`ExprSimplifier::coerce` / [`comparison_coercion`]): both sides are cast to the common +//! comparison type, and the persisted bound must be castable to that type with +//! [`ScalarValue::cast_to`]. A bound that would fail on scan is rejected here as well. + +use std::collections::HashMap; + +use common_telemetry::warn; +use datafusion_common::ScalarValue; +use datafusion_expr::type_coercion::binary::comparison_coercion; +use datatypes::data_type::{ConcreteDataType, DataType}; +use datatypes::schema::Schema; +use datatypes::value::{ + Value, duration_to_scalar_value, time_to_scalar_value, timestamp_to_scalar_value, +}; +use serde::Deserialize; +use store_api::storage::RegionId; + +use crate::error::{PartitionExprIncompatibleSnafu, Result}; +use crate::rpc::router::RegionRoute; + +/// Mirrors `partition::partition::PartitionBound`. +#[derive(Debug, Deserialize)] +enum PersistedPartitionBound { + /// Deprecated since 0.9.0. Only deserialized to keep unknown old formats parseable. + #[allow(dead_code)] + Value(Value), + /// Deprecated since 0.15.0. + MaxValue, + Expr(PersistedPartitionExpr), +} + +/// Mirrors `partition::expr::PartitionExpr`. +#[derive(Debug, Deserialize)] +struct PersistedPartitionExpr { + lhs: Box, + op: PersistedOp, + rhs: Box, +} + +/// Mirrors `partition::expr::Operand`. +#[derive(Debug, Deserialize)] +enum PersistedOperand { + Column(String), + Value(Value), + Expr(PersistedPartitionExpr), +} + +/// Mirrors `partition::expr::RestrictedOp`. +#[derive(Debug, Deserialize)] +enum PersistedOp { + Eq, + NotEq, + Lt, + LtEq, + Gt, + GtEq, + And, + Or, +} + +/// Returns the columns whose data type differs between `old_schema` and `new_schema`, +/// mapped to (old type, new type). Columns only present in one of the schemas (added or +/// dropped) are not included; dropping a column referenced by a partition expression is +/// already rejected by the table metadata validation. +pub(crate) fn changed_column_types( + old_schema: &Schema, + new_schema: &Schema, +) -> HashMap { + let mut changed = HashMap::new(); + for new_column in new_schema.column_schemas() { + let Some(old_column) = old_schema.column_schema_by_name(&new_column.name) else { + continue; + }; + if old_column.data_type != new_column.data_type { + changed.insert( + new_column.name.clone(), + (old_column.data_type.clone(), new_column.data_type.clone()), + ); + } + } + + changed +} + +/// Validates that the persisted partition bounds of every region are still representable +/// with the types of `changed_columns`. Returns an error naming the first offending +/// region without modifying anything. +/// +/// Public so that round-trip tests in the `partition` crate can run real serialized +/// partition expressions through the same mirror used here. +pub fn validate_region_partition_bounds( + changed_columns: &HashMap, + region_routes: &[RegionRoute], +) -> Result<()> { + if changed_columns.is_empty() { + return Ok(()); + } + + for route in region_routes { + let expr_json = route.region.partition_expr(); + if expr_json.is_empty() { + continue; + } + + let bound: PersistedPartitionBound = match serde_json::from_str(&expr_json) { + Ok(bound) => bound, + Err(err) => { + // A malformed persisted expression already fails every scan of this + // region. It is unrelated to the schema change, so don't block the ALTER. + warn!( + "Skipping partition expression validation for region {}, \ + the persisted expression is not parseable: {}, expr: {}", + route.region.id, err, expr_json + ); + continue; + } + }; + let PersistedPartitionBound::Expr(expr) = bound else { + // Deprecated bound formats don't carry expressions and are not used + // when the region builds its partition predicate. + continue; + }; + + validate_expr(&expr, changed_columns, route.region.id)?; + } + + Ok(()) +} + +fn validate_expr( + expr: &PersistedPartitionExpr, + changed_columns: &HashMap, + region_id: RegionId, +) -> Result<()> { + match expr.op { + PersistedOp::And | PersistedOp::Or => { + validate_operand(&expr.lhs, changed_columns, region_id)?; + validate_operand(&expr.rhs, changed_columns, region_id)?; + } + PersistedOp::Eq + | PersistedOp::NotEq + | PersistedOp::Lt + | PersistedOp::LtEq + | PersistedOp::Gt + | PersistedOp::GtEq => { + // Comparison between a column and a bound literal in either order. + validate_comparison(&expr.lhs, &expr.rhs, changed_columns, region_id)?; + validate_comparison(&expr.rhs, &expr.lhs, changed_columns, region_id)?; + } + } + + Ok(()) +} + +fn validate_operand( + operand: &PersistedOperand, + changed_columns: &HashMap, + region_id: RegionId, +) -> Result<()> { + match operand { + PersistedOperand::Expr(expr) => validate_expr(expr, changed_columns, region_id), + PersistedOperand::Column(_) | PersistedOperand::Value(_) => Ok(()), + } +} + +fn validate_comparison( + operand: &PersistedOperand, + other: &PersistedOperand, + changed_columns: &HashMap, + region_id: RegionId, +) -> Result<()> { + let (PersistedOperand::Column(column), PersistedOperand::Value(value)) = (operand, other) + else { + return Ok(()); + }; + let Some((old_type, target_type)) = changed_columns.get(column) else { + return Ok(()); + }; + // Values not representable as a partition bound (lists, structs, json) are + // rejected by the partition parser and cannot appear in a persisted expression. + let Some(scalar) = value_to_scalar_value(value) else { + return Ok(()); + }; + + // The write path (`MultiDimPartitionRule::find_region`) compares the persisted + // bound against row values directly: `Value::cmp` panics on mismatched variants + // and equality misroutes silently. Only a timestamp unit widening keeps the + // bound and the rows comparable (`Timestamp` comparisons convert units first), + // so every other type change of a referenced column is rejected here. + if !old_type.is_timestamp_unit_widening_to(target_type) { + warn!( + "Persisted partition bound would be compared against values of a different \ + type after the ALTER, region_id: {}, column: '{}', target_type: {}", + region_id, column, target_type + ); + return PartitionExprIncompatibleSnafu { + region_id, + column: column.clone(), + target_type: target_type.to_string(), + } + .fail(); + } + + // For the widening, mirror `ExprSimplifier::coerce` in `PredicateGroup::new`: + // the region casts both the column and the persisted bound to the common + // comparison type. The bound is not cast to the column type itself, so the + // widening is accepted as long as the bound survives the common type; a bound + // that cannot be cast to it — or a pair of types without a common comparison + // type — fails every scan. + let target_arrow_type = target_type.as_arrow_type(); + let Some(common_type) = comparison_coercion(&target_arrow_type, &scalar.data_type()) else { + warn!( + "Persisted partition bound has no common comparison type with the altered \ + column type, region_id: {}, column: '{}', target_type: {}", + region_id, column, target_type + ); + return PartitionExprIncompatibleSnafu { + region_id, + column: column.clone(), + target_type: target_type.to_string(), + } + .fail(); + }; + if scalar.data_type() != common_type + && let Err(error) = scalar.cast_to(&common_type) + { + warn!( + error; + "Persisted partition bound is not representable with the altered column type, \ + region_id: {}, column: '{}', target_type: {}", + region_id, + column, + target_type + ); + return PartitionExprIncompatibleSnafu { + region_id, + column: column.clone(), + target_type: target_type.to_string(), + } + .fail(); + } + + Ok(()) +} + +/// Converts a persisted bound value to a [`ScalarValue`], mirroring +/// `partition::expr::Operand::try_as_logical_expr`. Returns `None` for values that +/// cannot appear in a persisted partition expression. +fn value_to_scalar_value(value: &Value) -> Option { + Some(match value { + Value::Boolean(v) => ScalarValue::Boolean(Some(*v)), + Value::UInt8(v) => ScalarValue::UInt8(Some(*v)), + Value::UInt16(v) => ScalarValue::UInt16(Some(*v)), + Value::UInt32(v) => ScalarValue::UInt32(Some(*v)), + Value::UInt64(v) => ScalarValue::UInt64(Some(*v)), + Value::Int8(v) => ScalarValue::Int8(Some(*v)), + Value::Int16(v) => ScalarValue::Int16(Some(*v)), + Value::Int32(v) => ScalarValue::Int32(Some(*v)), + Value::Int64(v) => ScalarValue::Int64(Some(*v)), + Value::Float32(v) => ScalarValue::Float32(Some(v.0)), + Value::Float64(v) => ScalarValue::Float64(Some(v.0)), + Value::String(v) => ScalarValue::Utf8(Some(v.as_utf8().to_string())), + Value::Binary(v) => ScalarValue::Binary(Some(v.to_vec())), + Value::Date(v) => ScalarValue::Date32(Some(v.val())), + Value::Null => ScalarValue::Null, + Value::Timestamp(t) => timestamp_to_scalar_value(t.unit(), Some(t.value())), + Value::Time(t) => time_to_scalar_value(*t.unit(), Some(t.value())).ok()?, + Value::IntervalYearMonth(v) => ScalarValue::IntervalYearMonth(Some(v.to_i32())), + Value::IntervalDayTime(v) => ScalarValue::IntervalDayTime(Some((*v).into())), + Value::IntervalMonthDayNano(v) => ScalarValue::IntervalMonthDayNano(Some((*v).into())), + Value::Duration(d) => duration_to_scalar_value(d.unit(), Some(d.value())), + Value::Decimal128(d) => { + let (v, p, s) = d.to_scalar_value(); + ScalarValue::Decimal128(v, p, s) + } + Value::List(_) | Value::Struct(_) | Value::Json(_) => return None, + }) +} + +#[cfg(test)] +mod tests { + use datatypes::schema::ColumnSchema; + use store_api::storage::RegionId; + + use super::*; + use crate::rpc::router::Region; + + fn schema(ts_type: ConcreteDataType) -> Schema { + Schema::new(vec![ + ColumnSchema::new("ts", ts_type, false), + ColumnSchema::new("host", ConcreteDataType::string_datatype(), true), + ]) + } + + fn ts_bound(millis: i64) -> String { + format!( + r#"{{"Expr":{{"lhs":{{"Column":"ts"}},"op":"Lt","rhs":{{"Value":{{"Timestamp":{{"value":{millis},"unit":"Millisecond"}}}}}}}}}}"# + ) + } + + fn routes(exprs: &[&str]) -> Vec { + exprs + .iter() + .enumerate() + .map(|(index, expr)| RegionRoute { + region: Region { + id: RegionId::new(1024, index as u32 + 1), + partition_expr: expr.to_string(), + ..Default::default() + }, + ..Default::default() + }) + .collect() + } + + fn changed( + column: &str, + old_type: ConcreteDataType, + new_type: ConcreteDataType, + ) -> HashMap { + HashMap::from([(column.to_string(), (old_type, new_type))]) + } + + #[test] + fn test_changed_column_types() { + let old = schema(ConcreteDataType::timestamp_millisecond_datatype()); + let new = schema(ConcreteDataType::timestamp_nanosecond_datatype()); + let changed = changed_column_types(&old, &new); + assert_eq!(changed.len(), 1); + assert_eq!( + changed["ts"], + ( + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype() + ) + ); + + // Added and dropped columns are not type changes. + let mut new_columns = old.column_schemas().to_vec(); + new_columns.push(ColumnSchema::new( + "extra", + ConcreteDataType::int64_datatype(), + true, + )); + let new = Schema::new(new_columns); + assert!(changed_column_types(&old, &new).is_empty()); + } + + #[test] + fn test_overflowing_bound_is_rejected() { + // Year 3000 fits milliseconds but overflows nanoseconds. + let routes = routes(&[&ts_bound(32_503_680_000_000)]); + let err = validate_region_partition_bounds( + &changed( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype(), + ), + &routes, + ) + .unwrap_err(); + assert!( + err.to_string().contains("region 4398046511105"), + "unexpected error: {err}" + ); + assert!(err.to_string().contains("'ts'"), "unexpected error: {err}"); + } + + #[test] + fn test_representable_bound_is_accepted() { + // 2024-01-01T00:00:00Z fits both microseconds and nanoseconds. + let routes = routes(&[&ts_bound(1_704_067_200_000)]); + validate_region_partition_bounds( + &changed( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype(), + ), + &routes, + ) + .unwrap(); + validate_region_partition_bounds( + &changed( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_microsecond_datatype(), + ), + &routes, + ) + .unwrap(); + } + + #[test] + fn test_later_region_rejection_names_the_region() { + let routes = routes(&[&ts_bound(1_704_067_200_000), &ts_bound(32_503_680_000_000)]); + let err = validate_region_partition_bounds( + &changed( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype(), + ), + &routes, + ) + .unwrap_err(); + assert!( + err.to_string().contains("region 4398046511106"), + "unexpected error: {err}" + ); + } + + #[test] + fn test_unrelated_columns_are_skipped() { + let host_bound = + r#"{"Expr":{"lhs":{"Column":"host"},"op":"Eq","rhs":{"Value":{"String":"a"}}}}"#; + // The changed column is not referenced by this bound. + validate_region_partition_bounds( + &changed( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype(), + ), + &routes(&[host_bound]), + ) + .unwrap(); + // The referenced column did not change. + validate_region_partition_bounds( + &changed( + "host", + ConcreteDataType::int32_datatype(), + ConcreteDataType::string_datatype(), + ), + &routes(&[&ts_bound(32_503_680_000_000)]), + ) + .unwrap(); + } + + #[test] + fn test_nested_expression_is_validated() { + let expr = r#"{"Expr":{"lhs":{"Expr":{"lhs":{"Column":"ts"},"op":"GtEq","rhs":{"Value":{"Timestamp":{"value":0,"unit":"Millisecond"}}}}},"op":"And","rhs":{"Expr":{"lhs":{"Column":"ts"},"op":"Lt","rhs":{"Value":{"Timestamp":{"value":32503680000000,"unit":"Millisecond"}}}}}}}"#.to_string(); + let err = validate_region_partition_bounds( + &changed( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype(), + ), + &routes(&[&expr]), + ) + .unwrap_err(); + assert!(err.to_string().contains("'ts'"), "unexpected error: {err}"); + } + + #[test] + fn test_deprecated_and_empty_bounds_are_skipped() { + validate_region_partition_bounds( + &changed( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype(), + ), + &routes(&["", r#""MaxValue""#, r#"{"Value":{"UInt32":1}}"#, "not-json"]), + ) + .unwrap(); + } + + #[test] + fn test_field_narrowing_is_rejected_for_persisted_bounds() { + // BIGINT bound outside the INT range with the column narrowed to INT: the + // scan would coerce both sides to Int64, but the region's write path + // compares the persisted Int64 bound against Int32 rows directly and + // panics, so the ALTER must be rejected. + let bound = + r#"{"Expr":{"lhs":{"Column":"v"},"op":"Lt","rhs":{"Value":{"Int64":3000000000}}}}"#; + let err = validate_region_partition_bounds( + &changed( + "v", + ConcreteDataType::int64_datatype(), + ConcreteDataType::int32_datatype(), + ), + &routes(&[bound]), + ) + .unwrap_err(); + assert!(err.to_string().contains("'v'"), "unexpected error: {err}"); + } + + #[test] + fn test_non_timestamp_widening_is_rejected_for_persisted_bounds() { + // Widening INT to BIGINT still changes the variant rows carry: the + // region's write path compares the persisted Int32 bound against Int64 + // rows directly and panics, so the ALTER must be rejected as well. + let bound = r#"{"Expr":{"lhs":{"Column":"v"},"op":"Lt","rhs":{"Value":{"Int32":10}}}}"#; + let err = validate_region_partition_bounds( + &changed( + "v", + ConcreteDataType::int32_datatype(), + ConcreteDataType::int64_datatype(), + ), + &routes(&[bound]), + ) + .unwrap_err(); + assert!(err.to_string().contains("'v'"), "unexpected error: {err}"); + } + + #[test] + fn test_bound_without_common_comparison_type_is_rejected() { + // A boolean bound has no comparison coercion with a timestamp column, so + // even a widening whose bounds are normally representable fails every + // predicate build; the ALTER must be rejected. + let bound = + r#"{"Expr":{"lhs":{"Column":"ts"},"op":"Lt","rhs":{"Value":{"Boolean":true}}}}"#; + let err = validate_region_partition_bounds( + &changed( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype(), + ), + &routes(&[bound]), + ) + .unwrap_err(); + assert!(err.to_string().contains("'ts'"), "unexpected error: {err}"); + } +} diff --git a/src/common/meta/src/ddl/tests/alter_table.rs b/src/common/meta/src/ddl/tests/alter_table.rs index 8233d63b1e5..c67034b1da4 100644 --- a/src/common/meta/src/ddl/tests/alter_table.rs +++ b/src/common/meta/src/ddl/tests/alter_table.rs @@ -1478,3 +1478,176 @@ async fn test_annotation_alter_is_metadata_only() { } } } + +/// Builds the persisted partition expression `ts < ` in the format the +/// partition crate writes to region metadata. +fn test_ts_partition_bound(millis: i64) -> String { + format!( + r#"{{"Expr":{{"lhs":{{"Column":"ts"}},"op":"Lt","rhs":{{"Value":{{"Timestamp":{{"value":{millis},"unit":"Millisecond"}}}}}}}}}}"# + ) +} + +fn test_partitioned_table_route(table_id: u32, region_bounds: &[i64]) -> TableRouteValue { + TableRouteValue::physical( + region_bounds + .iter() + .enumerate() + .map(|(index, bound)| RegionRoute { + region: Region { + id: RegionId::new(table_id, index as u32 + 1), + partition_expr: test_ts_partition_bound(*bound), + ..Default::default() + }, + leader_peer: Some(Peer::empty(index as u64 + 1)), + ..Default::default() + }) + .collect(), + ) +} + +fn test_widen_ts_unit_task(table_name: &str, target_type: ColumnDataType) -> AlterTableTask { + AlterTableTask { + alter_table: AlterTableExpr { + catalog_name: DEFAULT_CATALOG_NAME.to_string(), + schema_name: DEFAULT_SCHEMA_NAME.to_string(), + table_name: table_name.to_string(), + kind: Some(Kind::ModifyColumnTypes(api::v1::ModifyColumnTypes { + modify_column_types: vec![api::v1::ModifyColumnType { + column_name: "ts".to_string(), + target_type: target_type as i32, + target_type_extension: None, + }], + })), + }, + } +} + +#[tokio::test] +async fn test_widen_ts_unit_rejects_incompatible_partition_bounds_before_altering() { + let node_manager = Arc::new(MockDatanodeManager::new(())); + let ddl_context = new_ddl_context(node_manager); + let table_name = "foo"; + let table_id = 1024; + let task = test_create_table_task(table_name, table_id); + + // Region 1 holds 2024-01-01, which fits nanoseconds. Region 2 holds 3000-01-01, + // which fits milliseconds but overflows nanoseconds. + let routes = test_partitioned_table_route(table_id, &[1_704_067_200_000, 32_503_680_000_000]); + ddl_context + .table_metadata_manager + .create_table_metadata(task.table_info.clone(), routes.clone(), HashMap::new()) + .await + .unwrap(); + + let task = test_widen_ts_unit_task(table_name, ColumnDataType::TimestampNanosecond); + let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap(); + let err = procedure.on_prepare().await.unwrap_err(); + + assert_matches!(err, Error::PartitionExprIncompatible { .. }); + assert_eq!(err.status_code(), StatusCode::InvalidArguments); + // The rejection names the offending region and column. + assert!( + err.to_string().contains("4398046511106"), + "unexpected error: {err}" + ); + assert!(err.to_string().contains("'ts'"), "unexpected error: {err}"); + + // The rejected ALTER left the table schema and the whole route unchanged. + let table_info = ddl_context + .table_metadata_manager + .table_info_manager() + .get(table_id) + .await + .unwrap() + .unwrap() + .into_inner() + .table_info; + assert_eq!( + table_info + .meta + .schema + .column_schema_by_name("ts") + .unwrap() + .data_type, + ConcreteDataType::timestamp_millisecond_datatype() + ); + let (physical_table_id, physical_table_route) = ddl_context + .table_metadata_manager + .table_route_manager() + .get_physical_table_route(table_id) + .await + .unwrap(); + assert_eq!(physical_table_id, table_id); + assert_eq!( + routes, + TableRouteValue::physical(physical_table_route.region_routes) + ); +} + +#[tokio::test] +async fn test_widen_ts_unit_with_compatible_partition_bounds_completes() { + let node_manager = Arc::new(MockDatanodeManager::new(())); + let ddl_context = new_ddl_context(node_manager); + let table_name = "foo"; + let table_id = 1024; + let task = test_create_table_task(table_name, table_id); + + // Both persisted bounds fit nanoseconds. + let routes = test_partitioned_table_route(table_id, &[0, 1_704_067_200_000]); + ddl_context + .table_metadata_manager + .create_table_metadata(task.table_info.clone(), routes, HashMap::new()) + .await + .unwrap(); + + let task = test_widen_ts_unit_task(table_name, ColumnDataType::TimestampNanosecond); + let procedure_id = ProcedureId::random(); + let provider = Arc::new(MockContextProvider::default()); + let mut procedure = AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap(); + procedure.on_prepare().await.unwrap(); + procedure + .submit_alter_region_requests(procedure_id, provider.as_ref()) + .await + .unwrap(); + procedure.mut_data().set_column_metadatas(vec![ + ColumnMetadata { + column_schema: ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_nanosecond_datatype(), + false, + ), + semantic_type: SemanticType::Timestamp, + column_id: 0, + }, + ColumnMetadata { + column_schema: ColumnSchema::new("host", ConcreteDataType::string_datatype(), false), + semantic_type: SemanticType::Tag, + column_id: 1, + }, + ColumnMetadata { + column_schema: ColumnSchema::new("cpu", ConcreteDataType::float64_datatype(), false), + semantic_type: SemanticType::Field, + column_id: 2, + }, + ]); + procedure.on_update_metadata().await.unwrap(); + + let table_info = ddl_context + .table_metadata_manager + .table_info_manager() + .get(table_id) + .await + .unwrap() + .unwrap() + .into_inner() + .table_info; + assert_eq!( + table_info + .meta + .schema + .column_schema_by_name("ts") + .unwrap() + .data_type, + ConcreteDataType::timestamp_nanosecond_datatype() + ); +} diff --git a/src/common/meta/src/error.rs b/src/common/meta/src/error.rs index fcc8c0cddde..e76e766294e 100644 --- a/src/common/meta/src/error.rs +++ b/src/common/meta/src/error.rs @@ -1161,6 +1161,20 @@ pub enum Error { actual_column_id: u32, }, + #[snafu(display( + "Partition expression of region {} is incompatible with the altered column '{}': the persisted bound can no longer be compared with the new column type {}", + region_id, + column, + target_type + ))] + PartitionExprIncompatible { + region_id: RegionId, + column: String, + target_type: String, + #[snafu(implicit)] + location: Location, + }, + #[cfg(feature = "enterprise")] #[snafu(display("Too large duration"))] TooLargeDuration { @@ -1284,7 +1298,8 @@ impl ErrorExt for Error { | InvalidFileName { .. } | InvalidFlowRequestBody { .. } | InvalidFilePath { .. } - | ConflictingSchemaOptions { .. } => StatusCode::InvalidArguments, + | ConflictingSchemaOptions { .. } + | PartitionExprIncompatible { .. } => StatusCode::InvalidArguments, #[cfg(feature = "enterprise")] MissingInterval { .. } | NegativeDuration { .. } | TooLargeDuration { .. } => { diff --git a/src/common/meta/src/key/flow.rs b/src/common/meta/src/key/flow.rs index 50c7e0ac38a..5ef0bbf39fb 100644 --- a/src/common/meta/src/key/flow.rs +++ b/src/common/meta/src/key/flow.rs @@ -19,7 +19,7 @@ pub mod flow_state; mod flownode_addr_helper; pub(crate) mod flownode_flow; pub(crate) mod table_flow; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use std::ops::Deref; use std::sync::Arc; @@ -42,7 +42,7 @@ pub use crate::key::flow::table_flow::{TableFlowManager, TableFlowManagerRef}; use crate::key::txn_helper::TxnOpGetResponseSet; use crate::key::{DeserializedValueWithBytes, FlowId, FlowPartitionId, MetadataKey}; use crate::kv_backend::KvBackendRef; -use crate::kv_backend::txn::Txn; +use crate::kv_backend::txn::{Txn, TxnOp}; use crate::rpc::store::BatchDeleteRequest; pub const FLOW_KEY_PREFIX: &str = "__flow"; @@ -134,6 +134,31 @@ impl<'a, T: MetadataKey<'a, T>> MetadataKey<'a, FlowScoped> for FlowScoped } } +/// Builds txn ops that replace the `old_keys` with the `new_kvs`. +/// +/// Only deletes the old keys absent from `new_kvs`: etcd rejects a txn that +/// writes the same key more than once ("duplicate key given in txn request"). +fn build_replace_ops( + old_keys: impl IntoIterator>, + new_kvs: Vec<(Vec, Vec)>, +) -> Vec { + let new_keys = new_kvs + .iter() + .map(|(key, _)| key.as_slice()) + .collect::>(); + let mut ops = old_keys + .into_iter() + .filter(|key| !new_keys.contains(key.as_slice())) + .map(TxnOp::Delete) + .collect::>(); + ops.extend( + new_kvs + .into_iter() + .map(|(key, value)| TxnOp::Put(key, value)), + ); + ops +} + pub type FlowMetadataManagerRef = Arc; /// The manager of metadata, provides ability to: @@ -456,6 +481,7 @@ mod tests { use futures::TryStreamExt; use table::metadata::TableId; use table::table_name::TableName; + use uuid::Uuid; use super::*; use crate::FlownodeId; @@ -463,8 +489,10 @@ mod tests { use crate::key::flow::table_flow::TableFlowKey; use crate::key::node_address::{NodeAddressKey, NodeAddressValue}; use crate::key::{FlowPartitionId, MetadataValue}; - use crate::kv_backend::KvBackend; + use crate::kv_backend::chroot::ChrootKvBackend; + use crate::kv_backend::etcd::EtcdStore; use crate::kv_backend::memory::MemoryKvBackend; + use crate::kv_backend::{KvBackend, KvBackendRef}; use crate::peer::Peer; use crate::rpc::store::PutRequest; @@ -1189,4 +1217,151 @@ mod tests { err ); } + + #[test] + fn test_build_replace_ops() { + let ops = build_replace_ops( + vec![b"a".to_vec(), b"b".to_vec()], + vec![ + (b"b".to_vec(), b"1".to_vec()), + (b"c".to_vec(), b"2".to_vec()), + ], + ); + assert_eq!( + ops, + vec![ + TxnOp::Delete(b"a".to_vec()), + TxnOp::Put(b"b".to_vec(), b"1".to_vec()), + TxnOp::Put(b"c".to_vec(), b"2".to_vec()), + ] + ); + } + + /// Returns an etcd backend isolated by a chroot if `GT_ETCD_ENDPOINTS` is set, + /// because only etcd rejects txns that write the same key more than once. + async fn build_kv_backend() -> KvBackendRef { + let endpoints = std::env::var("GT_ETCD_ENDPOINTS").unwrap_or_default(); + if endpoints.is_empty() { + return Arc::new(MemoryKvBackend::default()); + } + let endpoints = endpoints.split(',').collect::>(); + let backend = EtcdStore::with_endpoints(endpoints, 128).await.unwrap(); + let chroot = format!("test_update_flow_metadata/{}/", Uuid::new_v4()); + Arc::new(ChrootKvBackend::new(chroot.into(), backend)) + } + + async fn assert_flow_metadata( + manager: &FlowMetadataManager, + flow_id: FlowId, + flow_info: &FlowInfoValue, + ) { + let got = manager.flow_info_manager().get(flow_id).await.unwrap(); + assert_eq!(got.as_ref(), Some(flow_info)); + + let routes = manager.flow_route_manager().routes(flow_id).await.unwrap(); + let expected_routes = flow_info + .flownode_ids() + .iter() + .map(|(partition_id, flownode_id)| { + ( + FlowRouteKey::new(flow_id, *partition_id), + FlowRouteValue { + peer: Peer::empty(*flownode_id), + }, + ) + }) + .collect::>(); + assert_eq!(routes, expected_routes); + + for flownode_id in 1..=4 { + let flows = manager + .flownode_flow_manager() + .flows(flownode_id) + .try_collect::>() + .await + .unwrap(); + let expected = flow_info + .flownode_ids() + .iter() + .filter(|(_, id)| **id == flownode_id) + .map(|(partition_id, _)| (flow_id, *partition_id)) + .collect::>(); + assert_eq!(flows, expected, "flownode {flownode_id}"); + } + + for table_id in 1024..=1027 { + let flows = manager + .table_flow_manager() + .flows(table_id) + .await + .unwrap() + .into_iter() + .map(|(key, _)| (key.flownode_id(), key.partition_id())) + .collect::>(); + let expected = if flow_info.source_table_ids().contains(&table_id) { + flow_info + .flownode_ids() + .iter() + .map(|(partition_id, flownode_id)| (*flownode_id, *partition_id)) + .collect() + } else { + vec![] + }; + assert_eq!(flows, expected, "table {table_id}"); + } + } + + #[tokio::test] + async fn test_update_flow_metadata_with_overlapping_keys() { + let flow_metadata_manager = FlowMetadataManager::new(build_kv_backend().await); + let flow_id = 10; + let to_routes = |flow_info: &FlowInfoValue| { + flow_info + .flownode_ids() + .iter() + .map(|(partition_id, flownode_id)| { + ( + *partition_id, + FlowRouteValue { + peer: Peer::empty(*flownode_id), + }, + ) + }) + .collect::>() + }; + + let mut current = + test_flow_info_value("flow", [(0, 1u64), (1, 2u64)].into(), vec![1024, 1025]); + flow_metadata_manager + .create_flow_metadata(flow_id, current.clone(), to_routes(¤t)) + .await + .unwrap(); + assert_flow_metadata(&flow_metadata_manager, flow_id, ¤t).await; + + let updates = [ + // Unchanged partitions, flownodes and source tables. + ([(0, 1u64), (1, 2u64)].into(), vec![1024, 1025]), + // Partially overlapping flownodes and source tables. + ([(0, 1u64), (1, 3u64)].into(), vec![1025, 1026]), + // Fully changed partitions, flownodes and source tables. + ([(2, 4u64)].into(), vec![1027]), + ]; + for (i, (flownode_ids, source_table_ids)) in updates.into_iter().enumerate() { + let mut new_flow_info = current.clone(); + new_flow_info.raw_sql = format!("update {i}"); + new_flow_info.flownode_ids = flownode_ids; + new_flow_info.source_table_ids = source_table_ids; + flow_metadata_manager + .update_flow_metadata( + flow_id, + &DeserializedValueWithBytes::from_inner(current.clone()), + &new_flow_info, + to_routes(&new_flow_info), + ) + .await + .unwrap(); + assert_flow_metadata(&flow_metadata_manager, flow_id, &new_flow_info).await; + current = new_flow_info; + } + } } diff --git a/src/common/meta/src/key/flow/flow_route.rs b/src/common/meta/src/key/flow/flow_route.rs index d402e1acef7..0cd62a3b909 100644 --- a/src/common/meta/src/key/flow/flow_route.rs +++ b/src/common/meta/src/key/flow/flow_route.rs @@ -20,7 +20,7 @@ use snafu::OptionExt; use crate::error::{self, Result}; use crate::key::flow::flow_info::FlowInfoValue; -use crate::key::flow::{FlowScoped, flownode_addr_helper}; +use crate::key::flow::{FlowScoped, build_replace_ops, flownode_addr_helper}; use crate::key::node_address::NodeAddressKey; use crate::key::{ BytesAdapter, DeserializedValueWithBytes, FlowId, FlowPartitionId, MetadataKey, MetadataValue, @@ -216,25 +216,26 @@ impl FlowRouteManager { /// Builds a update flow routes transaction. /// /// Puts `__flow/route/{flow_id}/{partition_id}` keys. - /// Also removes `__flow/route/{flow_id}/{old_partition_id}` keys. + /// Also removes the `__flow/route/{flow_id}/{old_partition_id}` keys that are not put. pub(crate) fn build_update_txn>( &self, flow_id: FlowId, current_flow_info: &DeserializedValueWithBytes, flow_routes: I, ) -> Result { - let del_txns = current_flow_info.flownode_ids().keys().map(|partition_id| { - let key = FlowRouteKey::new(flow_id, *partition_id).to_bytes(); - Ok(TxnOp::Delete(key)) - }); + let old_keys = current_flow_info + .flownode_ids() + .keys() + .map(|partition_id| FlowRouteKey::new(flow_id, *partition_id).to_bytes()); + let new_kvs = flow_routes + .into_iter() + .map(|(partition_id, route)| { + let key = FlowRouteKey::new(flow_id, partition_id).to_bytes(); + Ok((key, route.try_as_raw_value()?)) + }) + .collect::>>()?; - let put_txns = flow_routes.into_iter().map(|(partition_id, route)| { - let key = FlowRouteKey::new(flow_id, partition_id).to_bytes(); - - Ok(TxnOp::Put(key, route.try_as_raw_value()?)) - }); - let txns = del_txns.chain(put_txns).collect::>>()?; - Ok(Txn::new().and_then(txns)) + Ok(Txn::new().and_then(build_replace_ops(old_keys, new_kvs))) } async fn remap_flow_route_addresses( diff --git a/src/common/meta/src/key/flow/flownode_flow.rs b/src/common/meta/src/key/flow/flownode_flow.rs index 2ba690c167e..83d49229fac 100644 --- a/src/common/meta/src/key/flow/flownode_flow.rs +++ b/src/common/meta/src/key/flow/flownode_flow.rs @@ -20,8 +20,8 @@ use snafu::OptionExt; use crate::FlownodeId; use crate::error::{self, Result}; -use crate::key::flow::FlowScoped; use crate::key::flow::flow_info::FlowInfoValue; +use crate::key::flow::{FlowScoped, build_replace_ops}; use crate::key::{BytesAdapter, DeserializedValueWithBytes, FlowId, FlowPartitionId, MetadataKey}; use crate::kv_backend::KvBackendRef; use crate::kv_backend::txn::{Txn, TxnOp}; @@ -218,28 +218,30 @@ impl FlownodeFlowManager { /// Builds a update flownode flow transaction. /// /// Puts `__flownode_flow/{flownode_id}/{flow_id}/{partition_id}` keys. - /// Remove the old `__flownode_flow/{old_flownode_id}/{flow_id}/{old_partition_id}` keys. + /// Removes the old `__flownode_flow/{old_flownode_id}/{flow_id}/{old_partition_id}` + /// keys that are not put. pub(crate) fn build_update_txn>( &self, flow_id: FlowId, current_flow_info: &DeserializedValueWithBytes, flownode_ids: I, ) -> Txn { - let del_txns = + let old_keys = current_flow_info .flownode_ids() .iter() .map(|(partition_id, flownode_id)| { - let key = FlownodeFlowKey::new(*flownode_id, flow_id, *partition_id).to_bytes(); - TxnOp::Delete(key) + FlownodeFlowKey::new(*flownode_id, flow_id, *partition_id).to_bytes() }); - let put_txns = flownode_ids.into_iter().map(|(partition_id, flownode_id)| { - let key = FlownodeFlowKey::new(flownode_id, flow_id, partition_id).to_bytes(); - TxnOp::Put(key, vec![]) - }); - let txns = del_txns.chain(put_txns).collect::>(); + let new_kvs = flownode_ids + .into_iter() + .map(|(partition_id, flownode_id)| { + let key = FlownodeFlowKey::new(flownode_id, flow_id, partition_id).to_bytes(); + (key, vec![]) + }) + .collect(); - Txn::new().and_then(txns) + Txn::new().and_then(build_replace_ops(old_keys, new_kvs)) } } diff --git a/src/common/meta/src/key/flow/table_flow.rs b/src/common/meta/src/key/flow/table_flow.rs index d334ca02181..ae0fcb525f1 100644 --- a/src/common/meta/src/key/flow/table_flow.rs +++ b/src/common/meta/src/key/flow/table_flow.rs @@ -24,7 +24,7 @@ use table::metadata::TableId; use crate::FlownodeId; use crate::error::{self, Result}; use crate::key::flow::flow_info::FlowInfoValue; -use crate::key::flow::{FlowScoped, flownode_addr_helper}; +use crate::key::flow::{FlowScoped, build_replace_ops, flownode_addr_helper}; use crate::key::node_address::NodeAddressKey; use crate::key::{ BytesAdapter, DeserializedValueWithBytes, FlowId, FlowPartitionId, MetadataKey, MetadataValue, @@ -245,8 +245,9 @@ impl TableFlowManager { /// Builds a update table flow transaction. /// /// Puts `__flow/source_table/{table_id}/{node_id}/{flow_id}/{partition_id}` keys, - /// Also remove previous - /// `__flow/source_table/{table_id}/{old_node_id}/{flow_id}/{partition_id}` keys. + /// Also removes the previous + /// `__flow/source_table/{table_id}/{old_node_id}/{flow_id}/{partition_id}` keys + /// that are not put. pub fn build_update_txn( &self, flow_id: FlowId, @@ -254,22 +255,24 @@ impl TableFlowManager { table_flow_values: Vec<(FlowPartitionId, TableFlowValue)>, source_table_ids: &[TableId], ) -> Result { - let mut txns = Vec::with_capacity(2 * source_table_ids.len() * table_flow_values.len()); - - // first remove the old keys - for (part_id, node_id) in current_flow_info.flownode_ids() { - for source_table_id in current_flow_info.source_table_ids() { - txns.push(TxnOp::Delete( - TableFlowKey::new(*source_table_id, *node_id, flow_id, *part_id).to_bytes(), - )); - } - } + let old_keys = current_flow_info + .flownode_ids() + .iter() + .flat_map(|(part_id, node_id)| { + current_flow_info + .source_table_ids() + .iter() + .map(move |source_table_id| { + TableFlowKey::new(*source_table_id, *node_id, flow_id, *part_id).to_bytes() + }) + }); + let mut new_kvs = Vec::with_capacity(source_table_ids.len() * table_flow_values.len()); for (partition_id, table_flow_value) in table_flow_values { let flownode_id = table_flow_value.peer.id; let value = table_flow_value.try_as_raw_value()?; for source_table_id in source_table_ids { - txns.push(TxnOp::Put( + new_kvs.push(( TableFlowKey::new(*source_table_id, flownode_id, flow_id, partition_id) .to_bytes(), value.clone(), @@ -277,7 +280,7 @@ impl TableFlowManager { } } - Ok(Txn::new().and_then(txns)) + Ok(Txn::new().and_then(build_replace_ops(old_keys, new_kvs))) } async fn remap_table_flow_addresses( diff --git a/src/common/recordbatch/src/filter.rs b/src/common/recordbatch/src/filter.rs index 60ff399601f..0893d2aa3ad 100644 --- a/src/common/recordbatch/src/filter.rs +++ b/src/common/recordbatch/src/filter.rs @@ -19,10 +19,11 @@ mod predicate; mod tests; mod timestamp; +use std::borrow::Borrow; use std::sync::Arc; use datafusion::error::Result as DfResult; -use datafusion::logical_expr::{Expr, Literal, Operator}; +use datafusion::logical_expr::{Expr, Literal, Operator, utils}; use datafusion::physical_plan::PhysicalExpr; use datafusion_common::arrow::array::{ArrayRef, Datum}; use datafusion_common::arrow::buffer::BooleanBuffer; @@ -41,6 +42,41 @@ pub use self::timestamp::timestamp_scalar_value; use crate::error::{Result, ToArrowScalarSnafu}; use crate::filter::predicate::SimplePredicate; +/// Prunes candidates using known metadata columns. Missing columns and unsupported +/// predicates cannot exclude a candidate. Complete SQL predicates must still run +/// after pruning. +#[derive(Debug, Clone, Default)] +pub struct MetadataFilter { + filters: Arc<[SimpleFilterEvaluator]>, +} + +impl MetadataFilter { + /// Compiles supported conjuncts once for reuse across scan stages. OR is + /// never split: an unknown alternative might match the candidate. + pub fn new(expressions: &[Expr]) -> Self { + Self { + filters: expressions + .iter() + .flat_map(utils::split_conjunction) + .filter_map(SimpleFilterEvaluator::try_new) + .collect(), + } + } + + /// Returns false only when known metadata proves a conjunct cannot match. + /// `None` means unavailable metadata, distinct from a known SQL NULL scalar. + /// Values may be borrowed so repeated predicates can reuse allocated strings. + pub fn matches>( + &self, + mut value: impl FnMut(&str) -> Option, + ) -> bool { + self.filters.iter().all(|filter| { + value(filter.column_name()) + .is_none_or(|value| filter.evaluate_scalar(value.borrow()).unwrap_or(true)) + }) + } +} + /// Evaluates supported single-column predicates with precompiled regular expressions. /// Compound predicates preserve SQL null semantics until conversion to a scan mask. #[derive(Debug, Clone)] diff --git a/src/common/recordbatch/src/filter/tests.rs b/src/common/recordbatch/src/filter/tests.rs index 77abf59b404..f48b4a3c126 100644 --- a/src/common/recordbatch/src/filter/tests.rs +++ b/src/common/recordbatch/src/filter/tests.rs @@ -70,3 +70,91 @@ fn test_nullable_boolean_predicate_becomes_scan_mask() { boolean_array_to_scan_mask(&predicate) ); } + +#[test] +fn test_partial_metadata() { + use datafusion_common::ScalarValue; + + use crate::filter::MetadataFilter; + + let node = col("node_id").eq(lit(7u64)); + let path = col("file_path").eq(lit("table/region/file.parquet")); + let filters = MetadataFilter::new(&[node.clone().and(path.clone())]); + assert!(!filters.matches(|name| (name == "node_id").then_some(ScalarValue::UInt64(Some(8))))); + assert!(filters.matches(|name| (name == "node_id").then_some(ScalarValue::UInt64(Some(7))))); + assert!(!filters.matches(|name| { + (name == "file_path").then(|| ScalarValue::Utf8(Some("other/file.parquet".into()))) + })); + assert!(filters.matches(|name| { + (name == "file_path").then(|| ScalarValue::Utf8(Some("table/region/file.parquet".into()))) + })); + // Unknown alternatives cannot be discarded from OR, nor extracted from NOT. + let disjunction = MetadataFilter::new(&[node.clone().or(path.clone())]); + assert!( + disjunction.matches(|name| (name == "node_id").then_some(ScalarValue::UInt64(Some(8)))) + ); + let negation = MetadataFilter::new(&[datafusion::logical_expr::Expr::Not(Box::new( + node.and(path), + ))]); + assert!(negation.matches(|_| None::)); + // Unsupported LIKE remains a residual even when its column is available. + let residual = MetadataFilter::new(&[col("file_path").like(lit("table/%"))]); + assert!(residual.matches(|_| Some(ScalarValue::Utf8(Some("other/file".into()))))); + // Missing metadata differs from known NULL; type errors conservatively pass. + let null_filter = MetadataFilter::new(&[col("node_id").is_not_null()]); + assert!(null_filter.matches(|_| None::)); + assert!(!null_filter.matches(|_| Some(ScalarValue::UInt64(None)))); + let typed = MetadataFilter::new(&[col("node_id").eq(lit(7u64))]); + assert!(typed.matches(|_| Some(ScalarValue::Utf8(Some("invalid".into()))))); +} + +#[test] +fn test_borrowed_and_owned_values() { + use datafusion_common::ScalarValue; + + use crate::filter::MetadataFilter; + + let pruner = MetadataFilter::new(&[col("path").gt_eq(lit("a")).and(col("path").lt(lit("z")))]); + for (value, expected) in [ + (ScalarValue::Utf8(Some("a".into())), true), + (ScalarValue::Utf8(Some("z".into())), false), + (ScalarValue::Utf8(None), false), + (ScalarValue::UInt64(Some(1)), true), + ] { + assert_eq!(pruner.matches(|_| Some(&value)), expected); + assert_eq!(pruner.matches(|_| Some(value.clone())), expected); + } +} + +#[test] +fn test_borrowed_lazy_metadata() { + use std::cell::OnceCell; + + use datafusion_common::ScalarValue; + + use crate::filter::MetadataFilter; + + let pruner = MetadataFilter::new(&[col("path").gt(lit("a")).and(col("path").lt(lit("z")))]); + let path = OnceCell::new(); + let mut builds = 0; + assert!(pruner.matches(|name| { + (name == "path").then(|| { + path.get_or_init(|| { + builds += 1; + ScalarValue::Utf8(Some("region/file.parquet".into())) + }) + }) + })); + assert_eq!(builds, 1); + assert_eq!( + path.into_inner(), + Some(ScalarValue::Utf8(Some("region/file.parquet".into()))) + ); + + let path = OnceCell::new(); + assert!( + MetadataFilter::default() + .matches(|_| { Some(path.get_or_init(|| ScalarValue::Utf8(Some("unused".into())))) }) + ); + assert!(path.get().is_none()); +} diff --git a/src/common/sql/src/convert.rs b/src/common/sql/src/convert.rs index fdd90ebe3df..8b84d591ef0 100644 --- a/src/common/sql/src/convert.rs +++ b/src/common/sql/src/convert.rs @@ -128,12 +128,29 @@ pub fn sql_value_to_value( column_schema: &ColumnSchema, sql_val: &SqlValue, timezone: Option<&Timezone>, - unary_op: Option, + mut unary_op: Option, auto_string_to_numeric: bool, ) -> Result { let column_name = &column_schema.name; let data_type = &column_schema.data_type; let mut value = match sql_val { + // Parse the sign with the digits, otherwise the minimum of a signed integer + // or timestamp overflows before it is negated. Other types keep the negation + // below: it accepts unsigned `-0`, and the decimal parser depends on length. + SqlValue::Number(n, _) + if unary_op == Some(UnaryOperator::Minus) + && matches!( + data_type, + ConcreteDataType::Int8(_) + | ConcreteDataType::Int16(_) + | ConcreteDataType::Int32(_) + | ConcreteDataType::Int64(_) + | ConcreteDataType::Timestamp(_) + ) => + { + unary_op = None; + sql_number_to_value(data_type, &format!("-{n}"))? + } SqlValue::Number(n, _) => sql_number_to_value(data_type, n)?, SqlValue::Null => Value::Null, SqlValue::Boolean(b) => { @@ -946,6 +963,55 @@ mod test { ); } + #[test] + fn test_negative_number_at_type_minimum() { + for (data_type, digits, expected) in [ + ( + ConcreteDataType::int8_datatype(), + "128", + Value::Int8(i8::MIN), + ), + ( + ConcreteDataType::int32_datatype(), + "2147483648", + Value::Int32(i32::MIN), + ), + ( + ConcreteDataType::int64_datatype(), + "9223372036854775808", + Value::Int64(i64::MIN), + ), + ] { + let value = call_sql_value_to_value!( + "test", + data_type, + &SqlValue::Number(digits.into(), false), + unary_op = UnaryOperator::Minus + ) + .unwrap(); + assert_eq!(value, expected); + } + let value = call_sql_value_to_value!( + "test", + ConcreteDataType::uint64_datatype(), + &SqlValue::Number("0".into(), false), + unary_op = UnaryOperator::Minus + ) + .unwrap(); + assert_eq!(value, Value::UInt64(0)); + + // 28 characters without the sign, 29 with it: the sign must not move the + // literal to the other decimal parser. + let value = call_sql_value_to_value!( + "test", + ConcreteDataType::decimal128_datatype(38, 30), + &SqlValue::Number("0.01234567890123456789012345".into(), false), + unary_op = UnaryOperator::Minus + ) + .unwrap(); + assert_eq!(value.to_string(), "-0.012345678901234567890123450000"); + } + #[test] fn test_auto_string_to_numeric() { // Test with auto_string_to_numeric=true diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs index 25d6b8698a2..73ec66b2109 100644 --- a/src/datanode/src/region_server.rs +++ b/src/datanode/src/region_server.rs @@ -458,7 +458,14 @@ impl RegionServer { region_id: RegionId, state: SettableRegionRoleState, ) -> Result { - match self.inner.region_map.get(®ion_id) { + // Cloning releases the DashMap guard before awaiting the engine operation. + let engine = self + .inner + .region_map + .get(®ion_id) + .map(|entry| entry.value().clone()); + + match engine { Some(engine) => Ok(engine .set_region_role_state_gracefully(region_id, state) .await diff --git a/src/datanode/src/region_server/catalog.rs b/src/datanode/src/region_server/catalog.rs index 264408342e9..789aa514785 100644 --- a/src/datanode/src/region_server/catalog.rs +++ b/src/datanode/src/region_server/catalog.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod inspect; + use std::collections::HashMap; use std::sync::Arc; @@ -33,6 +35,7 @@ use store_api::storage::RegionId; use crate::error::{DataFusionSnafu, ListStorageSstsSnafu, Result, UnexpectedSnafu}; use crate::region_server::RegionServer; +use crate::region_server::catalog::inspect::{InspectTable, ManifestSstSource}; /// Reserved internal table kinds used. /// These are recognized by reserved table names and mapped to providers. @@ -75,7 +78,7 @@ impl InternalTableKind { } impl RegionServer { - /// Expose SSTs listed in Manifest as an in-memory table for inspection. + /// Exposes manifest SSTs through a lazy table with metadata predicate pruning. pub async fn inspect_sst_manifest_provider(&self) -> Result> { let mito = { let guard = self.inner.mito_engine.read().unwrap(); @@ -84,14 +87,9 @@ impl RegionServer { })? }; - let entries = mito.all_ssts_from_manifest().await; - let schema = ManifestSstEntry::schema().arrow_schema().clone(); - let batch = ManifestSstEntry::to_record_batch(&entries) - .map_err(DataFusionError::from) - .context(DataFusionSnafu)?; - - let table = MemTable::try_new(schema, vec![vec![batch]]).context(DataFusionSnafu)?; - Ok(Arc::new(table)) + Ok(Arc::new(InspectTable::new(Arc::new( + ManifestSstSource::new(mito), + )))) } /// Expose SSTs found in storage as an in-memory table for inspection. @@ -355,6 +353,8 @@ impl TreeNodeRewriter for NameAwareDataSourceInjector { #[cfg(test)] mod tests { + mod manifest; + use std::sync::Arc; use datafusion::catalog::MemTable as DfMemTable; diff --git a/src/datanode/src/region_server/catalog/inspect.rs b/src/datanode/src/region_server/catalog/inspect.rs new file mode 100644 index 00000000000..307c8b42883 --- /dev/null +++ b/src/datanode/src/region_server/catalog/inspect.rs @@ -0,0 +1,311 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Streaming providers for internal inspection tables. + +use std::sync::Arc; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::catalog::{Session, TableProvider}; +use datafusion::error::Result; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::streaming::{PartitionStream, StreamingTableExec}; +use datafusion::physical_plan::{ExecutionPlan, SendableRecordBatchStream}; +use datafusion_common::DataFusionError; +use datafusion_expr::{Expr, TableProviderFilterPushDown, TableType}; +use futures::StreamExt; +use mito2::engine::MitoEngine; +use store_api::sst_entry::{ManifestSstBatchBuilder, ManifestSstEntry}; + +/// Supplies batches for an internal inspection table. Implementations may prune +/// conservatively; DataFusion retains complete residual predicates. +pub(crate) trait InspectSource: std::fmt::Debug + Send + Sync { + fn schema(&self) -> SchemaRef; + + /// Creates a fresh scan at execution time, including on repeated execution. + /// Projection includes columns required by residual predicates. + fn scan(&self, request: &InspectScan, ctx: Arc) -> SendableRecordBatchStream; +} + +/// Parameters shared by inspection sources. Limits are enforced by the physical +/// plan, after residual filters, rather than by individual sources. +#[derive(Debug, Clone)] +pub(crate) struct InspectScan { + schema: SchemaRef, + projection: Option>, + filters: Vec, + batch_size: usize, +} + +/// Adapts a lazy inspection source to DataFusion's streaming execution plan. +#[derive(Debug)] +pub(crate) struct InspectTable { + source: Arc, +} + +impl InspectTable { + /// Wraps a source without opening a scan. + pub(crate) fn new(source: Arc) -> Self { + Self { source } + } +} + +#[async_trait::async_trait] +impl TableProvider for InspectTable { + fn schema(&self) -> SchemaRef { + self.source.schema() + } + + fn table_type(&self) -> TableType { + TableType::View + } + + fn supports_filters_pushdown( + &self, + filters: &[&Expr], + ) -> Result> { + Ok(vec![TableProviderFilterPushDown::Inexact; filters.len()]) + } + + async fn scan( + &self, + state: &dyn Session, + projection: Option<&Vec>, + filters: &[Expr], + limit: Option, + ) -> Result> { + // An inexact predicate must be checked before selecting a limited prefix. + let limit = limit.filter(|_| filters.is_empty()); + let configured_batch_size: usize = state.config_options().execution.batch_size.into(); + let schema = match projection { + Some(projection) => Arc::new(self.schema().project(projection)?), + None => self.schema(), + }; + let partition = InspectPartition { + source: self.source.clone(), + request: InspectScan { + schema: schema.clone(), + projection: projection.cloned(), + filters: filters.to_vec(), + batch_size: limit + .unwrap_or(configured_batch_size) + .min(configured_batch_size) + .max(1), + }, + }; + Ok(Arc::new(StreamingTableExec::try_new( + schema, + vec![Arc::new(partition)], + None, + None, + false, + limit, + )?)) + } +} + +/// Reads manifest snapshots without materializing the full SST inventory. +pub(crate) struct ManifestSstSource { + engine: MitoEngine, +} + +impl ManifestSstSource { + /// Creates a source backed by the engine's open-region manifests. + pub(crate) fn new(engine: MitoEngine) -> Self { + Self { engine } + } +} + +impl std::fmt::Debug for ManifestSstSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ManifestSstSource").finish_non_exhaustive() + } +} + +impl InspectSource for ManifestSstSource { + fn schema(&self) -> SchemaRef { + ManifestSstEntry::schema().arrow_schema().clone() + } + + fn scan(&self, request: &InspectScan, _ctx: Arc) -> SendableRecordBatchStream { + let builder = ManifestSstBatchBuilder::new(request.schema.clone()); + let stream = self + .engine + .scan_ssts_from_manifest_projected( + &request.filters, + request.batch_size, + request.projection.as_deref(), + ) + .map(move |entries| builder.build(&entries).map_err(DataFusionError::from)); + Box::pin(RecordBatchStreamAdapter::new( + request.schema.clone(), + stream, + )) + } +} + +#[derive(Debug)] +struct InspectPartition { + source: Arc, + request: InspectScan, +} + +impl PartitionStream for InspectPartition { + fn schema(&self) -> &SchemaRef { + &self.request.schema + } + + fn execute(&self, ctx: Arc) -> SendableRecordBatchStream { + self.source.scan(&self.request, ctx) + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use datafusion::arrow::array::{ArrayRef, Int32Array}; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::arrow::record_batch::RecordBatch; + use datafusion::prelude::SessionContext; + + use super::*; + + #[derive(Debug, Default)] + struct CountingSource { + scans: Arc, + rows: Arc, + drops: Arc, + } + + struct ScanGuard(Arc); + + impl Drop for ScanGuard { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + impl InspectSource for CountingSource { + fn schema(&self) -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Int32, false), + ])) + } + + fn scan(&self, request: &InspectScan, _ctx: Arc) -> SendableRecordBatchStream { + self.scans.fetch_add(1, Ordering::SeqCst); + let guard = ScanGuard(self.drops.clone()); + let rows = self.rows.clone(); + let schema = request.schema.clone(); + let projection = request.projection.clone().unwrap_or_else(|| vec![0, 1]); + // Deliberately ignore source predicates: the provider must preserve + // residual filtering before limit, including hidden predicate columns. + let stream = futures::stream::iter(0..100).map(move |id| { + let _guard = &guard; + rows.fetch_add(1, Ordering::SeqCst); + let columns: Vec = projection + .iter() + .map(|column| { + Arc::new(Int32Array::from(vec![if *column == 0 { + id + } else { + id * 10 + }])) as ArrayRef + }) + .collect(); + RecordBatch::try_new(schema.clone(), columns).map_err(DataFusionError::from) + }); + Box::pin(RecordBatchStreamAdapter::new( + request.schema.clone(), + stream, + )) + } + } + + #[tokio::test] + async fn test_lazy_repeatable_scan_and_limit_cancellation() { + let source = Arc::new(CountingSource::default()); + let ctx = SessionContext::new(); + ctx.register_table("inspection", Arc::new(InspectTable::new(source.clone()))) + .unwrap(); + let frame = ctx + .sql("SELECT value FROM inspection LIMIT 2") + .await + .unwrap(); + let plan = frame.create_physical_plan().await.unwrap(); + assert_eq!(source.scans.load(Ordering::SeqCst), 0); + assert_eq!(source.rows.load(Ordering::SeqCst), 0); + for run in 1..=2 { + let batches = datafusion::physical_plan::collect(plan.clone(), ctx.task_ctx()) + .await + .unwrap(); + let values = batches + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect::>(); + assert_eq!(values, vec![0, 10]); + assert_eq!(source.scans.load(Ordering::SeqCst), run); + assert_eq!(source.rows.load(Ordering::SeqCst), run * 2); + assert_eq!(source.drops.load(Ordering::SeqCst), run); + } + } + + #[tokio::test] + async fn test_hidden_residual_filter_before_limit() { + let source = Arc::new(CountingSource::default()); + let ctx = SessionContext::new(); + ctx.register_table("inspection", Arc::new(InspectTable::new(source.clone()))) + .unwrap(); + let batches = ctx + .sql("SELECT value FROM inspection WHERE id >= 8 LIMIT 2") + .await + .unwrap() + .collect() + .await + .unwrap(); + let values = batches + .iter() + .flat_map(|batch| { + assert_eq!(batch.num_columns(), 1); + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect::>(); + // No ORDER BY: repartitioning may choose any two qualifying rows. + assert_eq!(values.len(), 2); + assert!( + values + .iter() + .all(|value| (80..1000).contains(value) && value % 10 == 0) + ); + assert_eq!(source.scans.load(Ordering::SeqCst), 1); + assert_eq!(source.drops.load(Ordering::SeqCst), 1); + } +} diff --git a/src/datanode/src/region_server/catalog/tests/manifest.rs b/src/datanode/src/region_server/catalog/tests/manifest.rs new file mode 100644 index 00000000000..91b9f757a3d --- /dev/null +++ b/src/datanode/src/region_server/catalog/tests/manifest.rs @@ -0,0 +1,257 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use common_time::Timestamp; +use datafusion::arrow::compute::concat_batches; +use datafusion::arrow::record_batch::RecordBatch; +use datafusion::catalog::{MemTable, TableProvider}; +use datafusion::prelude::{SessionConfig, SessionContext}; +use mito2::config::MitoConfig; +use mito2::engine::MitoEngine; +use mito2::manifest::action::RegionEdit; +use mito2::sst::file::FileMeta; +use mito2::test_util::{CreateRequestBuilder, TestEnv}; +use store_api::region_engine::RegionEngine; +use store_api::region_request::RegionRequest; +use store_api::sst_entry::ManifestSstEntry; +use store_api::storage::{FileId, RegionId}; + +use crate::region_server::catalog::inspect::{InspectTable, ManifestSstSource}; + +/// Installs deterministic metadata only. No SST object is read by inspection. +async fn manifest_engine(regions: u32, files_per_region: usize) -> (TestEnv, MitoEngine) { + let mut env = TestEnv::new().await; + let engine = env + .create_engine(MitoConfig { + schedule_compaction_after_edit: false, + ..Default::default() + }) + .await; + for number in 0..regions { + let region_id = RegionId::new(11, number); + engine + .handle_request( + region_id, + RegionRequest::Create(CreateRequestBuilder::new().build()), + ) + .await + .unwrap(); + engine + .edit_region( + region_id, + RegionEdit { + files_to_add: (0..files_per_region) + .map(|index| FileMeta { + region_id, + file_id: FileId::from_bytes( + (((number as u128) << 64) | index as u128).to_be_bytes(), + ), + time_range: ( + Timestamp::new_millisecond(index as i64), + Timestamp::new_millisecond(index as i64 + 1), + ), + file_size: 1024 + index as u64, + sequence: None, + ..Default::default() + }) + .collect(), + files_to_remove: vec![], + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id: None, + flushed_sequence: None, + committed_sequence: None, + }, + ) + .await + .unwrap(); + } + (env, engine) +} + +/// Baseline preserves eager full-row materialization, then applies SQL. Both +/// variants use the current engine enumerator, so this does not measure removal +/// of the old manifest-map cloning independently. +async fn inspect_query(engine: &MitoEngine, streaming: bool, sql: &str) -> RecordBatch { + let schema = ManifestSstEntry::schema().arrow_schema().clone(); + let table: Arc = if streaming { + Arc::new(InspectTable::new(Arc::new(ManifestSstSource::new( + engine.clone(), + )))) + } else { + let entries = engine.all_ssts_from_manifest().await; + let batch = ManifestSstEntry::to_record_batch(&entries).unwrap(); + Arc::new(MemTable::try_new(schema, vec![vec![batch]]).unwrap()) + }; + let context = SessionContext::new_with_config( + SessionConfig::new() + .with_batch_size(17) + .with_target_partitions(1), + ); + context.register_table("manifest", table).unwrap(); + let frame = context.sql(sql).await.unwrap(); + let schema = frame.schema().as_arrow().clone(); + let batches = frame.collect().await.unwrap(); + concat_batches(&Arc::new(schema), &batches).unwrap() +} + +#[tokio::test] +async fn test_manifest_provider_matches_eager_scan() { + let (_env, engine) = manifest_engine(3, 64).await; + let region = RegionId::new(11, 1).as_u64(); + for sql in [ + "SELECT file_id, min_ts FROM manifest WHERE table_id = 11 ORDER BY min_ts, file_id LIMIT 7" + .to_string(), + "SELECT file_id FROM manifest WHERE table_id = 12 LIMIT 7".to_string(), + format!( + "SELECT file_id FROM manifest WHERE table_id = 11 AND region_id = {region} ORDER BY min_ts DESC LIMIT 7" + ), + format!("SELECT * FROM manifest WHERE region_id = {region} ORDER BY file_id"), + format!( + "SELECT min_ts, max_ts FROM manifest WHERE region_id = {region} ORDER BY min_ts ASC NULLS LAST, file_id LIMIT 7" + ), + format!( + "SELECT file_size FROM manifest WHERE region_id = {region} ORDER BY max_ts DESC NULLS FIRST, file_id LIMIT 7 OFFSET 3" + ), + // Arithmetic is deliberately left as a residual predicate. Limiting + // metadata enumeration before it would return too few rows. + format!( + "SELECT file_id FROM manifest WHERE region_id = {region} AND file_size % 3 = 0 ORDER BY min_ts, file_id LIMIT 7" + ), + format!( + "SELECT file_id FROM manifest WHERE region_id = {region} AND min_ts >= TIMESTAMP '1970-01-01 00:00:00.020' ORDER BY min_ts, file_id LIMIT 7" + ), + // Variable-sized metadata needed only by a residual filter must still + // be materialized even when it is absent from the SELECT list. + format!( + "SELECT file_id FROM manifest WHERE region_id = {region} AND length(file_path) > 0 ORDER BY min_ts, file_id LIMIT 7" + ), + // Repeated path predicates reuse metadata while both path columns stay + // hidden from the output. The range selects only region 1's 64 files. + "SELECT file_id FROM manifest WHERE table_dir >= 'test' AND table_dir < 'tesu' AND file_path >= 'test/11_0000000001/' AND file_path < 'test/11_0000000002/' ORDER BY file_id LIMIT 7".to_string(), + // Visibility snapshots are needed for residual predicates even when + // the SELECT list does not include visible. + format!( + "SELECT file_id FROM manifest WHERE region_id = {region} AND visible ORDER BY file_id LIMIT 7" + ), + format!( + "SELECT file_id FROM manifest WHERE region_id = {region} AND NOT visible ORDER BY file_id LIMIT 7" + ), + "SELECT file_id FROM manifest WHERE region_id = 0 ORDER BY min_ts LIMIT 1".to_string(), + "SELECT min_ts FROM manifest ORDER BY min_ts LIMIT 0".to_string(), + "SELECT count(*) FROM manifest".to_string(), + // Unsupported compound predicates must not prune matching regions. + format!( + "SELECT file_id FROM manifest WHERE region_id = {region} OR file_size = 1024 ORDER BY file_id" + ), + ] { + let expected = inspect_query(&engine, false, &sql).await; + let actual = inspect_query(&engine, true, &sql).await; + assert_eq!(actual, expected, "{sql}"); + } +} + +#[tokio::test] +async fn test_manifest_referring_region_and_origin_path() { + use datafusion::arrow::array::{StringArray, UInt64Array}; + + let (_env, engine) = manifest_engine(2, 2).await; + let referring = RegionId::new(11, 1); + // The creating region need not be open on this datanode. + let origin = RegionId::new(11, 99); + let file_id = FileId::from_bytes(999u128.to_be_bytes()); + engine + .edit_region( + referring, + RegionEdit { + files_to_add: vec![FileMeta { + region_id: origin, + file_id, + file_size: 4096, + ..Default::default() + }], + files_to_remove: vec![], + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id: None, + flushed_sequence: None, + committed_sequence: None, + }, + ) + .await + .unwrap(); + let sql = format!( + "SELECT region_id, origin_region_id, file_path FROM manifest WHERE table_id = 11 AND region_id = {} AND origin_region_id = {} AND file_path LIKE '%/11_0000000099/%' LIMIT 1", + referring.as_u64(), + origin.as_u64(), + ); + // Check source pruning without SQL residuals: the origin is not a loaded + // region, but its file must be found in the referring region's manifest. + use datafusion_expr::{col, lit}; + use futures::StreamExt; + for (predicate, expected) in [ + (col("origin_region_id").eq(lit(origin.as_u64())), 1), + ( + col("file_path").eq(lit(format!("test/11_0000000099/{file_id}.parquet"))), + 1, + ), + (col("file_path").eq(lit("missing/file.parquet")), 0), + (col("table_dir").eq(lit("missing/")), 0), + (col("node_id").is_not_null(), 0), + ] { + let batches = engine + .scan_ssts_from_manifest_projected(std::slice::from_ref(&predicate), 2, None) + .collect::>() + .await; + assert_eq!( + batches.iter().map(Vec::len).sum::(), + expected, + "{predicate}" + ); + } + let actual = inspect_query(&engine, true, &sql).await; + assert_eq!(actual, inspect_query(&engine, false, &sql).await); + assert_eq!(actual.num_rows(), 1); + assert_eq!( + actual + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + referring.as_u64() + ); + assert_eq!( + actual + .column(1) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + origin.as_u64() + ); + assert!( + actual + .column(2) + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + .ends_with(&format!("11_0000000099/{file_id}.parquet")) + ); + let missing = sql.replace("%/11_0000000099/%", "%/11_0000000001/%"); + assert_eq!(inspect_query(&engine, true, &missing).await.num_rows(), 0); +} diff --git a/src/frontend/src/instance/jaeger.rs b/src/frontend/src/instance/jaeger.rs index c9b5bd9e988..3349b784caf 100644 --- a/src/frontend/src/instance/jaeger.rs +++ b/src/frontend/src/instance/jaeger.rs @@ -35,6 +35,7 @@ use datafusion::dataframe::DataFrame; use datafusion::execution::SessionStateBuilder; use datafusion::execution::context::SessionContext; use datafusion::functions::core::expr_fn::coalesce; +use datafusion::functions_aggregate::expr_fn::max; use datafusion::functions_window::expr_fn::row_number; use datafusion_expr::select_expr::SelectExpr; use datafusion_expr::{Expr, ExprFunctionExt, SortExpr, col, lit, lit_timestamp_nano, wildcard}; @@ -59,6 +60,7 @@ use crate::instance::Instance; const DEFAULT_LIMIT: usize = 2000; const KEY_RN: &str = "greptime_rn"; +const KEY_LATEST_MATCH: &str = "greptime_latest_match"; impl Instance { async fn check_jaeger_query_permission(&self, ctx: &QueryContextRef) -> ServerResult<()> { @@ -225,11 +227,11 @@ impl JaegerQueryHandler for Instance { filters.push(col(DURATION_NANO_COLUMN).lt_eq(lit(max_duration))); } - // Get all distinct trace ids that match the filters. + // Get the ids of the most recent traces that match the filters. // It's equivalent to the following SQL query: // // ``` - // SELECT DISTINCT trace_id + // SELECT trace_id // FROM // {db}.{trace_table} // WHERE @@ -239,17 +241,16 @@ impl JaegerQueryHandler for Instance { // timestamp <= {end_time} AND // duration >= {min_duration} AND // duration <= {max_duration} + // GROUP BY trace_id + // ORDER BY max(timestamp) DESC, trace_id // LIMIT {limit} // ```. - let output = query_trace_table( + let output = find_latest_trace_ids( ctx.clone(), self, - vec![wildcard()], filters, - vec![], - Some(query_params.limit.unwrap_or(DEFAULT_LIMIT)), + query_params.limit.unwrap_or(DEFAULT_LIMIT), query_params.tags, - vec![col(TRACE_ID_COLUMN)], ) .await?; @@ -332,6 +333,79 @@ async fn query_trace_table( tags: Option>, distincts: Vec, ) -> ServerResult { + let dataframe = + build_trace_dataframe(&ctx, instance, selects, filters, tags, &distincts).await?; + + // Apply the distinct if needed. + let dataframe = if !distincts.is_empty() { + dataframe + .distinct_on(distincts.clone(), distincts, None) + .context(DataFusionSnafu)? + } else { + dataframe + }; + + // Apply the sorts if needed. + let dataframe = if !sorts.is_empty() { + dataframe.sort(sorts).context(DataFusionSnafu)? + } else { + dataframe + }; + + // Apply the limit if needed. + let dataframe = if let Some(limit) = limit { + dataframe.limit(0, Some(limit)).context(DataFusionSnafu)? + } else { + dataframe + }; + + execute_trace_dataframe(dataframe).await +} + +/// Returns the ids of the `limit` traces whose latest span matching `filters` +/// and `tags` is the most recent, newest first. +/// +/// Ordering before the limit keeps a search that matches more traces than the +/// limit deterministic. +async fn find_latest_trace_ids( + ctx: QueryContextRef, + instance: &Instance, + filters: Vec, + limit: usize, + tags: Option>, +) -> ServerResult { + let dataframe = + build_trace_dataframe(&ctx, instance, vec![wildcard()], filters, tags, &[]).await?; + let dataframe = dataframe + .aggregate( + vec![col(TRACE_ID_COLUMN)], + vec![max(col(TIMESTAMP_COLUMN)).alias(KEY_LATEST_MATCH)], + ) + .context(DataFusionSnafu)? + .sort(vec![ + col(KEY_LATEST_MATCH).sort(false, false), + col(TRACE_ID_COLUMN).sort(true, false), + ]) + .context(DataFusionSnafu)? + .limit(0, Some(limit)) + .context(DataFusionSnafu)? + .select(vec![col(TRACE_ID_COLUMN)]) + .context(DataFusionSnafu)?; + + execute_trace_dataframe(dataframe).await +} + +/// Reads the trace table, or the services or operations lookup table that +/// answers `selects` and `distincts`, then projects `selects` and applies +/// `filters` and the Jaeger tag filters. +async fn build_trace_dataframe( + ctx: &QueryContextRef, + instance: &Instance, + selects: Vec, + filters: Vec, + tags: Option>, + distincts: &[Expr], +) -> ServerResult { let trace_table_name = ctx .extension(JAEGER_QUERY_TABLE_NAME_KEY) .unwrap_or(TRACE_TABLE_NAME); @@ -360,7 +434,7 @@ async fn query_trace_table( ctx.current_catalog(), &ctx.current_schema(), table_name, - Some(&ctx), + Some(ctx), ) .await? .with_context(|| TableNotFoundSnafu { @@ -397,30 +471,12 @@ async fn query_trace_table( df.filter(expr).context(DataFusionSnafu) })?; - // Apply the distinct if needed. - let dataframe = if !distincts.is_empty() { - dataframe - .distinct_on(distincts.clone(), distincts, None) - .context(DataFusionSnafu)? - } else { - dataframe - }; + Ok(dataframe) +} - // Apply the sorts if needed. - let dataframe = if !sorts.is_empty() { - dataframe.sort(sorts).context(DataFusionSnafu)? - } else { - dataframe - }; - - // Apply the limit if needed. - let dataframe = if let Some(limit) = limit { - dataframe.limit(0, Some(limit)).context(DataFusionSnafu)? - } else { - dataframe - }; - - // Execute the query and collect the result. +/// Executes `dataframe` and wraps its stream into an [`Output`] with dictionary +/// columns decoded to values. +async fn execute_trace_dataframe(dataframe: DataFrame) -> ServerResult { let stream = dataframe.execute_stream().await.context(DataFusionSnafu)?; let output = Output::new_with_stream(Box::pin( diff --git a/src/frontend/src/instance/prom_store.rs b/src/frontend/src/instance/prom_store.rs index d972abc9112..875cbaa873d 100644 --- a/src/frontend/src/instance/prom_store.rs +++ b/src/frontend/src/instance/prom_store.rs @@ -61,7 +61,7 @@ use tracing::instrument; use crate::error::{ AmbiguousValueColumnSnafu, CatalogSnafu, ColumnNotFoundSnafu, ExecLogicalPlanSnafu, - PromStoreRemoteQueryPlanSnafu, ReadTableSnafu, Result, TableNotFoundSnafu, + PromStoreRemoteQueryPlanSnafu, ReadTableSnafu, Result, }; use crate::instance::Instance; @@ -207,15 +207,21 @@ impl Instance { schema_name: &str, table_name: &str, query: &Query, - ) -> Result { - let table = self + ) -> Result> { + let Some(table) = self .catalog_manager .table(catalog_name, schema_name, table_name, Some(ctx)) .await .context(CatalogSnafu)? - .with_context(|| TableNotFoundSnafu { - table_name: format_full_table_name(catalog_name, schema_name, table_name), - })?; + else { + debug!( + catalog = catalog_name, + schema = schema_name, + table = table_name, + "Table not found during remote read, returning empty result" + ); + return Ok(None); + }; let timestamp_column_name = table .schema() @@ -250,12 +256,12 @@ impl Instance { .await .context(ExecLogicalPlanSnafu)?; - Ok(RemoteQueryOutput { + Ok(Some(RemoteQueryOutput { table_name: table_name.to_string(), timestamp_column_name, value_column_name, output, - }) + })) } #[tracing::instrument(skip_all)] @@ -264,7 +270,7 @@ impl Instance { ctx: QueryContextRef, queries: &[Query], query_targets: &[PermissionTableTarget], - ) -> ServerResult> { + ) -> ServerResult>> { let mut results = Vec::with_capacity(queries.len()); for (query, target) in queries.iter().zip(query_targets) { @@ -599,6 +605,10 @@ impl PromStoreProtocolHandler for Instance { let response_type = negotiate_response_type(&request.accepted_response_types)?; + for query in &request.queries { + prom_store::validate_matchers(&query.matchers)?; + } + // TODO(dennis): use read_hints to speedup query if possible let results = self .handle_remote_queries(ctx, &request.queries, &query_targets) @@ -609,6 +619,10 @@ impl PromStoreProtocolHandler for Instance { let mut query_results = Vec::with_capacity(results.len()); let mut map = HashMap::new(); for result in results { + let Some(result) = result else { + query_results.push(QueryResult { timeseries: vec![] }); + continue; + }; let RemoteQueryOutput { table_name, timestamp_column_name, diff --git a/src/meta-srv/src/metasrv/builder.rs b/src/meta-srv/src/metasrv/builder.rs index 57e7deb3db6..dc9035ddbd6 100644 --- a/src/meta-srv/src/metasrv/builder.rs +++ b/src/meta-srv/src/metasrv/builder.rs @@ -38,6 +38,7 @@ use common_meta::key::runtime_switch::{RuntimeSwitchManager, RuntimeSwitchManage use common_meta::kv_backend::memory::MemoryKvBackend; use common_meta::kv_backend::{KvBackendRef, ResettableKvBackendRef}; use common_meta::node_manager::NodeManagerRef; +use common_meta::peer::PeerDiscoveryRef; use common_meta::reconciliation::manager::ReconciliationManager; use common_meta::region_keeper::MemoryRegionKeeper; use common_meta::region_registry::LeaderRegionRegistry; @@ -461,7 +462,7 @@ impl MetasrvBuilder { { let ctx = DdlManagerConfigureContext { kv_backend: kv_backend.clone(), - meta_peer_client: meta_peer_client.clone(), + peer_discovery: selector_ctx.peer_discovery.clone(), }; configurator .configure(ddl_manager, ctx) @@ -722,7 +723,7 @@ impl Default for MetasrvBuilder { /// The context for [`DdlManagerConfiguratorRef`]. pub struct DdlManagerConfigureContext { pub kv_backend: KvBackendRef, - pub meta_peer_client: MetaPeerClientRef, + pub peer_discovery: PeerDiscoveryRef, } #[cfg(test)] @@ -801,4 +802,45 @@ mod tests { .await .unwrap(); } + + #[tokio::test] + async fn test_builder_shares_peer_discovery_with_ddl_configurator() { + use common_meta::ddl_manager::DdlManagerConfigurator; + + struct RecordingConfigurator { + discovery: Arc>>, + } + + #[async_trait::async_trait] + impl DdlManagerConfigurator for RecordingConfigurator { + async fn configure( + &self, + ddl_manager: DdlManager, + ctx: DdlManagerConfigureContext, + ) -> std::result::Result { + *self.discovery.lock().unwrap() = Some(ctx.peer_discovery); + Ok(ddl_manager) + } + } + + let discovery = Arc::new(std::sync::Mutex::new(None)); + let plugins = Plugins::new(); + plugins.insert(Arc::new(RecordingConfigurator { + discovery: discovery.clone(), + }) + as DdlManagerConfiguratorRef); + let metasrv = MetasrvBuilder::new() + .options(MetasrvOptions { + enable_telemetry: false, + ..Default::default() + }) + .plugins(plugins) + .build() + .await + .unwrap(); + assert!(Arc::ptr_eq( + discovery.lock().unwrap().as_ref().unwrap(), + &metasrv.selector_ctx().peer_discovery, + )); + } } diff --git a/src/metric-engine/src/engine/bulk_insert.rs b/src/metric-engine/src/engine/bulk_insert.rs index b7ad7646e3c..6467c710501 100644 --- a/src/metric-engine/src/engine/bulk_insert.rs +++ b/src/metric-engine/src/engine/bulk_insert.rs @@ -16,9 +16,11 @@ use std::collections::HashSet; use api::v1::{ArrowIpc, SemanticType}; use bytes::Bytes; +use common_error::ext::BoxedError; use common_grpc::flight::{FlightEncoder, FlightMessage}; use datatypes::arrow::record_batch::RecordBatch; -use snafu::{OptionExt, ensure}; +use mito2::request::check_bulk_insert_batch; +use snafu::{OptionExt, ResultExt, ensure}; use store_api::codec::PrimaryKeyEncoding; use store_api::metadata::RegionMetadataRef; use store_api::region_request::{AffectedRows, RegionBulkInsertsRequest, RegionRequest}; @@ -111,6 +113,11 @@ impl MetricEngineInner { &batch, &logical_metadata, )?; + // The rewrite below copies columns without checking them, and would encode a + // duplicated tag into the primary key twice. + check_bulk_insert_batch(&logical_metadata, &batch) + .map_err(BoxedError::new) + .context(error::MitoWriteOperationSnafu)?; let modified_batch = modify_batch_sparse( batch.clone(), region_id.table_id(), @@ -307,6 +314,92 @@ mod tests { }) } + #[tokio::test] + async fn test_bulk_insert_rejects_duplicate_tag() { + use common_error::status_code::StatusCode; + + let env = TestEnv::new().await; + env.init_metric_region().await; + let logical_region_id = env.default_logical_region_id(); + + let batch = build_logical_batch(0, 2); + let mut fields = batch.schema().fields().to_vec(); + let mut columns = batch.columns().to_vec(); + fields.push(fields[2].clone()); + columns.push(columns[2].clone()); + let batch = RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns).unwrap(); + let err = env + .metric() + .handle_request( + logical_region_id, + build_bulk_request(logical_region_id, batch, false), + ) + .await + .unwrap_err(); + assert_eq!(err.status_code(), StatusCode::InvalidArguments, "{err:?}"); + } + + #[tokio::test] + async fn test_bulk_insert_physical_region_rejects_malformed_primary_key() { + use common_error::status_code::StatusCode; + use datatypes::arrow::array::{ArrayRef, BinaryArray, Int64Array}; + use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME; + + let env = TestEnv::new().await; + env.init_metric_region().await; + let physical_region_id = env.default_physical_region_id(); + let logical_region_id = env.default_logical_region_id(); + let tag_columns = vec![TagColumnInfo { + name: "job".to_string(), + index: 2, + column_id: 2, + }]; + let batch = modify_batch_sparse( + build_logical_batch(0, 2), + logical_region_id.table_id(), + &tag_columns, + &[0, 1], + ) + .unwrap(); + let primary_key_index = batch.schema().index_of(PRIMARY_KEY_COLUMN_NAME).unwrap(); + let with_primary_key = |primary_key: Option, extra: Option| { + let mut fields = batch.schema().fields().to_vec(); + let mut columns = batch.columns().to_vec(); + fields.remove(primary_key_index); + columns.remove(primary_key_index); + for array in [primary_key, extra].into_iter().flatten() { + fields.push(Arc::new(Field::new( + PRIMARY_KEY_COLUMN_NAME, + array.data_type().clone(), + true, + ))); + columns.push(array); + } + RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns).unwrap() + }; + let encoded = batch.column(primary_key_index).clone(); + let rejected = [ + with_primary_key(None, None), + with_primary_key(Some(Arc::new(Int64Array::from(vec![1, 2]))), None), + with_primary_key( + Some(Arc::new(BinaryArray::from_opt_vec(vec![Some(b"a"), None]))), + None, + ), + with_primary_key(Some(encoded.clone()), Some(encoded)), + ]; + for batch in rejected { + let err = env + .metric() + .handle_request( + physical_region_id, + build_bulk_request(physical_region_id, batch, false), + ) + .await + .unwrap_err(); + assert_eq!(err.status_code(), StatusCode::InvalidArguments, "{err:?}"); + } + } + async fn init_dense_metric_region(env: &TestEnv) -> RegionId { let physical_region_id = env.default_physical_region_id(); env.create_physical_region( @@ -675,11 +768,16 @@ mod tests { .unwrap(); let region = env.mito().find_region(data_region_id).unwrap(); - let entry = region - .manifest_sst_entries() + let entry = env + .mito() + .all_ssts_from_manifest() .await .into_iter() - .find(|entry| entry.visible && entry.file_path.ends_with(".parquet")) + .find(|entry| { + entry.region_id == data_region_id + && entry.visible + && entry.file_path.ends_with(".parquet") + }) .unwrap(); let mut cache_metrics = MetadataCacheMetrics::default(); let footer = MetadataLoader::new( diff --git a/src/mito2/AGENTS.md b/src/mito2/AGENTS.md index 6d5f77546a8..c729071f47d 100644 --- a/src/mito2/AGENTS.md +++ b/src/mito2/AGENTS.md @@ -92,6 +92,9 @@ Tests live next to the code as `*_test.rs` (e.g. `src/mito2/src/engine/flush_tes - Lock ordering: take the manifest lock before updating `version_control`; the reverse deadlocks against concurrent flush/compaction. - All region I/O runs on tokio workers; never `block_on` inside a worker. +- In flat batches, equal `__primary_key` dictionary keys imply the same series, but + not the reverse: `wrap_pk_binary_to_dict` gives every row its own key. Compare key + bytes when grouping rows by series. ## Maintenance contract diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs index 85ae5c39d1a..02dcf8594d2 100644 --- a/src/mito2/src/engine.rs +++ b/src/mito2/src/engine.rs @@ -25,6 +25,8 @@ mod batch_catchup_test; #[cfg(test)] mod batch_open_test; #[cfg(test)] +mod bulk_insert_validation_test; +#[cfg(test)] mod bump_committed_sequence_test; #[cfg(test)] mod catchup_test; @@ -98,10 +100,13 @@ use common_base::Plugins; use common_error::ext::BoxedError; use common_meta::error::UnexpectedSnafu; use common_meta::key::SchemaMetadataManagerRef; +use common_recordbatch::filter::MetadataFilter; use common_recordbatch::{QueryMemoryTracker, SendableRecordBatchStream}; use common_stat::get_total_memory_bytes; use common_telemetry::{debug, info, tracing, warn}; use common_wal::options::WalOptions; +use datafusion_common::ScalarValue; +use datafusion_expr::Expr; use futures::future::{join_all, try_join_all}; use futures::stream::{self, Stream, StreamExt}; use object_store::manager::ObjectStoreManagerRef; @@ -125,7 +130,7 @@ use store_api::region_info::RegionInfoEntry; use store_api::region_request::{ AffectedRows, RegionCatchupRequest, RegionOpenRequest, RegionRequest, }; -use store_api::sst_entry::{ManifestSstEntry, PuffinIndexMetaEntry, StorageSstEntry}; +use store_api::sst_entry::{self, ManifestSstEntry, PuffinIndexMetaEntry, StorageSstEntry}; use store_api::storage::{FileId, FileRefsManifest, RegionId, ScanRequest, SequenceNumber}; use tokio::sync::{Semaphore, oneshot}; @@ -571,19 +576,65 @@ impl MitoEngine { /// Lists all SSTs from the manifest of all regions in the engine. pub async fn all_ssts_from_manifest(&self) -> Vec { - let node_id = self.inner.workers.file_ref_manager().node_id(); - let regions = self.inner.workers.all_regions(); + self.scan_ssts_from_manifest(&[], 1024) + .flat_map(stream::iter) + .collect() + .await + } - let mut results = Vec::new(); - for region in regions { - let mut entries = region.manifest_sst_entries().await; - for e in &mut entries { - e.node_id = node_id; + /// Lazily emits bounded batches of manifest entries, pruning regions before + /// opening their manifest snapshots and files before constructing entries. + /// Filters are conservative; callers must retain their residual predicates. + pub(crate) fn scan_ssts_from_manifest( + &self, + filters: &[Expr], + batch_size: usize, + ) -> futures::stream::BoxStream<'static, Vec> { + self.scan_ssts_from_manifest_projected(filters, batch_size, None) + } + + /// Like [`Self::scan_ssts_from_manifest`], but avoids allocating unrequested + /// variable-sized fields. The projection must include residual filter columns. + pub fn scan_ssts_from_manifest_projected( + &self, + filters: &[Expr], + batch_size: usize, + projection: Option<&[usize]>, + ) -> futures::stream::BoxStream<'static, Vec> { + let engine = self.clone(); + let projection = crate::region::ManifestSstProjection::new(projection); + let filters = MetadataFilter::new(filters); + Box::pin(async_stream::stream! { + let node_id = engine.inner.workers.file_ref_manager().node_id(); + if !filters.matches(|column| match column { + sst_entry::NODE_ID => Some(ScalarValue::UInt64(node_id)), + _ => None, + }) { + return; } - results.extend(entries); - } - - results + let mut batch = Vec::with_capacity(batch_size.max(1)); + for region in engine.inner.workers.all_regions() { + if !manifest_region_matches(region.region_id, node_id, &filters) + || !filters.matches(|column| match column { + sst_entry::TABLE_DIR => Some(ScalarValue::Utf8(Some(region.table_dir().to_string()))), + _ => None, + }) + { + continue; + } + let mut entries = region.manifest_sst_stream(filters.clone(), projection); + while let Some(mut entry) = entries.next().await { + entry.node_id = node_id; + batch.push(entry); + if batch.len() == batch_size.max(1) { + yield std::mem::replace(&mut batch, Vec::with_capacity(batch_size.max(1))); + } + } + } + if !batch.is_empty() { + yield batch; + } + }) } /// Lists metadata about all puffin index targets stored in the engine. @@ -1653,6 +1704,25 @@ impl MitoEngine { } } +/// Evaluates region-constant predicates before visiting any of its files. +fn manifest_region_matches( + region_id: RegionId, + node_id: Option, + filters: &MetadataFilter, +) -> bool { + filters.matches(|column| { + Some(match column { + sst_entry::REGION_ID => ScalarValue::UInt64(Some(region_id.as_u64())), + sst_entry::TABLE_ID => ScalarValue::UInt32(Some(region_id.table_id())), + sst_entry::REGION_NUMBER => ScalarValue::UInt32(Some(region_id.region_number())), + sst_entry::REGION_GROUP => ScalarValue::UInt8(Some(region_id.region_group())), + sst_entry::REGION_SEQUENCE => ScalarValue::UInt32(Some(region_id.region_sequence())), + sst_entry::NODE_ID => ScalarValue::UInt64(node_id), + _ => return None, + }) + }) +} + #[cfg(test)] mod tests { use std::time::Duration; @@ -1660,6 +1730,30 @@ mod tests { use super::*; use crate::sst::file::FileMeta; + #[test] + fn test_manifest_region_predicates() { + use super::manifest_region_matches; + use common_recordbatch::filter::MetadataFilter; + use datafusion_expr::{col, lit}; + let region = store_api::storage::RegionId::new(42, 3); + for (expr, expected) in [ + (col("region_id").eq(lit(region.as_u64())), true), + (col("region_id").eq(lit(region.as_u64() + 1)), false), + (col("table_id").eq(lit(42u32)), true), + (col("node_id").is_null(), true), + (col("file_size").gt(lit(100u64)), true), + // A type mismatch is left to the residual filter, never pruned. + (col("region_id").eq(lit("invalid")), true), + ] { + let filter = MetadataFilter::new(std::slice::from_ref(&expr)); + assert_eq!( + manifest_region_matches(region, None, &filter), + expected, + "{expr}" + ); + } + } + #[test] fn test_is_valid_region_edit() { // Valid: has only "files_to_add" diff --git a/src/mito2/src/engine/alter_test.rs b/src/mito2/src/engine/alter_test.rs index 9ff56d0ddab..aee05880125 100644 --- a/src/mito2/src/engine/alter_test.rs +++ b/src/mito2/src/engine/alter_test.rs @@ -37,7 +37,9 @@ use datatypes::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; use datatypes::extension::json::Json2ExtensionType; use datatypes::json::value::JsonValue; use datatypes::prelude::ConcreteDataType; -use datatypes::schema::{ColumnSchema, FulltextAnalyzer, FulltextBackend, FulltextOptions}; +use datatypes::schema::{ + ColumnDefaultConstraint, ColumnSchema, FulltextAnalyzer, FulltextBackend, FulltextOptions, +}; use datatypes::types::json_type::{JsonNativeType, JsonObjectType}; use store_api::codec::PrimaryKeyEncoding; use store_api::logstore::provider::Provider; @@ -46,8 +48,8 @@ use store_api::metric_engine_consts::{PRIMARY_KEY_ENCODING, TABLE_COLUMN_METADAT use store_api::region_engine::{RegionEngine, RegionManifestInfo, RegionRole}; use store_api::region_request::{ AddColumn, AddColumnLocation, AlterKind, ModifyColumnType, PathType, RegionAlterRequest, - RegionBulkInsertsRequest, RegionCompactRequest, RegionOpenRequest, RegionPutRequest, - RegionRequest, SetIndexOption, SetRegionOption, UnsetRegionOption, + RegionBulkInsertsRequest, RegionCompactRequest, RegionCreateRequest, RegionOpenRequest, + RegionPutRequest, RegionRequest, SetIndexOption, SetRegionOption, UnsetRegionOption, }; use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME; use store_api::storage::{ColumnId, RegionId, ScanRequest}; @@ -56,6 +58,7 @@ use crate::config::MitoConfig; use crate::engine::MitoEngine; use crate::engine::listener::{AlterFlushListener, NotifyRegionChangeResultListener}; use crate::error; +use crate::request::WorkerRequest; use crate::sst::FormatType; use crate::test_util::batch_util::sort_batches_and_print; use crate::test_util::sst_util::{new_sparse_primary_key, sst_region_metadata_with_encoding}; @@ -3904,3 +3907,157 @@ async fn test_alter_time_index_widen_strict_window_compaction() { // Each instant exactly once, rescaled to microseconds. assert_eq!(vec![0, 1_000_000, 2_000_000, 3_000_000, 4_000_000], values); } + +/// Returns a create request with a field `field_1` that defaults to 42, and the +/// schema of rows that leave `field_1` out, so the engine fills it on puts. +fn create_request_with_default_field() -> (RegionCreateRequest, Vec) { + let mut request = CreateRequestBuilder::new().build(); + let column_schemas = rows_schema(&request); + request.column_metadatas.push(ColumnMetadata { + column_schema: ColumnSchema::new("field_1", ConcreteDataType::float64_datatype(), true) + .with_default_constraint(Some(ColumnDefaultConstraint::Value( + datatypes::value::Value::Float64(42.0.into()), + ))) + .unwrap(), + semantic_type: SemanticType::Field, + column_id: 3, + }); + (request, column_schemas) +} + +fn add_nullable_field2() -> RegionAlterRequest { + RegionAlterRequest { + kind: AlterKind::AddColumns { + columns: vec![AddColumn { + column_metadata: ColumnMetadata { + column_schema: ColumnSchema::new( + "field_2", + ConcreteDataType::float64_datatype(), + true, + ), + semantic_type: SemanticType::Field, + column_id: 4, + }, + location: None, + }], + }, + } +} + +fn build_put_request( + schema: Vec, + start: usize, + end: usize, +) -> RegionRequest { + RegionRequest::Put(RegionPutRequest { + rows: Rows { + schema, + rows: build_rows(start, end), + }, + hint: None, + skip_wal: false, + partition_expr_version: None, + }) +} + +/// Sends a put that the engine boundary checked and filled with `metadata`, +/// the region metadata before an alter, and returns the worker's answer. +async fn put_filled_with_stale_metadata( + engine: &MitoEngine, + region_id: RegionId, + metadata: store_api::metadata::RegionMetadataRef, + schema: Vec, +) -> error::Result { + let (request, receiver) = WorkerRequest::try_from_region_request( + region_id, + build_put_request(schema, 0, 2), + Some(metadata), + ) + .unwrap(); + engine + .inner + .workers + .submit_to_worker(region_id, request) + .await + .unwrap(); + receiver + .await + .expect("the worker dropped the request without an answer") +} + +/// A put filled with a default value at the engine boundary is checked again by +/// the worker after a schema change; the filled column must not break the check. +#[tokio::test] +async fn test_put_filled_at_boundary_rechecked_after_add_column() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::new().await; + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let (request, column_schemas) = create_request_with_default_field(); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + let metadata = engine.get_region(region_id).unwrap().metadata(); + engine + .handle_request(region_id, RegionRequest::Alter(add_nullable_field2())) + .await + .unwrap(); + + let result = + put_filled_with_stale_metadata(&engine, region_id, metadata, column_schemas.clone()).await; + assert_eq!(2, result.unwrap()); + + // The worker still serves the region. + let response = engine + .handle_request(region_id, build_put_request(column_schemas, 2, 3)) + .await + .unwrap(); + assert_eq!(1, response.affected_rows); + let scanner = engine + .scanner(region_id, ScanRequest::default()) + .await + .unwrap(); + let batches = RecordBatches::try_collect(scanner.scan().await.unwrap()) + .await + .unwrap(); + let expected = "\ ++-------+---------+---------------------+---------+---------+ +| tag_0 | field_0 | ts | field_1 | field_2 | ++-------+---------+---------------------+---------+---------+ +| 0 | 0.0 | 1970-01-01T00:00:00 | 42.0 | | +| 1 | 1.0 | 1970-01-01T00:00:01 | 42.0 | | +| 2 | 2.0 | 1970-01-01T00:00:02 | 42.0 | | ++-------+---------+---------------------+---------+---------+"; + assert_eq!(expected, batches.pretty_print().unwrap()); +} + +/// A put that left out a column which is dropped before the worker handles the +/// put is still valid. +#[tokio::test] +async fn test_put_filled_at_boundary_rechecked_after_drop_column() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::new().await; + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let (request, column_schemas) = create_request_with_default_field(); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + let metadata = engine.get_region(region_id).unwrap().metadata(); + engine + .handle_request( + region_id, + RegionRequest::Alter(RegionAlterRequest { + kind: AlterKind::DropColumns { + names: vec!["field_1".to_string()], + }, + }), + ) + .await + .unwrap(); + + let result = put_filled_with_stale_metadata(&engine, region_id, metadata, column_schemas).await; + assert_eq!(2, result.unwrap()); +} diff --git a/src/mito2/src/engine/basic_test.rs b/src/mito2/src/engine/basic_test.rs index c11ec4324d6..d16df1457a5 100644 --- a/src/mito2/src/engine/basic_test.rs +++ b/src/mito2/src/engine/basic_test.rs @@ -16,16 +16,22 @@ use std::collections::HashMap; use std::sync::atomic::Ordering; +use std::time::Duration; use api::v1::helper::row; use api::v1::value::ValueData; -use api::v1::{Rows, SemanticType}; +use api::v1::{ArrowIpc, Rows, SemanticType}; use common_base::readable_size::ReadableSize; use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; -use common_recordbatch::RecordBatches; +use common_recordbatch::{DfRecordBatch, RecordBatches}; +use common_test_util::flight::encode_to_flight_data; use common_time::Timestamp; use common_wal::options::WAL_OPTIONS_KEY; +use datatypes::arrow::array::{ + ArrayRef, DictionaryArray, Float64Array, StringArray, TimestampMillisecondArray, +}; +use datatypes::arrow::datatypes::{DataType, Field, Schema, TimeUnit, UInt32Type}; use datatypes::prelude::ConcreteDataType; use datatypes::schema::ColumnSchema; use futures::TryStreamExt; @@ -35,13 +41,14 @@ use rstest::rstest; use rstest_reuse::{self, apply}; use store_api::metadata::ColumnMetadata; use store_api::region_request::{ - PathType, RegionCreateRequest, RegionFlushRequest, RegionOpenRequest, RegionPutRequest, - RegionRequirements, + PathType, RegionBulkInsertsRequest, RegionCreateRequest, RegionFlushRequest, RegionOpenRequest, + RegionPutRequest, RegionRequirements, }; use store_api::storage::RegionId; use super::*; use crate::region::version::VersionControlData; +use crate::region_write_ctx::test_hooks::arm_bulk_install_barrier; use crate::test_util::{ CreateRequestBuilder, LogStoreFactory, TestEnv, build_delete_rows_for_key, build_rows, build_rows_for_key, delete_rows, delete_rows_schema, flush_region, kafka_log_store_factory, @@ -1872,3 +1879,415 @@ async fn restart_without_region_close( .unwrap(); engine } + +/// Builds a bulk insert request for a region created by [CreateRequestBuilder] +/// with its default columns `(tag_0, field_0, ts)`. +pub(crate) fn build_bulk_insert_request( + region_id: RegionId, + num_rows: usize, +) -> RegionBulkInsertsRequest { + let schema = Arc::new(Schema::new(vec![ + Field::new("tag_0", DataType::Utf8, true), + Field::new("field_0", DataType::Float64, true), + Field::new( + "ts", + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + ])); + let tag = Arc::new(StringArray::from_iter_values( + (0..num_rows).map(|v| format!("a{v}")), + )) as ArrayRef; + let field = Arc::new(Float64Array::from_iter_values( + (0..num_rows).map(|v| v as f64), + )) as ArrayRef; + let ts = Arc::new(TimestampMillisecondArray::from_iter_values( + (0..num_rows).map(|v| v as i64 * 1000), + )) as ArrayRef; + let payload = DfRecordBatch::try_new(schema, vec![tag, field, ts]).unwrap(); + let (schema, record_batch) = encode_to_flight_data(payload.clone()); + + RegionBulkInsertsRequest { + skip_wal: false, + region_id, + payload, + raw_data: ArrowIpc { + schema: schema.data_header, + data_header: record_batch.data_header, + payload: record_batch.data_body, + }, + partition_expr_version: None, + aligned_schema_version: None, + } +} + +/// Builds a bulk insert request whose WAL entry can't be encoded. The batch +/// misses `field_0`, so the worker fills it and clears `raw_data`, and its +/// all-null dictionary `tag_0` column can't be encoded into a single IPC +/// message. +fn build_unencodable_bulk_insert_request(region_id: RegionId) -> RegionBulkInsertsRequest { + let schema = Arc::new(Schema::new(vec![ + Field::new( + "tag_0", + DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)), + true, + ), + Field::new( + "ts", + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + ])); + let tag = Arc::new(DictionaryArray::::from_iter([None::<&str>])) as ArrayRef; + let ts = Arc::new(TimestampMillisecondArray::from_iter_values([0])) as ArrayRef; + let payload = DfRecordBatch::try_new(schema, vec![tag, ts]).unwrap(); + + RegionBulkInsertsRequest { + skip_wal: false, + region_id, + payload, + // The worker clears it after filling the missing column. + raw_data: ArrowIpc::default(), + partition_expr_version: None, + aligned_schema_version: None, + } +} + +async fn scan_num_rows(engine: &MitoEngine, region_id: RegionId) -> usize { + let stream = engine + .scan_to_stream(region_id, ScanRequest::default()) + .await + .unwrap(); + let batches = RecordBatches::try_collect(stream).await.unwrap(); + batches.iter().map(|batch| batch.num_rows()).sum() +} + +#[tokio::test] +async fn test_bulk_insert_encode_failure_keeps_worker_batch() { + common_telemetry::init_default_ut_logging(); + + let mut env = TestEnv::new().await; + // A single worker so requests of all regions share one batch. + let engine = env + .create_engine(MitoConfig { + num_workers: 1, + ..Default::default() + }) + .await; + + let blocker_region_id = RegionId::new(1, 1); + let region_id = RegionId::new(1, 2); + let other_region_id = RegionId::new(1, 3); + for id in [blocker_region_id, region_id, other_region_id] { + engine + .handle_request( + id, + RegionRequest::Create(CreateRequestBuilder::new().build()), + ) + .await + .unwrap(); + } + + // Pauses the worker in a bulk write of the blocker region so the requests + // below queue up and reach the worker as one batch. + let mut barrier = arm_bulk_install_barrier( + blocker_region_id, + engine + .get_region(blocker_region_id) + .unwrap() + .version_control + .clone(), + ); + let blocker_engine = engine.clone(); + let blocker = tokio::spawn(async move { + blocker_engine + .handle_request( + blocker_region_id, + RegionRequest::BulkInserts(build_bulk_insert_request(blocker_region_id, 1)), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_reached()) + .await + .expect("bulk write never reached the install barrier"); + + let mut receivers = Vec::new(); + for (id, request) in [ + (region_id, build_unencodable_bulk_insert_request(region_id)), + (region_id, build_bulk_insert_request(region_id, 3)), + ( + other_region_id, + build_bulk_insert_request(other_region_id, 2), + ), + ] { + let metadata = engine.inner.get_metadata(id).ok(); + let (request, receiver) = WorkerRequest::try_from_region_request( + id, + RegionRequest::BulkInserts(request), + metadata, + ) + .unwrap(); + engine + .inner + .workers + .submit_to_worker(id, request) + .await + .unwrap(); + receivers.push(receiver); + } + barrier.release(); + assert_eq!(1, blocker.await.unwrap().unwrap().affected_rows); + + let [failed, same_region, other_region] = receivers.try_into().unwrap(); + let err = failed.await.unwrap().unwrap_err(); + assert_eq!(StatusCode::Unsupported, err.status_code(), "{err:?}"); + // Requests after the failed one in the same batch are still written. + assert_eq!(3, same_region.await.unwrap().unwrap()); + assert_eq!(2, other_region.await.unwrap().unwrap()); + assert_eq!(3, scan_num_rows(&engine, region_id).await); + assert_eq!(2, scan_num_rows(&engine, other_region_id).await); +} + +#[tokio::test] +async fn test_manifest_scan_region_filter() { + use datafusion_expr::{col, lit}; + use futures::StreamExt; + + let regions = 3; + let files_per_region = 64; + let mut env = TestEnv::new().await; + let engine = env.create_engine(MitoConfig::default()).await; + for number in 0..regions { + add_manifest_scan_region(&engine, RegionId::new(11, number), files_per_region).await; + } + + let target = RegionId::new(11, 1); + let batch_size = 17; + let mut reference = None; + for pushed in [false, true] { + let filters = if pushed { + vec![col("region_id").eq(lit(target.as_u64()))] + } else { + vec![] + }; + let mut stream = engine.scan_ssts_from_manifest(&filters, batch_size); + let mut examined_entries = 0; + let mut selected = Vec::new(); + while let Some(batch) = stream.next().await { + assert!(batch.len() <= batch_size); + examined_entries += batch.len(); + selected.extend(batch.into_iter().filter(|entry| entry.region_id == target)); + } + assert_eq!(selected.len(), files_per_region); + assert_eq!( + examined_entries, + if pushed { + files_per_region + } else { + regions as usize * files_per_region + } + ); + selected.sort_unstable_by(|a, b| a.file_id.cmp(&b.file_id)); + if let Some(reference) = &reference { + assert_eq!(&selected, reference); + } else { + reference = Some(selected); + } + } + // Region pruning must happen before acquiring a manifest snapshot. Holding + // another region's write lock would deadlock an unpruned enumeration. + let unrelated = engine.get_region(RegionId::new(11, 0)).unwrap(); + let guard = unrelated.manifest_ctx.manifest_manager.write().await; + // Node and directory predicates also reject candidates before opening any + // manifest; otherwise the locked region would block these empty scans. + for predicate in [ + col("node_id").is_not_null(), + col("table_dir").eq(lit("missing/")), + ] { + let batches = tokio::time::timeout( + Duration::from_secs(5), + engine + .scan_ssts_from_manifest(&[predicate], batch_size) + .collect::>(), + ) + .await + .expect("pruned scan must not acquire a manifest lock"); + assert!(batches.is_empty()); + } + let filtered = + engine.scan_ssts_from_manifest(&[col("region_id").eq(lit(target.as_u64()))], batch_size); + let batches = tokio::time::timeout(Duration::from_secs(5), filtered.collect::>()) + .await + .expect("filtered scan must not acquire an unrelated manifest lock"); + assert_eq!( + batches.iter().map(Vec::len).sum::(), + files_per_region + ); + drop(guard); + + let schema = store_api::sst_entry::ManifestSstEntry::schema(); + let projection = ["file_id", "min_ts"] + .map(|name| { + schema + .column_schemas() + .iter() + .position(|c| c.name == name) + .unwrap() + }) + .to_vec(); + let mut projected = engine.scan_ssts_from_manifest_projected( + &[col("region_id").eq(lit(target.as_u64()))], + batch_size, + Some(&projection), + ); + let mut projected_entries = Vec::new(); + while let Some(batch) = projected.next().await { + assert!( + batch + .iter() + .all(|entry| entry.table_dir.is_empty() && entry.file_path.is_empty()) + ); + projected_entries.extend(batch); + } + projected_entries.sort_unstable_by(|a, b| a.file_id.cmp(&b.file_id)); + let builder = store_api::sst_entry::ManifestSstBatchBuilder::new(Arc::new( + store_api::sst_entry::ManifestSstEntry::schema() + .arrow_schema() + .project(&projection) + .unwrap(), + )); + let expected = builder.build(&reference.unwrap()).unwrap(); + let actual = builder.build(&projected_entries).unwrap(); + assert_eq!(actual, expected); + assert!( + engine + .scan_ssts_from_manifest(&[col("region_id").eq(lit(0u64))], batch_size) + .next() + .await + .is_none() + ); +} + +/// Installs synthetic immutable file metadata without creating physical SSTs. +async fn add_manifest_scan_region( + engine: &MitoEngine, + region_id: RegionId, + files_per_region: usize, +) { + use crate::manifest::action::{RegionEdit, RegionMetaAction, RegionMetaActionList}; + use crate::sst::file::FileMeta; + use store_api::storage::FileId; + engine + .handle_request( + region_id, + RegionRequest::Create(CreateRequestBuilder::new().build()), + ) + .await + .unwrap(); + let region = engine.get_region(region_id).unwrap(); + let edit = RegionEdit { + files_to_add: (0..files_per_region) + .map(|index| FileMeta { + region_id, + file_id: FileId::from_bytes( + (((region_id.region_number() as u128) << 64) | index as u128).to_be_bytes(), + ), + time_range: ( + Timestamp::new_millisecond(index as i64), + Timestamp::new_millisecond(index as i64 + 1), + ), + file_size: 1024 + index as u64, + ..Default::default() + }) + .collect(), + files_to_remove: vec![], + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id: None, + flushed_sequence: None, + committed_sequence: None, + }; + region + .manifest_ctx + .manifest_manager + .write() + .await + .update( + RegionMetaActionList::new(vec![RegionMetaAction::Edit(edit)]), + false, + ) + .await + .unwrap(); +} + +#[tokio::test] +async fn test_manifest_scan_table_filter() { + use datafusion_expr::{col, lit}; + use futures::StreamExt; + use store_api::sst_entry::ManifestSstEntry; + + let mut env = TestEnv::new().await; + let engine = env.create_engine(MitoConfig::default()).await; + let target_regions = [RegionId::new(11, 0), RegionId::new(11, 1)]; + let unrelated_id = RegionId::new(12, 0); + for region_id in target_regions.into_iter().chain([unrelated_id]) { + add_manifest_scan_region(&engine, region_id, 3).await; + } + let mut expected = engine.all_ssts_from_manifest().await; + expected.retain(|entry| entry.table_id == 11); + expected.sort_unstable_by_key(|entry| (entry.region_id, entry.file_id.clone())); + assert_eq!(expected.len(), 6); + + // A table predicate must exclude other tables before locking their manifests, + // while keeping every region of the selected table (including the same number). + let unrelated = engine.get_region(unrelated_id).unwrap(); + let guard = unrelated.manifest_ctx.manifest_manager.write().await; + let batches = tokio::time::timeout( + Duration::from_secs(5), + engine + .scan_ssts_from_manifest(&[col("table_id").eq(lit(11u32))], 2) + .collect::>(), + ) + .await + .expect("table filtering must not acquire another table's manifest lock"); + assert!(batches.iter().all(|batch| batch.len() <= 2)); + let mut actual = batches.into_iter().flatten().collect::>(); + actual.sort_unstable_by_key(|entry| (entry.region_id, entry.file_id.clone())); + assert_eq!(actual, expected); + + let schema = ManifestSstEntry::schema(); + let projection = ["region_id", "file_id"] + .map(|name| { + schema + .column_schemas() + .iter() + .position(|column| column.name == name) + .unwrap() + }) + .to_vec(); + let batches = tokio::time::timeout( + Duration::from_secs(5), + engine + .scan_ssts_from_manifest_projected( + &[col("table_id").eq(lit(11u32))], + 2, + Some(&projection), + ) + .collect::>(), + ) + .await + .expect("hidden table predicate must prune unrelated manifests"); + let mut actual = batches.into_iter().flatten().collect::>(); + actual.sort_unstable_by_key(|entry| (entry.region_id, entry.file_id.clone())); + let builder = store_api::sst_entry::ManifestSstBatchBuilder::new(Arc::new( + ManifestSstEntry::schema() + .arrow_schema() + .project(&projection) + .unwrap(), + )); + assert_eq!( + builder.build(&actual).unwrap(), + builder.build(&expected).unwrap() + ); + drop(guard); +} diff --git a/src/mito2/src/engine/bulk_insert_validation_test.rs b/src/mito2/src/engine/bulk_insert_validation_test.rs new file mode 100644 index 00000000000..9652152031f --- /dev/null +++ b/src/mito2/src/engine/bulk_insert_validation_test.rs @@ -0,0 +1,329 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Bulk inserts checked against the region schema. + +use std::sync::Arc; + +use api::v1::{ArrowIpc, Rows}; +use common_error::ext::ErrorExt; +use common_error::status_code::StatusCode; +use common_recordbatch::{DfRecordBatch, RecordBatches}; +use common_test_util::flight::encode_to_flight_data; +use datatypes::arrow::array::{ + ArrayRef, Float64Array, Int64Array, LargeStringArray, ListArray, StringArray, StringViewArray, + TimestampMillisecondArray, TimestampSecondArray, +}; +use datatypes::arrow::datatypes::{Field, Int64Type, Schema}; +use datatypes::prelude::ConcreteDataType; +use store_api::region_engine::RegionEngine; +use store_api::region_request::{ + AlterKind, ModifyColumnType, RegionAlterRequest, RegionBulkInsertsRequest, RegionPutRequest, + RegionRequest, +}; +use store_api::storage::{RegionId, ScanRequest}; + +use crate::config::MitoConfig; +use crate::engine::MitoEngine; +use crate::request::WorkerRequest; +use crate::test_util::{CreateRequestBuilder, TestEnv, build_rows, reopen_region, rows_schema}; + +const SST_FORMATS: [&str; 2] = ["flat", "primary_key"]; + +fn column(name: &str, array: ArrayRef) -> (Field, ArrayRef) { + (Field::new(name, array.data_type().clone(), true), array) +} + +fn tag() -> (Field, ArrayRef) { + column("tag_0", Arc::new(StringArray::from(vec!["a", "b"]))) +} + +fn field() -> (Field, ArrayRef) { + column("field_0", Arc::new(Float64Array::from(vec![1.0, 2.0]))) +} + +fn ts() -> (Field, ArrayRef) { + column( + "ts", + Arc::new(TimestampMillisecondArray::from(vec![0, 1000])), + ) +} + +fn bulk_request(region_id: RegionId, columns: Vec<(Field, ArrayRef)>) -> RegionRequest { + RegionRequest::BulkInserts(bulk_insert(region_id, columns)) +} + +fn bulk_insert(region_id: RegionId, columns: Vec<(Field, ArrayRef)>) -> RegionBulkInsertsRequest { + let (fields, arrays): (Vec<_>, Vec<_>) = columns.into_iter().unzip(); + let payload = DfRecordBatch::try_new(Arc::new(Schema::new(fields)), arrays).unwrap(); + let (schema, record_batch) = encode_to_flight_data(payload.clone()); + RegionBulkInsertsRequest { + skip_wal: false, + region_id, + payload, + raw_data: ArrowIpc { + schema: schema.data_header, + data_header: record_batch.data_header, + payload: record_batch.data_body, + }, + partition_expr_version: None, + aligned_schema_version: None, + } +} + +async fn scan_num_rows(engine: &MitoEngine, region_id: RegionId) -> usize { + let stream = engine + .scan_to_stream(region_id, ScanRequest::default()) + .await + .unwrap(); + let batches = RecordBatches::try_collect(stream).await.unwrap(); + batches.iter().map(|batch| batch.num_rows()).sum() +} + +/// Batches that do not match the region schema are rejected before they are +/// appended to the WAL, and the region keeps accepting writes. +#[tokio::test] +async fn test_bulk_insert_rejects_batches_not_matching_schema() { + common_telemetry::init_default_ut_logging(); + let list = ListArray::from_iter_primitive::(vec![ + Some(vec![Some(1)]), + Some(vec![Some(2)]), + ]); + let rejected = vec![ + vec![ + tag(), + field(), + column("ts", Arc::new(Int64Array::from(vec![0, 1000]))), + ], + vec![ + tag(), + field(), + column( + "ts", + Arc::new(TimestampMillisecondArray::from(vec![Some(0), None])), + ), + ], + vec![ + tag(), + field(), + column( + "ts", + Arc::new(TimestampMillisecondArray::from(vec![None, None])), + ), + ], + vec![ + tag(), + field(), + column("ts", Arc::new(TimestampSecondArray::from(vec![0, 1]))), + ], + vec![tag(), field()], + vec![ + tag(), + column("field_0", Arc::new(StringArray::from(vec!["1", "2"]))), + ts(), + ], + vec![tag(), column("field_0", Arc::new(list)), ts()], + vec![ + tag(), + column( + "field_0", + Arc::new(Float64Array::from(vec![Some(1.0), None])), + ), + ts(), + ], + vec![tag(), field(), field(), ts()], + vec![ + tag(), + field(), + ts(), + column("unknown", Arc::new(Int64Array::from(vec![1, 2]))), + ], + ]; + for (i, sst_format) in SST_FORMATS.into_iter().enumerate() { + let mut env = TestEnv::new().await; + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, i as u32 + 1); + let request = CreateRequestBuilder::new() + .all_not_null(true) + .insert_option("sst_format", sst_format) + .build(); + let column_schemas = rows_schema(&request); + let table_dir = request.table_dir.clone(); + let options = request.options.clone(); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + + for columns in &rejected { + let err = engine + .handle_request(region_id, bulk_request(region_id, columns.clone())) + .await + .unwrap_err(); + assert_eq!( + StatusCode::InvalidArguments, + err.status_code(), + "{sst_format}: {err:?}" + ); + } + + let put = RegionRequest::Put(RegionPutRequest { + rows: Rows { + schema: column_schemas, + rows: build_rows(0, 3), + }, + hint: None, + skip_wal: false, + partition_expr_version: None, + }); + engine.handle_request(region_id, put).await.unwrap(); + // Reopening replays the WAL, which holds none of the rejected batches. + reopen_region(&engine, region_id, table_dir, true, options).await; + assert_eq!(3, scan_num_rows(&engine, region_id).await, "{sst_format}"); + } +} + +/// Tags may use any string encoding, which the memtables convert. +#[tokio::test] +async fn test_bulk_insert_accepts_tag_encodings() { + common_telemetry::init_default_ut_logging(); + let tags: [ArrayRef; 3] = [ + Arc::new(StringArray::from(vec!["a", "b"])), + Arc::new(LargeStringArray::from(vec!["a", "b"])), + Arc::new(StringViewArray::from(vec!["a", "b"])), + ]; + for (i, sst_format) in SST_FORMATS.into_iter().enumerate() { + let mut env = TestEnv::new().await; + let engine = env.create_engine(MitoConfig::default()).await; + for (j, tag) in tags.iter().enumerate() { + let region_id = RegionId::new(i as u32 + 1, j as u32 + 1); + let request = CreateRequestBuilder::new() + .insert_option("sst_format", sst_format) + .build(); + let table_dir = request.table_dir.clone(); + let options = request.options.clone(); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + + let columns = vec![column("tag_0", tag.clone()), field(), ts()]; + let response = engine + .handle_request(region_id, bulk_request(region_id, columns)) + .await + .unwrap(); + assert_eq!(2, response.affected_rows, "{sst_format}: {tag:?}"); + reopen_region(&engine, region_id, table_dir, true, options).await; + assert_eq!( + 2, + scan_num_rows(&engine, region_id).await, + "{sst_format}: {tag:?}" + ); + } + } +} + +/// The worker checks a batch again if the region schema changed after the +/// request was checked. +#[tokio::test] +async fn test_bulk_insert_rechecks_batch_after_schema_change() { + common_telemetry::init_default_ut_logging(); + for (i, sst_format) in SST_FORMATS.into_iter().enumerate() { + let mut env = TestEnv::new().await; + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, i as u32 + 1); + let request = CreateRequestBuilder::new() + .insert_option("sst_format", sst_format) + .build(); + let table_dir = request.table_dir.clone(); + let options = request.options.clone(); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + let metadata = engine.get_region(region_id).unwrap().metadata(); + + // The batch matches the schema before `field_0` becomes a string. + let alter = RegionAlterRequest { + kind: AlterKind::ModifyColumnTypes { + columns: vec![ModifyColumnType { + column_name: "field_0".to_string(), + target_type: ConcreteDataType::string_datatype(), + }], + }, + }; + engine + .handle_request(region_id, RegionRequest::Alter(alter)) + .await + .unwrap(); + let (request, receiver) = WorkerRequest::try_from_region_request( + region_id, + bulk_request(region_id, vec![tag(), field(), ts()]), + Some(metadata), + ) + .unwrap(); + engine + .inner + .workers + .submit_to_worker(region_id, request) + .await + .unwrap(); + let err = receiver.await.unwrap().unwrap_err(); + assert_eq!( + StatusCode::InvalidArguments, + err.status_code(), + "{sst_format}: {err:?}" + ); + + reopen_region(&engine, region_id, table_dir, true, options).await; + assert_eq!(0, scan_num_rows(&engine, region_id).await, "{sst_format}"); + } +} + +/// A batch that claims to match the current schema but misses a column still +/// gets the column filled. +#[tokio::test] +async fn test_bulk_insert_fills_columns_missing_from_aligned_batch() { + common_telemetry::init_default_ut_logging(); + for (i, sst_format) in SST_FORMATS.into_iter().enumerate() { + let mut env = TestEnv::new().await; + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, i as u32 + 1); + let request = CreateRequestBuilder::new() + .insert_option("sst_format", sst_format) + .build(); + let table_dir = request.table_dir.clone(); + let options = request.options.clone(); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + + let mut request = bulk_insert(region_id, vec![tag(), ts()]); + request.aligned_schema_version = Some( + engine + .get_region(region_id) + .unwrap() + .metadata() + .schema_version, + ); + let response = engine + .handle_request(region_id, RegionRequest::BulkInserts(request)) + .await + .unwrap(); + assert_eq!(2, response.affected_rows, "{sst_format}"); + reopen_region(&engine, region_id, table_dir, true, options).await; + assert_eq!(2, scan_num_rows(&engine, region_id).await, "{sst_format}"); + } +} diff --git a/src/mito2/src/engine/set_role_state_test.rs b/src/mito2/src/engine/set_role_state_test.rs index 53aa3511459..8407f53d209 100644 --- a/src/mito2/src/engine/set_role_state_test.rs +++ b/src/mito2/src/engine/set_role_state_test.rs @@ -12,23 +12,35 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; use std::time::Duration; -use api::v1::Rows; -use common_error::ext::ErrorExt; +use api::v1::{Rows, SemanticType}; +use common_error::ext::{ErrorExt, RetryHint}; use common_error::status_code::StatusCode; +use common_recordbatch::RecordBatches; +use datatypes::prelude::ConcreteDataType; +use datatypes::schema::ColumnSchema; +use store_api::metadata::ColumnMetadata; use store_api::region_engine::{ RegionEngine, RegionRole, SetRegionRoleStateResponse, SetRegionRoleStateSuccess, SettableRegionRoleState, }; use store_api::region_request::{ - EnterStagingRequest, RegionFlushRequest, RegionPutRequest, RegionRequest, + AddColumn, AffectedRows, AlterKind, EnterStagingRequest, PathType, RegionAlterRequest, + RegionCatchupRequest, RegionFlushRequest, RegionOpenRequest, RegionPutRequest, RegionRequest, StagingPartitionDirective, }; -use store_api::storage::RegionId; +use store_api::storage::{RegionId, ScanRequest}; +use tokio::sync::oneshot; use crate::config::MitoConfig; -use crate::region::{RegionLeaderState, RegionRoleState}; +use crate::engine::MitoEngine; +use crate::engine::basic_test::build_bulk_insert_request; +use crate::engine::listener::NotifyRegionChangeResultListener; +use crate::region::{MitoRegionRef, RegionLeaderState, RegionRoleState}; +use crate::region_write_ctx::test_hooks::arm_bulk_install_barrier; +use crate::request::WorkerRequest; use crate::test_util::{ CheckpointTaskBlocker, CreateRequestBuilder, TestEnv, build_rows, put_rows, rows_schema, }; @@ -1001,3 +1013,470 @@ async fn test_restricted_state_transitions_with_format(flat_format: bool) { .unwrap(); assert_success_response(&result, 0); } + +const WAIT: Duration = Duration::from_secs(10); + +type WriteReceiver = oneshot::Receiver>; + +/// Submits a graceful role change straight to the region's worker. Requests +/// submitted after this returns reach the worker after it. +async fn submit_role_change( + engine: &MitoEngine, + region_id: RegionId, + state: SettableRegionRoleState, +) -> oneshot::Receiver { + let (request, receiver) = WorkerRequest::new_set_readonly_gracefully(region_id, state); + engine + .inner + .workers + .submit_to_worker(region_id, request) + .await + .unwrap(); + receiver +} + +/// Submits a region request straight to the region's worker. +async fn submit_request( + engine: &MitoEngine, + region_id: RegionId, + request: RegionRequest, +) -> WriteReceiver { + let metadata = engine.inner.get_metadata(region_id).ok(); + let (request, receiver) = + WorkerRequest::try_from_region_request(region_id, request, metadata).unwrap(); + engine + .inner + .workers + .submit_to_worker(region_id, request) + .await + .unwrap(); + receiver +} + +fn put_request(rows: Rows) -> RegionRequest { + RegionRequest::Put(RegionPutRequest { + skip_wal: false, + rows, + hint: None, + partition_expr_version: None, + }) +} + +fn reported_last_entry_id(response: &SetRegionRoleStateResponse) -> u64 { + match response { + SetRegionRoleStateResponse::Success(success) => success.last_entry_id().unwrap(), + _ => panic!("Expected success response, got: {:?}", response), + } +} + +/// Lets the role change task waiting behind `manifest_guard` run and returns +/// the `last_entry_id` it reports. +async fn finish_role_change( + manifest_guard: G, + role_change: oneshot::Receiver, +) -> u64 { + drop(manifest_guard); + let response = tokio::time::timeout(WAIT, role_change) + .await + .expect("role change did not finish") + .unwrap(); + reported_last_entry_id(&response) +} + +/// Waits for the write's result if it is still in flight. +async fn write_result( + early: Option>, + write: WriteReceiver, +) -> crate::error::Result { + match early { + Some(result) => result, + None => tokio::time::timeout(WAIT, write) + .await + .expect("write did not finish") + .unwrap(), + } +} + +/// Checks the promise of `set_region_role_state_gracefully`: an acknowledged +/// write must be covered by the `last_entry_id` the call returned. +fn assert_write_covered( + region: &MitoRegionRef, + result: crate::error::Result, + reported: u64, +) { + let last_entry_id = region.version_control.current().last_entry_id; + assert!( + result.is_err() || last_entry_id <= reported, + "a write was acknowledged ({result:?}) after the role change returned last_entry_id {reported}, \ + but the region's last_entry_id is now {last_entry_id}" + ); +} + +/// The role change task waits for the manifest lock before it changes the +/// role, while the worker keeps handling requests. A bulk write handled after +/// the role change request must be rejected, or be covered by the returned +/// `last_entry_id` if it was accepted. +#[tokio::test(flavor = "multi_thread")] +async fn test_set_follower_gracefully_covers_inflight_bulk_write() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::with_prefix("follower-inflight-bulk-write").await; + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new().build(); + let column_schemas = rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + put_rows( + &engine, + region_id, + Rows { + schema: column_schemas, + rows: build_rows(0, 3), + }, + ) + .await; + + let region = engine.get_region(region_id).unwrap(); + let manifest_guard = region.manifest_ctx.manifest_manager.read().await; + let role_change = + submit_role_change(&engine, region_id, SettableRegionRoleState::Follower).await; + // An accepted write pauses after its WAL append, before installing its rows. + let mut barrier = arm_bulk_install_barrier(region_id, region.version_control.clone()); + let mut write = submit_request( + &engine, + region_id, + RegionRequest::BulkInserts(build_bulk_insert_request(region_id, 2)), + ) + .await; + let early = tokio::select! { + _ = barrier.wait_until_reached() => None, + result = &mut write => Some(result.unwrap()), + _ = tokio::time::sleep(WAIT) => panic!("bulk write never reached the install barrier"), + }; + let reported = finish_role_change(manifest_guard, role_change).await; + assert_eq!(RegionRoleState::Follower, region.state()); + + barrier.release(); + let result = write_result(early, write).await; + assert_write_covered(®ion, result, reported); +} + +/// A lease renewal sets Downgrading outside the worker, so a write that passed +/// the state check before it can still be in flight. The datanode's downgrade +/// steps (DowngradingLeader, Flush, Follower) must still flush that write and +/// report its entry. +#[tokio::test(flavor = "multi_thread")] +async fn test_downgrade_flush_follower_covers_inflight_write() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::with_prefix("downgrade-flush-follower").await; + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + engine + .handle_request( + region_id, + RegionRequest::Create(CreateRequestBuilder::new().build()), + ) + .await + .unwrap(); + + let region = engine.get_region(region_id).unwrap(); + let mut barrier = arm_bulk_install_barrier(region_id, region.version_control.clone()); + let write = submit_request( + &engine, + region_id, + RegionRequest::BulkInserts(build_bulk_insert_request(region_id, 2)), + ) + .await; + tokio::time::timeout(WAIT, barrier.wait_until_reached()) + .await + .expect("bulk write never reached the install barrier"); + engine + .set_region_role(region_id, RegionRole::DowngradingLeader) + .unwrap(); + + let downgrade = submit_role_change( + &engine, + region_id, + SettableRegionRoleState::DowngradingLeader, + ) + .await; + barrier.release(); + assert_eq!(2, write.await.unwrap().unwrap()); + reported_last_entry_id(&downgrade.await.unwrap()); + engine + .handle_request( + region_id, + RegionRequest::Flush(RegionFlushRequest::default()), + ) + .await + .unwrap(); + + let response = engine + .set_region_role_state_gracefully(region_id, SettableRegionRoleState::Follower) + .await + .unwrap(); + let reported = reported_last_entry_id(&response); + let current = region.version_control.current(); + assert_eq!(current.last_entry_id, reported); + assert_eq!(current.version.flushed_entry_id, reported); + assert!(current.version.memtables.is_empty()); +} + +fn add_nullable_field1() -> RegionAlterRequest { + RegionAlterRequest { + kind: AlterKind::AddColumns { + columns: vec![AddColumn { + column_metadata: ColumnMetadata { + column_schema: ColumnSchema::new( + "field_1", + ConcreteDataType::float64_datatype(), + true, + ), + semantic_type: SemanticType::Field, + column_id: 3, + }, + location: None, + }], + }, + } +} + +async fn wait_for_state(region: &MitoRegionRef, state: RegionRoleState) { + tokio::time::timeout(WAIT, async { + while region.state() != state { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("region never reached {state:?}, now {:?}", region.state())); +} + +async fn count_rows(engine: &MitoEngine, region_id: RegionId) -> usize { + let stream = engine + .scan_to_stream(region_id, ScanRequest::default()) + .await + .unwrap(); + let batches = RecordBatches::try_collect(stream).await.unwrap(); + batches.iter().map(|batch| batch.num_rows()).sum() +} + +/// Creates a region with rows `[0, 3)` in an SST and starts an ALTER whose +/// result the worker does not handle until `listener` is woken, so the region +/// stays in Altering. Returns the ALTER task. +async fn create_region_and_start_alter( + env: &TestEnv, + engine: &MitoEngine, + region_id: RegionId, + request: store_api::region_request::RegionCreateRequest, +) -> tokio::task::JoinHandle<()> { + env.get_schema_metadata_manager() + .register_region_table_info( + region_id.table_id(), + "test_table", + "test_catalog", + "test_schema", + None, + env.get_kv_backend(), + ) + .await; + let column_schemas = rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + put_rows( + engine, + region_id, + Rows { + schema: column_schemas, + rows: build_rows(0, 3), + }, + ) + .await; + engine + .handle_request( + region_id, + RegionRequest::Flush(RegionFlushRequest::default()), + ) + .await + .unwrap(); + + let cloned_engine = engine.clone(); + let alter = tokio::spawn(async move { + cloned_engine + .handle_request(region_id, RegionRequest::Alter(add_nullable_field1())) + .await + .unwrap(); + }); + let region = engine.get_region(region_id).unwrap(); + wait_for_state( + ®ion, + RegionRoleState::Leader(RegionLeaderState::Altering), + ) + .await; + alter +} + +/// DowngradingLeader while an ALTER keeps the region in Altering must fail +/// with a retryable error, because the region becomes writable again once the +/// ALTER finishes. A retry after the ALTER fences writes. +#[tokio::test(flavor = "multi_thread")] +async fn test_downgrading_leader_during_alter_stops_writes() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::with_prefix("downgrading-during-alter").await; + let listener = Arc::new(NotifyRegionChangeResultListener::default()); + let engine = env + .create_engine_with(MitoConfig::default(), None, Some(listener.clone()), None) + .await; + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new().build(); + let column_schemas = rows_schema(&request); + let alter = create_region_and_start_alter(&env, &engine, region_id, request).await; + let region = engine.get_region(region_id).unwrap(); + + let response = engine + .set_region_role_state_gracefully(region_id, SettableRegionRoleState::DowngradingLeader) + .await + .unwrap(); + let state_after_downgrade = region.state(); + listener.wake_notify(); + tokio::time::timeout(WAIT, alter).await.unwrap().unwrap(); + match &response { + SetRegionRoleStateResponse::InvalidTransition(err) => { + assert_eq!(RetryHint::Retryable, err.retry_hint(), "{err:?}") + } + _ => panic!( + "DowngradingLeader returned {response:?} with the region in {state_after_downgrade:?}" + ), + } + + let response = engine + .set_region_role_state_gracefully(region_id, SettableRegionRoleState::DowngradingLeader) + .await + .unwrap(); + reported_last_entry_id(&response); + let err = engine + .handle_request( + region_id, + put_request(Rows { + schema: column_schemas, + rows: build_rows(3, 5), + }), + ) + .await + .unwrap_err(); + assert_eq!(StatusCode::RegionNotReady, err.status_code()); +} + +/// The datanode's downgrade steps (DowngradingLeader, Flush, Follower) with +/// the DowngradingLeader step overlapping an ALTER. The region accepts writes +/// until a downgrade takes effect. A migration candidate opens the region from +/// the object store with its own local WAL, so the final flush must carry +/// every write the old leader acknowledged. +#[tokio::test(flavor = "multi_thread")] +async fn test_downgrade_during_alter_keeps_acknowledged_writes_with_local_wal() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::with_prefix("downgrade-during-alter-local-wal").await; + let listener = Arc::new(NotifyRegionChangeResultListener::default()); + let engine = env + .create_engine_with(MitoConfig::default(), None, Some(listener.clone()), None) + .await; + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new().build(); + let column_schemas = rows_schema(&request); + let table_dir = request.table_dir.clone(); + let options = request.options.clone(); + let alter = create_region_and_start_alter(&env, &engine, region_id, request).await; + + // Mirrors `DowngradeRegionsHandler::handle_downgrade_region` in the + // datanode. Metasrv retries the downgrade on a retryable error. + let mut response = engine + .set_region_role_state_gracefully(region_id, SettableRegionRoleState::DowngradingLeader) + .await + .unwrap(); + listener.wake_notify(); + tokio::time::timeout(WAIT, alter).await.unwrap().unwrap(); + put_rows( + &engine, + region_id, + Rows { + schema: column_schemas.clone(), + rows: build_rows(3, 5), + }, + ) + .await; + let mut acknowledged_rows = 5; + if let SetRegionRoleStateResponse::InvalidTransition(err) = &response { + assert_eq!(RetryHint::Retryable, err.retry_hint(), "{err:?}"); + response = engine + .set_region_role_state_gracefully(region_id, SettableRegionRoleState::DowngradingLeader) + .await + .unwrap(); + } + reported_last_entry_id(&response); + engine + .handle_request( + region_id, + RegionRequest::Flush(RegionFlushRequest::default()), + ) + .await + .unwrap(); + if let Ok(output) = engine + .handle_request( + region_id, + put_request(Rows { + schema: column_schemas, + rows: build_rows(5, 8), + }), + ) + .await + { + acknowledged_rows += output.affected_rows; + } + let response = engine + .set_region_role_state_gracefully(region_id, SettableRegionRoleState::Follower) + .await + .unwrap(); + let reported = reported_last_entry_id(&response); + + // The candidate shares the object store but not the local WAL. + let mut candidate_env = TestEnv::with_prefix("downgrade-during-alter-candidate").await; + candidate_env.object_store_manager = env.get_object_store_manager(); + candidate_env.log_store = Some(candidate_env.create_log_store().await); + let candidate = candidate_env.new_mito_engine(MitoConfig::default()).await; + candidate + .handle_request( + region_id, + RegionRequest::Open(RegionOpenRequest { + engine: String::new(), + table_dir, + options, + skip_wal_replay: true, + path_type: PathType::Bare, + checkpoint: None, + requirements: Default::default(), + }), + ) + .await + .unwrap(); + candidate + .handle_request( + region_id, + RegionRequest::Catchup(RegionCatchupRequest { + set_writable: true, + entry_id: Some(reported), + metadata_entry_id: None, + location_id: None, + checkpoint: None, + }), + ) + .await + .unwrap(); + assert_eq!( + acknowledged_rows, + count_rows(&candidate, region_id).await, + "the candidate misses rows the old leader acknowledged; Follower reported last_entry_id {reported}" + ); +} diff --git a/src/mito2/src/engine/staging_test.rs b/src/mito2/src/engine/staging_test.rs index 1b2a7ffee13..98973de4f30 100644 --- a/src/mito2/src/engine/staging_test.rs +++ b/src/mito2/src/engine/staging_test.rs @@ -843,6 +843,104 @@ async fn test_staging_exit_success_with_manifests_with_format(flat_format: bool) assert_eq!(sst_entries.iter().filter(|e| e.visible).count(), 1); assert_eq!(sst_entries.iter().filter(|e| !e.visible).count(), 2); + // Streaming inspection must retain staging entries and their visibility. + { + use futures::StreamExt; + + let mut batches = engine.scan_ssts_from_manifest( + &[datafusion_expr::col("region_id").eq(datafusion_expr::lit(region_id.as_u64()))], + 1, + ); + let mut actual = Vec::new(); + while let Some(batch) = batches.next().await { + assert_eq!(batch.len(), 1); + actual.extend(batch); + } + actual.sort_unstable_by(|a, b| a.file_id.cmp(&b.file_id)); + let mut expected = sst_entries; + expected.sort_unstable_by(|a, b| a.file_id.cmp(&b.file_id)); + assert_eq!(actual, expected); + } + + // Staging metadata overrides the same file in the normal manifest. Filter + // evaluation must use that winning metadata, even if it is not projected. + { + use futures::StreamExt; + + let region = engine.get_region(region_id).unwrap(); + let original = region + .manifest_ctx + .manifest_manager + .read() + .await + .manifest() + .files + .values() + .next() + .unwrap() + .clone(); + let mut overridden = original.clone(); + overridden.file_size = u64::MAX - 1; + for metadata in [overridden, original.clone()] { + region + .manifest_ctx + .manifest_manager + .write() + .await + .update( + RegionMetaActionList::new(vec![RegionMetaAction::Edit(RegionEdit { + files_to_add: vec![metadata.clone()], + files_to_remove: vec![], + timestamp_ms: None, + compaction_time_window: None, + flushed_entry_id: None, + flushed_sequence: None, + committed_sequence: None, + })]), + true, + ) + .await + .unwrap(); + let entries = engine.all_ssts_from_manifest().await; + assert_eq!(entries.len(), 3); + let matches = entries + .iter() + .filter(|entry| entry.file_id == original.file_id.to_string()) + .collect::>(); + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].file_size, metadata.file_size); + assert!(matches[0].visible); + + let schema = store_api::sst_entry::ManifestSstEntry::schema(); + let projection = ["file_id", "visible"] + .map(|name| { + schema + .column_schemas() + .iter() + .position(|c| c.name == name) + .unwrap() + }) + .to_vec(); + let mut stream = engine.scan_ssts_from_manifest_projected( + &[datafusion_expr::col("file_size").eq(datafusion_expr::lit(metadata.file_size))], + 1, + Some(&projection), + ); + let mut filtered = Vec::new(); + while let Some(batch) = stream.next().await { + filtered.extend(batch); + } + assert!( + filtered + .iter() + .any(|entry| entry.file_id == original.file_id.to_string()) + ); + if metadata.file_size == u64::MAX - 1 { + assert_eq!(filtered.len(), 1); + } + } + } + // Exit staging mode successfully engine .set_region_role_state_gracefully(region_id, SettableRegionRoleState::Leader) diff --git a/src/mito2/src/error.rs b/src/mito2/src/error.rs index cc97d2df51a..e9003b65c9b 100644 --- a/src/mito2/src/error.rs +++ b/src/mito2/src/error.rs @@ -1421,7 +1421,9 @@ impl Error { /// Returns whether a failed manifest update may have been persisted. /// /// Unknown errors are treated conservatively because deleting output SSTs after an ambiguous - /// manifest write could leave the manifest referencing missing files. + /// manifest write could leave the manifest referencing missing files. The retry layer returns + /// the last attempt's error, so an error that can follow a stored attempt (throttling, or a 403 + /// from a revoked key, a changed bucket policy or clock skew) never proves nothing was written. pub(crate) fn may_have_persisted_manifest_update(&self) -> bool { match self { Error::UpdateManifest { .. } @@ -1435,11 +1437,9 @@ impl Error { ErrorKind::Unsupported | ErrorKind::ConfigInvalid | ErrorKind::NotFound - | ErrorKind::PermissionDenied | ErrorKind::IsADirectory | ErrorKind::NotADirectory | ErrorKind::AlreadyExists - | ErrorKind::RateLimited | ErrorKind::IsSameFile | ErrorKind::ConditionNotMatch | ErrorKind::RangeNotSatisfied @@ -1817,11 +1817,9 @@ mod tests { ErrorKind::Unsupported, ErrorKind::ConfigInvalid, ErrorKind::NotFound, - ErrorKind::PermissionDenied, ErrorKind::IsADirectory, ErrorKind::NotADirectory, ErrorKind::AlreadyExists, - ErrorKind::RateLimited, ErrorKind::IsSameFile, ErrorKind::ConditionNotMatch, ErrorKind::RangeNotSatisfied, @@ -1834,9 +1832,18 @@ mod tests { ); } - let error = - OpenDalSnafu {}.into_error(object_store::Error::new(ErrorKind::Unexpected, "test")); - assert!(error.may_have_persisted_manifest_update()); + // The last retry can fail with these kinds even though an earlier attempt was stored. + for kind in [ + ErrorKind::Unexpected, + ErrorKind::RateLimited, + ErrorKind::PermissionDenied, + ] { + let error = OpenDalSnafu {}.into_error(object_store::Error::new(kind, "test")); + assert!( + error.may_have_persisted_manifest_update(), + "error kind {kind:?} should not prove the manifest was not persisted" + ); + } let region_id = RegionId::new(1, 1); let error = RegionTruncatedSnafu { region_id }.build(); diff --git a/src/mito2/src/manifest/storage/staging.rs b/src/mito2/src/manifest/storage/staging.rs index a0b350a76cd..e4e9e7f177f 100644 --- a/src/mito2/src/manifest/storage/staging.rs +++ b/src/mito2/src/manifest/storage/staging.rs @@ -23,7 +23,7 @@ use store_api::ManifestVersion; use crate::error::{OpenDalSnafu, Result}; use crate::manifest::storage::delta::DeltaStorage; use crate::manifest::storage::size_tracker::NoopTracker; -use crate::manifest::storage::utils::sort_manifests; +use crate::manifest::storage::utils::{sort_manifests, write_and_put_cache}; use crate::manifest::storage::{file_version, is_delta_file}; /// A simple blob storage for arbitrary binary data in the staging directory. @@ -79,10 +79,7 @@ impl StagingBlobStorage { self.object_store.info().root(), bytes.len() ); - self.object_store - .write(&path, bytes) - .await - .context(OpenDalSnafu)?; + write_and_put_cache(&self.object_store, None, &path, bytes).await?; Ok(()) } @@ -222,13 +219,75 @@ impl StagingStorage { #[cfg(test)] mod tests { + use std::sync::{Arc, Mutex}; + use common_datasource::compression::CompressionType; use common_test_util::temp_dir::create_temp_dir; use futures::TryStreamExt; + use object_store::layers::mock::{MockLayerBuilder, ObservedWriter, WriterObservations, oio}; use object_store::services::Fs; use object_store::{ErrorKind, ObjectStore}; - use super::{StagingStorage, staging_blob_path, staging_manifest_path}; + use super::{StagingBlobStorage, StagingStorage, staging_blob_path, staging_manifest_path}; + + // A test-only per-write limit, not a storage provider's default limit. + const WRITE_LIMIT: usize = 64 * 1024 * 1024; + + struct LimitedWriter { + inner: oio::Writer, + } + + impl oio::Write for LimitedWriter { + async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> { + if bytes.len() > WRITE_LIMIT { + return Err(object_store::Error::new( + ErrorKind::Unexpected, + "test write size limit exceeded", + )); + } + self.inner.write(bytes).await + } + + async fn close(&mut self) -> object_store::Result { + self.inner.close().await + } + + async fn abort(&mut self) -> object_store::Result<()> { + self.inner.abort().await + } + } + + #[tokio::test] + async fn test_staging_blob_put_chunked() { + let tmp_dir = create_temp_dir("test_staging_blob_put_chunked"); + let observations = Arc::new(Mutex::new(WriterObservations::default())); + let captured = observations.clone(); + let layer = MockLayerBuilder::default() + .writer_factory(Arc::new(move |_, _, inner| { + Box::new(ObservedWriter::new( + Box::new(LimitedWriter { inner }), + captured.clone(), + )) + })) + .build() + .unwrap(); + let store = ObjectStore::new(Fs::default().root(&tmp_dir.path().to_string_lossy())) + .unwrap() + .layer(layer); + let storage = StagingBlobStorage::new("staging/blob/".to_string(), store); + + for size in [17, WRITE_LIMIT, WRITE_LIMIT + 17] { + let bytes = (0..size).map(|i| (i % 251) as u8).collect::>(); + *observations.lock().unwrap() = WriterObservations::default(); + storage.put("manifest", bytes.clone()).await.unwrap(); + assert_eq!(storage.get("manifest").await.unwrap(), bytes); + let observed = observations.lock().unwrap(); + assert_eq!(observed.write_sizes.iter().sum::(), size); + assert!(observed.write_sizes.iter().all(|size| *size <= WRITE_LIMIT)); + assert_eq!(observed.write_sizes.len(), size.div_ceil(WRITE_LIMIT)); + assert_eq!(observed.close_calls, 1); + } + } #[test] fn test_staging_path() { diff --git a/src/mito2/src/memtable/bulk.rs b/src/mito2/src/memtable/bulk.rs index 6a09f1d228e..e323fb6b340 100644 --- a/src/mito2/src/memtable/bulk.rs +++ b/src/mito2/src/memtable/bulk.rs @@ -1681,9 +1681,10 @@ mod tests { use datatypes::schema::ColumnSchema; use datatypes::types::json_type::{JsonNativeType, JsonObjectType}; use mito_codec::row_converter::build_primary_key_codec; - use parquet::basic::{Encoding, Type as PhysicalType}; + use parquet::basic::{Compression, Encoding, Type as PhysicalType}; use serde_json::json; use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder, RegionMetadataRef}; + use store_api::storage::consts::{OP_TYPE_COLUMN_NAME, SEQUENCE_COLUMN_NAME}; use super::*; use crate::memtable::bulk::part::BulkPartConverter; @@ -3049,6 +3050,45 @@ mod tests { Ok(()) } + #[test] + fn test_bulk_part_encoder_matches_sst_writer_internal_encodings() -> WhateverResult<()> { + let metadata = metadata_for_test(); + let bulk_part = create_bulk_part_with_converter( + "key", + 0, + vec![1000, 2000, 3000], + vec![Some(1.0), Some(2.0), Some(3.0)], + 100, + )?; + let encoder = BulkPartEncoder::new(metadata, DEFAULT_ROW_GROUP_SIZE)?; + let encoded_part = encoder.encode_part(&bulk_part)?.unwrap(); + let columns = encoded_part + .metadata() + .parquet_metadata + .row_group(0) + .columns(); + + for name in ["ts", SEQUENCE_COLUMN_NAME] { + let column = columns + .iter() + .find(|column| column.column_path().string() == name) + .unwrap(); + assert!( + column + .encodings() + .any(|encoding| encoding == Encoding::DELTA_BINARY_PACKED), + "{name}" + ); + assert!(column.dictionary_page_offset().is_none(), "{name}"); + } + let op_type = columns + .iter() + .find(|column| column.column_path().string() == OP_TYPE_COLUMN_NAME) + .unwrap(); + assert_eq!(Compression::UNCOMPRESSED, op_type.compression()); + Ok(()) + } + #[test] fn test_encoded_part_batch_size_uses_largest_uncompressed_row_group() { const NUM_ROWS: usize = 14; diff --git a/src/mito2/src/memtable/bulk/part.rs b/src/mito2/src/memtable/bulk/part.rs index fecfceafeee..362b687100f 100644 --- a/src/mito2/src/memtable/bulk/part.rs +++ b/src/mito2/src/memtable/bulk/part.rs @@ -74,6 +74,7 @@ use crate::sst::parquet::flat_format::primary_key_column_index; use crate::sst::parquet::format::{PrimaryKeyArray, PrimaryKeyArrayBuilder}; use crate::sst::parquet::{ COLUMN_INDEX_TRUNCATE_LENGTH, PARQUET_METADATA_KEY, SstInfo, apply_float_field_encoding, + apply_internal_column_encoding, }; const INIT_DICT_VALUE_CAPACITY: usize = 8; @@ -1338,7 +1339,6 @@ impl BulkPartEncoder { let key_value_meta = parquet::file::metadata::KeyValue::new(PARQUET_METADATA_KEY.to_string(), json); - // TODO(yingwen): Do we need compression? let mut props = WriterProperties::builder() .set_key_value_metadata(Some(vec![key_value_meta])) .set_write_batch_size(row_group_size) @@ -1346,6 +1346,7 @@ impl BulkPartEncoder { .set_compression(Compression::ZSTD(ZstdLevel::default())) .set_column_index_truncate_length(COLUMN_INDEX_TRUNCATE_LENGTH) .set_statistics_truncate_length(None); + props = apply_internal_column_encoding(props, &metadata); props = apply_float_field_encoding(props, &metadata, float_field_encoding); let writer_props = Some(props.build()); diff --git a/src/mito2/src/memtable/time_series.rs b/src/mito2/src/memtable/time_series.rs index fad746927d4..1fa6cb24b15 100644 --- a/src/mito2/src/memtable/time_series.rs +++ b/src/mito2/src/memtable/time_series.rs @@ -17,7 +17,7 @@ use std::collections::{BTreeMap, Bound, HashSet}; use std::fmt::{Debug, Formatter}; use std::iter; use std::sync::atomic::{AtomicI64, AtomicU64, AtomicUsize, Ordering}; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, PoisonError, RwLock}; use std::time::{Duration, Instant}; use api::v1::OpType; @@ -474,7 +474,13 @@ impl Drop for SeriesMap { let num_field_builders = self .0 .values() - .map(|v| v.read().unwrap().active.num_field_builders()) + // The map may be dropped while unwinding from a panic that poisoned a lock. + .map(|v| { + v.read() + .unwrap_or_else(PoisonError::into_inner) + .active + .num_field_builders() + }) .sum::(); MEMTABLE_ACTIVE_SERIES_COUNT.sub(num_series as i64); MEMTABLE_ACTIVE_FIELD_BUILDER_COUNT.sub(num_field_builders as i64); @@ -1493,6 +1499,20 @@ mod tests { assert_eq!(expect, &read); } + #[test] + fn test_series_map_drop_with_poisoned_lock() { + let region_metadata = schema_for_test(); + let series = Arc::new(RwLock::new(Series::new(®ion_metadata))); + let map = SeriesMap(BTreeMap::from([(vec![1], series.clone())])); + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = series.write().unwrap(); + panic!("poison the series lock"); + })) + .unwrap_err(); + assert!(series.is_poisoned()); + drop(map); + } + #[test] fn test_series() { let region_metadata = schema_for_test(); diff --git a/src/mito2/src/read/last_row.rs b/src/mito2/src/read/last_row.rs index 2bafe02dbbe..b7da3e1f70c 100644 --- a/src/mito2/src/read/last_row.rs +++ b/src/mito2/src/read/last_row.rs @@ -16,8 +16,8 @@ use std::sync::Arc; -use datatypes::arrow::array::{Array, BinaryArray}; -use datatypes::arrow::compute::concat_batches; +use datatypes::arrow::array::{Array, BinaryArray, UInt32Array}; +use datatypes::arrow::compute::{concat_batches, take_record_batch}; use datatypes::arrow::record_batch::RecordBatch; use futures::{Stream, TryStreamExt}; use snafu::ResultExt; @@ -155,6 +155,11 @@ impl BatchBuffer { self.batches.extend_from_slice(batches); } + fn push(&mut self, batch: RecordBatch) { + self.num_rows += batch.num_rows(); + self.batches.push(batch); + } + /// Returns true if the buffer has no batches. fn is_empty(&self) -> bool { self.batches.is_empty() @@ -306,41 +311,70 @@ impl FlatLastTimestampSelector { return Ok(()); } + // Dictionary keys may differ for the same primary key, e.g. readers that wrap binary + // keys into a dictionary with identity keys, so merge adjacent ranges by key bytes. + let mut series_ranges: Vec<(usize, usize)> = Vec::with_capacity(offsets.len() - 1); + for window in offsets.windows(2) { + let (start, end) = (window[0], window[1]); + match series_ranges.last_mut() { + Some(last) + if primary_key_bytes_at(&batch, pk_col_idx, last.0) + == primary_key_bytes_at(&batch, pk_col_idx, start) => + { + last.1 = end; + } + _ => series_ranges.push((start, end)), + } + } + let ts_values = timestamp_array_to_i64_slice(batch.column(ts_col_idx)); - for i in 0..offsets.len() - 1 { - let range_start = offsets[i]; - let range_end = offsets[i + 1]; + let num_ranges = series_ranges.len(); + // Rows of keys that start and end inside this batch. They are gathered with one + // `take` instead of buffering a slice per key: concatenating thousands of single-row + // slices merges their dictionaries once per slice. + let mut completed_rows = Vec::new(); + for (i, &(range_start, range_end)) in series_ranges.iter().enumerate() { let range_key = primary_key_bytes_at(&batch, pk_col_idx, range_start); let range_last_ts = ts_values[range_end - 1]; let range_last_ts_start = last_timestamp_start(ts_values, range_start, range_end); - let range_slice = batch.slice(range_last_ts_start, range_end - range_last_ts_start); + let is_last_range = i + 1 == num_ranges; - match self.current_key.as_mut() { - Some(state) if state.key.as_slice() == range_key => { - if range_last_ts > state.last_timestamp { - state.last_timestamp = range_last_ts; - state.slices.clear(); - state.slices.push(range_slice); - } else if range_last_ts == state.last_timestamp { - state.slices.push(range_slice); - } + // Only the first range can continue the key carried over from previous batches. + if let Some(state) = self.current_key.as_mut() + && state.key.as_slice() == range_key + { + let range_slice = batch.slice(range_last_ts_start, range_end - range_last_ts_start); + if range_last_ts > state.last_timestamp { + state.last_timestamp = range_last_ts; + state.slices.clear(); + state.slices.push(range_slice); + } else if range_last_ts == state.last_timestamp { + state.slices.push(range_slice); } - Some(_) => { + if !is_last_range { self.flush_current_key(output_buffer); - self.current_key = Some(LastKeyState::new( - range_key.to_vec(), - range_last_ts, - range_slice, - )); - } - None => { - self.current_key = Some(LastKeyState::new( - range_key.to_vec(), - range_last_ts, - range_slice, - )); } + continue; } + + // Flushes the carried-over key before rows of this batch to keep the output sorted. + self.flush_current_key(output_buffer); + if is_last_range { + // The key may continue in the next batch. + self.current_key = Some(LastKeyState::new( + range_key.to_vec(), + range_last_ts, + batch.slice(range_last_ts_start, range_end - range_last_ts_start), + )); + } else { + completed_rows.extend(range_last_ts_start as u32..range_end as u32); + } + } + + if !completed_rows.is_empty() { + let indices = UInt32Array::from(completed_rows); + let selected = take_record_batch(&batch, &indices).context(ComputeArrowSnafu)?; + output_buffer.push(selected); } Ok(()) @@ -429,8 +463,8 @@ mod tests { use std::sync::Arc; use datatypes::arrow::array::{ - ArrayRef, BinaryDictionaryBuilder, Int64Array, TimestampMillisecondArray, UInt8Array, - UInt64Array, + ArrayRef, BinaryDictionaryBuilder, DictionaryArray, Int64Array, TimestampMillisecondArray, + UInt8Array, UInt64Array, }; use datatypes::arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit, UInt32Type}; use datatypes::arrow::record_batch::RecordBatch; @@ -630,6 +664,79 @@ mod tests { assert_eq!(vec![(b"k1".to_vec(), 4), (b"k1".to_vec(), 4)], results); } + #[test] + fn test_flat_concat_across_parent_dictionaries() { + let mut selector = FlatLastTimestampSelector::default(); + let mut output_buffer = BatchBuffer::new(); + let batches = vec![ + new_flat_batch(&[b"a", b"b", b"b", b"c"], &[1, 1, 2, 3], &[10, 20, 21, 30]), + new_flat_batch( + &[b"c", b"d", b"d", b"e", b"e"], + &[3, 4, 4, 5, 5], + &[31, 40, 41, 50, 51], + ), + new_flat_batch(&[b"f", b"g"], &[6, 7], &[60, 70]), + ]; + for batch in batches { + selector.on_next(batch, &mut output_buffer).unwrap(); + } + selector.finish(&mut output_buffer).unwrap(); + let merged = output_buffer.concat().unwrap(); + + let mut rows = Vec::new(); + extract_flat_rows(&merged, &mut rows); + let fields = merged + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + assert_eq!( + vec![ + (b"a".to_vec(), 1), + (b"b".to_vec(), 2), + (b"c".to_vec(), 3), + (b"c".to_vec(), 3), + (b"d".to_vec(), 4), + (b"d".to_vec(), 4), + (b"e".to_vec(), 5), + (b"e".to_vec(), 5), + (b"f".to_vec(), 6), + (b"g".to_vec(), 7), + ], + rows + ); + assert_eq!(vec![10, 21, 30, 31, 40, 41, 50, 51, 60, 70], fields); + } + + /// Rebuilds the primary key column with one dictionary key per row, as + /// `wrap_pk_binary_to_dict` does for binary primary keys. + fn with_identity_pk_keys(batch: RecordBatch) -> RecordBatch { + let pk_col_idx = primary_key_column_index(batch.num_columns()); + let values = (0..batch.num_rows()) + .map(|i| Some(primary_key_bytes_at(&batch, pk_col_idx, i))) + .collect::(); + let keys = UInt32Array::from_iter_values(0..batch.num_rows() as u32); + let mut columns = batch.columns().to_vec(); + columns[pk_col_idx] = Arc::new(DictionaryArray::new(keys, Arc::new(values))); + RecordBatch::try_new(batch.schema(), columns).unwrap() + } + + #[test] + fn test_flat_identity_dictionary_keys() { + let mut selector = FlatLastTimestampSelector::default(); + let batches = vec![ + with_identity_pk_keys(new_flat_batch(&[b"a", b"a", b"b"], &[1, 2, 3], &[0; 3])), + with_identity_pk_keys(new_flat_batch(&[b"b", b"c", b"c"], &[4, 5, 6], &[0; 3])), + ]; + let results = collect_flat_results(&mut selector, batches); + assert_eq!( + vec![(b"a".to_vec(), 2), (b"b".to_vec(), 4), (b"c".to_vec(), 6)], + results + ); + } + #[test] fn test_flat_finish_is_one_shot() { let mut selector = FlatLastTimestampSelector::default(); diff --git a/src/mito2/src/read/scan_region.rs b/src/mito2/src/read/scan_region.rs index d2bc53167be..da4e1c82bf9 100644 --- a/src/mito2/src/read/scan_region.rs +++ b/src/mito2/src/read/scan_region.rs @@ -1703,9 +1703,14 @@ impl ScanInput { ); } - /// Whether physical source row counts equal the rows visible to this region, - /// before applying query predicates. + /// Whether physical source row counts are known and equal the rows visible + /// to this region, before applying query predicates. pub(crate) fn total_rows_is_exact(&self) -> bool { + // Legacy SST metadata uses zero for unknown row counts. + if self.files.iter().any(|file| file.num_rows() == 0) { + return false; + } + if self.region_partition_expr.is_none() { return true; } @@ -1964,6 +1969,41 @@ pub(crate) fn exact_sequence_range( Ok((files, sequence_range)) } +/// Metadata totals for distinct SST files, not post-filter rows or actual read bytes. +/// Older files may have unknown row and row-group counts represented by zero. +#[derive(Default, serde::Serialize)] +struct FileSummary { + files: usize, + rows: u64, + row_groups: u64, + size: u64, + index_size: u64, +} + +impl FileSummary { + fn new<'a>(files: impl IntoIterator) -> Self { + let mut summary = Self::default(); + for file in files { + summary.files += 1; + summary.rows += file.meta_ref().num_rows; + summary.row_groups += file.meta_ref().num_row_groups; + summary.size += file.size(); + summary.index_size += file.index_size(); + } + summary + } +} + +/// Keeps the existing debug-list spacing while escaping strings as JSON, including +/// control characters for which Rust's Debug escaping is not valid JSON. +struct ExplainString<'a>(&'a str); + +impl fmt::Debug for ExplainString<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&serde_json::to_string(self.0).map_err(|_| fmt::Error)?) + } +} + /// Context shared by different streams from a scanner. /// It contains the input and ranges to scan. pub struct StreamContext { @@ -2064,6 +2104,7 @@ impl StreamContext { write!(f, ", \"distribution\":\"{}\"", distribution)?; } + self.format_scan_details(f)?; if verbose { self.format_verbose_content(f)?; } @@ -2071,6 +2112,94 @@ impl StreamContext { Ok(()) } + fn format_scan_details(&self, f: &mut fmt::Formatter) -> fmt::Result { + let output_schema = self.input.mapper.output_schema(); + if !output_schema.is_empty() { + write!(f, ", \"projection\": ")?; + f.debug_list() + .entries( + output_schema + .column_schemas() + .iter() + .map(|col| ExplainString(&col.name)), + ) + .finish()?; + } + if let Some(predicate) = self.input.predicate.predicate() { + if !predicate.exprs().is_empty() { + let exprs: Vec<_> = predicate.exprs().iter().map(|e| e.to_string()).collect(); + write!(f, ", \"filters\": ")?; + f.debug_list() + .entries(exprs.iter().map(|e| ExplainString(e))) + .finish()?; + } + if !predicate.dyn_filters().is_empty() { + let filters: Vec<_> = predicate + .dyn_filters() + .iter() + .map(|filter| filter.to_string()) + .collect(); + write!(f, ", \"dyn_filters\": ")?; + f.debug_list() + .entries(filters.iter().map(|filter| ExplainString(filter))) + .finish()?; + } + } + if let Some(time_range) = &self.input.time_range { + write!( + f, + ", \"time_range\": {:?}", + ExplainString(&time_range.to_string()) + )?; + } + let summary = FileSummary::new(&self.input.files); + write!( + f, + ", \"file_summary\": {}", + serde_json::to_string(&summary).map_err(|_| fmt::Error)? + )?; + + // Source indices are unique within a range even when a file is split into + // row groups. Memtables and extension sources must not count as files. + let file_offset = self.input.num_memtables(); + let file_end = file_offset + self.input.num_files(); + let mut max_range = None; + let mut max_files = 0; + for range in &self.ranges { + let files = range + .indices + .iter() + .filter(|source| (file_offset..file_end).contains(&source.index)) + .count(); + if files > max_files { + max_files = files; + max_range = Some(range); + } + } + if let Some(range) = max_range { + let summary = FileSummary::new(range.indices.iter().filter_map(|source| { + source + .index + .checked_sub(file_offset) + .and_then(|index| self.input.files.get(index)) + })); + let (start, end) = range.time_range; + // RangeMeta bounds are inclusive, just like the verbose file list. + let summary = serde_json::json!({ + "time_range_start": format!("{}::{}", start.value(), start.unit()), + "time_range_end": format!("{}::{}", end.value(), end.unit()), + "files": summary.files, + "rows": summary.rows, + "row_groups": summary.row_groups, + "size": summary.size, + "index_size": summary.index_size, + "estimated_rows": range.num_rows, + }); + write!(f, ", \"max_part_range\": {summary}")?; + } + Ok(()) + } + fn format_verbose_content(&self, f: &mut fmt::Formatter) -> fmt::Result { struct FileWrapper<'a> { file: &'a FileHandle, @@ -2118,30 +2247,6 @@ impl StreamContext { impl fmt::Debug for InputWrapper<'_> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let output_schema = self.input.mapper.output_schema(); - if !output_schema.is_empty() { - let names: Vec<_> = output_schema - .column_schemas() - .iter() - .map(|col| &col.name) - .collect(); - write!(f, ", \"projection\": {:?}", names)?; - } - if let Some(predicate) = &self.input.predicate.predicate() { - if !predicate.exprs().is_empty() { - let exprs: Vec<_> = - predicate.exprs().iter().map(|e| e.to_string()).collect(); - write!(f, ", \"filters\": {:?}", exprs)?; - } - if !predicate.dyn_filters().is_empty() { - let dyn_filters: Vec<_> = predicate - .dyn_filters() - .iter() - .map(|f| format!("{}", f)) - .collect(); - write!(f, ", \"dyn_filters\": {:?}", dyn_filters)?; - } - } if !self.input.files.is_empty() { write!(f, ", \"files\": ")?; f.debug_list() @@ -2442,6 +2547,39 @@ mod tests { ); } + #[tokio::test] + async fn test_total_rows_is_exact_with_legacy_file() { + let file_meta = FileMeta { + num_rows: 10, + ..Default::default() + }; + let mut json = serde_json::to_value(&file_meta).unwrap(); + json.as_object_mut().unwrap().remove("num_rows"); + let legacy_meta = serde_json::from_value::(json).unwrap(); + assert_eq!(legacy_meta.num_rows, 0); + + let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false)); + for (file_metas, expected_rows, exact) in [ + (vec![file_meta.clone()], 10, true), + (vec![legacy_meta.clone()], 0, false), + (vec![file_meta, legacy_meta], 10, false), + ] { + let files = file_metas + .into_iter() + .map(|meta| { + FileHandle::new(meta, Arc::new(crate::sst::file_purger::NoopFilePurger)) + }) + .collect(); + let input = new_scan_input(metadata.clone(), vec![]) + .await + .with_append_mode(true) + .with_files(files) + .build(); + assert_eq!(input.total_rows(), expected_rows); + assert_eq!(input.total_rows_is_exact(), exact); + } + } + #[tokio::test] async fn test_total_rows_is_exact_after_partition_filter() { let expr = partition_col("k0").gt_eq(Value::String("foo".into())); @@ -2464,6 +2602,7 @@ mod tests { .map(|partition_expr| { FileHandle::new( FileMeta { + num_rows: 1, partition_expr, ..Default::default() }, diff --git a/src/mito2/src/region.rs b/src/mito2/src/region.rs index 521a0e08a21..cc4b5bce852 100644 --- a/src/mito2/src/region.rs +++ b/src/mito2/src/region.rs @@ -20,6 +20,7 @@ pub mod options; pub mod utils; pub(crate) mod version; +use std::cell::OnceCell; use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; @@ -27,8 +28,12 @@ use std::sync::{Arc, Mutex, RwLock}; use common_base::hash::partition_expr_version; use common_recordbatch::adapter::RegionQueryStatCounters; +use common_recordbatch::filter::MetadataFilter; use common_telemetry::{error, info, warn}; +use common_time::timestamp::TimeUnit; use crossbeam_utils::atomic::AtomicCell; +use datafusion_common::ScalarValue; +use futures::stream::{BoxStream, StreamExt}; use object_store::ObjectStore; use partition::expr::PartitionExpr; use snafu::{OptionExt, ResultExt, ensure}; @@ -42,7 +47,7 @@ use store_api::region_engine::{ }; use store_api::region_info::RegionInfoEntry; use store_api::region_request::{PathType, StagingPartitionDirective}; -use store_api::sst_entry::ManifestSstEntry; +use store_api::sst_entry::{self, ManifestSstEntry}; use store_api::storage::{FileId, RegionId, SequenceNumber}; use tokio::sync::RwLockWriteGuard; pub use utils::*; @@ -594,12 +599,24 @@ impl MitoRegion { info!("Region {} already in downgrading mode", self.region_id); wait_for_checkpoint = true; } - _ => { + RegionRoleState::Follower => { warn!( "Cannot start downgrade for region {} from state {:?}", self.region_id, current_state ); } + RegionRoleState::Leader(_) => { + // The region may become writable again when the + // current operation, such as an ALTER, finishes. Fail + // with a retryable error instead of reporting a + // downgrade that does not fence writes. + return Err(RegionStateSnafu { + region_id: self.region_id, + state: current_state, + expect: RegionRoleState::Leader(RegionLeaderState::Writable), + } + .build()); + } } None } @@ -806,58 +823,85 @@ impl MitoRegion { } /// Returns the SST entries of the region. - pub async fn manifest_sst_entries(&self) -> Vec { - let table_dir = self.table_dir(); - let path_type = self.access_layer.path_type(); - - let visible_ssts = self - .version() - .ssts - .levels() - .iter() - .flat_map(|level| level.files().map(|file| file.file_id().file_id())) - .collect::>(); - - let manifest_files = self.manifest_ctx.manifest().await.files.clone(); - let staging_files = self - .manifest_ctx - .staging_manifest() + pub(crate) async fn manifest_sst_entries(&self) -> Vec { + self.manifest_sst_stream(MetadataFilter::default(), ManifestSstProjection::new(None)) + .collect() .await - .map(|m| m.files.clone()) - .unwrap_or_default(); - let files = manifest_files - .into_iter() - .chain(staging_files) - .collect::>(); + } - files - .values() - .map(|meta| { + /// Traverses manifest snapshots without copying the file maps. Staging metadata + /// overrides the normal manifest for the same file. + pub(crate) fn manifest_sst_stream( + &self, + filters: MetadataFilter, + projection: ManifestSstProjection, + ) -> BoxStream<'_, ManifestSstEntry> { + Box::pin(async_stream::stream! { + let table_dir = self.table_dir(); + let path_type = self.access_layer.path_type(); + let visible_files = self.manifest_visible_file_ids(projection); + let (manifest, staging) = { + let manager = self.manifest_ctx.manifest_manager.read().await; + (manager.manifest(), manager.staging_manifest()) + }; + let normal_files = manifest.files.values().map(|meta| (false, meta)); + let staging_files = staging.iter().flat_map(|s| s.files.values()).map(|meta| (true, meta)); + for (index, (is_staging, meta)) in normal_files.chain(staging_files).enumerate() { + // A selective filter may otherwise traverse millions of files in + // a single poll without yielding any rows. + if index > 0 && index % 1024 == 0 { + tokio::task::yield_now().await; + } + if !is_staging && staging.as_ref().is_some_and(|s| s.files.contains_key(&meta.file_id)) { + continue; + } + if !manifest_file_matches(meta, &filters) { + continue; + } + // Paths describe the creating region, not the referring region. + // Construct them only when a path predicate or projection needs them. + let file_path = OnceCell::new(); + if !filters.matches(|column| match column { + sst_entry::FILE_PATH => Some(file_path.get_or_init(|| { + ScalarValue::Utf8(Some(sst_file_path(table_dir, meta.file_id(), path_type))) + })), + _ => None, + }) { + continue; + } let region_id = self.region_id; let origin_region_id = meta.region_id; let (index_version, index_file_path, index_file_size) = if meta.index_file_size > 0 { - let index_file_path = index_file_path(table_dir, meta.index_id(), path_type); + let index_file_path = projection.index_file_path.then(|| index_file_path(table_dir, meta.index_id(), path_type)); ( meta.index_version, - Some(index_file_path), + index_file_path, Some(meta.index_file_size), ) } else { (0, None, None) }; - let visible = visible_ssts.contains(&meta.file_id); - ManifestSstEntry { - table_dir: table_dir.to_string(), + let visible = visible_files.as_ref().is_some_and(|files| files.contains(&meta.file_id)); + let file_path = if projection.file_path { + match file_path.into_inner() { + Some(ScalarValue::Utf8(Some(path))) => path, + _ => sst_file_path(table_dir, meta.file_id(), path_type), + } + } else { + String::new() + }; + yield ManifestSstEntry { + table_dir: if projection.table_dir { table_dir.to_string() } else { String::new() }, region_id, table_id: region_id.table_id(), region_number: region_id.region_number(), region_group: region_id.region_group(), region_sequence: region_id.region_sequence(), - file_id: meta.file_id.to_string(), + file_id: if projection.file_id { meta.file_id.to_string() } else { String::new() }, index_version, level: meta.level, - file_path: sst_file_path(table_dir, meta.file_id(), path_type), + file_path, file_size: meta.file_size, max_row_group_uncompressed_size: meta.max_row_group_uncompressed_size, index_file_path, @@ -868,15 +912,31 @@ impl MitoRegion { min_ts: meta.time_range.0, max_ts: meta.time_range.1, sequence: meta.sequence.map(|s| s.get()), - partition_expr: meta.partition_expr.as_ref().map(ToString::to_string), + partition_expr: if projection.partition_expr { meta.partition_expr.as_ref().map(ToString::to_string) } else { None }, origin_region_id, node_id: None, visible, - primary_key_min: meta.primary_key_min.clone(), - primary_key_max: meta.primary_key_max.clone(), - } - }) - .collect() + primary_key_min: if projection.primary_key_min { meta.primary_key_min.clone() } else { None }, + primary_key_max: if projection.primary_key_max { meta.primary_key_max.clone() } else { None }, + }; + } + }) + } + + /// Copies only file IDs so a paused inspection cannot delay SST purging. + /// Hidden residual columns are included in the query's projection. + fn manifest_visible_file_ids( + &self, + projection: ManifestSstProjection, + ) -> Option> { + projection.visible.then(|| { + self.version() + .ssts + .levels() + .iter() + .flat_map(|level| level.files().map(|file| file.file_id().file_id())) + .collect() + }) } /// Returns the file metas of the region by file ids. @@ -1976,6 +2036,75 @@ pub fn parse_partition_expr(partition_expr_str: Option<&str>) -> Result) -> Self { + let schema = ManifestSstEntry::schema(); + let included = |name: &str| { + projection.is_none_or(|indices| { + indices.iter().any(|index| { + schema + .column_schemas() + .get(*index) + .is_some_and(|column| column.name == name) + }) + }) + }; + Self { + visible: included(sst_entry::VISIBLE), + table_dir: included(sst_entry::TABLE_DIR), + file_id: included(sst_entry::FILE_ID), + file_path: included(sst_entry::FILE_PATH), + index_file_path: included(sst_entry::INDEX_FILE_PATH), + partition_expr: included(sst_entry::PARTITION_EXPR), + primary_key_min: included(sst_entry::PRIMARY_KEY_MIN), + primary_key_max: included(sst_entry::PRIMARY_KEY_MAX), + } + } +} + +/// Applies supported file predicates before allocating an inspection row. Unknown +/// predicates and evaluation failures are left to the query's residual filter. +fn manifest_file_matches(meta: &FileMeta, filters: &MetadataFilter) -> bool { + filters.matches(|column| { + Some(match column { + sst_entry::FILE_SIZE => ScalarValue::UInt64(Some(meta.file_size)), + sst_entry::LEVEL => ScalarValue::UInt8(Some(meta.level)), + sst_entry::NUM_ROWS => ScalarValue::UInt64(Some(meta.num_rows)), + sst_entry::NUM_ROW_GROUPS => ScalarValue::UInt64(Some(meta.num_row_groups)), + sst_entry::ORIGIN_REGION_ID => ScalarValue::UInt64(Some(meta.region_id.as_u64())), + sst_entry::MIN_TS => ScalarValue::TimestampNanosecond( + meta.time_range + .0 + .convert_to(TimeUnit::Nanosecond) + .map(|ts| ts.value()), + None, + ), + sst_entry::MAX_TS => ScalarValue::TimestampNanosecond( + meta.time_range + .1 + .convert_to(TimeUnit::Nanosecond) + .map(|ts| ts.value()), + None, + ), + _ => return None, + }) + }) +} + #[cfg(test)] mod tests { use std::sync::Arc; @@ -2007,6 +2136,66 @@ mod tests { use crate::test_util::version_util::VersionControlBuilder; use crate::time_provider::StdTimeProvider; + #[test] + fn test_manifest_file_predicates() { + use common_recordbatch::filter::MetadataFilter; + use common_time::Timestamp; + use datafusion_common::ScalarValue; + use datafusion_expr::{col, lit}; + let meta = super::FileMeta { + file_size: 100, + time_range: ( + Timestamp::new_millisecond(10), + Timestamp::new_millisecond(20), + ), + ..Default::default() + }; + for (expr, expected) in [ + (col("file_size").lt(lit(101u64)), true), + (col("file_size").lt(lit(100u64)), false), + ( + col("min_ts").eq(lit(ScalarValue::TimestampNanosecond( + Some(10_000_000), + None, + ))), + true, + ), + ( + col("max_ts").lt(lit(ScalarValue::TimestampNanosecond( + Some(20_000_000), + None, + ))), + false, + ), + (col("file_id").eq(lit("unknown")), true), + ] { + let filter = MetadataFilter::new(std::slice::from_ref(&expr)); + assert_eq!( + super::manifest_file_matches(&meta, &filter), + expected, + "{expr}" + ); + } + } + + #[test] + fn test_manifest_timestamp_overflow_is_null() { + use common_recordbatch::filter::MetadataFilter; + use common_time::Timestamp; + use datafusion_expr::col; + let meta = super::FileMeta { + time_range: ( + Timestamp::new_second(i64::MAX), + Timestamp::new_second(i64::MAX), + ), + ..Default::default() + }; + let filter = MetadataFilter::new(&[col("min_ts").is_null()]); + assert!(super::manifest_file_matches(&meta, &filter)); + let filter = MetadataFilter::new(&[col("max_ts").is_not_null()]); + assert!(!super::manifest_file_matches(&meta, &filter)); + } + #[test] fn test_region_state_lock_free() { assert!(AtomicCell::::is_lock_free()); @@ -2075,6 +2264,106 @@ mod tests { } } + #[tokio::test] + async fn test_manifest_stream_does_not_pin_sst_handles() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use common_recordbatch::filter::MetadataFilter; + use datafusion_expr::{col, lit}; + use futures::StreamExt; + + use super::ManifestSstProjection; + use crate::sst::file::FileMeta; + use crate::sst::file_purger::FilePurger; + + #[derive(Debug, Default)] + struct CountingPurger(AtomicUsize); + + impl FilePurger for CountingPurger { + fn remove_file(&self, _: FileMeta, is_delete: bool, _: bool) { + if is_delete { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + } + + for include_visible in [false, true] { + let env = SchedulerEnv::new().await; + let region = build_test_region(&env).await; + let purger = Arc::new(CountingPurger::default()); + let files = (0..2) + .map(|_| FileMeta { + region_id: region.region_id, + file_id: FileId::random(), + ..Default::default() + }) + .collect::>(); + let mut edit = empty_edit(); + edit.files_to_add = files.clone(); + region + .manifest_ctx + .manifest_manager + .write() + .await + .update( + RegionMetaActionList::with_action(RegionMetaAction::Edit(edit.clone())), + false, + ) + .await + .unwrap(); + region + .version_control + .apply_edit(Some(edit), &[], purger.clone()); + + let schema = store_api::sst_entry::ManifestSstEntry::schema(); + let columns = if include_visible { + vec!["file_id", "visible"] + } else { + vec!["file_id"] + }; + let projection = columns + .iter() + .map(|name| { + schema + .column_schemas() + .iter() + .position(|column| column.name == *name) + .unwrap() + }) + .collect::>(); + let projection = ManifestSstProjection::new(Some(&projection)); + let snapshot = region.manifest_visible_file_ids(projection); + assert_eq!(snapshot.is_some(), include_visible); + if let Some(snapshot) = snapshot { + assert_eq!(snapshot, files.iter().map(|file| file.file_id).collect()); + } + // A direct caller may omit a residual column. The source must not + // interpret an unmaterialized visible value as grounds for pruning. + let filters = MetadataFilter::new(&[col("visible").eq(lit(true))]); + let mut stream = region.manifest_sst_stream(filters, projection); + let first = stream.next().await.unwrap(); + if include_visible { + assert!(first.visible); + } + let mut removal = empty_edit(); + removal.files_to_remove = files; + region + .version_control + .apply_edit(Some(removal), &[], purger.clone()); + // Both final FileHandles must reach the purger while the stream is + // suspended at its first yield, rather than waiting for cancellation. + assert_eq!(purger.0.load(Ordering::SeqCst), 2); + let second = stream.next().await.unwrap(); + if include_visible { + assert!( + second.visible, + "visibility must retain the original snapshot" + ); + } + assert!(stream.next().await.is_none()); + } + } + fn empty_edit() -> RegionEdit { RegionEdit { files_to_add: Vec::new(), diff --git a/src/mito2/src/region_write_ctx.rs b/src/mito2/src/region_write_ctx.rs index f8d7eb9c840..b0b781eb43c 100644 --- a/src/mito2/src/region_write_ctx.rs +++ b/src/mito2/src/region_write_ctx.rs @@ -23,7 +23,7 @@ use store_api::logstore::LogStore; use store_api::logstore::provider::Provider; use store_api::storage::{RegionId, SequenceNumber}; -use crate::error::{Error, Result, WriteGroupSnafu}; +use crate::error::{Error, Result, UnexpectedSnafu, WriteGroupSnafu}; use crate::memtable::KeyValues; use crate::memtable::bulk::part::BulkPart; use crate::metrics; @@ -66,6 +66,15 @@ impl WriteNotify { impl Drop for WriteNotify { fn drop(&mut self) { + // A panic may have interrupted the write, so it must not be reported as written. + if self.err.is_none() && std::thread::panicking() { + self.err = Some(Arc::new( + UnexpectedSnafu { + reason: "the write was interrupted by a panic", + } + .build(), + )); + } self.notify_result(); } } @@ -470,13 +479,13 @@ pub(crate) mod test_hooks { release: watch::Sender, } - static ACTIVE_BARRIER: Mutex> = Mutex::new(None); + static ACTIVE_BARRIERS: Mutex> = Mutex::new(Vec::new()); static NEXT_BARRIER_ID: AtomicU64 = AtomicU64::new(1); - fn lock_active_barrier() -> std::sync::MutexGuard<'static, Option> { + fn lock_active_barriers() -> std::sync::MutexGuard<'static, Vec> { // Never let a poisoned mutex (e.g. a panic in another test while // holding the lock) hang or break unrelated tests. - ACTIVE_BARRIER + ACTIVE_BARRIERS .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) } @@ -513,8 +522,8 @@ pub(crate) mod test_hooks { } } - /// Arms the bulk-install barrier and returns the owning guard; any - /// previously armed barrier is replaced. + /// Arms a bulk-install barrier for the target and returns the owning + /// guard; barriers of concurrently running tests stay independent. pub(crate) fn arm_bulk_install_barrier( target_region_id: RegionId, target_version_control: VersionControlRef, @@ -522,8 +531,7 @@ pub(crate) mod test_hooks { let (reached_tx, reached_rx) = watch::channel(false); let (release_tx, _release_rx) = watch::channel(false); let id = NEXT_BARRIER_ID.fetch_add(1, Ordering::Relaxed); - let mut active = lock_active_barrier(); - *active = Some(ActiveBarrier { + lock_active_barriers().push(ActiveBarrier { id, target_region_id, target_version_control, @@ -539,10 +547,7 @@ pub(crate) mod test_hooks { } fn disarm_barrier(id: u64) { - let mut active = lock_active_barrier(); - if active.as_ref().is_some_and(|barrier| barrier.id == id) { - *active = None; - } + lock_active_barriers().retain(|barrier| barrier.id != id); } /// Pauses a bulk write before installing its parts until the barrier is @@ -552,19 +557,14 @@ pub(crate) mod test_hooks { version_control: &VersionControlRef, ) { let (reached_tx, release_rx) = { - let active = lock_active_barrier(); - match active.as_ref() { - Some(barrier) - if barrier.target_region_id == region_id - && std::sync::Arc::ptr_eq( - &barrier.target_version_control, - version_control, - ) => - { - (barrier.reached.clone(), barrier.release.subscribe()) - } - _ => return, - } + let active = lock_active_barriers(); + let Some(barrier) = active.iter().find(|barrier| { + barrier.target_region_id == region_id + && std::sync::Arc::ptr_eq(&barrier.target_version_control, version_control) + }) else { + return; + }; + (barrier.reached.clone(), barrier.release.subscribe()) }; let _ = reached_tx.send(true); let mut release_rx = release_rx; @@ -592,6 +592,18 @@ mod tests { use crate::memtable::bulk::part::BulkPart; use crate::test_util::version_util::VersionControlBuilder; + #[test] + fn test_write_notify_dropped_by_panic_reports_error() { + let (tx, rx) = oneshot::channel(); + let notify = WriteNotify::new(OptionOutputTx::from(tx), 3); + std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || { + let _notify = notify; + panic!("interrupt the write"); + })) + .unwrap_err(); + assert!(rx.blocking_recv().unwrap().is_err()); + } + #[test] fn test_request_skip_wal_preserves_sequences_and_other_writes() { // Ablate only the request flag: the workload and sequence allocation stay identical. diff --git a/src/mito2/src/request.rs b/src/mito2/src/request.rs index 0a743b2557e..efbe2d88065 100644 --- a/src/mito2/src/request.rs +++ b/src/mito2/src/request.rs @@ -25,6 +25,9 @@ use api::helper::{ use api::v1::column_def::options_from_column_schema; use api::v1::{ColumnDataType, ColumnSchema, OpType, Rows, SemanticType, Value, WriteHint}; use common_telemetry::info; +use datatypes::arrow::array::{Array, ArrayRef}; +use datatypes::arrow::datatypes::DataType as ArrowDataType; +use datatypes::arrow::record_batch::RecordBatch; use datatypes::prelude::DataType; use partition::expr::PartitionExpr; use prometheus::HistogramTimer; @@ -44,6 +47,7 @@ use store_api::region_request::{ RegionFlushRequest, RegionOpenRequest, RegionRequest, RegionTruncateRequest, StagingPartitionDirective, }; +use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME; use store_api::storage::{FileId, RegionId, SequenceNumber}; use tokio::sync::oneshot::{self, Receiver, Sender}; @@ -193,85 +197,22 @@ impl WriteRequest { let region_id = self.region_id; // Index all columns in rows. - let mut rows_columns: HashMap<_, _> = self + let rows_columns = self .rows .schema .iter() - .map(|column| (&column.column_name, column)) + .map(|schema| { + let column = RowColumn { + request: self, + schema, + }; + (schema.column_name.as_str(), column) + }) .collect(); - - let mut need_fill_default = false; - // Checks all columns in this region. - for column in &metadata.column_metadatas { - if let Some(input_col) = rows_columns.remove(&column.column_schema.name) { - // Check data type. - ensure!( - is_column_type_value_eq( - input_col.datatype, - input_col.datatype_extension.clone(), - &column.column_schema.data_type - ), - InvalidRequestSnafu { - region_id, - reason: format!( - "column {} expect type {:?}, given: {}({})", - column.column_schema.name, - column.column_schema.data_type, - ColumnDataType::try_from(input_col.datatype) - .map(|v| v.as_str_name()) - .unwrap_or("Unknown"), - input_col.datatype, - ) - } - ); - - // Check semantic type. - ensure!( - is_semantic_type_eq(input_col.semantic_type, column.semantic_type), - InvalidRequestSnafu { - region_id, - reason: format!( - "column {} has semantic type {:?}, given: {}({})", - column.column_schema.name, - column.semantic_type, - api::v1::SemanticType::try_from(input_col.semantic_type) - .map(|v| v.as_str_name()) - .unwrap_or("Unknown"), - input_col.semantic_type - ), - } - ); - - // Check nullable. - // Safety: `rows_columns` ensures this column exists. - let has_null = self.has_null[self.name_to_index[&column.column_schema.name]]; - ensure!( - !has_null || column.column_schema.is_nullable(), - InvalidRequestSnafu { - region_id, - reason: format!( - "column {} is not null but input has null", - column.column_schema.name - ), - } - ); - } else { - // Rows don't have this column. - self.check_missing_column(column)?; - - need_fill_default = true; - } - } - - // Checks all columns in rows exist in the region. - if !rows_columns.is_empty() { - let names: Vec<_> = rows_columns.into_keys().collect(); - return InvalidRequestSnafu { - region_id, - reason: format!("unknown columns: {:?}", names), - } - .fail(); - } + let need_fill_default = + check_request_columns(region_id, metadata, rows_columns, |column| { + self.check_missing_column(column) + })?; // If we need to fill default values, return a special error. ensure!(!need_fill_default, FillDefaultSnafu { region_id }); @@ -299,6 +240,15 @@ impl WriteRequest { /// Checks the schema and fill missing columns. pub(crate) fn maybe_fill_missing_columns(&mut self, metadata: &RegionMetadata) -> Result<()> { + // Removes the columns filled for an older schema, so the request is checked + // as sent and filled again for this schema. + let num_columns = self.name_to_index.len(); + if self.rows.schema.len() > num_columns { + self.rows.schema.truncate(num_columns); + for row in &mut self.rows.rows { + row.values.truncate(num_columns); + } + } if let Err(e) = self.check_schema(metadata) { if e.is_fill_default() { // TODO(yingwen): Add metrics for this case. @@ -368,16 +318,7 @@ impl WriteRequest { } // Not a delete request. Checks whether they have default value. - ensure!( - column.column_schema.is_nullable() - || column.column_schema.default_constraint().is_some(), - InvalidRequestSnafu { - region_id: self.region_id, - reason: format!("missing column {}", column.column_schema.name), - } - ); - - Ok(()) + check_column_can_be_filled(self.region_id, column) } /// Returns the default value for specific column. @@ -441,6 +382,242 @@ impl WriteRequest { } } +/// A column of a write request, as [check_request_columns] sees it. +trait RequestColumn { + /// Returns whether the column has the data type of `column`. + fn has_type_of(&self, column: &ColumnMetadata) -> bool; + + /// Describes the data type of the column in errors. + fn type_name(&self) -> String; + + /// Returns the semantic type of the column if the request carries one. + fn semantic_type(&self) -> Option; + + /// Returns whether the column contains nulls. + fn has_null(&self) -> bool; +} + +/// Checks the columns of a write request against the region schema. Each column +/// must be a column of the region, have its data type and, if the request carries +/// one, its semantic type, and contain nulls only if the region column is +/// nullable. `check_missing` checks the region columns that the request doesn't +/// have. Returns whether there are such columns. +fn check_request_columns( + region_id: RegionId, + metadata: &RegionMetadata, + mut columns: HashMap<&str, C>, + mut check_missing: impl FnMut(&ColumnMetadata) -> Result<()>, +) -> Result { + let mut has_missing = false; + for column in &metadata.column_metadatas { + let Some(input_col) = columns.remove(column.column_schema.name.as_str()) else { + check_missing(column)?; + has_missing = true; + continue; + }; + // Check data type. + ensure!( + input_col.has_type_of(column), + InvalidRequestSnafu { + region_id, + reason: format!( + "column {} expect type {:?}, given: {}", + column.column_schema.name, + column.column_schema.data_type, + input_col.type_name(), + ) + } + ); + + // Check semantic type. + if let Some(semantic_type) = input_col.semantic_type() { + ensure!( + is_semantic_type_eq(semantic_type, column.semantic_type), + InvalidRequestSnafu { + region_id, + reason: format!( + "column {} has semantic type {:?}, given: {}({})", + column.column_schema.name, + column.semantic_type, + api::v1::SemanticType::try_from(semantic_type) + .map(|v| v.as_str_name()) + .unwrap_or("Unknown"), + semantic_type + ), + } + ); + } + + // Check nullable. + ensure!( + !input_col.has_null() || column.column_schema.is_nullable(), + InvalidRequestSnafu { + region_id, + reason: format!( + "column {} is not null but input has null", + column.column_schema.name + ), + } + ); + } + + // Checks all columns in the request exist in the region. + if !columns.is_empty() { + let names: Vec<_> = columns.into_keys().collect(); + return InvalidRequestSnafu { + region_id, + reason: format!("unknown columns: {:?}", names), + } + .fail(); + } + + Ok(has_missing) +} + +/// Checks that a write may leave out `column`, which is then filled with its +/// default value or null. +fn check_column_can_be_filled(region_id: RegionId, column: &ColumnMetadata) -> Result<()> { + ensure!( + column.column_schema.is_nullable() || column.column_schema.default_constraint().is_some(), + InvalidRequestSnafu { + region_id, + reason: format!("missing column {}", column.column_schema.name), + } + ); + Ok(()) +} + +/// A column of the rows of a [WriteRequest]. +struct RowColumn<'a> { + request: &'a WriteRequest, + schema: &'a ColumnSchema, +} + +impl RequestColumn for RowColumn<'_> { + fn has_type_of(&self, column: &ColumnMetadata) -> bool { + is_column_type_value_eq( + self.schema.datatype, + self.schema.datatype_extension.clone(), + &column.column_schema.data_type, + ) + } + + fn type_name(&self) -> String { + format!( + "{}({})", + ColumnDataType::try_from(self.schema.datatype) + .map(|v| v.as_str_name()) + .unwrap_or("Unknown"), + self.schema.datatype, + ) + } + + fn semantic_type(&self) -> Option { + Some(self.schema.semantic_type) + } + + fn has_null(&self) -> bool { + // Safety: `name_to_index` has every column of the request. + self.request.has_null[self.request.name_to_index[&self.schema.column_name]] + } +} + +/// A column of the batch of a bulk insert. +struct BulkColumn<'a> { + array: &'a ArrayRef, +} + +impl RequestColumn for BulkColumn<'_> { + fn has_type_of(&self, column: &ColumnMetadata) -> bool { + let actual = self.array.data_type(); + let data_type = &column.column_schema.data_type; + // The memtables convert tags of any string encoding. + if column.semantic_type == SemanticType::Tag && data_type.is_string() { + return match actual { + ArrowDataType::Dictionary(key, value) => key.is_integer() && value.is_string(), + actual => actual.is_string(), + }; + } + *actual == data_type.as_arrow_type() + } + + fn type_name(&self) -> String { + self.array.data_type().to_string() + } + + fn semantic_type(&self) -> Option { + None + } + + fn has_null(&self) -> bool { + self.array.logical_null_count() > 0 + } +} + +/// Checks the batch of a bulk insert against the region schema with the rules +/// [WriteRequest::check_schema] applies to rows: column names are unique, every +/// column is a column of the region with its Arrow type and contains nulls only +/// if the region column is nullable, and a missing column is nullable or has a +/// default value. Tags may use any string encoding. Sparse batches carry the +/// tags in one encoded primary key column instead. Returns whether the batch +/// misses columns that the writer fills. +pub fn check_bulk_insert_batch(metadata: &RegionMetadata, batch: &RecordBatch) -> Result { + let region_id = metadata.region_id; + let sparse = metadata.primary_key_encoding == PrimaryKeyEncoding::Sparse; + let mut primary_key = None; + let mut columns = HashMap::with_capacity(batch.num_columns()); + for (field, array) in batch.schema_ref().fields().iter().zip(batch.columns()) { + let duplicate = if sparse && field.name() == PRIMARY_KEY_COLUMN_NAME { + primary_key.replace(array).is_some() + } else { + columns + .insert(field.name().as_str(), BulkColumn { array }) + .is_some() + }; + ensure!( + !duplicate, + InvalidRequestSnafu { + region_id, + reason: format!("duplicate column {}", field.name()), + } + ); + } + if sparse { + let primary_key = primary_key.with_context(|| InvalidRequestSnafu { + region_id, + reason: format!("missing column {PRIMARY_KEY_COLUMN_NAME}"), + })?; + // The writer casts the encoded primary key to a dictionary of binary. + let type_matches = match primary_key.data_type() { + ArrowDataType::Binary => true, + ArrowDataType::Dictionary(key, value) => { + key.is_integer() && **value == ArrowDataType::Binary + } + _ => false, + }; + ensure!( + type_matches && primary_key.logical_null_count() == 0, + InvalidRequestSnafu { + region_id, + reason: format!( + "column {PRIMARY_KEY_COLUMN_NAME} must be binary without nulls, given: {}", + primary_key.data_type() + ), + } + ); + } + let mut needs_fill = false; + check_request_columns(region_id, metadata, columns, |column| { + if sparse && column.semantic_type == SemanticType::Tag { + Ok(()) + } else { + needs_fill = true; + check_column_can_be_filled(region_id, column) + } + })?; + Ok(needs_fill) +} + /// Validate proto value schema. pub(crate) fn validate_proto_value( region_id: RegionId, @@ -761,7 +938,18 @@ impl WorkerRequest { sender: sender.into(), request: DdlRequest::EnterStaging(v), }), - RegionRequest::BulkInserts(region_bulk_inserts_request) => { + RegionRequest::BulkInserts(mut region_bulk_inserts_request) => { + // The worker answers empty batches without writing them. + if let Some(region_metadata) = ®ion_metadata + && region_bulk_inserts_request.payload.num_rows() > 0 + && check_bulk_insert_batch( + region_metadata, + ®ion_bulk_inserts_request.payload, + )? + { + // Lets the worker fill the missing columns. + region_bulk_inserts_request.aligned_schema_version = None; + } WorkerRequest::BulkInserts(BulkInsertRequest { metadata: region_metadata, sender: sender.into(), diff --git a/src/mito2/src/sst/parquet.rs b/src/mito2/src/sst/parquet.rs index 9404d4484fc..f2712b7e442 100644 --- a/src/mito2/src/sst/parquet.rs +++ b/src/mito2/src/sst/parquet.rs @@ -20,11 +20,13 @@ use std::sync::Arc; use api::v1::SemanticType; use common_base::readable_size::ReadableSize; use datatypes::json::JsonSettings; +use parquet::basic::{Compression, Encoding}; use parquet::file::metadata::ParquetMetaData; use parquet::file::properties::WriterPropertiesBuilder; use parquet::schema::types::ColumnPath; use store_api::metadata::RegionMetadataRef; use store_api::mito_engine_options::FloatFieldEncoding; +use store_api::storage::consts::{OP_TYPE_COLUMN_NAME, SEQUENCE_COLUMN_NAME}; use store_api::storage::{ColumnId, FileId}; use crate::sst::DEFAULT_WRITE_BUFFER_SIZE; @@ -89,6 +91,24 @@ pub const DEFAULT_ROW_GROUP_SIZE: usize = 100 * 1024; pub(crate) const COLUMN_INDEX_TRUNCATE_LENGTH: Option = parquet::file::properties::DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH; +/// Applies the encodings of the time index and internal columns shared by every SST writer. +pub(crate) fn apply_internal_column_encoding( + builder: WriterPropertiesBuilder, + metadata: &RegionMetadataRef, +) -> WriterPropertiesBuilder { + let ts_col = ColumnPath::new(vec![ + metadata.time_index_column().column_schema.name.clone(), + ]); + let seq_col = ColumnPath::new(vec![SEQUENCE_COLUMN_NAME.to_string()]); + let op_type_col = ColumnPath::new(vec![OP_TYPE_COLUMN_NAME.to_string()]); + builder + .set_column_encoding(seq_col.clone(), Encoding::DELTA_BINARY_PACKED) + .set_column_dictionary_enabled(seq_col, false) + .set_column_encoding(ts_col.clone(), Encoding::DELTA_BINARY_PACKED) + .set_column_dictionary_enabled(ts_col, false) + .set_column_compression(op_type_col, Compression::UNCOMPRESSED) +} + /// Applies the configured encoding to direct floating-point field columns. pub(crate) fn apply_float_field_encoding( mut builder: WriterPropertiesBuilder, diff --git a/src/mito2/src/sst/parquet/writer.rs b/src/mito2/src/sst/parquet/writer.rs index 932654dfdd8..b883ab0304c 100644 --- a/src/mito2/src/sst/parquet/writer.rs +++ b/src/mito2/src/sst/parquet/writer.rs @@ -39,11 +39,9 @@ use parquet::arrow::AsyncArrowWriter; use parquet::basic::{Compression, Encoding, ZstdLevel}; use parquet::file::metadata::KeyValue; use parquet::file::properties::WriterProperties; -use parquet::schema::types::ColumnPath; use smallvec::smallvec; use snafu::{OptionExt, ResultExt}; use store_api::metadata::RegionMetadataRef; -use store_api::storage::consts::{OP_TYPE_COLUMN_NAME, SEQUENCE_COLUMN_NAME}; use store_api::storage::{FileId, SequenceNumber}; use tokio::io::AsyncWrite; use tokio_util::compat::{Compat, FuturesAsyncWriteCompatExt}; @@ -63,7 +61,7 @@ use crate::sst::parquet::flat_format::{ use crate::sst::parquet::format::{PrimaryKeyArray, PrimaryKeyWriteFormat}; use crate::sst::parquet::{ COLUMN_INDEX_TRUNCATE_LENGTH, PARQUET_METADATA_KEY, SstInfo, WriteOptions, - apply_float_field_encoding, + apply_float_field_encoding, apply_internal_column_encoding, }; use crate::sst::{ DEFAULT_WRITE_CONCURRENCY, FlatSchemaOptions, SeriesEstimator, maybe_wrap_schema, @@ -547,17 +545,7 @@ where .set_max_row_group_row_count(Some(opts.row_group_size)) .set_column_index_truncate_length(COLUMN_INDEX_TRUNCATE_LENGTH) .set_statistics_truncate_length(None); - let ts_col = ColumnPath::new(vec![ - self.metadata.time_index_column().column_schema.name.clone(), - ]); - let seq_col = ColumnPath::new(vec![SEQUENCE_COLUMN_NAME.to_string()]); - let op_type_col = ColumnPath::new(vec![OP_TYPE_COLUMN_NAME.to_string()]); - let props_builder = props_builder - .set_column_encoding(seq_col.clone(), Encoding::DELTA_BINARY_PACKED) - .set_column_dictionary_enabled(seq_col, false) - .set_column_encoding(ts_col.clone(), Encoding::DELTA_BINARY_PACKED) - .set_column_dictionary_enabled(ts_col, false) - .set_column_compression(op_type_col, Compression::UNCOMPRESSED); + let props_builder = apply_internal_column_encoding(props_builder, &self.metadata); let props_builder = apply_float_field_encoding( props_builder, &self.metadata, diff --git a/src/mito2/src/worker.rs b/src/mito2/src/worker.rs index e9239952115..a90768c7bab 100644 --- a/src/mito2/src/worker.rs +++ b/src/mito2/src/worker.rs @@ -52,7 +52,7 @@ use rand::{Rng, rng}; use snafu::{ResultExt, ensure}; use store_api::logstore::LogStore; use store_api::region_engine::{ - SetRegionRoleStateResponse, SetRegionRoleStateSuccess, SettableRegionRoleState, + RegionRole, SetRegionRoleStateResponse, SetRegionRoleStateSuccess, SettableRegionRoleState, }; use store_api::storage::{FileId, RegionId}; use tokio::sync::mpsc::{Receiver, Sender}; @@ -1385,6 +1385,15 @@ impl RegionWorkerLoop { sender: oneshot::Sender, ) { if let Some(region) = self.regions.get_region(region_id) { + if matches!( + region_role_state, + SettableRegionRoleState::Follower | SettableRegionRoleState::DowngradingLeader + ) { + // Fence writes here: writes the worker handles later see + // Downgrading, and earlier ones have finished. This only + // changes a writable or staging leader. + region.set_role(RegionRole::DowngradingLeader); + } // We need to do this in background as we need the manifest lock. common_runtime::spawn_global(async move { match region.set_role_state_gracefully(region_role_state).await { diff --git a/src/mito2/src/worker/handle_bulk_insert.rs b/src/mito2/src/worker/handle_bulk_insert.rs index 9ecbf537661..d48f06178c6 100644 --- a/src/mito2/src/worker/handle_bulk_insert.rs +++ b/src/mito2/src/worker/handle_bulk_insert.rs @@ -71,10 +71,12 @@ impl RegionWorkerLoop { return; } - // safety: ts data type must be a timestamp type. + // safety: `WorkerRequest::try_from_region_request` checked the batch against + // `region_metadata`, so the time index is a timestamp array without nulls. let (ts_primitive, _) = datatypes::timestamp::timestamp_array_to_primitive(ts).unwrap(); - // safety: we've checked ts.len() == batch.num_rows() and batch is not empty + // safety: we've checked ts.len() == batch.num_rows(), batch is not empty and + // the time index has no nulls. let min_ts = arrow::compute::min(&ts_primitive).unwrap(); let max_ts = arrow::compute::max(&ts_primitive).unwrap(); diff --git a/src/mito2/src/worker/handle_write.rs b/src/mito2/src/worker/handle_write.rs index 1383a3e9a5d..aa45793f668 100644 --- a/src/mito2/src/worker/handle_write.rs +++ b/src/mito2/src/worker/handle_write.rs @@ -34,7 +34,9 @@ use crate::metrics::{ }; use crate::region::{RegionLeaderState, RegionRoleState}; use crate::region_write_ctx::{RegionWriteCtx, WriteSource}; -use crate::request::{SenderBulkRequest, SenderWriteRequest, WriteRequest}; +use crate::request::{ + SenderBulkRequest, SenderWriteRequest, WriteRequest, check_bulk_insert_batch, +}; use crate::wal::Wal; use crate::worker::RegionWorkerLoop; @@ -552,19 +554,21 @@ impl RegionWorkerLoop { aligned_schema.schema_version == region_ctx.version().metadata.schema_version }); - // Fill missing columns if needed - if need_fill_missing_columns - && let Err(e) = bulk_req - .request - .fill_missing_columns(®ion_ctx.version().metadata) - { - bulk_req.sender.send(Err(e)); - continue; + // Fill missing columns if needed. The batch is checked again as the region + // schema may have changed since the request was checked. + if need_fill_missing_columns { + let metadata = ®ion_ctx.version().metadata; + if let Err(e) = check_bulk_insert_batch(metadata, &bulk_req.request.batch) + .and_then(|_| bulk_req.request.fill_missing_columns(metadata)) + { + bulk_req.sender.send(Err(e)); + continue; + } } // Collect requests by region. if !region_ctx.push_bulk(bulk_req.sender, bulk_req.request, None, bulk_req.skip_wal) { - return; + continue; } } } diff --git a/src/object-store/src/layers/hdfs.rs b/src/object-store/src/layers/hdfs.rs index 4e00098fbf7..ed8b2019450 100644 --- a/src/object-store/src/layers/hdfs.rs +++ b/src/object-store/src/layers/hdfs.rs @@ -15,6 +15,7 @@ use std::fmt::{self, Debug}; use std::sync::Arc; +use common_telemetry::warn; use hdfs_native::{Client, ClientBuilder}; use opendal::raw::oio::{Delete as _, Read as _, ReadStream as _, Write as _}; use opendal::raw::{ @@ -162,6 +163,9 @@ enum HdfsWriterInner { writer: Option, temporary_path: String, target_path: String, + /// Message of the write that failed and removed the writer, so later + /// `write`/`close` calls can explain why the writer is gone. + failure: Option, }, } @@ -169,12 +173,29 @@ impl oio::Write for HdfsWriter { async fn write(&mut self, buffer: Buffer) -> Result<()> { match &mut self.0 { HdfsWriterInner::Direct(writer) => writer.write(buffer).await, - HdfsWriterInner::Atomic { writer, .. } => { - writer + HdfsWriterInner::Atomic { + inner, + context, + writer, + temporary_path, + failure, + .. + } => { + let result = writer .as_mut() - .ok_or_else(writer_unavailable)? + .ok_or_else(|| writer_unavailable(failure.as_deref()))? .write(buffer) - .await + .await; + if let Err(error) = result { + // A failed write leaves the temporary file behind, so abort + // it before returning the original error to the caller. + if let Some(writer) = writer.take() { + abort_and_delete(inner, context, writer, temporary_path).await; + } + *failure = Some(error.to_string()); + return Err(error); + } + Ok(()) } } } @@ -189,8 +210,11 @@ impl oio::Write for HdfsWriter { writer, temporary_path, target_path, + failure, } => { - let mut writer = writer.take().ok_or_else(writer_unavailable)?; + let mut writer = writer + .take() + .ok_or_else(|| writer_unavailable(failure.as_deref()))?; let metadata = match writer.close().await { Ok(metadata) => metadata, Err(error) => { @@ -242,11 +266,17 @@ impl oio::Write for HdfsWriter { } } -fn writer_unavailable() -> opendal::Error { - opendal::Error::new( - ErrorKind::Unexpected, - "HDFS writer is unavailable after close or abort", - ) +fn writer_unavailable(failure: Option<&str>) -> opendal::Error { + match failure { + Some(failure) => opendal::Error::new( + ErrorKind::Unexpected, + format!("HDFS writer is unavailable: a previous write failed: {failure}"), + ), + None => opendal::Error::new( + ErrorKind::Unexpected, + "HDFS writer is unavailable after close or abort", + ), + } } impl Service for HdfsCompatibilityService { @@ -301,6 +331,7 @@ impl Service for HdfsCompatibilityService { writer: Some(writer), temporary_path, target_path: path.to_string(), + failure: None, })) } @@ -416,9 +447,13 @@ async fn abort_and_delete( mut writer: oio::Writer, path: &str, ) { - let _ = writer.abort().await; + if let Err(error) = writer.abort().await { + warn!(error; "Failed to abort HDFS writer for temporary file {path}"); + } drop(writer); - let _ = delete_path(inner, context, path).await; + if let Err(error) = delete_path(inner, context, path).await { + warn!(error; "Failed to delete temporary file {path} after a failed HDFS write"); + } } async fn delete_path(inner: &Servicer, context: &OperationContext, path: &str) -> Result<()> { @@ -443,6 +478,7 @@ mod tests { use tempfile::TempDir; use super::*; + use crate::layers::mock::{MockLayerBuilder, MockWriterFactory}; fn test_store() -> (TempDir, Operator) { let directory = tempfile::tempdir().unwrap(); @@ -548,4 +584,82 @@ mod tests { .as_ref() ); } + + /// A writer that starts failing once more than `fail_after` bytes have been + /// accepted, to simulate a failure in the middle of an atomic write. + struct FailingWriter { + inner: oio::Writer, + written: usize, + fail_after: usize, + } + + impl oio::Write for FailingWriter { + async fn write(&mut self, buffer: Buffer) -> Result<()> { + if self.written + buffer.len() > self.fail_after { + return Err(opendal::Error::new( + ErrorKind::Unexpected, + "injected write failure", + )); + } + self.written += buffer.len(); + self.inner.write(buffer).await + } + + async fn close(&mut self) -> Result { + self.inner.close().await + } + + async fn abort(&mut self) -> Result<()> { + self.inner.abort().await + } + } + + #[tokio::test] + async fn test_atomic_write_cleans_up_temporary_file_after_write_error() { + let directory = tempfile::tempdir().unwrap(); + let factory: MockWriterFactory = Arc::new(|path, _args, inner| { + if path.contains(".greptime-") { + Box::new(FailingWriter { + inner, + written: 0, + fail_after: 4, + }) + } else { + inner + } + }); + let mock_layer = MockLayerBuilder::default() + .writer_factory(factory) + .build() + .unwrap(); + let store = Operator::new(Fs::default().root(directory.path().to_str().unwrap())) + .unwrap() + .layer(mock_layer) + .layer(HdfsCompatibilityLayer::new_for_test()); + + store.write("data/target", "old").await.unwrap(); + + let mut writer: Writer = store.writer("data/target").await.unwrap(); + writer.write("part").await.unwrap(); + + let error = writer.write("ial").await.unwrap_err(); + assert_eq!(ErrorKind::Unexpected, error.kind()); + assert!(error.to_string().contains("injected write failure")); + + // The destination keeps its previous contents... + assert_eq!( + b"old", + store.read("data/target").await.unwrap().to_bytes().as_ref() + ); + // ...and the temporary file is removed even though the writer was not + // closed or aborted by the caller. + assert!( + store + .list("data/") + .await + .unwrap() + .iter() + .all(|entry| !entry.path().contains(".greptime-")) + ); + } } diff --git a/src/object-store/src/layers/mock.rs b/src/object-store/src/layers/mock.rs index 270f776f935..4807a401c95 100644 --- a/src/object-store/src/layers/mock.rs +++ b/src/object-store/src/layers/mock.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::fmt::Debug; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use derive_builder::Builder; pub use oio::*; @@ -225,6 +225,50 @@ impl oio::Read for MockReader { } } +/// Writer calls observed before forwarding, including calls that fail. +#[derive(Debug, Default)] +pub struct WriterObservations { + pub write_sizes: Vec, + pub close_calls: usize, + pub abort_calls: usize, +} + +/// Records calls without changing the inner writer's completion or abort semantics. +pub struct ObservedWriter { + inner: oio::Writer, + observations: Arc>, +} + +impl ObservedWriter { + pub fn new(inner: oio::Writer, observations: Arc>) -> Self { + Self { + inner, + observations, + } + } +} + +impl oio::Write for ObservedWriter { + async fn write(&mut self, bytes: Buffer) -> Result<()> { + self.observations + .lock() + .unwrap() + .write_sizes + .push(bytes.len()); + self.inner.write(bytes).await + } + + async fn close(&mut self) -> Result { + self.observations.lock().unwrap().close_calls += 1; + self.inner.close().await + } + + async fn abort(&mut self) -> Result<()> { + self.observations.lock().unwrap().abort_calls += 1; + self.inner.abort().await + } +} + pub struct MockWriter { inner: oio::Writer, } diff --git a/src/partition/src/expr.rs b/src/partition/src/expr.rs index 29b22b6f09c..03e7249797b 100644 --- a/src/partition/src/expr.rs +++ b/src/partition/src/expr.rs @@ -280,76 +280,43 @@ impl PartitionExpr { } pub fn try_as_logical_expr(&self) -> error::Result { - // Special handling for null equality. - // `col = NULL` -> `col IS NULL` to match SQL (DataFusion) semantics. - let lhs_is_null = matches!(self.lhs.as_ref(), Operand::Value(Value::Null)); - let rhs_is_null = matches!(self.rhs.as_ref(), Operand::Value(Value::Null)); - - match (self.op.clone(), lhs_is_null, rhs_is_null) { - (RestrictedOp::Eq, _, true) => { - return Ok(self.lhs.try_as_logical_expr()?.is_null()); + // Row routing compares `Value`s, which order NULL before every other value. + // Batch routing and region filters treat a NULL result as false, so only the + // comparisons that hold for NULL need an explicit `IS NULL`. + let comparison = match (self.lhs.as_ref(), self.rhs.as_ref()) { + (column @ Operand::Column(_), Operand::Value(value)) => { + Some((column, value, self.op.clone())) } - (RestrictedOp::Eq, true, _) => { - return Ok(self.rhs.try_as_logical_expr()?.is_null()); + (Operand::Value(value), column @ Operand::Column(_)) => { + Some((column, value, self.op.invert_for_swap())) } - (RestrictedOp::NotEq, _, true) => { - return Ok(self.lhs.try_as_logical_expr()?.is_not_null()); - } - (RestrictedOp::NotEq, true, _) => { - return Ok(self.rhs.try_as_logical_expr()?.is_not_null()); - } - _ => {} - } - - if matches!( - self.op, - RestrictedOp::Lt | RestrictedOp::LtEq | RestrictedOp::Gt | RestrictedOp::GtEq - ) { - // Keep filtering semantics aligned with direct PartitionExpr evaluation (null-first ordering). - // In DataFusion SQL semantics, range comparisons with NULL yield NULL, so we inject - // `OR col IS NULL` on the null-first side of the comparison. - if matches!(self.lhs.as_ref(), Operand::Column(_)) { - let column_expr = self.lhs.try_as_logical_expr()?; - let other_expr = self.rhs.try_as_logical_expr()?; - let base = match self.op { - RestrictedOp::Lt => { - column_expr.clone().lt(other_expr).or(column_expr.is_null()) + _ => None, + }; + if let Some((column, value, op)) = comparison { + let column = column.try_as_logical_expr()?; + if matches!(value, Value::Null) { + return match op { + RestrictedOp::Eq | RestrictedOp::LtEq => Ok(column.is_null()), + RestrictedOp::NotEq | RestrictedOp::Gt => Ok(column.is_not_null()), + RestrictedOp::Lt => Ok(datafusion_expr::lit(false)), + RestrictedOp::GtEq => Ok(datafusion_expr::lit(true)), + RestrictedOp::And | RestrictedOp::Or => { + error::InvalidExprSnafu { expr: self.clone() }.fail() } - RestrictedOp::LtEq => column_expr - .clone() - .lt_eq(other_expr) - .or(column_expr.is_null()), - RestrictedOp::Gt => column_expr - .clone() - .gt(other_expr) - .and(column_expr.is_not_null()), - RestrictedOp::GtEq => column_expr - .clone() - .gt_eq(other_expr) - .and(column_expr.is_not_null()), - _ => unreachable!(), }; - return Ok(base); - } else if matches!(self.rhs.as_ref(), Operand::Column(_)) { - let other_expr = self.lhs.try_as_logical_expr()?; - let column_expr = self.rhs.try_as_logical_expr()?; - let base = match self.op { - RestrictedOp::Lt => other_expr - .lt(column_expr.clone()) - .and(column_expr.is_not_null()), - RestrictedOp::LtEq => other_expr - .lt_eq(column_expr.clone()) - .and(column_expr.is_not_null()), - RestrictedOp::Gt => { - other_expr.gt(column_expr.clone()).or(column_expr.is_null()) - } - RestrictedOp::GtEq => other_expr - .gt_eq(column_expr.clone()) - .or(column_expr.is_null()), - _ => unreachable!(), - }; - return Ok(base); } + let bound = Operand::Value(value.clone()).try_as_logical_expr()?; + return match op { + RestrictedOp::Eq => Ok(column.eq(bound)), + RestrictedOp::NotEq => Ok(column.clone().not_eq(bound).or(column.is_null())), + RestrictedOp::Lt => Ok(column.clone().lt(bound).or(column.is_null())), + RestrictedOp::LtEq => Ok(column.clone().lt_eq(bound).or(column.is_null())), + RestrictedOp::Gt => Ok(column.gt(bound)), + RestrictedOp::GtEq => Ok(column.gt_eq(bound)), + RestrictedOp::And | RestrictedOp::Or => { + error::InvalidExprSnafu { expr: self.clone() }.fail() + } + }; } // Normal cases handling, without NULL @@ -478,6 +445,14 @@ impl Display for PartitionExpr { #[cfg(test)] mod tests { + use std::collections::HashMap; + + use common_meta::ddl::alter_table::validate_region_partition_bounds; + use common_meta::error::Error; + use common_meta::rpc::router::{Region, RegionRoute}; + use datatypes::data_type::ConcreteDataType; + use store_api::storage::RegionId; + use super::*; #[test] @@ -596,7 +571,7 @@ mod tests { .try_as_logical_expr() .unwrap() .to_string(), - "a > Int64(10) AND a IS NOT NULL" + "a > Int64(10)" ); // Test Gt with column on LHS @@ -607,7 +582,7 @@ mod tests { ); assert_eq!( gt_expr.try_as_logical_expr().unwrap().to_string(), - "a > Int64(10) AND a IS NOT NULL" + "a > Int64(10)" ); // Test Gt with column on RHS @@ -632,7 +607,7 @@ mod tests { ); assert_eq!( gteq_expr.try_as_logical_expr().unwrap().to_string(), - "a >= Int64(10) AND a IS NOT NULL" + "a >= Int64(10)" ); // Test LtEq with column on LHS @@ -669,7 +644,7 @@ mod tests { .try_as_logical_expr() .unwrap() .to_string(), - "a >= Int64(10) AND a IS NOT NULL" + "a >= Int64(10)" ); let and_expr = PartitionExpr::new( @@ -687,7 +662,7 @@ mod tests { ); assert_eq!( and_expr.try_as_logical_expr().unwrap().to_string(), - "(a <= Int64(10) OR a IS NULL) AND b > Int64(5) AND b IS NOT NULL" + "(a <= Int64(10) OR a IS NULL) AND b > Int64(5)" ); let and_expr = PartitionExpr::new( @@ -705,7 +680,7 @@ mod tests { ); assert_eq!( and_expr.try_as_logical_expr().unwrap().to_string(), - "(a <= Int64(10) OR a IS NULL) AND a > Int64(5) AND a IS NOT NULL" + "(a <= Int64(10) OR a IS NULL) AND a > Int64(5)" ); let and_expr_strict_lower = PartitionExpr::new( @@ -726,7 +701,7 @@ mod tests { .try_as_logical_expr() .unwrap() .to_string(), - "(a < Int64(10) OR a IS NULL) AND a >= Int64(5) AND a IS NOT NULL" + "(a < Int64(10) OR a IS NULL) AND a >= Int64(5)" ); let and_expr_rhs_column = PartitionExpr::new( @@ -747,7 +722,7 @@ mod tests { .try_as_logical_expr() .unwrap() .to_string(), - "(a <= Int64(10) OR a IS NULL) AND a > Int64(5) AND a IS NOT NULL" + "(a <= Int64(10) OR a IS NULL) AND a > Int64(5)" ); let or_expr_same_column = PartitionExpr::new( @@ -768,7 +743,7 @@ mod tests { .try_as_logical_expr() .unwrap() .to_string(), - "a <= Int64(10) OR a IS NULL OR a > Int64(5) AND a IS NOT NULL" + "a <= Int64(10) OR a IS NULL OR a > Int64(5)" ); } @@ -784,7 +759,7 @@ mod tests { .try_as_logical_expr() .unwrap() .to_string(), - "Int64(10) > a OR a IS NULL" + "a < Int64(10) OR a IS NULL" ); let gteq_expr_rhs_column = PartitionExpr { @@ -797,7 +772,7 @@ mod tests { .try_as_logical_expr() .unwrap() .to_string(), - "Int64(10) >= a OR a IS NULL" + "a <= Int64(10) OR a IS NULL" ); } @@ -878,4 +853,113 @@ mod tests { assert!(columns.contains("b")); assert!(columns.contains("c")); } + + /// Real serialized expressions must survive the round trip through + /// `common-meta`'s persisted-expression mirror: whenever that mirror drifts + /// from these types, deserialization fails, validation is skipped and the + /// overflowing cases below stop producing `PartitionExprIncompatible`. + #[test] + fn test_serialized_expressions_validate_against_type_changes() { + let overflow_ms = serde_json::from_str::( + r#"{"Timestamp":{"value":32503680000000,"unit":"Millisecond"}}"#, + ) + .unwrap(); + let start_ms = + serde_json::from_str::(r#"{"Timestamp":{"value":0,"unit":"Millisecond"}}"#) + .unwrap(); + let ts_ms = ConcreteDataType::timestamp_millisecond_datatype(); + let ts_ns = ConcreteDataType::timestamp_nanosecond_datatype(); + + let routes = |expr: &PartitionExpr| { + vec![RegionRoute { + region: Region { + id: RegionId::new(1024, 1), + partition_expr: expr.as_json_str().unwrap(), + ..Default::default() + }, + ..Default::default() + }] + }; + let changed = |column: &str, old_type: ConcreteDataType, new_type: ConcreteDataType| { + HashMap::from([(column.to_string(), (old_type, new_type))]) + }; + + // Compile-time tripwire: exhaustively matching the operand and operator + // variants makes a new variant break this test at compile time, so the + // `partition_compat::Persisted*` mirror and these cases must be extended + // together instead of silently skipping validation. + fn walk(expr: &PartitionExpr) { + walk_operand(&expr.lhs); + walk_operand(&expr.rhs); + match expr.op { + RestrictedOp::Eq + | RestrictedOp::NotEq + | RestrictedOp::Lt + | RestrictedOp::LtEq + | RestrictedOp::Gt + | RestrictedOp::GtEq + | RestrictedOp::And + | RestrictedOp::Or => {} + } + } + fn walk_operand(operand: &Operand) { + match operand { + Operand::Column(_) | Operand::Value(_) => {} + Operand::Expr(expr) => walk(expr), + } + } + + // Every expression shape must be rejected while the bound overflows + // nanoseconds, with the error naming the offending column. + let simple = col("ts").lt(overflow_ms.clone()); + let nested_and = col("ts") + .gt_eq(start_ms.clone()) + .and(col("ts").lt(overflow_ms.clone())); + let nested_or = PartitionExpr::new( + Operand::Expr(col("host").eq(Value::String("a".into()))), + RestrictedOp::Or, + Operand::Expr(col("ts").lt(overflow_ms.clone())), + ); + for expr in [&simple, &nested_and, &nested_or] { + walk(expr); + let err = validate_region_partition_bounds( + &changed("ts", ts_ms.clone(), ts_ns.clone()), + &routes(expr), + ) + .unwrap_err(); + assert!( + matches!( + err, + Error::PartitionExprIncompatible { ref column, .. } if column == "ts" + ), + "unexpected error: {err}" + ); + } + + // A representable bound passes. + let compatible = col("ts").lt(start_ms); + validate_region_partition_bounds(&changed("ts", ts_ms, ts_ns), &routes(&compatible)) + .unwrap(); + + // Narrowing a field column must be rejected as well: the region's write + // path compares the persisted Int64 bound against Int32 rows directly + // and panics, even though the scan would coerce both sides. + let narrowing = col("v").lt(Value::Int64(3_000_000_000)); + let err = validate_region_partition_bounds( + &changed( + "v", + ConcreteDataType::int64_datatype(), + ConcreteDataType::int32_datatype(), + ), + &routes(&narrowing), + ) + .unwrap_err(); + assert!( + matches!( + err, + Error::PartitionExprIncompatible { ref column, .. } if column == "v" + ), + "unexpected error: {err}" + ); + } } diff --git a/src/partition/src/multi_dim.rs b/src/partition/src/multi_dim.rs index 16db78ffff6..25dcc0fb89d 100644 --- a/src/partition/src/multi_dim.rs +++ b/src/partition/src/multi_dim.rs @@ -21,8 +21,9 @@ use std::sync::{Arc, RwLock}; use datafusion_expr::ColumnarValue; use datafusion_physical_expr::PhysicalExpr; use datatypes::arrow; -use datatypes::arrow::array::{BooleanArray, BooleanBufferBuilder, RecordBatch}; +use datatypes::arrow::array::{Array, BooleanArray, BooleanBufferBuilder, RecordBatch}; use datatypes::arrow::buffer::BooleanBuffer; +use datatypes::arrow::compute::prep_null_mask_filter; use datatypes::arrow::datatypes::Schema; use datatypes::prelude::Value; use datatypes::vectors::{Helper, VectorRef}; @@ -272,6 +273,9 @@ impl MultiDimPartitionRule { } }; let array = match columnar_value_to_boolean_array(col_val, num_rows) { + // A NULL result means the row is not in this region. Masks are merged + // with non-Kleene `or` below, where a NULL would drop the row. + Ok(array) if array.null_count() > 0 => prep_null_mask_filter(&array), Ok(array) => array, Err(e) => { return Some(Err(e)); @@ -1017,6 +1021,105 @@ mod test_split_record_batch { assert_eq!(total_rows, batch.num_rows()); } + #[test] + fn test_comparison_batch_matches_row() { + use datatypes::arrow::array::Array; + use datatypes::arrow::datatypes::{DataType, Field}; + + let inputs = [None, Some(4), Some(5), Some(6)]; + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, true)])), + vec![Arc::new(Int64Array::from(inputs.to_vec()))], + ) + .unwrap(); + let mut mismatches = vec![]; + for op in [ + RestrictedOp::Eq, + RestrictedOp::NotEq, + RestrictedOp::Lt, + RestrictedOp::LtEq, + RestrictedOp::Gt, + RestrictedOp::GtEq, + ] { + for bound in [Value::Null, Value::Int64(5)] { + // Both operand orders, since a rule restored without canonicalization may + // keep the literal on the left. + for expr in [ + PartitionExpr::new(col("a"), op.clone(), bound.clone().into()), + PartitionExpr { + lhs: Box::new(bound.clone().into()), + op: op.clone(), + rhs: Box::new(col("a")), + }, + ] { + let rule = MultiDimPartitionRule::try_new( + vec!["a".to_string()], + vec![1], + vec![expr.clone()], + false, + ) + .unwrap(); + let by_batch = match expr + .try_as_physical_expr(batch.schema_ref()) + .unwrap() + .evaluate(&batch) + { + Ok(by_batch) => { + columnar_value_to_boolean_array(by_batch, batch.num_rows()).unwrap() + } + Err(err) => { + mismatches.push(format!("{expr}: batch error {err}")); + continue; + } + }; + for (row, input) in inputs.iter().enumerate() { + let value = input.map_or(Value::Null, Value::Int64); + let by_row = rule.evaluate_expr(&expr, &[value]).unwrap(); + let by_batch = by_batch.is_valid(row) && by_batch.value(row); + if by_row != by_batch { + mismatches.push(format!( + "{expr} with a = {input:?}: row {by_row}, batch {by_batch}" + )); + } + } + } + } + } + assert!(mismatches.is_empty(), "{mismatches:#?}"); + } + + #[test] + fn test_split_null_rows_match_row_routing() { + use datatypes::arrow::datatypes::{DataType, Field}; + + let rule = MultiDimPartitionRule::try_new( + vec!["host".to_string()], + vec![0, 1], + vec![ + col("host").not_eq(Value::from("a")), + col("host").eq(Value::from("a")), + ], + true, + ) + .unwrap(); + for hosts in [vec![Some("a"), None, Some("z")], vec![Some("a"), None]] { + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("host", DataType::Utf8, true)])), + vec![Arc::new(StringArray::from(hosts.clone()))], + ) + .unwrap(); + let by_row = rule.split_record_batch_naive(&batch).unwrap(); + let by_batch = rule.split_record_batch(&batch).unwrap(); + for (region, expected) in by_row { + let actual = by_batch + .get(®ion) + .map(|mask| mask.array().clone()) + .unwrap_or_else(|| BooleanArray::from(vec![false; batch.num_rows()])); + assert_eq!(actual, expected, "{hosts:?}, region {region}"); + } + } + } + #[test] fn test_columnar_value_to_boolean_array_scalar_false() { let result = columnar_value_to_boolean_array( diff --git a/src/query/src/dist_plan/analyzer/test.rs b/src/query/src/dist_plan/analyzer/test.rs index 207de7d43fd..f644a9d93fb 100644 --- a/src/query/src/dist_plan/analyzer/test.rs +++ b/src/query/src/dist_plan/analyzer/test.rs @@ -32,7 +32,7 @@ use datafusion::functions_nested::expr_fn::make_array; use datafusion::prelude::SessionContext; use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_common::{ExprSchema, JoinType, ScalarValue, TableReference}; -use datafusion_expr::expr::{Exists, ScalarFunction}; +use datafusion_expr::expr::{Exists, NullTreatment, ScalarFunction}; use datafusion_expr::utils::split_conjunction; use datafusion_expr::{ AggregateUDF, Expr, ExprSchemable as _, Extension, LogicalPlanBuilder, Operator, Subquery, @@ -2484,6 +2484,75 @@ fn date_bin_ts_group_by() { assert_eq!(expected, result.to_string()); } +#[test] +fn ignore_nulls_aggregate_stays_on_frontend() { + use datafusion::functions_aggregate::first_last::last_value; + + let table = TestTable::table_with_name(0, "t".to_string()); + let source = Arc::new(DefaultTableSource::new(Arc::new( + DfTableProviderAdapter::new(table), + ))); + let input = LogicalPlanBuilder::scan_with_filters("t", source, None, vec![]) + .unwrap() + .build() + .unwrap(); + for (group_by, null_treatment) in [ + ( + vec![col("pk1"), col("pk2")], + Some(NullTreatment::IgnoreNulls), + ), + (vec![col("number")], Some(NullTreatment::IgnoreNulls)), + (vec![col("pk1"), col("pk2")], None), + ( + vec![col("pk1"), col("pk2")], + Some(NullTreatment::RespectNulls), + ), + ] { + let mut ignored = last_value(col("number"), vec![]); + let Expr::AggregateFunction(func) = &mut ignored else { + panic!("expected aggregate expression"); + }; + func.params.null_treatment = null_treatment; + let aggregate_exprs = if null_treatment == Some(NullTreatment::IgnoreNulls) { + let mut ordinary = last_value(col("number"), vec![]); + let Expr::AggregateFunction(func) = &mut ordinary else { + panic!("expected aggregate expression"); + }; + func.params.null_treatment = None; + vec![ordinary.alias("ordinary"), ignored.alias("ignored")] + } else { + vec![ignored] + }; + let plan = LogicalPlanBuilder::from(input.clone()) + .aggregate(group_by, aggregate_exprs) + .unwrap() + .build() + .unwrap(); + let analyzed = DistPlannerAnalyzer {} + .analyze(plan.clone(), &ConfigOptions::default()) + .unwrap(); + if null_treatment == Some(NullTreatment::IgnoreNulls) { + let LogicalPlan::Aggregate(aggregate) = &analyzed else { + panic!("IGNORE NULLS aggregate must remain on the frontend: {analyzed}"); + }; + let remote_input = find_merge_scan(&analyzed).unwrap().input(); + assert!( + !matches!(remote_input, LogicalPlan::Aggregate(_)), + "remote input contains aggregate: {remote_input}" + ); + let Some(func) = + common_function::aggrs::aggr_wrapper::get_aggr_func(&aggregate.aggr_expr[1]) + else { + unreachable!(); + }; + assert_eq!(Some(NullTreatment::IgnoreNulls), func.params.null_treatment); + } else { + let remote_input = find_merge_scan(&analyzed).unwrap().input(); + assert!(matches!(remote_input, LogicalPlan::Aggregate(_))); + } + } +} + #[test] fn test_last_value_order_by() { init_default_ut_logging(); diff --git a/src/query/src/dist_plan/commutativity.rs b/src/query/src/dist_plan/commutativity.rs index b69589a7cfe..ae7bec52297 100644 --- a/src/query/src/dist_plan/commutativity.rs +++ b/src/query/src/dist_plan/commutativity.rs @@ -15,10 +15,13 @@ use std::collections::HashSet; use std::sync::Arc; -use common_function::aggrs::aggr_wrapper::{StateMergeHelper, is_all_aggr_exprs_steppable}; +use common_function::aggrs::aggr_wrapper::{ + StateMergeHelper, get_aggr_func, is_all_aggr_exprs_steppable, +}; use common_telemetry::debug; use datafusion::error::Result as DfResult; use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion}; +use datafusion_expr::expr::NullTreatment; use datafusion_expr::{Expr, LogicalPlan, UserDefinedLogicalNode}; use promql::extension_plan::{ EmptyMetric, InstantManipulate, RangeManipulate, SeriesDivide, SeriesNormalize, @@ -116,6 +119,14 @@ impl Categorizer { LogicalPlan::Filter(filter) => Self::check_expr(&filter.predicate), LogicalPlan::Window(_) => Commutativity::Unimplemented, LogicalPlan::Aggregate(aggr) => { + // The pinned Substrait codec drops IGNORE NULLS, so keep these aggregates on the frontend. + if aggr.aggr_expr.iter().any(|expr| { + get_aggr_func(expr).is_some_and(|func| { + func.params.null_treatment == Some(NullTreatment::IgnoreNulls) + }) + }) { + return Ok(Commutativity::NonCommutative); + } // The state/merge split maps each group expression to one output column, // which doesn't hold for grouping sets. let has_grouping_set = aggr diff --git a/src/query/src/dist_plan/predicate_extractor.rs b/src/query/src/dist_plan/predicate_extractor.rs index 778f5b9fc94..020e6d169ef 100644 --- a/src/query/src/dist_plan/predicate_extractor.rs +++ b/src/query/src/dist_plan/predicate_extractor.rs @@ -18,10 +18,9 @@ use std::collections::HashSet; -use arrow::datatypes::DataType; use common_telemetry::debug; use datafusion_common::Result as DfResult; -use datafusion_expr::{Expr, LogicalPlan, Operator}; +use datafusion_expr::{Expr, LogicalPlan, Operator, utils::split_conjunction}; use datatypes::value::Value; use partition::expr::{Operand, PartitionExpr, RestrictedOp}; @@ -48,33 +47,46 @@ impl PredicateExtractor { let partition_set: HashSet = partition_columns.iter().cloned().collect(); for filter_expr in filter_exprs { - match DataFusionExprConverter::convert(&filter_expr) { - Ok(partition_expr) => { - // Check expression for safe partition pruning - match ExpressionChecker::check_expression_for_pruning( - &partition_expr, - &partition_set, - ) { - ExpressionCheckResult::UseAsIs(expr) => { - partition_exprs.push(expr); - } - ExpressionCheckResult::UsePartial(exprs) => { - partition_exprs.extend(exprs); - } - ExpressionCheckResult::Drop => { - debug!( - "Dropping mixed expression for correctness: {}", - partition_expr - ); - } - } - } + let partition_expr = match DataFusionExprConverter::convert(&filter_expr) { + Ok(expr) => Some(expr), Err(err) => { debug!( - "Failed to convert filter expression to PartitionExpr: {}, skipping", + "Failed to convert filter expression to PartitionExpr: {}, trying conjuncts", err ); - continue; + split_conjunction(&filter_expr) + .into_iter() + .filter_map(|conjunct| match DataFusionExprConverter::convert(conjunct) { + Ok(expr) => Some(expr), + Err(err) => { + debug!( + "Failed to convert filter conjunct to PartitionExpr: {}, skipping", + err + ); + None + } + }) + // Keep salvaged conjuncts in one AND expression, not as alternatives. + .reduce(PartitionExpr::and) + } + }; + let Some(partition_expr) = partition_expr else { + continue; + }; + + // Check expression for safe partition pruning + match ExpressionChecker::check_expression_for_pruning(&partition_expr, &partition_set) { + ExpressionCheckResult::UseAsIs(expr) => { + partition_exprs.push(expr); + } + ExpressionCheckResult::UsePartial(exprs) => { + partition_exprs.extend(exprs); + } + ExpressionCheckResult::Drop => { + debug!( + "Dropping mixed expression for correctness: {}", + partition_expr + ); } } } @@ -423,7 +435,11 @@ impl DataFusionExprConverter { Ok(Operand::Column(column_name)) } Expr::Literal(scalar_value, _) => { - let value = Value::try_from(scalar_value.clone()).unwrap(); + let value = Value::try_from(scalar_value.clone()).map_err(|err| { + datafusion_common::DataFusionError::Plan(format!( + "Failed to convert literal for partition pruning: {err}" + )) + })?; Ok(Operand::Value(value)) } Expr::Alias(alias_expr) => { @@ -431,19 +447,20 @@ impl DataFusionExprConverter { Self::convert_to_operand(&alias_expr.expr) } Expr::Cast(cast_expr) => { - // For safe casts, unwrap to the inner expression - // For unsafe casts, skip with debug logging - if Self::is_safe_cast_for_partition_pruning(cast_expr.field.data_type()) { - Self::convert_to_operand(&cast_expr.expr) + // Column casts need input type/value equivalence proof; literals must use the actual cast value. + if let Expr::Literal(value, _) = cast_expr.expr.as_ref() { + let value = value.cast_to(cast_expr.field.data_type())?; + let value = Value::try_from(value).map_err(|err| { + datafusion_common::DataFusionError::Plan(format!( + "Failed to convert cast literal for partition pruning: {err}" + )) + })?; + Ok(Operand::Value(value)) } else { - debug!( - "Skipping unsafe cast for partition pruning: {:?}", - cast_expr.field.data_type() - ); - Err(datafusion_common::DataFusionError::Plan(format!( - "Cast to {:?} not supported for partition pruning", - cast_expr.field.data_type() - ))) + debug!("Skipping cast expression for partition pruning: {:?}", expr); + Err(datafusion_common::DataFusionError::Plan( + "Cast expression not supported for partition pruning".to_string(), + )) } } other => { @@ -481,46 +498,6 @@ impl DataFusionExprConverter { }; Self::convert_operator(&negated) } - - /// Determine if a cast is safe for partition pruning - /// Safe casts don't change the logical meaning of constraints - fn is_safe_cast_for_partition_pruning(data_type: &DataType) -> bool { - match data_type { - // Integer widening casts are generally safe - DataType::Int8 => true, - DataType::Int16 => true, - DataType::Int32 => true, - DataType::Int64 => true, - DataType::UInt8 => true, - DataType::UInt16 => true, - DataType::UInt32 => true, - DataType::UInt64 => true, - - // Float casts are generally safe for equality/inequality comparisons - DataType::Float32 => true, - DataType::Float64 => true, - - // String casts might be safe in some cases - DataType::Utf8 => true, - DataType::LargeUtf8 => true, - // Dictionary encoding does not change the logical value. DataFusion inserts this cast - // when a predicate literal is compared with a dictionary-encoded scan column. - DataType::Dictionary(_, value_type) => { - Self::is_safe_cast_for_partition_pruning(value_type) - } - - // Date/time casts might be safe if they don't change precision significantly - DataType::Date32 => true, - DataType::Date64 => true, - DataType::Timestamp(_, _) => true, - - // Boolean casts are straightforward - DataType::Boolean => true, - - // For other types, be conservative and skip - _ => false, - } - } } #[cfg(test)] @@ -654,6 +631,190 @@ mod tests { ); } + #[test] + fn test_cast_literals_are_evaluated_and_column_casts_are_skipped() { + let casts = [ + (DataType::Int64, lit(1i64)), + (DataType::Float32, lit(1.0f32)), + (DataType::Utf8, lit("1")), + (DataType::Date32, lit(1i32)), + ( + DataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Nanosecond, None), + lit(1i64), + ), + ]; + for (data_type, value) in casts { + let cast = Expr::Cast(datafusion_expr::Cast::new( + Box::new(col("user_id")), + data_type, + )); + assert!(DataFusionExprConverter::convert(&cast.eq(value)).is_err()); + } + + let chain = Expr::Cast(datafusion_expr::Cast::new( + Box::new(Expr::Cast(datafusion_expr::Cast::new( + Box::new(col("user_id")), + DataType::Float64, + ))), + DataType::Int64, + )); + assert!(DataFusionExprConverter::convert(&chain.eq(lit(1i64))).is_err()); + + let cast_int = Expr::Cast(datafusion_expr::Cast::new( + Box::new(lit(1.5f64)), + DataType::Int64, + )); + assert_eq!( + DataFusionExprConverter::convert(&col("user_id").eq(cast_int)) + .unwrap() + .rhs(), + &Operand::Value(Value::Int64(1)) + ); + + let cast_string = Expr::Cast(datafusion_expr::Cast::new( + Box::new(lit("12")), + DataType::Int64, + )); + assert_eq!( + DataFusionExprConverter::convert(&col("user_id").eq(cast_string)) + .unwrap() + .rhs(), + &Operand::Value(Value::Int64(12)) + ); + + let unsupported = Expr::Cast(datafusion_expr::Cast::new( + Box::new(lit(1i64)), + DataType::Decimal256(10, 2), + )); + assert!(DataFusionExprConverter::convert_to_operand(&unsupported).is_err()); + } + + #[test] + fn test_cast_predicates_are_skipped_in_and_and_or() { + let cast = Expr::Cast(datafusion_expr::Cast::new( + Box::new(col("user_id")), + DataType::Float32, + )); + let lower = PartitionExpr::new( + Operand::Column("user_id".to_string()), + RestrictedOp::GtEq, + Operand::Value(Value::Int64(1)), + ); + let upper = PartitionExpr::new( + Operand::Column("user_id".to_string()), + RestrictedOp::Lt, + Operand::Value(Value::Int64(10)), + ); + let bounds = PartitionExpr::new( + Operand::Expr(lower.clone()), + RestrictedOp::And, + Operand::Expr(upper.clone()), + ); + check_partition_expressions(vec![ + FilterTestCase::new( + "cast_in_and_preserves_partition_bound", + col("user_id") + .eq(lit(1i64)) + .and(cast.clone().eq(lit(1.0f32))), + vec![PartitionExpr::new( + Operand::Column("user_id".to_string()), + RestrictedOp::Eq, + Operand::Value(Value::Int64(1)), + )], + vec!["user_id"], + ), + FilterTestCase::new( + "cast_between_two_bounds_preserves_both", + col("user_id") + .gt_eq(lit(1i64)) + .and(col("user_id").lt(lit(10i64))) + .and(cast.clone().eq(lit(1.0f32))), + vec![bounds.clone()], + vec!["user_id"], + ), + FilterTestCase::new( + "cast_left_of_bounds_preserves_both", + cast.clone() + .eq(lit(1.0f32)) + .and(col("user_id").gt_eq(lit(1i64))) + .and(col("user_id").lt(lit(10i64))), + vec![bounds.clone()], + vec!["user_id"], + ), + FilterTestCase::new( + "cast_between_bounds_preserves_both", + col("user_id") + .gt_eq(lit(1i64)) + .and(cast.clone().eq(lit(1.0f32))) + .and(col("user_id").lt(lit(10i64))), + vec![bounds], + vec!["user_id"], + ), + FilterTestCase::new( + "is_not_null_cast_preserves_partition_bound", + col("user_id").eq(lit(1i64)).and( + Expr::Cast(datafusion_expr::Cast::new( + Box::new(col("value")), + DataType::Int32, + )) + .is_not_null(), + ), + vec![PartitionExpr::new( + Operand::Column("user_id".to_string()), + RestrictedOp::Eq, + Operand::Value(Value::Int64(1)), + )], + vec!["user_id"], + ), + FilterTestCase::new( + "all_unsupported_conjunction_is_dropped", + Expr::Cast(datafusion_expr::Cast::new( + Box::new(col("value")), + DataType::Int32, + )) + .eq(lit(1i32)) + .and( + Expr::Cast(datafusion_expr::Cast::new( + Box::new(col("timestamp")), + DataType::Float32, + )) + .eq(lit(1.0f32)), + ), + vec![], + vec!["user_id"], + ), + FilterTestCase::new( + "nested_and_with_or_side_preserves_bound", + col("user_id").eq(lit(1i64)).and( + col("user_id") + .eq(lit(2i64)) + .or(cast.clone().eq(lit(1.0f32))), + ), + vec![PartitionExpr::new( + Operand::Column("user_id".to_string()), + RestrictedOp::Eq, + Operand::Value(Value::Int64(1)), + )], + vec!["user_id"], + ), + FilterTestCase::new( + "or_with_nested_and_cast_drops_whole_expression", + col("user_id") + .eq(lit(1i64)) + .and(cast.clone().eq(lit(1.0f32))) + .or(col("user_id").eq(lit(2i64))), + vec![], + vec!["user_id"], + ), + FilterTestCase::new( + "cast_in_or_drops_entire_predicate", + col("user_id").eq(lit(1i64)).or(cast.eq(lit(1.0f32))), + vec![], + vec!["user_id"], + ), + ]); + } + #[test] fn test_combines_table_scan_filters_as_conjunction() { let table_scan = create_test_table_scan(); @@ -663,32 +824,42 @@ mod tests { panic!("expected test table scan"); }; let plan = LogicalPlan::TableScan(datafusion_expr::logical_plan::TableScan { - filters: vec![filter_a, filter_b], + filters: vec![filter_a.clone(), filter_b.clone()], + ..scan.clone() + }); + let partition_columns = ["user_id".to_string(), "value".to_string()]; + let expected = vec![PartitionExpr::new( + Operand::Expr(PartitionExpr::new( + Operand::Column("user_id".to_string()), + RestrictedOp::Eq, + Operand::Value(Value::Int64(10)), + )), + RestrictedOp::And, + Operand::Expr(PartitionExpr::new( + Operand::Column("value".to_string()), + RestrictedOp::Eq, + Operand::Value(Value::Int64(20)), + )), + )]; + + let partition_exprs = + PredicateExtractor::extract_partition_expressions(&plan, &partition_columns).unwrap(); + assert_eq!(partition_exprs, expected); + + let unsupported = Expr::Cast(datafusion_expr::Cast::new( + Box::new(col("timestamp")), + DataType::Float32, + )) + .eq(lit(1.0f32)); + let plan = LogicalPlan::TableScan(datafusion_expr::logical_plan::TableScan { + filters: vec![filter_a, unsupported, filter_b], ..scan }); - let partition_exprs = PredicateExtractor::extract_partition_expressions( - &plan, - &["user_id".to_string(), "value".to_string()], - ) - .unwrap(); + let partition_exprs = + PredicateExtractor::extract_partition_expressions(&plan, &partition_columns).unwrap(); - assert_eq!( - partition_exprs, - vec![PartitionExpr::new( - Operand::Expr(PartitionExpr::new( - Operand::Column("user_id".to_string()), - RestrictedOp::Eq, - Operand::Value(Value::Int64(10)), - )), - RestrictedOp::And, - Operand::Expr(PartitionExpr::new( - Operand::Column("value".to_string()), - RestrictedOp::Eq, - Operand::Value(Value::Int64(20)), - )), - )] - ); + assert_eq!(partition_exprs, expected); } #[test] @@ -1010,31 +1181,23 @@ mod tests { fn test_cast_expressions() { let cases = vec![ FilterTestCase::new( - "safe_cast", + "column_cast_requires_type_proof", Expr::Cast(datafusion_expr::Cast::new( Box::new(col("user_id")), DataType::Int64, )) .eq(lit(100i64)), - vec![PartitionExpr::new( - Operand::Column("user_id".to_string()), - RestrictedOp::Eq, - Operand::Value(Value::Int64(100)), - )], + vec![], vec!["user_id"], ), FilterTestCase::new( - "cast_with_alias", + "cast_with_alias_requires_type_proof", Expr::Cast(datafusion_expr::Cast::new( Box::new(col("user_id").alias("uid")), DataType::Int64, )) .eq(lit(100i64)), - vec![PartitionExpr::new( - Operand::Column("user_id".to_string()), - RestrictedOp::Eq, - Operand::Value(Value::Int64(100)), - )], + vec![], vec!["user_id"], ), FilterTestCase::new( @@ -1122,11 +1285,7 @@ mod tests { let in_expr = col("user_id") .alias("uid") .in_list(vec![lit(100i64), lit(200i64)], false); - let cast_expr = Expr::Cast(datafusion_expr::Cast::new( - Box::new(col("user_id")), - DataType::Int64, - )); - let between_expr = cast_expr.between(lit(300i64), lit(400i64)); + let between_expr = col("user_id").between(lit(300i64), lit(400i64)); in_expr.or(between_expr) }, vec![PartitionExpr::new( diff --git a/src/query/src/optimizer/scan_hint.rs b/src/query/src/optimizer/scan_hint.rs index 4685889c998..bc3c04ec1bf 100644 --- a/src/query/src/optimizer/scan_hint.rs +++ b/src/query/src/optimizer/scan_hint.rs @@ -22,7 +22,7 @@ use common_recordbatch::filter::SimpleFilterEvaluator; use datafusion::datasource::DefaultTableSource; use datafusion_common::tree_node::{Transformed, TreeNodeRewriter}; use datafusion_common::{Column, Result}; -use datafusion_expr::expr::Sort; +use datafusion_expr::expr::{NullTreatment, Sort}; use datafusion_expr::{Expr, LogicalPlan, utils}; use datafusion_optimizer::{OptimizerConfig, OptimizerRule}; use datatypes::arrow::datatypes::{DataType, TimeUnit as ArrowTimeUnit}; @@ -413,6 +413,8 @@ impl ScanHintRewriter { && func.func.name() != aggr_state_func_name("last_value")) || func.params.filter.is_some() || func.params.distinct + // A shared final-row selector would discard earlier nonnull values. + || func.params.null_treatment == Some(NullTreatment::IgnoreNulls) { return None; } @@ -455,7 +457,7 @@ mod test { use datafusion::prelude::JoinType; use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_expr::expr::{ - AggregateFunction, AggregateFunctionParams, Cast, WindowFunctionParams, + AggregateFunction, AggregateFunctionParams, Cast, NullTreatment, WindowFunctionParams, }; use datafusion_expr::expr_fn::scalar_subquery; use datafusion_expr::{Extension, LogicalPlan, LogicalPlanBuilder, col, lit}; @@ -561,6 +563,13 @@ mod test { } fn last_value_aggregate(input: LogicalPlan) -> LogicalPlan { + last_value_aggregate_with_null_treatments(input, [None, None]) + } + + fn last_value_aggregate_with_null_treatments( + input: LogicalPlan, + null_treatments: [Option; 2], + ) -> LogicalPlan { let aggregate = LogicalPlanBuilder::from(input) .aggregate( vec![col("k0")], @@ -576,7 +585,7 @@ mod test { asc: true, nulls_first: true, }], - null_treatment: None, + null_treatment: null_treatments[0], }, }), Expr::AggregateFunction(AggregateFunction { @@ -590,7 +599,7 @@ mod test { asc: true, nulls_first: true, }], - null_treatment: None, + null_treatment: null_treatments[1], }, }), ], @@ -901,6 +910,43 @@ mod test { } } + #[test] + fn last_value_null_treatment_controls_selector() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let context = OptimizerContext::default(); + + for treatment in [None, Some(NullTreatment::RespectNulls)] { + let rewritten = ScanHintRule + .rewrite( + last_value_aggregate_with_null_treatments( + scan_plan(provider.clone(), "t"), + [treatment, treatment], + ), + &context, + ) + .unwrap() + .data; + assert_eq!( + scan_requests(&rewritten)[0].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: false }) + ); + } + + // One IGNORE NULLS member prevents the shared scan selector for every + // aggregate in the group, including the member that respects nulls. + let rewritten = ScanHintRule + .rewrite( + last_value_aggregate_with_null_treatments( + scan_plan(provider, "t"), + [None, Some(NullTreatment::IgnoreNulls)], + ), + &context, + ) + .unwrap() + .data; + assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None); + } + #[test] fn single_evaluation_last_value_aggregate_keeps_legacy_selector() { let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs index afe52c7b560..6cb0d671196 100644 --- a/src/query/src/promql/planner.rs +++ b/src/query/src/promql/planner.rs @@ -577,7 +577,7 @@ impl PromPlanner { range, step, offset, - .. + at, } = subquery_expr; // Shift the child window back; `RangeManipulate` restores the outer timeline. @@ -603,10 +603,18 @@ impl PromPlanner { end: current_end, } ); + // `@` makes the subquery step-invariant: like Prometheus (`setOffsetForAtModifier`), fold + // the anchor into the offset and evaluate once at `current_start`, then replay the result + // over the outer grid. See [`Self::at_modifier_offset`]. + let at_offset = self.at_modifier_offset(at, offset)?; + let (eval_end, offset_ms) = match at_offset { + Some(at_offset) => (current_start, at_offset), + None => (current_end, offset_ms), + }; // An empty child needs only its schema; plan it over the valid caller window. let child_window = Self::subquery_child_window( current_start, - current_end, + eval_end, current_interval, offset_ms, range.as_millis() as Millisecond, @@ -707,23 +715,31 @@ impl PromPlanner { }; let manipulate = RangeManipulate::new( - self.ctx.start, - self.ctx.end, + current_start, + eval_end, self.ctx.interval, offset_ms, range_ms, - time_index_column, + time_index_column.clone(), self.ctx.field_columns.clone(), divide_plan, ) .context(DataFusionPlanningSnafu)?; - // The payload timestamps are shifted by the subquery offset; see - // [`Self::create_range_eval_ts_expr`]. + // The payload timestamps are shifted by the subquery offset (the rewritten one under `@`); + // see [`Self::create_range_eval_ts_expr`]. self.ctx.range_fold_offset = Some(offset_ms); - - Ok(LogicalPlan::Extension(Extension { + let manipulate = LogicalPlan::Extension(Extension { node: Arc::new(manipulate), - })) + }); + + // An anchored subquery was folded once, at `current_start`; report that window at every + // step of the outer grid, like an anchored range selector. + Ok(match at_offset { + Some(_) => { + self.replay_over_grid(manipulate, current_start, current_end, time_index_column) + } + None => manipulate, + }) } async fn prom_aggr_expr_to_plan( diff --git a/src/query/src/promql/planner/at_modifier.rs b/src/query/src/promql/planner/at_modifier.rs index 545f1714a22..81b03d29ed3 100644 --- a/src/query/src/promql/planner/at_modifier.rs +++ b/src/query/src/promql/planner/at_modifier.rs @@ -21,7 +21,8 @@ use datafusion::logical_expr::{Extension, LogicalPlan, LogicalPlanBuilder}; use datafusion::prelude::{Column, Expr as DfExpr}; use promql::extension_plan::{InstantManipulate, Millisecond, SeriesDivide}; use promql_parser::parser::{ - AtModifier, Call, Expr as PromExpr, MatrixSelector, Offset, ParenExpr, + AtModifier, Call, Expr as PromExpr, MatrixSelector, Offset, ParenExpr, SubqueryExpr, + VectorSelector, }; use snafu::{OptionExt, ResultExt, ensure}; @@ -138,8 +139,14 @@ impl PromPlanner { arg = expr; } match arg { - // The window is pinned by `@`, so every step folds the same samples. - PromExpr::MatrixSelector(MatrixSelector { vs, .. }) if vs.at.is_some() => { + // The window is pinned by `@`, so every step folds the same samples. That holds + // for a subquery with `@` too, which Prometheus treats as step-invariant whatever + // its inner expression is. + PromExpr::MatrixSelector(MatrixSelector { + vs: VectorSelector { at: Some(_), .. }, + .. + }) + | PromExpr::Subquery(SubqueryExpr { at: Some(_), .. }) => { if anchored_range { return false; } diff --git a/src/query/src/promql/planner/test.rs b/src/query/src/promql/planner/test.rs index ae40640b50b..be3140a6a52 100644 --- a/src/query/src/promql/planner/test.rs +++ b/src/query/src/promql/planner/test.rs @@ -5754,6 +5754,143 @@ async fn subquery_rejects_a_child_window_out_of_the_representable_range() { assert_eq!(err.status_code(), StatusCode::InvalidArguments, "{err}"); } +/// `@` on a subquery pins its evaluation instant, like `@` on a range selector: Prometheus folds +/// the anchor into the subquery offset (`setOffsetForAtModifier`) and treats the subquery as +/// step-invariant, so every step reports the window `(anchor - offset - range, anchor - offset]`. +/// See . +/// +/// The probe samples are at 0s..60s every 10s with values 1..7, on the 10s subquery grid. +#[tokio::test] +async fn subquery_at_modifier_pins_the_evaluation_instant() { + // The issue reproducer: `@ 60` folds `(40s, 60s]` -- the points 50s and 60s, 6 + 7 -- at every + // step, the same as the range selector form `m[20s] @ 60`. + for query in [ + "sum_over_time(subquery_offset_probe[20s] @ 60)", + "sum_over_time(subquery_offset_probe[20s:10s] @ 60)", + "sum_over_time(subquery_offset_probe[20s:10s] @ end())", + ] { + assert_eq!( + run_subquery_offset_probe(query, 30, 60, 30).await, + vec![(30_000, 13.0), (60_000, 13.0)], + "{query}" + ); + } + // `@ start()` folds `(10s, 30s]` -- 3 + 4 -- at every step. + assert_eq!( + run_subquery_offset_probe( + "sum_over_time(subquery_offset_probe[20s:10s] @ start())", + 30, + 60, + 30 + ) + .await, + vec![(30_000, 7.0), (60_000, 7.0)], + ); + // `offset` is applied relative to the anchor: `@ 60 offset 30s` folds `(10s, 30s]`, in either + // order of the modifiers. + for query in [ + "sum_over_time(subquery_offset_probe[20s:10s] @ 60 offset 30s)", + "sum_over_time(subquery_offset_probe[20s:10s] offset 30s @ 60)", + ] { + assert_eq!( + run_subquery_offset_probe(query, 30, 60, 30).await, + vec![(30_000, 7.0), (60_000, 7.0)], + "{query}" + ); + } + // An instant query evaluated away from the anchor still folds the anchored window. + assert_eq!( + run_subquery_offset_probe( + "sum_over_time(subquery_offset_probe[20s:10s] @ 60)", + 0, + 0, + 1 + ) + .await, + vec![(0, 13.0)], + ); + // An anchor whose window starts before the first sample: `@ 0` folds `(-20s, 0s]`, where only + // the point 0s has a sample. + assert_eq!( + run_subquery_offset_probe( + "sum_over_time(subquery_offset_probe[20s:10s] @ 0)", + 30, + 60, + 30 + ) + .await, + vec![(30_000, 1.0), (60_000, 1.0)], + ); + // `predict_linear` is not promoted as a whole, so it consumes the window replayed by the + // subquery itself; that window holds the same points as the range selector form. + let anchored = run_subquery_offset_probe( + "predict_linear(subquery_offset_probe[20s:10s] @ 60, 0)", + 30, + 60, + 30, + ) + .await; + assert_eq!(anchored.len(), 2, "{anchored:?}"); + assert_eq!( + anchored, + run_subquery_offset_probe( + "predict_linear(subquery_offset_probe[20s] @ 60, 0)", + 30, + 60, + 30 + ) + .await, + ); +} + +/// An anchored subquery evaluates its child over the anchored window only, not over the whole +/// outer evaluation range, and an anchor that cannot be represented in milliseconds is rejected. +#[tokio::test] +async fn subquery_at_modifier_scans_only_the_anchored_window() { + let plan = plan_subquery_offset_probe( + "sum_over_time(subquery_offset_probe[20s:10s] @ 60)", + 0, + 100_000, + 30, + ) + .await + .unwrap(); + let plan_str = plan.display_indent_schema().to_string(); + // The child grid is the points 50s and 60s of `(40s, 60s]`. + assert!( + plan_str.contains("PromInstantManipulate: range=[50000..60000]"), + "{plan_str}" + ); + // The subquery is folded once, at the start of the evaluation, and only the outer replay spans + // the evaluation range. + assert!( + plan_str.contains("PromRangeManipulate: req range=[0..0]"), + "{plan_str}" + ); + assert!( + plan_str.contains("subquery_offset_probe.timestamp <= TimestampMillisecond(60000, None)"), + "the child must not be evaluated over the outer range:\n{plan_str}" + ); + + #[cfg(not(windows))] + { + let err = plan_subquery_offset_probe( + "sum_over_time(subquery_offset_probe[20s:10s] @ 1e16)", + 0, + 60, + 30, + ) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("Timestamp out of range for the `@` modifier"), + "{err}" + ); + assert_eq!(err.status_code(), StatusCode::InvalidArguments); + } +} + #[tokio::test] async fn test_hash_join() { let mut eval_stmt = EvalStmt { diff --git a/src/servers/src/prom_store.rs b/src/servers/src/prom_store.rs index ff82f8f933b..0568d967010 100644 --- a/src/servers/src/prom_store.rs +++ b/src/servers/src/prom_store.rs @@ -21,7 +21,7 @@ use std::hash::{Hash, Hasher}; use std::sync::Arc; use api::prom_store::remote::label_matcher::Type as MatcherType; -use api::prom_store::remote::{Label, Query, Sample, TimeSeries, WriteRequest}; +use api::prom_store::remote::{Label, LabelMatcher, Query, Sample, TimeSeries, WriteRequest}; use api::v1::RowInsertRequests; use arrow::array::{ Array, ArrayRef, AsArray, DictionaryArray, LargeStringArray, StringArray, StringViewArray, @@ -138,6 +138,30 @@ pub fn extract_schema_from_query(query: &Query) -> Option { .map(|matcher| matcher.value.clone()) } +/// Validate matcher syntax independently of whether the metric exists. +pub fn validate_matchers(matchers: &[LabelMatcher]) -> Result<()> { + for matcher in matchers { + let matcher_type = MatcherType::try_from(matcher.r#type).map_err(|e| { + error::InvalidPromRemoteRequestSnafu { + msg: format!("invalid LabelMatcher type, decode error: {e}"), + } + .build() + })?; + if matches!(matcher_type, MatcherType::Re | MatcherType::Nre) { + // Validate using the same regex engine as query execution. + // Its syntax differs from Prometheus's; e.g. `(?x) a` is accepted here + // but rejected by Prometheus. + regex::Regex::new(&matcher.value).map_err(|e| { + error::InvalidPromRemoteRequestSnafu { + msg: format!("invalid regex for label '{}': {e}", matcher.name), + } + .build() + })?; + } + } + Ok(()) +} + /// Create a DataFrame from a remote Query #[tracing::instrument(skip_all)] pub fn query_to_plan( diff --git a/src/store-api/src/sst_entry.rs b/src/store-api/src/sst_entry.rs index c70e909ebd6..6017f26bed3 100644 --- a/src/store-api/src/sst_entry.rs +++ b/src/store-api/src/sst_entry.rs @@ -24,6 +24,7 @@ use datatypes::arrow::array::{ ArrayRef, BinaryArray, BooleanArray, TimestampMillisecondArray, TimestampNanosecondArray, UInt8Array, UInt32Array, UInt64Array, }; +use datatypes::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datatypes::arrow::error::ArrowError; use datatypes::arrow_array::StringArray; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; @@ -31,6 +32,34 @@ use serde::{Deserialize, Serialize}; use crate::storage::{RegionGroup, RegionId, RegionNumber, RegionSeq, ScanRequest, TableId}; +// Manifest SST column names. +pub const TABLE_DIR: &str = "table_dir"; +pub const REGION_ID: &str = "region_id"; +pub const TABLE_ID: &str = "table_id"; +pub const REGION_NUMBER: &str = "region_number"; +pub const REGION_GROUP: &str = "region_group"; +pub const REGION_SEQUENCE: &str = "region_sequence"; +pub const FILE_ID: &str = "file_id"; +pub const INDEX_VERSION: &str = "index_version"; +pub const LEVEL: &str = "level"; +pub const FILE_PATH: &str = "file_path"; +pub const FILE_SIZE: &str = "file_size"; +pub const INDEX_FILE_PATH: &str = "index_file_path"; +pub const INDEX_FILE_SIZE: &str = "index_file_size"; +pub const NUM_ROWS: &str = "num_rows"; +pub const NUM_ROW_GROUPS: &str = "num_row_groups"; +pub const NUM_SERIES: &str = "num_series"; +pub const MIN_TS: &str = "min_ts"; +pub const MAX_TS: &str = "max_ts"; +pub const SEQUENCE: &str = "sequence"; +pub const ORIGIN_REGION_ID: &str = "origin_region_id"; +pub const NODE_ID: &str = "node_id"; +pub const VISIBLE: &str = "visible"; +pub const PRIMARY_KEY_MIN: &str = "primary_key_min"; +pub const PRIMARY_KEY_MAX: &str = "primary_key_max"; +pub const MAX_ROW_GROUP_UNCOMPRESSED_SIZE: &str = "max_row_group_uncompressed_size"; +pub const PARTITION_EXPR: &str = "partition_expr"; + /// Puffin index type for bloom filter indexes. pub const PUFFIN_INDEX_TYPE_BLOOM_FILTER: &str = "bloom_filter"; /// Puffin index type for bloom-backed full-text indexes. @@ -102,110 +131,55 @@ impl ManifestSstEntry { pub fn schema() -> SchemaRef { use datatypes::prelude::ConcreteDataType as Ty; Arc::new(Schema::new(vec![ - ColumnSchema::new("table_dir", Ty::string_datatype(), false), - ColumnSchema::new("region_id", Ty::uint64_datatype(), false), - ColumnSchema::new("table_id", Ty::uint32_datatype(), false), - ColumnSchema::new("region_number", Ty::uint32_datatype(), false), - ColumnSchema::new("region_group", Ty::uint8_datatype(), false), - ColumnSchema::new("region_sequence", Ty::uint32_datatype(), false), - ColumnSchema::new("file_id", Ty::string_datatype(), false), - ColumnSchema::new("index_version", Ty::uint64_datatype(), false), - ColumnSchema::new("level", Ty::uint8_datatype(), false), - ColumnSchema::new("file_path", Ty::string_datatype(), false), - ColumnSchema::new("file_size", Ty::uint64_datatype(), false), - ColumnSchema::new("index_file_path", Ty::string_datatype(), true), - ColumnSchema::new("index_file_size", Ty::uint64_datatype(), true), - ColumnSchema::new("num_rows", Ty::uint64_datatype(), false), - ColumnSchema::new("num_row_groups", Ty::uint64_datatype(), false), - ColumnSchema::new("num_series", Ty::uint64_datatype(), true), - ColumnSchema::new("min_ts", Ty::timestamp_nanosecond_datatype(), true), - ColumnSchema::new("max_ts", Ty::timestamp_nanosecond_datatype(), true), - ColumnSchema::new("sequence", Ty::uint64_datatype(), true), - ColumnSchema::new("origin_region_id", Ty::uint64_datatype(), false), - ColumnSchema::new("node_id", Ty::uint64_datatype(), true), - ColumnSchema::new("visible", Ty::boolean_datatype(), false), - ColumnSchema::new("primary_key_min", Ty::binary_datatype(), true), - ColumnSchema::new("primary_key_max", Ty::binary_datatype(), true), + ColumnSchema::new(TABLE_DIR, Ty::string_datatype(), false), + ColumnSchema::new(REGION_ID, Ty::uint64_datatype(), false), + ColumnSchema::new(TABLE_ID, Ty::uint32_datatype(), false), + ColumnSchema::new(REGION_NUMBER, Ty::uint32_datatype(), false), + ColumnSchema::new(REGION_GROUP, Ty::uint8_datatype(), false), + ColumnSchema::new(REGION_SEQUENCE, Ty::uint32_datatype(), false), + ColumnSchema::new(FILE_ID, Ty::string_datatype(), false), + ColumnSchema::new(INDEX_VERSION, Ty::uint64_datatype(), false), + ColumnSchema::new(LEVEL, Ty::uint8_datatype(), false), + ColumnSchema::new(FILE_PATH, Ty::string_datatype(), false), + ColumnSchema::new(FILE_SIZE, Ty::uint64_datatype(), false), + ColumnSchema::new(INDEX_FILE_PATH, Ty::string_datatype(), true), + ColumnSchema::new(INDEX_FILE_SIZE, Ty::uint64_datatype(), true), + ColumnSchema::new(NUM_ROWS, Ty::uint64_datatype(), false), + ColumnSchema::new(NUM_ROW_GROUPS, Ty::uint64_datatype(), false), + ColumnSchema::new(NUM_SERIES, Ty::uint64_datatype(), true), + ColumnSchema::new(MIN_TS, Ty::timestamp_nanosecond_datatype(), true), + ColumnSchema::new(MAX_TS, Ty::timestamp_nanosecond_datatype(), true), + ColumnSchema::new(SEQUENCE, Ty::uint64_datatype(), true), + ColumnSchema::new(ORIGIN_REGION_ID, Ty::uint64_datatype(), false), + ColumnSchema::new(NODE_ID, Ty::uint64_datatype(), true), + ColumnSchema::new(VISIBLE, Ty::boolean_datatype(), false), + ColumnSchema::new(PRIMARY_KEY_MIN, Ty::binary_datatype(), true), + ColumnSchema::new(PRIMARY_KEY_MAX, Ty::binary_datatype(), true), ColumnSchema::new( - "max_row_group_uncompressed_size", + MAX_ROW_GROUP_UNCOMPRESSED_SIZE, Ty::uint64_datatype(), false, ), - ColumnSchema::new("partition_expr", Ty::string_datatype(), true), + ColumnSchema::new(PARTITION_EXPR, Ty::string_datatype(), true), ])) } /// Converts a list of manifest sst entries to a record batch. pub fn to_record_batch(entries: &[Self]) -> std::result::Result { - let schema = Self::schema(); - let table_dirs = entries.iter().map(|e| e.table_dir.as_str()); - let region_ids = entries.iter().map(|e| e.region_id.as_u64()); - let table_ids = entries.iter().map(|e| e.table_id); - let region_numbers = entries.iter().map(|e| e.region_number); - let region_groups = entries.iter().map(|e| e.region_group); - let region_sequences = entries.iter().map(|e| e.region_sequence); - let file_ids = entries.iter().map(|e| e.file_id.as_str()); - let index_versions = entries.iter().map(|e| e.index_version); - let levels = entries.iter().map(|e| e.level); - let file_paths = entries.iter().map(|e| e.file_path.as_str()); - let file_sizes = entries.iter().map(|e| e.file_size); - let max_row_group_uncompressed_sizes = - entries.iter().map(|e| e.max_row_group_uncompressed_size); - let index_file_paths = entries.iter().map(|e| e.index_file_path.as_ref()); - let index_file_sizes = entries.iter().map(|e| e.index_file_size); - let num_rows = entries.iter().map(|e| e.num_rows); - let num_row_groups = entries.iter().map(|e| e.num_row_groups); - let num_series = entries.iter().map(|e| e.num_series); - let min_ts = entries.iter().map(|e| { - e.min_ts - .convert_to(TimeUnit::Nanosecond) - .map(|ts| ts.value()) - }); - let max_ts = entries.iter().map(|e| { - e.max_ts - .convert_to(TimeUnit::Nanosecond) - .map(|ts| ts.value()) - }); - let sequences = entries.iter().map(|e| e.sequence); - let partition_exprs = entries.iter().map(|e| e.partition_expr.as_ref()); - let origin_region_ids = entries.iter().map(|e| e.origin_region_id.as_u64()); - let node_ids = entries.iter().map(|e| e.node_id); - let visible_flags = entries.iter().map(|e| Some(e.visible)); - let primary_key_min = entries.iter().map(|e| e.primary_key_min.as_deref()); - let primary_key_max = entries.iter().map(|e| e.primary_key_max.as_deref()); + Self::to_projected_record_batch(entries, None) + } - let columns: Vec = vec![ - Arc::new(StringArray::from_iter_values(table_dirs)), - Arc::new(UInt64Array::from_iter_values(region_ids)), - Arc::new(UInt32Array::from_iter_values(table_ids)), - Arc::new(UInt32Array::from_iter_values(region_numbers)), - Arc::new(UInt8Array::from_iter_values(region_groups)), - Arc::new(UInt32Array::from_iter_values(region_sequences)), - Arc::new(StringArray::from_iter_values(file_ids)), - Arc::new(UInt64Array::from_iter(index_versions)), - Arc::new(UInt8Array::from_iter_values(levels)), - Arc::new(StringArray::from_iter_values(file_paths)), - Arc::new(UInt64Array::from_iter_values(file_sizes)), - Arc::new(StringArray::from_iter(index_file_paths)), - Arc::new(UInt64Array::from_iter(index_file_sizes)), - Arc::new(UInt64Array::from_iter_values(num_rows)), - Arc::new(UInt64Array::from_iter_values(num_row_groups)), - Arc::new(UInt64Array::from_iter(num_series)), - Arc::new(TimestampNanosecondArray::from_iter(min_ts)), - Arc::new(TimestampNanosecondArray::from_iter(max_ts)), - Arc::new(UInt64Array::from_iter(sequences)), - Arc::new(UInt64Array::from_iter_values(origin_region_ids)), - Arc::new(UInt64Array::from_iter(node_ids)), - Arc::new(BooleanArray::from_iter(visible_flags)), - Arc::new(BinaryArray::from_iter(primary_key_min)), - Arc::new(BinaryArray::from_iter(primary_key_max)), - Arc::new(UInt64Array::from_iter_values( - max_row_group_uncompressed_sizes, - )), - Arc::new(StringArray::from_iter(partition_exprs)), - ]; - - DfRecordBatch::try_new(schema.arrow_schema().clone(), columns) + /// Builds only the requested columns, including empty and repeated projections. + fn to_projected_record_batch( + entries: &[Self], + projection: Option<&[usize]>, + ) -> std::result::Result { + let schema = Self::schema().arrow_schema().clone(); + let schema = match projection { + Some(projection) => Arc::new(schema.project(projection)?), + None => schema, + }; + ManifestSstBatchBuilder::new(schema).build(entries) } /// Reserved internal inspect table name. @@ -227,6 +201,140 @@ impl ManifestSstEntry { } } +/// Converts manifest SST batches using a schema prepared once per scan. +/// Repeated projected columns share the same Arrow array. +pub struct ManifestSstBatchBuilder { + schema: ArrowSchemaRef, + duplicates: Vec>, +} + +impl ManifestSstBatchBuilder { + /// Creates a builder for a projected manifest SST schema. + pub fn new(schema: ArrowSchemaRef) -> Self { + let duplicates = schema + .fields() + .iter() + .enumerate() + .map(|(index, field)| { + schema.fields()[..index] + .iter() + .position(|previous| previous == field) + }) + .collect(); + Self { schema, duplicates } + } + + /// Builds the projected columns, copying string contents only into Arrow buffers. + pub fn build( + &self, + entries: &[ManifestSstEntry], + ) -> std::result::Result { + let mut columns: Vec = Vec::with_capacity(self.schema.fields().len()); + for (index, field) in self.schema.fields().iter().enumerate() { + if let Some(previous) = self.duplicates[index] { + columns.push(columns[previous].clone()); + continue; + } + let array: ArrayRef = match field.name().as_str() { + TABLE_DIR => Arc::new(StringArray::from_iter_values( + entries.iter().map(|e| e.table_dir.as_str()), + )), + REGION_ID => Arc::new(UInt64Array::from_iter_values( + entries.iter().map(|e| e.region_id.as_u64()), + )), + TABLE_ID => Arc::new(UInt32Array::from_iter_values( + entries.iter().map(|e| e.table_id), + )), + REGION_NUMBER => Arc::new(UInt32Array::from_iter_values( + entries.iter().map(|e| e.region_number), + )), + REGION_GROUP => Arc::new(UInt8Array::from_iter_values( + entries.iter().map(|e| e.region_group), + )), + REGION_SEQUENCE => Arc::new(UInt32Array::from_iter_values( + entries.iter().map(|e| e.region_sequence), + )), + FILE_ID => Arc::new(StringArray::from_iter_values( + entries.iter().map(|e| e.file_id.as_str()), + )), + INDEX_VERSION => Arc::new(UInt64Array::from_iter( + entries.iter().map(|e| e.index_version), + )), + LEVEL => Arc::new(UInt8Array::from_iter_values( + entries.iter().map(|e| e.level), + )), + FILE_PATH => Arc::new(StringArray::from_iter_values( + entries.iter().map(|e| e.file_path.as_str()), + )), + FILE_SIZE => Arc::new(UInt64Array::from_iter_values( + entries.iter().map(|e| e.file_size), + )), + INDEX_FILE_PATH => Arc::new(StringArray::from_iter( + entries.iter().map(|e| e.index_file_path.as_ref()), + )), + INDEX_FILE_SIZE => Arc::new(UInt64Array::from_iter( + entries.iter().map(|e| e.index_file_size), + )), + NUM_ROWS => Arc::new(UInt64Array::from_iter_values( + entries.iter().map(|e| e.num_rows), + )), + NUM_ROW_GROUPS => Arc::new(UInt64Array::from_iter_values( + entries.iter().map(|e| e.num_row_groups), + )), + NUM_SERIES => { + Arc::new(UInt64Array::from_iter(entries.iter().map(|e| e.num_series))) + } + MIN_TS => Arc::new(TimestampNanosecondArray::from_iter(entries.iter().map( + |e| { + e.min_ts + .convert_to(TimeUnit::Nanosecond) + .map(|ts| ts.value()) + }, + ))), + MAX_TS => Arc::new(TimestampNanosecondArray::from_iter(entries.iter().map( + |e| { + e.max_ts + .convert_to(TimeUnit::Nanosecond) + .map(|ts| ts.value()) + }, + ))), + SEQUENCE => Arc::new(UInt64Array::from_iter(entries.iter().map(|e| e.sequence))), + ORIGIN_REGION_ID => Arc::new(UInt64Array::from_iter_values( + entries.iter().map(|e| e.origin_region_id.as_u64()), + )), + NODE_ID => Arc::new(UInt64Array::from_iter(entries.iter().map(|e| e.node_id))), + VISIBLE => Arc::new(BooleanArray::from_iter( + entries.iter().map(|e| Some(e.visible)), + )), + PRIMARY_KEY_MIN => Arc::new(BinaryArray::from_iter( + entries.iter().map(|e| e.primary_key_min.as_deref()), + )), + PRIMARY_KEY_MAX => Arc::new(BinaryArray::from_iter( + entries.iter().map(|e| e.primary_key_max.as_deref()), + )), + MAX_ROW_GROUP_UNCOMPRESSED_SIZE => Arc::new(UInt64Array::from_iter_values( + entries.iter().map(|e| e.max_row_group_uncompressed_size), + )), + PARTITION_EXPR => Arc::new(StringArray::from_iter( + entries.iter().map(|e| e.partition_expr.as_ref()), + )), + name => { + return Err(ArrowError::SchemaError(format!( + "Unknown manifest SST column: {name}" + ))); + } + }; + columns.push(array); + } + DfRecordBatch::try_new_with_options( + self.schema.clone(), + columns, + &datatypes::arrow::record_batch::RecordBatchOptions::new() + .with_row_count(Some(entries.len())), + ) + } +} + /// An entry describing a SST file listed from storage layer directly. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct StorageSstEntry { @@ -421,13 +529,25 @@ fn build_plan_helper( ) -> Result { let table_source = LogicalTableSource::new(schema.arrow_schema().clone()); + // Filters may reference columns omitted from the requested output. Apply + // them to the full schema and let the optimizer push the minimal projection. let projection = scan_request.projection; - let mut builder = LogicalPlanBuilder::scan(table_name, Arc::new(table_source), projection)?; + let mut builder = LogicalPlanBuilder::scan(table_name, Arc::new(table_source), None)?; for filter in scan_request.filters { builder = builder.filter(filter)?; } + if let Some(projection) = projection { + let projected = schema.arrow_schema().project(&projection)?; + builder = builder.project( + projected + .fields() + .iter() + .map(|field| datafusion_expr::col(field.name())), + )?; + } + if let Some(limit) = scan_request.limit { builder = builder.limit(0, Some(limit))?; } @@ -438,7 +558,7 @@ fn build_plan_helper( #[cfg(test)] mod tests { use datafusion_common::TableReference; - use datafusion_expr::{LogicalPlan, Operator, binary_expr, col, lit}; + use datafusion_expr::{Expr, LogicalPlan, Operator, binary_expr, col, lit}; use datatypes::arrow::array::{ Array, BinaryArray, TimestampMillisecondArray, TimestampNanosecondArray, UInt8Array, UInt32Array, UInt64Array, @@ -523,6 +643,29 @@ mod tests { let schema = ManifestSstEntry::schema(); let batch = ManifestSstEntry::to_record_batch(&entries).unwrap(); + for projection in [vec![], vec![16, 6, 10], vec![1, 1], (0..26).rev().collect()] { + let projected = + ManifestSstEntry::to_projected_record_batch(&entries, Some(&projection)).unwrap(); + assert_eq!(projected, batch.project(&projection).unwrap()); + assert_eq!(projected.num_rows(), entries.len()); + } + assert!(ManifestSstEntry::to_projected_record_batch(&entries, Some(&[26])).is_err()); + + let projected_schema = Arc::new(batch.schema().project(&[6, 9, 6, 11, 25, 9]).unwrap()); + let builder = ManifestSstBatchBuilder::new(projected_schema.clone()); + for rows in [entries.as_slice(), &entries[..1], &entries[..0]] { + let projected = builder.build(rows).unwrap(); + assert_eq!( + projected, + ManifestSstEntry::to_record_batch(rows) + .unwrap() + .project(&[6, 9, 6, 11, 25, 9]) + .unwrap() + ); + assert!(Arc::ptr_eq(&projected.schema(), &projected_schema)); + assert!(Arc::ptr_eq(projected.column(0), projected.column(2))); + assert!(Arc::ptr_eq(projected.column(1), projected.column(5))); + } // Schema checks assert_eq!(schema.arrow_schema().fields().len(), batch.num_columns()); @@ -1021,35 +1164,46 @@ mod tests { #[test] fn test_manifest_build_plan() { - // Note: filter must reference a column in the projected schema - let projection = Some(vec![0, 1, 2]); let request = ScanRequest { - projection, - filters: vec![binary_expr(col("table_id"), Operator::Gt, lit(0))], + projection: Some(vec![0, 1]), + filters: vec![col("table_id").gt(lit(0u32))], limit: Some(5), ..Default::default() }; let plan = ManifestSstEntry::build_plan(request).unwrap(); - // Expect plan to be Filter -> Limit -> TableScan or Filter+Limit wrapped. - // We'll pattern match to reach TableScan and verify key fields. - let (scan, has_filter, has_limit) = extract_scan(&plan); - - assert!(has_filter); - assert!(has_limit); + // The hidden table_id predicate must run before projection and limit. + let LogicalPlan::Limit(limit) = &plan else { + panic!("expected limit") + }; + let LogicalPlan::Projection(projection) = limit.input.as_ref() else { + panic!("expected projection") + }; + let LogicalPlan::Filter(filter) = projection.input.as_ref() else { + panic!("expected filter") + }; + let LogicalPlan::TableScan(scan) = filter.input.as_ref() else { + panic!("expected table scan") + }; assert_eq!( scan.table_name, TableReference::bare(ManifestSstEntry::reserved_table_name_for_inspection()) ); - assert_eq!(scan.projection, Some(vec![0, 1, 2])); + assert_eq!(scan.projection, None); + assert_eq!( + filter.predicate, + Expr::Column(datafusion_common::Column::new( + Some(scan.table_name.clone()), + "table_id", + )) + .gt(lit(0u32)) + ); - // projected schema should match projection - let fields = scan.projected_schema.fields(); - assert_eq!(fields.len(), 3); + let fields = plan.schema().fields(); + assert_eq!(fields.len(), 2); assert_eq!(fields[0].name(), "table_dir"); assert_eq!(fields[1].name(), "region_id"); - assert_eq!(fields[2].name(), "table_id"); } #[test] @@ -1070,9 +1224,9 @@ mod tests { scan.table_name, TableReference::bare(StorageSstEntry::reserved_table_name_for_inspection()) ); - assert_eq!(scan.projection, Some(vec![0, 2])); + assert_eq!(scan.projection, None); - let fields = scan.projected_schema.fields(); + let fields = plan.schema().fields(); assert_eq!(fields.len(), 2); assert_eq!(fields[0].name(), "file_path"); assert_eq!(fields[1].name(), "last_modified_ms"); @@ -1091,6 +1245,7 @@ mod tests { let (scan, has_filter, _) = extract_scan(input); (scan, has_filter, true) } + LogicalPlan::Projection(projection) => extract_scan(&projection.input), LogicalPlan::TableScan(scan) => (scan, false, false), other => panic!("unexpected plan: {other:?}"), } diff --git a/tests-integration/src/prom_store.rs b/tests-integration/src/prom_store.rs index af901120454..80da82c0e89 100644 --- a/tests-integration/src/prom_store.rs +++ b/tests-integration/src/prom_store.rs @@ -34,6 +34,8 @@ mod tests { use frontend::instance::Instance; use prost::Message; use servers::http::prom_store::PHYSICAL_TABLE_PARAM; + use servers::http::test_helpers::{TestClient, TestResponse}; + use servers::http::{HttpOptions, HttpServerBuilder}; use servers::prom_store; use servers::prom_store::to_grpc_row_insert_requests; use servers::query_handler::PromStoreProtocolHandler; @@ -663,10 +665,135 @@ mod tests { ] ); + instance + .do_query( + "CREATE TABLE invalid_metric (ts TIMESTAMP TIME INDEX, a DOUBLE, b DOUBLE)", + ctx.clone(), + ) + .await[0] + .as_ref() + .unwrap(); + let server = HttpServerBuilder::new(HttpOptions::default()) + .with_prom_handler(instance.clone(), None, true, Default::default(), None) + .build(); + let client = TestClient::new(server.build(server.make_app()).unwrap()).await; + + // Cover missing schemas/catalogs and each database label alias. + for (selector, database) in [ + ("__schema__", "unknown_schema"), + ("__database__", "unknown_catalog-prometheus"), + ("x_greptime_database", "unknown_catalog-information_schema"), + ] { + let mut query = remote_query("missing_metric"); + query.matchers.push(LabelMatcher { + name: selector.to_string(), + value: database.to_string(), + r#type: MatcherType::Eq as i32, + }); + let response = post_remote_read(&client, vec![query]).await; + assert_eq!(response.status(), 200, "{selector}={database}"); + let body = prom_store::snappy_decompress(&response.bytes().await).unwrap(); + let response = ReadResponse::decode(&body[..]).unwrap(); + assert_eq!(response.results.len(), 1); + assert!(response.results[0].timeseries.is_empty()); + } + + // Matcher syntax must be checked even when the metric does not exist. + for metric in ["metric1", "missing_metric"] { + for (matcher_type, value, expected_status) in [ + (99, "spark", 400), + (MatcherType::Re as i32, "[", 400), + (MatcherType::Nre as i32, "[", 400), + (MatcherType::Eq as i32, "[", 200), + (MatcherType::Neq as i32, "[", 200), + (MatcherType::Re as i32, "spark", 200), + (MatcherType::Nre as i32, "api.*", 200), + ] { + let mut query = remote_query(metric); + query.matchers.push(LabelMatcher { + name: "job".to_string(), + value: value.to_string(), + r#type: matcher_type, + }); + let response = post_remote_read(&client, vec![query]).await; + assert_eq!( + response.status().as_u16(), + expected_status, + "metric={metric}, type={matcher_type}, value={value}" + ); + } + } + + // Missing metrics must retain their result slots in multi-query responses. + for metrics in [ + vec!["missing_metric"], + vec!["missing_metric", "another_missing_metric"], + vec![ + "missing_metric", + "metric1", + "missing_metric", + "metric3", + "missing_metric", + ], + vec!["missing_metric", "invalid_metric"], + ] { + let queries = metrics.iter().map(|metric| remote_query(metric)).collect(); + let response = post_remote_read(&client, queries).await; + if metrics.contains(&"invalid_metric") { + assert!(!response.status().is_success()); + assert!(response.text().await.contains("Ambiguous value column")); + continue; + } + assert_eq!(response.status(), 200, "{}", response.text().await); + assert_eq!(response.headers()["content-type"], "application/x-protobuf"); + assert_eq!(response.headers()["content-encoding"], "snappy"); + let body = prom_store::snappy_decompress(&response.bytes().await).unwrap(); + let response = ReadResponse::decode(&body[..]).unwrap(); + assert_eq!(response.results.len(), metrics.len()); + for (result, metric) in response.results.iter().zip(metrics) { + if metric.ends_with("missing_metric") { + assert!(result.timeseries.is_empty()); + continue; + } + assert!(!result.timeseries.is_empty()); + for series in &result.timeseries { + assert!(series.labels.iter().any(|label| { + label.name == prom_store::METRIC_NAME_LABEL && label.value == metric + })); + assert_eq!(series.samples.len(), 2); + } + } + } + // check physical table if provided if let Some(physical_table) = physical_table { let sql = format!("DESC TABLE {physical_table};"); instance.do_query(&sql, ctx).await[0].as_ref().unwrap(); } } + + fn remote_query(metric: &str) -> Query { + Query { + start_timestamp_ms: 1000, + end_timestamp_ms: 2000, + matchers: vec![LabelMatcher { + name: prom_store::METRIC_NAME_LABEL.to_string(), + value: metric.to_string(), + r#type: MatcherType::Eq as i32, + }], + ..Default::default() + } + } + + async fn post_remote_read(client: &TestClient, queries: Vec) -> TestResponse { + let request = ReadRequest { + queries, + ..Default::default() + }; + client + .post("/v1/prometheus/read?db=prometheus") + .body(prom_store::snappy_compress(&request.encode_to_vec()).unwrap()) + .send() + .await + } } diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index f95a3396e83..863e2d3c63e 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -163,6 +163,7 @@ macro_rules! http_tests { test_http_auth, test_sql_api, test_http_analyze_stream_tql, + test_http_analyze_stream_scan_diagnostics, test_http_sql_slow_query, test_prometheus_promql_api, test_promql_over_non_millisecond_physical_tables, @@ -230,6 +231,7 @@ macro_rules! http_tests { test_jaeger_query_api, test_jaeger_query_api_for_trace_v1, test_jaeger_query_api_for_trace_v2, + test_jaeger_search_limit_returns_latest_traces, test_influxdb_write, test_influxdb_write_with_hints, @@ -882,6 +884,133 @@ async fn test_sql_format_api() { guard.remove_all().await; } +pub async fn test_http_analyze_stream_scan_diagnostics(store_type: StorageType) { + // Check both levels of JSON: the analyze node schema and the verbose scan + // object inside its string-valued `param`. + fn scan_details(node: &Value, scans: &mut Vec) { + assert!(node["name"].is_string()); + assert!(node["param"].is_string()); + assert!(node["output_rows"].is_u64()); + assert!(node["elapsed_compute"].is_u64()); + assert!(node["metrics"].is_object()); + if matches!( + node["name"].as_str(), + Some("SeqScan" | "UnorderedScan" | "SeriesScan") + ) { + let (_, fields) = node["param"].as_str().unwrap().split_once("), ").unwrap(); + let fields = if fields.starts_with('{') { + fields.to_string() + } else { + format!("{{{fields}}}") + }; + let details: Value = serde_json::from_str(&fields).unwrap(); + assert_eq!(details["projection"], json!(["reading", "ts"])); + assert!( + details["filters"] + .as_array() + .is_some_and(|filters| !filters.is_empty()) + ); + assert_eq!(details["time_range"], "TimestampRange{[0ms,6000ms)}"); + assert_eq!(details["file_summary"]["files"], 2); + assert_eq!(details["file_summary"]["rows"], 4); + assert_eq!(details["file_summary"]["row_groups"], 2); + assert_eq!(details["max_part_range"]["files"], 2); + assert!(details["max_part_range"].get("identifier").is_none()); + scans.push(details); + } + for child in node["children"].as_array().unwrap() { + scan_details(child, scans); + } + } + fn assert_verbose(scans: &[Value]) { + assert!(!scans.is_empty()); + for scan in scans { + let files = scan["files"].as_array().unwrap(); + assert_eq!(files.len(), 2); + for key in ["rows", "size", "index_size"] { + assert_eq!( + scan["file_summary"][key], + files + .iter() + .map(|file| file[key].as_u64().unwrap()) + .sum::() + ); + } + } + } + + let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "scan_diagnostics").await; + let client = TestClient::new(app).await; + for sql in [ + "CREATE TABLE scan_diagnostics (host STRING PRIMARY KEY, reading DOUBLE, ts TIMESTAMP TIME INDEX)", + "INSERT INTO scan_diagnostics VALUES ('a', 1, 0), ('b', 2, 3000)", + "ADMIN FLUSH_TABLE('scan_diagnostics')", + "INSERT INTO scan_diagnostics VALUES ('c', 3, 2000), ('d', 4, 5000)", + "ADMIN FLUSH_TABLE('scan_diagnostics')", + "INSERT INTO scan_diagnostics VALUES ('outside', 5, 10000)", + "ADMIN FLUSH_TABLE('scan_diagnostics')", + "INSERT INTO scan_diagnostics VALUES ('mem', 6, 1000)", + ] { + let response = execute_sql(&client, sql).await; + let status = response.status(); + let body = response.text().await; + assert_eq!(status, StatusCode::OK, "{sql}: {body}"); + let body: Value = serde_json::from_str(&body).unwrap(); + assert!(body.get("error").is_none(), "{body}"); + } + let query = "SELECT reading, ts FROM scan_diagnostics WHERE ts >= 0 AND ts < 6000"; + let sql = format!("EXPLAIN ANALYZE FORMAT JSON {query}"); + let response = execute_sql(&client, &sql).await; + let status = response.status(); + let body = response.text().await; + assert_eq!(status, StatusCode::OK, "{body}"); + let body: Value = serde_json::from_str(&body).unwrap(); + let mut explain_scans = vec![]; + for row in body["output"][0]["records"]["rows"].as_array().unwrap() { + if row[0].is_null() { + assert_eq!(row[2], "Total rows: 5"); + continue; + } + let plan: Value = serde_json::from_str(row[2].as_str().unwrap()).unwrap(); + scan_details(&plan, &mut explain_scans); + } + assert!(!explain_scans.is_empty()); + assert!(explain_scans.iter().all(|scan| scan.get("files").is_none())); + + let sql = format!("EXPLAIN ANALYZE VERBOSE FORMAT JSON {query}"); + let response = client + .post("/v1/sql/analyze/stream?snapshot_interval_ms=100") + .header("Accept", "text/event-stream") + .form(&BTreeMap::from([("sql", sql)])) + .send() + .await; + assert_eq!(response.status(), StatusCode::OK); + let body = response.text().await; + let mut first_scan_seen = false; + let mut final_seen = false; + for event in body.split("\n\n") { + let Some(data) = event.lines().find_map(|line| line.strip_prefix("data: ")) else { + continue; + }; + let payload: Value = serde_json::from_str(data).unwrap(); + let mut scans = vec![]; + for stage in payload["metrics"].as_array().unwrap() { + assert!(stage["stage"].is_u64()); + assert!(stage["node"].is_u64()); + scan_details(&stage["plan"], &mut scans); + } + // Every available scan is checked, including its first appearance. A fast + // query may finish before the first periodic snapshot. + first_scan_seen |= !scans.is_empty(); + if payload["state"] == "final" { + assert_verbose(&scans); + final_seen = true; + } + } + assert!(first_scan_seen && final_seen, "{body}"); + guard.remove_all().await; +} + pub async fn test_http_analyze_stream_tql(store_type: StorageType) { common_telemetry::init_default_ut_logging(); let (app, mut guard) = @@ -5009,7 +5138,11 @@ pub async fn test_prometheus_remote_schema_labels(store_type: StorageType) { .body(compressed_unknown_read_request) .send() .await; - assert_eq!(unknown_result.status(), StatusCode::BAD_REQUEST); + assert_eq!(unknown_result.status(), StatusCode::OK); + let body = prom_store::snappy_decompress(&unknown_result.bytes().await).unwrap(); + let response = ReadResponse::decode(&body[..]).unwrap(); + assert_eq!(response.results.len(), 1); + assert!(response.results[0].timeseries.is_empty()); guard.remove_all().await; } @@ -11777,6 +11910,88 @@ pub async fn test_jaeger_query_api_for_trace_v2( Ok(()) } +pub async fn test_jaeger_search_limit_returns_latest_traces(store_type: StorageType) { + common_telemetry::init_default_ut_logging(); + let (app, mut guard) = + setup_test_http_app_with_frontend(store_type, "test_jaeger_search_limit").await; + let client = TestClient::new(app).await; + + // Twelve single-span traces spread over all trace partitions. The oldest + // traces get the smallest ids and are written first, so neither id nor + // write order points at the newest ones. + let now = Utc::now().timestamp_nanos_opt().unwrap() as u64; + let traces = (0..12u64) + .map(|i| { + let start = now - (60 - i) * 1_000_000_000; + (format!("{:x}{:031x}", i, i + 1), start) + }) + .collect::>(); + let spans = traces + .iter() + .map(|(trace_id, start)| { + json!({ + "traceId": trace_id, + "spanId": &trace_id[16..], + "name": "op", + "kind": 2, + "startTimeUnixNano": start.to_string(), + "endTimeUnixNano": (start + 1_000).to_string(), + }) + }) + .collect::>(); + let content = json!({ + "resourceSpans": [{ + "resource": {"attributes": [{"key": "service.name", "value": {"stringValue": "svc"}}]}, + "scopeSpans": [{"scope": {"name": "s"}, "spans": spans}], + }] + }); + let req: ExportTraceServiceRequest = serde_json::from_value(content).unwrap(); + let res = send_req( + &client, + vec![ + ( + HeaderName::from_static("content-type"), + HeaderValue::from_static("application/x-protobuf"), + ), + ( + HeaderName::from_static("x-greptime-pipeline-name"), + HeaderValue::from_static(GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME), + ), + ], + "/v1/otlp/v1/traces", + req.encode_to_vec(), + false, + ) + .await; + assert_eq!(StatusCode::OK, res.status()); + + let mut expected = traces[9..] + .iter() + .map(|(trace_id, _)| trace_id.clone()) + .collect::>(); + expected.sort(); + let path = format!( + "/v1/jaeger/api/traces?service=svc&operation=op&start={}&end={}&limit=3", + (now - 120_000_000_000) / 1_000, + now / 1_000 + ); + for _ in 0..5 { + let res = client.get(&path).send().await; + assert_eq!(StatusCode::OK, res.status()); + let resp: Value = serde_json::from_str(&res.text().await).unwrap(); + let mut found = resp["data"] + .as_array() + .unwrap() + .iter() + .map(|trace| trace["traceID"].as_str().unwrap().to_string()) + .collect::>(); + found.sort(); + assert_eq!(found, expected); + } + + guard.remove_all().await; +} + pub async fn test_jaeger_query_api_for_trace_v1(store_type: StorageType) { common_telemetry::init_default_ut_logging(); let (app, mut guard) = diff --git a/tests/cases/distributed/explain/analyze.result b/tests/cases/distributed/explain/analyze.result index 77f6cff8b59..0fcdab8b704 100644 --- a/tests/cases/distributed/explain/analyze.result +++ b/tests/cases/distributed/explain/analyze.result @@ -33,6 +33,7 @@ SELECT count(*) FROM system_metrics; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze SELECT count(*) FROM system_metrics; +-+-+-+ @@ -45,7 +46,7 @@ explain analyze SELECT count(*) FROM system_metrics; |_|_|_AggregateExec: mode=Final, gby=[], aggr=[count(system_REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(system_REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ diff --git a/tests/cases/distributed/explain/analyze.sql b/tests/cases/distributed/explain/analyze.sql index bdf51cbca49..1e52a00236c 100644 --- a/tests/cases/distributed/explain/analyze.sql +++ b/tests/cases/distributed/explain/analyze.sql @@ -23,6 +23,7 @@ SELECT count(*) FROM system_metrics; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze SELECT count(*) FROM system_metrics; drop table system_metrics; diff --git a/tests/cases/distributed/explain/order_by.result b/tests/cases/distributed/explain/order_by.result index e5781659521..062b5b26ad0 100644 --- a/tests/cases/distributed/explain/order_by.result +++ b/tests/cases/distributed/explain/order_by.result @@ -116,6 +116,7 @@ SELECT i, t AS alias_ts FROM test_pk ORDER BY alias_ts DESC LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY t DESC LIMIT 5; +-+-+-+ @@ -127,7 +128,7 @@ EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY t DESC LIMIT 5; | 1_| 0_|_ProjectionExec: expr=[i@0 as i, t@1 as alias_ts] REDACTED |_|_|_SortPreservingMergeExec: [t@1 DESC], fetch=5 REDACTED |_|_|_SortExec: TopK(fetch=5), expr=[t@1 DESC], preserve_partitioning=[true], filter=[t@1 IS NULL OR t@1 > 2] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["i", "t"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ @@ -138,6 +139,7 @@ EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY t DESC LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY alias_ts DESC LIMIT 5; +-+-+-+ @@ -149,7 +151,7 @@ EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY alias_ts DESC LIMI | 1_| 0_|_ProjectionExec: expr=[i@0 as i, t@1 as alias_ts] REDACTED |_|_|_SortPreservingMergeExec: [t@1 DESC], fetch=5 REDACTED |_|_|_SortExec: TopK(fetch=5), expr=[t@1 DESC], preserve_partitioning=[true], filter=[t@1 IS NULL OR t@1 > 2] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["i", "t"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ diff --git a/tests/cases/distributed/explain/order_by.sql b/tests/cases/distributed/explain/order_by.sql index 3819e8061fa..152b70147f6 100644 --- a/tests/cases/distributed/explain/order_by.sql +++ b/tests/cases/distributed/explain/order_by.sql @@ -44,6 +44,7 @@ SELECT i, t AS alias_ts FROM test_pk ORDER BY alias_ts DESC LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY t DESC LIMIT 5; -- SQLNESS REPLACE (-+) - @@ -52,6 +53,7 @@ EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY t DESC LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY alias_ts DESC LIMIT 5; DROP TABLE test_pk; diff --git a/tests/cases/distributed/explain/step_aggr.result b/tests/cases/distributed/explain/step_aggr.result index a69a82352b3..ed5dad360b2 100644 --- a/tests/cases/distributed/explain/step_aggr.result +++ b/tests/cases/distributed/explain/step_aggr.result @@ -81,6 +81,7 @@ FROM -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT count(i), @@ -103,17 +104,17 @@ FROM | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i"], "filters": ["host < Utf8(\"550-A\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i"], "filters": ["host >= Utf8(\"550-A\") AND (host < Utf8(\"550-W\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i"], "filters": ["host >= Utf8(\"550-W\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -164,6 +165,7 @@ FROM -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT avg(i) FROM @@ -181,17 +183,17 @@ FROM | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__avg_state(integers.i)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__avg_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i"], "filters": ["host < Utf8(\"550-A\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__avg_state(integers.i)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__avg_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i"], "filters": ["host >= Utf8(\"550-A\") AND (host < Utf8(\"550-W\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__avg_state(integers.i)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__avg_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i"], "filters": ["host >= Utf8(\"550-W\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -267,6 +269,7 @@ ORDER BY -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT ts, @@ -296,17 +299,17 @@ ORDER BY | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[ts@1 as ts], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host < Utf8(\"550-A\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[ts@1 as ts], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host >= Utf8(\"550-A\") AND (host < Utf8(\"550-W\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[ts@1 as ts], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host >= Utf8(\"550-W\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -382,6 +385,7 @@ ORDER BY -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT date_bin('2s'::INTERVAL, ts) as time_window, @@ -411,17 +415,17 @@ ORDER BY | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("2 seconds"),integers.ts)@0 as date_bin(Utf8("2 seconds"),integers.ts)], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 2000000000 }, ts@1) as date_bin(Utf8("2 seconds"),integers.ts)], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host < Utf8(\"550-A\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("2 seconds"),integers.ts)@0 as date_bin(Utf8("2 seconds"),integers.ts)], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 2000000000 }, ts@1) as date_bin(Utf8("2 seconds"),integers.ts)], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host >= Utf8(\"550-A\") AND (host < Utf8(\"550-W\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("2 seconds"),integers.ts)@0 as date_bin(Utf8("2 seconds"),integers.ts)], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 2000000000 }, ts@1) as date_bin(Utf8("2 seconds"),integers.ts)], aggr=[__count_state(integers.i), __sum_state(integers.i), __uddsketch_state_state(Int64(128),Float64(0.01),integers.i), __hll_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host >= Utf8(\"550-W\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ diff --git a/tests/cases/distributed/explain/step_aggr.sql b/tests/cases/distributed/explain/step_aggr.sql index cb6cb3b8834..b8b8b0d6ffb 100644 --- a/tests/cases/distributed/explain/step_aggr.sql +++ b/tests/cases/distributed/explain/step_aggr.sql @@ -53,6 +53,7 @@ FROM -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT count(i), @@ -86,6 +87,7 @@ FROM -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT avg(i) FROM @@ -132,6 +134,7 @@ ORDER BY -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT ts, @@ -188,6 +191,7 @@ ORDER BY -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT date_bin('2s'::INTERVAL, ts) as time_window, diff --git a/tests/cases/distributed/explain/step_aggr_advance.result b/tests/cases/distributed/explain/step_aggr_advance.result index 11adf22f866..4186cb18c3a 100644 --- a/tests/cases/distributed/explain/step_aggr_advance.result +++ b/tests/cases/distributed/explain/step_aggr_advance.result @@ -49,6 +49,7 @@ tql explain (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr_optimize_not [2m])); +-+-+-+ @@ -68,7 +69,7 @@ tql analyze (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr |_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[greptime_timestamp], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(1752591744001, None)", "greptime_timestamp <= TimestampMillisecond(1752592164000, None)", "a < Utf8(\"b\") OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp, prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value))@4 as max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED @@ -81,7 +82,7 @@ tql analyze (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr |_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[greptime_timestamp], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(1752591744001, None)", "greptime_timestamp <= TimestampMillisecond(1752592164000, None)", "a >= Utf8(\"b\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -125,6 +126,7 @@ tql explain (1752591864, 1752592164, '30s') sum by (a, b) (max_over_time(aggr_op -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1752591864, 1752592164, '30s') sum by (a, b) (max_over_time(aggr_optimize_not [2m])); +-+-+-+ @@ -146,7 +148,7 @@ tql analyze (1752591864, 1752592164, '30s') sum by (a, b) (max_over_time(aggr_op |_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[greptime_timestamp], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(1752591744001, None)", "greptime_timestamp <= TimestampMillisecond(1752592164000, None)", "a < Utf8(\"b\") OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, greptime_timestamp@2 as greptime_timestamp], aggr=[__sum_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED @@ -156,7 +158,7 @@ tql analyze (1752591864, 1752592164, '30s') sum by (a, b) (max_over_time(aggr_op |_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[greptime_timestamp], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(1752591744001, None)", "greptime_timestamp <= TimestampMillisecond(1752592164000, None)", "a >= Utf8(\"b\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -200,6 +202,7 @@ tql explain (1752591864, 1752592164, '30s') avg by (a) (max_over_time(aggr_optim -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1752591864, 1752592164, '30s') avg by (a) (max_over_time(aggr_optimize_not [2m])); +-+-+-+ @@ -221,7 +224,7 @@ tql analyze (1752591864, 1752592164, '30s') avg by (a) (max_over_time(aggr_optim |_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[greptime_timestamp], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(1752591744001, None)", "greptime_timestamp <= TimestampMillisecond(1752592164000, None)", "a < Utf8(\"b\") OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, greptime_timestamp@1 as greptime_timestamp], aggr=[__avg_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED @@ -231,7 +234,7 @@ tql analyze (1752591864, 1752592164, '30s') avg by (a) (max_over_time(aggr_optim |_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[greptime_timestamp], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(1752591744001, None)", "greptime_timestamp <= TimestampMillisecond(1752592164000, None)", "a >= Utf8(\"b\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -270,6 +273,7 @@ tql explain (1752591864, 1752592164, '30s') count by (a, b, c, d) (max_over_time -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1752591864, 1752592164, '30s') count by (a, b, c, d) (max_over_time(aggr_optimize_not [2m])); +-+-+-+ @@ -286,7 +290,7 @@ tql analyze (1752591864, 1752592164, '30s') count by (a, b, c, d) (max_over_time |_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[greptime_timestamp], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(1752591744001, None)", "greptime_timestamp <= TimestampMillisecond(1752592164000, None)", "a < Utf8(\"b\") OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, d@3 ASC NULLS LAST, greptime_timestamp@4 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, d@3 ASC NULLS LAST, greptime_timestamp@4 ASC NULLS LAST], preserve_partitioning=[true] REDACTED @@ -296,7 +300,7 @@ tql analyze (1752591864, 1752592164, '30s') count by (a, b, c, d) (max_over_time |_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[greptime_timestamp], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(1752591744001, None)", "greptime_timestamp <= TimestampMillisecond(1752592164000, None)", "a >= Utf8(\"b\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -340,6 +344,7 @@ tql explain (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr_optimize_not [2m])); +-+-+-+ @@ -361,7 +366,7 @@ tql analyze (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr |_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[greptime_timestamp], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(1752591744001, None)", "greptime_timestamp <= TimestampMillisecond(1752592164000, None)", "a < Utf8(\"b\") OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED @@ -371,7 +376,7 @@ tql analyze (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr |_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[greptime_timestamp], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(1752591744001, None)", "greptime_timestamp <= TimestampMillisecond(1752592164000, None)", "a >= Utf8(\"b\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -412,6 +417,7 @@ tql explain sum(aggr_optimize_not); -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze sum(aggr_optimize_not); +-+-+-+ @@ -431,7 +437,7 @@ tql analyze sum(aggr_optimize_not); |_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, greptime_value@5 as greptime_value] REDACTED |_|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(-299999, None)", "greptime_timestamp <= TimestampMillisecond(0, None)", "a < Utf8(\"b\") OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[__sum_state(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED @@ -439,7 +445,7 @@ tql analyze sum(aggr_optimize_not); |_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, greptime_value@5 as greptime_value] REDACTED |_|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(-299999, None)", "greptime_timestamp <= TimestampMillisecond(0, None)", "a >= Utf8(\"b\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -459,6 +465,24 @@ CREATE TABLE IF NOT EXISTS aggr_optimize_not_count ( Affected Rows: 0 -- Case 6: Test average rate (sum/count like) +-- Populate both partitions on both sides so the join cannot short-circuit on +-- empty input and omit remote ANALYZE plans depending on execution timing. +INSERT INTO aggr_optimize_not VALUES + ('a', 'b', 'c', 'd', 1752591840000, 10), + ('a', 'b', 'c', 'd', 1752591860000, 20), + ('c', 'b', 'c', 'd', 1752591840000, 30), + ('c', 'b', 'c', 'd', 1752591860000, 40); + +Affected Rows: 4 + +INSERT INTO aggr_optimize_not_count VALUES + ('a', 'b', 'c', 'd', 1752591840000, 1), + ('a', 'b', 'c', 'd', 1752591860000, 2), + ('c', 'b', 'c', 'd', 1752591840000, 3), + ('c', 'b', 'c', 'd', 1752591860000, 4); + +Affected Rows: 4 + -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED @@ -524,6 +548,7 @@ tql explain (1752591864, 1752592164, '30s') sum by (a, b, c) (rate(aggr_optimize -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1752591864, 1752592164, '30s') sum by (a, b, c) (rate(aggr_optimize_not [2m])) / sum by (a, b, c) (rate(aggr_optimize_not_count [2m])); +-+-+-+ @@ -549,15 +574,49 @@ tql analyze (1752591864, 1752592164, '30s') sum by (a, b, c) (rate(aggr_optimize |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(1752591744001, None)", "greptime_timestamp <= TimestampMillisecond(1752592164000, None)", "a < Utf8(\"b\") OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(1752591744001, None)", "greptime_timestamp <= TimestampMillisecond(1752592164000, None)", "a >= Utf8(\"b\")"], "time_range":REDACTED |_|_|_| -|_|_| Total rows: 0_| +| 1_| 0_|_ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp, sum(prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)))@4 as sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))] REDACTED +|_|_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED +|_|_|_SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp], aggr=[sum(prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)))] REDACTED +|_|_|_RepartitionExec: partitioning=REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[a@2 as a, b@3 as b, c@4 as c, greptime_timestamp@0 as greptime_timestamp], aggr=[sum(prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)))] REDACTED +|_|_|_FilterExec: prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000))@1 IS NOT NULL REDACTED +|_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, prom_rate(greptime_timestamp_range@6, greptime_value@5, greptime_timestamp@4, 120000) as prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)), a@0 as a, b@1 as b, c@2 as c] REDACTED +|_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED +|_|_|_PromSeriesNormalizeExec: offset=[0], time index=[greptime_timestamp], filter NaN: [true] REDACTED +|_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(1752591744001, None)", "greptime_timestamp <= TimestampMillisecond(1752592164000, None)", "a < Utf8(\"b\") OR a IS NULL"], "time_range":REDACTED +|_|_|_| +| 1_| 1_|_ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp, sum(prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)))@4 as sum(prom_rate(greptime_timestamp_range,greptime_value,greptime_timestamp,Int64(120000)))] REDACTED +|_|_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED +|_|_|_SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp], aggr=[sum(prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)))] REDACTED +|_|_|_RepartitionExec: partitioning=REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[a@2 as a, b@3 as b, c@4 as c, greptime_timestamp@0 as greptime_timestamp], aggr=[sum(prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)))] REDACTED +|_|_|_FilterExec: prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000))@1 IS NOT NULL REDACTED +|_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, prom_rate(greptime_timestamp_range@6, greptime_value@5, greptime_timestamp@4, 120000) as prom_rate(greptime_timestamp_range,greptime_value,aggr_optimize_not.greptime_timestamp,Int64(120000)), a@0 as a, b@1 as b, c@2 as c] REDACTED +|_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED +|_|_|_PromSeriesNormalizeExec: offset=[0], time index=[greptime_timestamp], filter NaN: [true] REDACTED +|_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c", "d", "greptime_timestamp", "greptime_value"], "filters": ["greptime_timestamp >= TimestampMillisecond(1752591744001, None)", "greptime_timestamp <= TimestampMillisecond(1752592164000, None)", "a >= Utf8(\"b\")"], "time_range":REDACTED +|_|_|_| +|_|_| Total rows: 8_| +-+-+-+ -- Case 7: aggregate without sort should be pushed down. This one push down for include all partition columns. +TRUNCATE TABLE aggr_optimize_not; + +Affected Rows: 0 + +TRUNCATE TABLE aggr_optimize_not_count; + +Affected Rows: 0 + -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED @@ -591,6 +650,7 @@ GROUP BY -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT min(greptime_value) @@ -611,13 +671,13 @@ GROUP BY |_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c], aggr=[min(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b, c@2 as c], aggr=[min(aggr_optimize_not.greptime_value)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "c", "greptime_value"], "filters": ["a < Utf8(\"b\") OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_ProjectionExec: expr=[min(aggr_optimize_not.greptime_value)@3 as min(aggr_optimize_not.greptime_value)] REDACTED |_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c], aggr=[min(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b, c@2 as c], aggr=[min(aggr_optimize_not.greptime_value)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "c", "greptime_value"], "filters": ["a >= Utf8(\"b\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -657,6 +717,7 @@ GROUP BY -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT min(greptime_value) @@ -678,13 +739,13 @@ GROUP BY |_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, d@3 as d], aggr=[min(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b, c@2 as c, d@3 as d], aggr=[min(aggr_optimize_not.greptime_value)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "c", "d", "greptime_value"], "filters": ["a < Utf8(\"b\") OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_ProjectionExec: expr=[min(aggr_optimize_not.greptime_value)@4 as min(aggr_optimize_not.greptime_value)] REDACTED |_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, d@3 as d], aggr=[min(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b, c@2 as c, d@3 as d], aggr=[min(aggr_optimize_not.greptime_value)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "c", "d", "greptime_value"], "filters": ["a >= Utf8(\"b\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -727,6 +788,7 @@ GROUP BY -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT min(greptime_value) @@ -749,12 +811,12 @@ GROUP BY | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[__min_state(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[__min_state(aggr_optimize_not.greptime_value)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "greptime_value"], "filters": ["a < Utf8(\"b\") OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[__min_state(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[__min_state(aggr_optimize_not.greptime_value)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "greptime_value"], "filters": ["a >= Utf8(\"b\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -797,6 +859,7 @@ GROUP BY -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT min(greptime_value) + max(greptime_value) @@ -819,12 +882,12 @@ GROUP BY | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[__min_state(aggr_optimize_not.greptime_value), __max_state(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[__min_state(aggr_optimize_not.greptime_value), __max_state(aggr_optimize_not.greptime_value)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "greptime_value"], "filters": ["a < Utf8(\"b\") OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[__min_state(aggr_optimize_not.greptime_value), __max_state(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[__min_state(aggr_optimize_not.greptime_value), __max_state(aggr_optimize_not.greptime_value)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "greptime_value"], "filters": ["a >= Utf8(\"b\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -876,6 +939,7 @@ GROUP BY -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT a, @@ -907,12 +971,12 @@ GROUP BY | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[__min_state(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[__min_state(aggr_optimize_not.greptime_value)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "greptime_value"], "filters": ["a < Utf8(\"b\") OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[__min_state(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[__min_state(aggr_optimize_not.greptime_value)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "greptime_value"], "filters": ["a >= Utf8(\"b\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ diff --git a/tests/cases/distributed/explain/step_aggr_advance.sql b/tests/cases/distributed/explain/step_aggr_advance.sql index e9a6a3363e3..adf932d9aa4 100644 --- a/tests/cases/distributed/explain/step_aggr_advance.sql +++ b/tests/cases/distributed/explain/step_aggr_advance.sql @@ -26,6 +26,7 @@ tql explain (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr_optimize_not [2m])); -- Case 1: group by columns are prefix of partition columns. @@ -41,6 +42,7 @@ tql explain (1752591864, 1752592164, '30s') sum by (a, b) (max_over_time(aggr_op -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1752591864, 1752592164, '30s') sum by (a, b) (max_over_time(aggr_optimize_not [2m])); -- Case 2: group by columns are prefix of partition columns. @@ -56,6 +58,7 @@ tql explain (1752591864, 1752592164, '30s') avg by (a) (max_over_time(aggr_optim -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1752591864, 1752592164, '30s') avg by (a) (max_over_time(aggr_optimize_not [2m])); -- Case 3: group by columns are superset of partition columns. @@ -71,6 +74,7 @@ tql explain (1752591864, 1752592164, '30s') count by (a, b, c, d) (max_over_time -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1752591864, 1752592164, '30s') count by (a, b, c, d) (max_over_time(aggr_optimize_not [2m])); -- Case 4: group by columns are not prefix of partition columns. @@ -86,6 +90,7 @@ tql explain (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr_optimize_not [2m])); -- Case 5: a simple sum @@ -101,6 +106,7 @@ tql explain sum(aggr_optimize_not); -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze sum(aggr_optimize_not); -- TODO(discord9): more cases for aggr push down interacting with partitioning&tql @@ -116,6 +122,19 @@ CREATE TABLE IF NOT EXISTS aggr_optimize_not_count ( ) PARTITION ON COLUMNS (a, b, c) (a < 'b', a >= 'b',); -- Case 6: Test average rate (sum/count like) +-- Populate both partitions on both sides so the join cannot short-circuit on +-- empty input and omit remote ANALYZE plans depending on execution timing. +INSERT INTO aggr_optimize_not VALUES + ('a', 'b', 'c', 'd', 1752591840000, 10), + ('a', 'b', 'c', 'd', 1752591860000, 20), + ('c', 'b', 'c', 'd', 1752591840000, 30), + ('c', 'b', 'c', 'd', 1752591860000, 40); +INSERT INTO aggr_optimize_not_count VALUES + ('a', 'b', 'c', 'd', 1752591840000, 1), + ('a', 'b', 'c', 'd', 1752591860000, 2), + ('c', 'b', 'c', 'd', 1752591840000, 3), + ('c', 'b', 'c', 'd', 1752591860000, 4); + -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED @@ -128,9 +147,13 @@ tql explain (1752591864, 1752592164, '30s') sum by (a, b, c) (rate(aggr_optimize -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1752591864, 1752592164, '30s') sum by (a, b, c) (rate(aggr_optimize_not [2m])) / sum by (a, b, c) (rate(aggr_optimize_not_count [2m])); -- Case 7: aggregate without sort should be pushed down. This one push down for include all partition columns. +TRUNCATE TABLE aggr_optimize_not; +TRUNCATE TABLE aggr_optimize_not_count; + -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED @@ -151,6 +174,7 @@ GROUP BY -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT min(greptime_value) @@ -183,6 +207,7 @@ GROUP BY -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT min(greptime_value) @@ -214,6 +239,7 @@ GROUP BY -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT min(greptime_value) @@ -243,6 +269,7 @@ GROUP BY -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT min(greptime_value) + max(greptime_value) @@ -283,6 +310,7 @@ GROUP BY -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT a, diff --git a/tests/cases/distributed/explain/step_aggr_basic.result b/tests/cases/distributed/explain/step_aggr_basic.result index 39880450875..282d0f441da 100644 --- a/tests/cases/distributed/explain/step_aggr_basic.result +++ b/tests/cases/distributed/explain/step_aggr_basic.result @@ -74,6 +74,7 @@ FROM -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT count(i) @@ -92,17 +93,17 @@ FROM | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(integers.i)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i"], "filters": ["host < Utf8(\"550-A\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(integers.i)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i"], "filters": ["host >= Utf8(\"550-A\") AND (host < Utf8(\"550-W\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(integers.i)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i"], "filters": ["host >= Utf8(\"550-W\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -172,6 +173,7 @@ ORDER BY -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT ts, @@ -198,17 +200,17 @@ ORDER BY | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[__count_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[ts@1 as ts], aggr=[__count_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host < Utf8(\"550-A\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[__count_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[ts@1 as ts], aggr=[__count_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host >= Utf8(\"550-A\") AND (host < Utf8(\"550-W\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=FinalPartitioned, gby=[ts@0 as ts], aggr=[__count_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[ts@1 as ts], aggr=[__count_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host >= Utf8(\"550-W\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -280,6 +282,7 @@ ORDER BY -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT date_bin('1 hour' :: INTERVAL, ts) as time_window, @@ -307,17 +310,17 @@ ORDER BY | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("1 hour"),integers.ts)@0 as date_bin(Utf8("1 hour"),integers.ts)], aggr=[__count_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 3600000000000 }, ts@1) as date_bin(Utf8("1 hour"),integers.ts)], aggr=[__count_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host < Utf8(\"550-A\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("1 hour"),integers.ts)@0 as date_bin(Utf8("1 hour"),integers.ts)], aggr=[__count_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 3600000000000 }, ts@1) as date_bin(Utf8("1 hour"),integers.ts)], aggr=[__count_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host >= Utf8(\"550-A\") AND (host < Utf8(\"550-W\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("1 hour"),integers.ts)@0 as date_bin(Utf8("1 hour"),integers.ts)], aggr=[__count_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 3600000000000 }, ts@1) as date_bin(Utf8("1 hour"),integers.ts)], aggr=[__count_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host >= Utf8(\"550-W\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -397,6 +400,7 @@ ORDER BY -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT ts::BIGINT + 1, @@ -426,17 +430,17 @@ ORDER BY | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[integers.ts + Int64(1)@0 as integers.ts + Int64(1), integers.i / Int64(2)@1 as integers.i / Int64(2)], aggr=[__count_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[CAST(ts@1 AS Int64) + 1 as integers.ts + Int64(1), i@0 / 2 as integers.i / Int64(2)], aggr=[__count_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host < Utf8(\"550-A\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[integers.ts + Int64(1)@0 as integers.ts + Int64(1), integers.i / Int64(2)@1 as integers.i / Int64(2)], aggr=[__count_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[CAST(ts@1 AS Int64) + 1 as integers.ts + Int64(1), i@0 / 2 as integers.i / Int64(2)], aggr=[__count_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host >= Utf8(\"550-A\") AND (host < Utf8(\"550-W\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=FinalPartitioned, gby=[integers.ts + Int64(1)@0 as integers.ts + Int64(1), integers.i / Int64(2)@1 as integers.i / Int64(2)], aggr=[__count_state(integers.i)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[CAST(ts@1 AS Int64) + 1 as integers.ts + Int64(1), i@0 / 2 as integers.i / Int64(2)], aggr=[__count_state(integers.i)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["i", "ts"], "filters": ["host >= Utf8(\"550-W\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 8_| +-+-+-+ @@ -522,6 +526,7 @@ FROM -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT uddsketch_calc(0.5, uddsketch_merge(128, 0.01, udd_state)) as udd_result, @@ -542,17 +547,17 @@ FROM | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__uddsketch_merge_state(Int64(128),Float64(0.01),sink_table.udd_state), __hll_merge_state(sink_table.hll_state)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__uddsketch_merge_state(Int64(128),Float64(0.01),sink_table.udd_state), __hll_merge_state(sink_table.hll_state)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["udd_state", "hll_state"], "filters": ["host < Utf8(\"550-A\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__uddsketch_merge_state(Int64(128),Float64(0.01),sink_table.udd_state), __hll_merge_state(sink_table.hll_state)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__uddsketch_merge_state(Int64(128),Float64(0.01),sink_table.udd_state), __hll_merge_state(sink_table.hll_state)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["udd_state", "hll_state"], "filters": ["host >= Utf8(\"550-A\") AND (host < Utf8(\"550-W\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__uddsketch_merge_state(Int64(128),Float64(0.01),sink_table.udd_state), __hll_merge_state(sink_table.hll_state)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__uddsketch_merge_state(Int64(128),Float64(0.01),sink_table.udd_state), __hll_merge_state(sink_table.hll_state)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["udd_state", "hll_state"], "filters": ["host >= Utf8(\"550-W\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ diff --git a/tests/cases/distributed/explain/step_aggr_basic.sql b/tests/cases/distributed/explain/step_aggr_basic.sql index 9a0d226a130..d693ccbaabd 100644 --- a/tests/cases/distributed/explain/step_aggr_basic.sql +++ b/tests/cases/distributed/explain/step_aggr_basic.sql @@ -48,6 +48,7 @@ FROM -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT count(i) @@ -91,6 +92,7 @@ ORDER BY -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT ts, @@ -140,6 +142,7 @@ ORDER BY -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT date_bin('1 hour' :: INTERVAL, ts) as time_window, @@ -195,6 +198,7 @@ ORDER BY -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT ts::BIGINT + 1, @@ -264,6 +268,7 @@ FROM -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT uddsketch_calc(0.5, uddsketch_merge(128, 0.01, udd_state)) as udd_result, diff --git a/tests/cases/distributed/explain/step_aggr_massive.result b/tests/cases/distributed/explain/step_aggr_massive.result index 5aa969c6741..3a8e9634653 100644 --- a/tests/cases/distributed/explain/step_aggr_massive.result +++ b/tests/cases/distributed/explain/step_aggr_massive.result @@ -269,6 +269,7 @@ GROUP BY -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT env, @@ -466,112 +467,112 @@ GROUP BY |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key < Int64(4) OR shard_key IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(4) AND (shard_key < Int64(8) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(8) AND (shard_key < Int64(12) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 3_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(12) AND (shard_key < Int64(16) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 4_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(16) AND (shard_key < Int64(20) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 5_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(20) AND (shard_key < Int64(24) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 6_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(24) AND (shard_key < Int64(28) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 7_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(28) AND (shard_key < Int64(32) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 8_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(32) AND (shard_key < Int64(36) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 9_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(36) AND (shard_key < Int64(40) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 10_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(40) AND (shard_key < Int64(44) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 11_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(44) AND (shard_key < Int64(48) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 12_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(48) AND (shard_key < Int64(52) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 13_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(52) AND (shard_key < Int64(56) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 14_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(56) AND (shard_key < Int64(60) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 15_|_AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, service_name@1 as service_name, city@2 as city, page@3 as page, arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))@4 as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[env@7 as env, service_name@8 as service_name, city@9 as city, page@10 as page, CAST(date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, time@6) AS Timestamp(s)) as arrow_cast(date_bin(Utf8("60 seconds"),base_table.time),Utf8("Timestamp(s)"))], aggr=[__uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.lcp > Int64(0) AND base_table.lcp < Int64(3000000) THEN base_table.lcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fmp > Int64(0) AND base_table.fmp < Int64(3000000) THEN base_table.fmp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fcp > Int64(0) AND base_table.fcp < Int64(3000000) THEN base_table.fcp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fp > Int64(0) AND base_table.fp < Int64(3000000) THEN base_table.fp ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.tti > Int64(0) AND base_table.tti < Int64(3000000) THEN base_table.tti ELSE Int64(NULL) END), __uddsketch_state_state(Int64(128),Float64(0.01),CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __min_state(CASE WHEN base_table.fid > Int64(0) AND base_table.fid < Int64(3000000) THEN base_table.fid ELSE Int64(NULL) END), __max_state(base_table.shard_key)] REDACTED |_|_|_ProjectionExec: expr=[CASE WHEN lcp@5 > 0 AND lcp@5 < 3000000 THEN lcp@5 END as __common_expr_1, CASE WHEN fmp@6 > 0 AND fmp@6 < 3000000 THEN fmp@6 END as __common_expr_2, CASE WHEN fcp@7 > 0 AND fcp@7 < 3000000 THEN fcp@7 END as __common_expr_3, CASE WHEN fp@8 > 0 AND fp@8 < 3000000 THEN fp@8 END as __common_expr_4, CASE WHEN tti@9 > 0 AND tti@9 < 3000000 THEN tti@9 END as __common_expr_5, CASE WHEN fid@10 > 0 AND fid@10 < 3000000 THEN fid@10 END as __common_expr_6, time@0 as time, env@1 as env, service_name@2 as service_name, city@3 as city, page@4 as page, shard_key@11 as shard_key] REDACTED |_|_|_FilterExec: lcp@5 > 0 AND lcp@5 < 3000000 OR fmp@6 > 0 AND fmp@6 < 3000000 OR fcp@7 > 0 AND fcp@7 < 3000000 OR fp@8 > 0 AND fp@8 < 3000000 OR tti@9 > 0 AND tti@9 < 3000000 OR fid@10 > 0 AND fid@10 < 3000000 REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time", "env", "service_name", "city", "page", "lcp", "fmp", "fcp", "fp", "tti", "fid", "shard_key"], "filters": ["lcp > Int64(0) AND lcp < Int64(3000000) OR fmp > Int64(0) AND fmp < Int64(3000000) OR fcp > Int64(0) AND fcp < Int64(3000000) OR fp > Int64(0) AND fp < Int64(3000000) OR tti > Int64(0) AND tti < Int64(3000000) OR fid > Int64(0) AND fid < Int64(3000000)", "time >= TimestampMillisecond(0, None)", "shard_key >= Int64(60)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -616,6 +617,7 @@ where -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT count(*) @@ -636,82 +638,82 @@ where | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key < Int64(4) OR shard_key IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(4) AND (shard_key < Int64(8) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(8) AND (shard_key < Int64(12) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 3_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(12) AND (shard_key < Int64(16) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 4_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(16) AND (shard_key < Int64(20) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 5_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(20) AND (shard_key < Int64(24) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 6_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(24) AND (shard_key < Int64(28) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 7_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(28) AND (shard_key < Int64(32) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 8_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(32) AND (shard_key < Int64(36) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 9_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(36) AND (shard_key < Int64(40) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 10_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(40) AND (shard_key < Int64(44) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 11_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(44) AND (shard_key < Int64(48) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 12_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(48) AND (shard_key < Int64(52) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 13_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(52) AND (shard_key < Int64(56) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 14_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(56) AND (shard_key < Int64(60) OR shard_key IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 15_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(base_table.time)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(base_table.time)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["time"], "filters": ["time >= TimestampMillisecond(0, None)", "shard_key >= Int64(60)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ diff --git a/tests/cases/distributed/explain/step_aggr_massive.sql b/tests/cases/distributed/explain/step_aggr_massive.sql index e30bf79a670..490ee255602 100644 --- a/tests/cases/distributed/explain/step_aggr_massive.sql +++ b/tests/cases/distributed/explain/step_aggr_massive.sql @@ -249,6 +249,7 @@ GROUP BY -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT env, @@ -454,6 +455,7 @@ where -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT count(*) diff --git a/tests/cases/distributed/information_schema/ssts_limit.result b/tests/cases/distributed/information_schema/ssts_limit.result index 1b21a84c7af..67c6abf3cb1 100644 --- a/tests/cases/distributed/information_schema/ssts_limit.result +++ b/tests/cases/distributed/information_schema/ssts_limit.result @@ -95,6 +95,59 @@ FROM ( | | | +---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +-- The frontend orders results from all datanodes before applying the global offset. +SELECT min_ts, max_ts +FROM information_schema.ssts_manifest +WHERE region_id > 0 +ORDER BY min_ts DESC +LIMIT 2 OFFSET 1; + ++-------------------------+-------------------------+ +| min_ts | max_ts | ++-------------------------+-------------------------+ +| 1970-01-01T00:00:00.002 | 1970-01-01T00:00:00.002 | +| 1970-01-01T00:00:00.001 | 1970-01-01T00:00:00.001 | ++-------------------------+-------------------------+ + +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT min_ts +FROM information_schema.ssts_manifest +WHERE region_id > 0 +ORDER BY min_ts DESC +LIMIT 1; + ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Limit: skip=0, fetch=1 | +| | Sort: information_schema.ssts_manifest.min_ts DESC NULLS FIRST | +| | Projection: information_schema.ssts_manifest.min_ts | +| | Filter: information_schema.ssts_manifest.region_id > UInt64(0) | +| | TableScan: information_schema.ssts_manifest, partial_filters=[information_schema.ssts_manifest.region_id > UInt64(0)] | +| | ]] | +| physical_plan | SortPreservingMergeExec: [min_ts@0 DESC], fetch=1 | +| | SortExec: TopK(fetch=1), expr=[min_ts@0 DESC], preserve_partitioning=[true] | +| | FilterExec: region_id@0 > 0, projection=[min_ts@1] | +| | RepartitionExec: REDACTED +| | DistributedInspectExec: kind=SstManifest, scan=ScanRequest { projection: [1, 16], filters: [region_id > UInt64(0)] }, schema=Schema { fields: [Field { name: "region_id", data_type: UInt64 }, Field { name: "min_ts", data_type: Timestamp(Nanosecond, None), nullable: true }], metadata: {"greptime:version": "0"} } | +| | | ++---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- Timestamp filtering happens before global ordering and offset. The sort key +-- must remain available even when it is absent from the output projection. +SELECT max_ts +FROM information_schema.ssts_manifest +WHERE region_id > 0 AND min_ts >= '1970-01-01T00:00:00.002Z'::TIMESTAMP_NS +ORDER BY min_ts DESC +LIMIT 1 OFFSET 1; + ++-------------------------+ +| max_ts | ++-------------------------+ +| 1970-01-01T00:00:00.002 | ++-------------------------+ + DROP TABLE ssts_limit_case; Affected Rows: 0 diff --git a/tests/cases/distributed/information_schema/ssts_limit.sql b/tests/cases/distributed/information_schema/ssts_limit.sql index d548bd283a5..e634f7a2378 100644 --- a/tests/cases/distributed/information_schema/ssts_limit.sql +++ b/tests/cases/distributed/information_schema/ssts_limit.sql @@ -44,4 +44,26 @@ FROM ( LIMIT 1 ); +-- The frontend orders results from all datanodes before applying the global offset. +SELECT min_ts, max_ts +FROM information_schema.ssts_manifest +WHERE region_id > 0 +ORDER BY min_ts DESC +LIMIT 2 OFFSET 1; + +-- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED +EXPLAIN SELECT min_ts +FROM information_schema.ssts_manifest +WHERE region_id > 0 +ORDER BY min_ts DESC +LIMIT 1; + +-- Timestamp filtering happens before global ordering and offset. The sort key +-- must remain available even when it is absent from the output projection. +SELECT max_ts +FROM information_schema.ssts_manifest +WHERE region_id > 0 AND min_ts >= '1970-01-01T00:00:00.002Z'::TIMESTAMP_NS +ORDER BY min_ts DESC +LIMIT 1 OFFSET 1; + DROP TABLE ssts_limit_case; diff --git a/tests/cases/distributed/optimizer/count.result b/tests/cases/distributed/optimizer/count.result index 7c2d419b8cf..706340006cf 100644 --- a/tests/cases/distributed/optimizer/count.result +++ b/tests/cases/distributed/optimizer/count.result @@ -89,6 +89,7 @@ Affected Rows: 5 -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (\{count\[count\]:)\d+(\}) {count[count]:REDACTED} -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug; @@ -126,6 +127,7 @@ select count(1) from count_where_bug; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where `tag` = 'b'; @@ -139,7 +141,7 @@ select count(1) from count_where_bug where `tag` = 'b'; |_|_|_AggregateExec: mode=Final, gby=[], aggr=[count(count_where_bug.ts)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(count_where_bug.ts)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts"], "filters": ["tag = Dictionary(UInt32, Utf8(\"b\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -168,6 +170,7 @@ select count(1) from count_where_bug where `tag` = 'b'; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (\{count\[count\]:)\d+(\}) {count[count]:REDACTED} -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; @@ -181,7 +184,7 @@ select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; |_|_|_AggregateExec: mode=Final, gby=[], aggr=[count(count_where_bug.ts)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(count_where_bug.ts)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["ts > TimestampMillisecond(1725602404000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -208,6 +211,7 @@ select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where num != 3; @@ -222,7 +226,7 @@ select count(1) from count_where_bug where num != 3; |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(count_where_bug.ts)] REDACTED |_|_|_FilterExec: num@1 != 3, projection=[ts@0] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "num"], "filters": ["num != Int64(3)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -272,6 +276,7 @@ Affected Rows: 5 -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (\{count\[count\]:)\d+(\}) {count[count]:REDACTED} -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug; @@ -308,6 +313,7 @@ select count(1) from count_where_bug; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where `tag` = 'b'; @@ -323,7 +329,7 @@ select count(1) from count_where_bug where `tag` = 'b'; | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts"], "filters": ["tag = Dictionary(UInt32, Utf8(\"b\"))", "tag > Utf8(\"a\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -335,6 +341,7 @@ select count(1) from count_where_bug where `tag` = 'b'; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where `tag` = 'b'; @@ -350,7 +357,7 @@ select count(1) from count_where_bug where `tag` = 'b'; | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts"], "filters": ["tag = Dictionary(UInt32, Utf8(\"b\"))", "tag > Utf8(\"a\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -372,6 +379,7 @@ select count(1) from count_where_bug where `tag` = 'b'; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (\{count\[count]:)\d+(\}) {count[count]:REDACTED} -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; @@ -387,12 +395,12 @@ select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["ts > TimestampMillisecond(1725602404000, None)", "tag <= Utf8(\"a\") OR tag IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["ts > TimestampMillisecond(1725602404000, None)", "tag > Utf8(\"a\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -412,6 +420,7 @@ select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where num != 3; @@ -428,13 +437,13 @@ select count(1) from count_where_bug where num != 3; |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED |_|_|_FilterExec: num@1 != 3, projection=[ts@0] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "num"], "filters": ["num != Int64(3)", "tag <= Utf8(\"a\") OR tag IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED |_|_|_FilterExec: num@1 != 3, projection=[ts@0] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "num"], "filters": ["num != Int64(3)", "tag > Utf8(\"a\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ diff --git a/tests/cases/distributed/optimizer/first_value_advance.result b/tests/cases/distributed/optimizer/first_value_advance.result index 78aa8bcf6ab..59de599197c 100644 --- a/tests/cases/distributed/optimizer/first_value_advance.result +++ b/tests/cases/distributed/optimizer/first_value_advance.result @@ -95,6 +95,8 @@ explain select -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts), @@ -113,7 +115,7 @@ explain analyze |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "not_pk", "val"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -153,6 +155,8 @@ explain select first_value(ts order by ts) from t; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(ts order by ts) from t; @@ -165,7 +169,7 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -255,6 +259,8 @@ order by ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts) as ordered_host, @@ -277,7 +283,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "not_pk", "val"], "filters": ["host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_ProjectionExec: expr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@0 as ordered_host, first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST]@1 as first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_SortPreservingMergeExec: [first_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@0 ASC NULLS LAST] REDACTED @@ -286,7 +292,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "not_pk", "val"], "filters": ["host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_ProjectionExec: expr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@0 as ordered_host, first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST]@1 as first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_SortPreservingMergeExec: [first_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@0 ASC NULLS LAST] REDACTED @@ -295,7 +301,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "not_pk", "val"], "filters": ["host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -340,6 +346,8 @@ explain select first_value(ts order by ts) from t; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(ts order by ts) from t; @@ -355,17 +363,17 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -431,6 +439,8 @@ order by time_window, ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts) as ordered_host, @@ -456,17 +466,17 @@ order by time_window, ordered_host; | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@0) as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "val"], "filters": ["host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@0) as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "val"], "filters": ["host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@0) as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "val"], "filters": ["host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -544,6 +554,8 @@ explain -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(ts order by ts) from t1; @@ -559,17 +571,17 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["__table_id=UInt32(REDACTED)", "host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -627,6 +639,8 @@ order by ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts) as ordered_host, @@ -648,7 +662,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_ProjectionExec: expr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@0 as ordered_host, first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@1 as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_SortPreservingMergeExec: [first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@0 ASC NULLS LAST] REDACTED @@ -657,7 +671,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_ProjectionExec: expr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@0 as ordered_host, first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@1 as first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_SortPreservingMergeExec: [first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@0 ASC NULLS LAST] REDACTED @@ -666,7 +680,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -732,6 +746,8 @@ order by time_window, ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts) as ordered_host, @@ -757,17 +773,17 @@ order by time_window, ordered_host; | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@1) as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@1) as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@1) as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ diff --git a/tests/cases/distributed/optimizer/last_value_advance.result b/tests/cases/distributed/optimizer/last_value_advance.result index 795f34a582e..c9c1f3887ef 100644 --- a/tests/cases/distributed/optimizer/last_value_advance.result +++ b/tests/cases/distributed/optimizer/last_value_advance.result @@ -95,6 +95,8 @@ explain select -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts), @@ -113,7 +115,7 @@ explain analyze |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow { after_merge: false }", "projection": ["ts", "host", "not_pk", "val"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -153,6 +155,8 @@ explain select last_value(ts order by ts) from t; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(ts order by ts) from t; @@ -165,7 +169,7 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -255,6 +259,8 @@ order by ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts) as ordered_host, @@ -277,7 +283,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts", "host", "not_pk", "val"], "filters": ["host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_ProjectionExec: expr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@0 as ordered_host, last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST]@1 as last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_SortPreservingMergeExec: [last_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@0 ASC NULLS LAST] REDACTED @@ -286,7 +292,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts", "host", "not_pk", "val"], "filters": ["host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_ProjectionExec: expr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@0 as ordered_host, last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST]@1 as last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_SortPreservingMergeExec: [last_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@0 ASC NULLS LAST] REDACTED @@ -295,7 +301,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts", "host", "not_pk", "val"], "filters": ["host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -340,6 +346,8 @@ explain select last_value(ts order by ts) from t; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(ts order by ts) from t; @@ -355,17 +363,17 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "filters": ["host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "filters": ["host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "filters": ["host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -431,6 +439,8 @@ order by time_window, ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts) as ordered_host, @@ -456,17 +466,17 @@ order by time_window, ordered_host; | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@0) as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "val"], "filters": ["host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@0) as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "val"], "filters": ["host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@0) as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "val"], "filters": ["host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -544,6 +554,8 @@ explain -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(ts order by ts) from t1; @@ -559,17 +571,17 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "filters": ["__table_id=UInt32(REDACTED)", "host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -627,6 +639,8 @@ order by ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts) as ordered_host, @@ -648,7 +662,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_ProjectionExec: expr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@0 as ordered_host, last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@1 as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_SortPreservingMergeExec: [last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@0 ASC NULLS LAST] REDACTED @@ -657,7 +671,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_ProjectionExec: expr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@0 as ordered_host, last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@1 as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_SortPreservingMergeExec: [last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@0 ASC NULLS LAST] REDACTED @@ -666,7 +680,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -732,6 +746,8 @@ order by time_window, ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts) as ordered_host, @@ -757,17 +773,17 @@ order by time_window, ordered_host; | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@1) as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@1) as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@1) as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ diff --git a/tests/cases/distributed/optimizer/partition_pruning_cast.result b/tests/cases/distributed/optimizer/partition_pruning_cast.result new file mode 100644 index 00000000000..2fda5e7a12a --- /dev/null +++ b/tests/cases/distributed/optimizer/partition_pruning_cast.result @@ -0,0 +1,249 @@ +CREATE TABLE partition_pruning_cast_partitioned ( + ts TIMESTAMP TIME INDEX, + k FLOAT PRIMARY KEY, + v BIGINT, +) +PARTITION ON COLUMNS (k) ( + k < 1.25, + k >= 1.25 +); + +Affected Rows: 0 + +CREATE TABLE partition_pruning_cast_control ( + ts TIMESTAMP TIME INDEX, + k FLOAT PRIMARY KEY, + v BIGINT, +); + +Affected Rows: 0 + +INSERT INTO partition_pruning_cast_partitioned VALUES + (1000, 1.0, 10), + (1000, 1.5, 15), + (1000, 2.5, 25); + +Affected Rows: 3 + +INSERT INTO partition_pruning_cast_control VALUES + (1000, 1.0, 10), + (1000, 1.5, 15), + (1000, 2.5, 25); + +Affected Rows: 3 + +SELECT k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_partitioned ORDER BY k; + ++-----+----+--------+ +| k | v | cast_k | ++-----+----+--------+ +| 1.0 | 10 | 1 | +| 1.5 | 15 | 1 | +| 2.5 | 25 | 2 | ++-----+----+--------+ + +SELECT k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_control ORDER BY k; + ++-----+----+--------+ +| k | v | cast_k | ++-----+----+--------+ +| 1.0 | 10 | 1 | +| 1.5 | 15 | 1 | +| 2.5 | 25 | 2 | ++-----+----+--------+ + +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (partitioning.*) REDACTED +EXPLAIN SELECT k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_partitioned +WHERE CAST(k AS BIGINT) = 1; + ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Projection: partition_pruning_cast_partitioned.k, partition_pruning_cast_partitioned.v, CAST(partition_pruning_cast_partitioned.k AS Int64) AS cast_k | +| | Filter: CAST(partition_pruning_cast_partitioned.k AS Int64) = Int64(1) | +| | TableScan: partition_pruning_cast_partitioned, partial_filters=[CAST(partition_pruning_cast_partitioned.k AS Int64) = Int64(1)] | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+ + +SELECT ts, k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_partitioned +WHERE CAST(k AS BIGINT) = 1 +ORDER BY k; + ++---------------------+-----+----+--------+ +| ts | k | v | cast_k | ++---------------------+-----+----+--------+ +| 1970-01-01T00:00:01 | 1.0 | 10 | 1 | +| 1970-01-01T00:00:01 | 1.5 | 15 | 1 | ++---------------------+-----+----+--------+ + +SELECT ts, k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_control +WHERE CAST(k AS BIGINT) = 1 +ORDER BY k; + ++---------------------+-----+----+--------+ +| ts | k | v | cast_k | ++---------------------+-----+----+--------+ +| 1970-01-01T00:00:01 | 1.0 | 10 | 1 | +| 1970-01-01T00:00:01 | 1.5 | 15 | 1 | ++---------------------+-----+----+--------+ + +-- The same rows must also come back in descending order: a filter on a lossy +-- cast must not turn `k` into a constant and drop the requested sort. +SELECT ts, k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_partitioned +WHERE CAST(k AS BIGINT) = 1 +ORDER BY k DESC; + ++---------------------+-----+----+--------+ +| ts | k | v | cast_k | ++---------------------+-----+----+--------+ +| 1970-01-01T00:00:01 | 1.5 | 15 | 1 | +| 1970-01-01T00:00:01 | 1.0 | 10 | 1 | ++---------------------+-----+----+--------+ + +SELECT ts, k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_control +WHERE CAST(k AS BIGINT) = 1 +ORDER BY k DESC; + ++---------------------+-----+----+--------+ +| ts | k | v | cast_k | ++---------------------+-----+----+--------+ +| 1970-01-01T00:00:01 | 1.5 | 15 | 1 | +| 1970-01-01T00:00:01 | 1.0 | 10 | 1 | ++---------------------+-----+----+--------+ + +-- A table whose matching rows are stored in neither ascending nor descending +-- order, so a dropped sort cannot accidentally satisfy the ASC assertion. +CREATE TABLE partition_pruning_cast_unsorted ( + ts TIMESTAMP TIME INDEX, + k FLOAT PRIMARY KEY, + v BIGINT, +); + +Affected Rows: 0 + +INSERT INTO partition_pruning_cast_unsorted VALUES + (1000, 1.5, 15), + (1000, 2.5, 25), + (1000, 1.0, 10), + (1000, 1.25, 12); + +Affected Rows: 4 + +SELECT ts, k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_unsorted +WHERE CAST(k AS BIGINT) = 1 +ORDER BY k; + ++---------------------+------+----+--------+ +| ts | k | v | cast_k | ++---------------------+------+----+--------+ +| 1970-01-01T00:00:01 | 1.0 | 10 | 1 | +| 1970-01-01T00:00:01 | 1.25 | 12 | 1 | +| 1970-01-01T00:00:01 | 1.5 | 15 | 1 | ++---------------------+------+----+--------+ + +SELECT ts, k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_unsorted +WHERE CAST(k AS BIGINT) = 1 +ORDER BY k DESC; + ++---------------------+------+----+--------+ +| ts | k | v | cast_k | ++---------------------+------+----+--------+ +| 1970-01-01T00:00:01 | 1.5 | 15 | 1 | +| 1970-01-01T00:00:01 | 1.25 | 12 | 1 | +| 1970-01-01T00:00:01 | 1.0 | 10 | 1 | ++---------------------+------+----+--------+ + +DROP TABLE partition_pruning_cast_unsorted; + +Affected Rows: 0 + +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (partitioning.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED +EXPLAIN ANALYZE SELECT k, v +FROM partition_pruning_cast_partitioned +WHERE k >= 1.25::FLOAT AND k < 2.5::FLOAT; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_CooperativeExec REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_CooperativeExec REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["k", "v"], "filters": ["k >= Float32(1.25)", "k < Float32(2.5)", "k >= Float32(1.25)"], "time_range":REDACTED +|_|_|_| +|_|_| Total rows: 1_| ++-+-+-+ + +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (partitioning.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED +EXPLAIN ANALYZE SELECT k, v +FROM partition_pruning_cast_partitioned +WHERE k >= 1.25::FLOAT AND k < 2.5::FLOAT AND CAST(v AS INT) IS NOT NULL; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_CooperativeExec REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_FilterExec: CAST(v@1 AS Int32) IS NOT NULL REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["k", "v"], "filters": ["k >= Float32(1.25)", "k < Float32(2.5)", "CAST(v AS Int32) IS NOT NULL", "k >= Float32(1.25)"], "time_range":REDACTED +|_|_|_| +|_|_| Total rows: 1_| ++-+-+-+ + +SELECT k, v FROM partition_pruning_cast_partitioned +WHERE k >= 1.25::FLOAT AND k < 2.5::FLOAT AND CAST(v AS INT) IS NOT NULL; + ++-----+----+ +| k | v | ++-----+----+ +| 1.5 | 15 | ++-----+----+ + +SELECT k FROM partition_pruning_cast_partitioned +WHERE k = 1.0 OR CAST(k AS BIGINT) = 1 ORDER BY k; + ++-----+ +| k | ++-----+ +| 1.0 | +| 1.5 | ++-----+ + +DROP TABLE partition_pruning_cast_partitioned; + +Affected Rows: 0 + +DROP TABLE partition_pruning_cast_control; + +Affected Rows: 0 + diff --git a/tests/cases/distributed/optimizer/partition_pruning_cast.sql b/tests/cases/distributed/optimizer/partition_pruning_cast.sql new file mode 100644 index 00000000000..f72355b892d --- /dev/null +++ b/tests/cases/distributed/optimizer/partition_pruning_cast.sql @@ -0,0 +1,111 @@ +CREATE TABLE partition_pruning_cast_partitioned ( + ts TIMESTAMP TIME INDEX, + k FLOAT PRIMARY KEY, + v BIGINT, +) +PARTITION ON COLUMNS (k) ( + k < 1.25, + k >= 1.25 +); + +CREATE TABLE partition_pruning_cast_control ( + ts TIMESTAMP TIME INDEX, + k FLOAT PRIMARY KEY, + v BIGINT, +); + +INSERT INTO partition_pruning_cast_partitioned VALUES + (1000, 1.0, 10), + (1000, 1.5, 15), + (1000, 2.5, 25); +INSERT INTO partition_pruning_cast_control VALUES + (1000, 1.0, 10), + (1000, 1.5, 15), + (1000, 2.5, 25); + +SELECT k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_partitioned ORDER BY k; +SELECT k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_control ORDER BY k; + +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (partitioning.*) REDACTED +EXPLAIN SELECT k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_partitioned +WHERE CAST(k AS BIGINT) = 1; + +SELECT ts, k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_partitioned +WHERE CAST(k AS BIGINT) = 1 +ORDER BY k; +SELECT ts, k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_control +WHERE CAST(k AS BIGINT) = 1 +ORDER BY k; + +-- The same rows must also come back in descending order: a filter on a lossy +-- cast must not turn `k` into a constant and drop the requested sort. +SELECT ts, k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_partitioned +WHERE CAST(k AS BIGINT) = 1 +ORDER BY k DESC; +SELECT ts, k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_control +WHERE CAST(k AS BIGINT) = 1 +ORDER BY k DESC; + +-- A table whose matching rows are stored in neither ascending nor descending +-- order, so a dropped sort cannot accidentally satisfy the ASC assertion. +CREATE TABLE partition_pruning_cast_unsorted ( + ts TIMESTAMP TIME INDEX, + k FLOAT PRIMARY KEY, + v BIGINT, +); + +INSERT INTO partition_pruning_cast_unsorted VALUES + (1000, 1.5, 15), + (1000, 2.5, 25), + (1000, 1.0, 10), + (1000, 1.25, 12); + +SELECT ts, k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_unsorted +WHERE CAST(k AS BIGINT) = 1 +ORDER BY k; +SELECT ts, k, v, CAST(k AS BIGINT) AS cast_k +FROM partition_pruning_cast_unsorted +WHERE CAST(k AS BIGINT) = 1 +ORDER BY k DESC; + +DROP TABLE partition_pruning_cast_unsorted; + +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (partitioning.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED +EXPLAIN ANALYZE SELECT k, v +FROM partition_pruning_cast_partitioned +WHERE k >= 1.25::FLOAT AND k < 2.5::FLOAT; +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (partitioning.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED +EXPLAIN ANALYZE SELECT k, v +FROM partition_pruning_cast_partitioned +WHERE k >= 1.25::FLOAT AND k < 2.5::FLOAT AND CAST(v AS INT) IS NOT NULL; +SELECT k, v FROM partition_pruning_cast_partitioned +WHERE k >= 1.25::FLOAT AND k < 2.5::FLOAT AND CAST(v AS INT) IS NOT NULL; + +SELECT k FROM partition_pruning_cast_partitioned +WHERE k = 1.0 OR CAST(k AS BIGINT) = 1 ORDER BY k; + +DROP TABLE partition_pruning_cast_partitioned; +DROP TABLE partition_pruning_cast_control; diff --git a/tests/cases/distributed/optimizer/range_select_projection.result b/tests/cases/distributed/optimizer/range_select_projection.result index 28d8fff50a3..7541d929073 100644 --- a/tests/cases/distributed/optimizer/range_select_projection.result +++ b/tests/cases/distributed/optimizer/range_select_projection.result @@ -29,6 +29,7 @@ Affected Rows: 5 -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE Total\s+rows:\s+\d+ Total rows: REDACTED -- SQLNESS REPLACE Dictionary\(UInt32,\s*Utf8\(([^)]*)\)\) Utf8($1) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT ts, station, @@ -52,7 +53,7 @@ ORDER BY station, "channel", ts; |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "station", "channel", "value_a", "value_b"], "filters": ["station = Utf8(\"station-a\")", "unused_tag = Utf8(\"unused-a\")", "ts >= TimestampMillisecond(0, None)", "ts < TimestampMillisecond(15000, None)"], "flat_format": true, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "station", "channel", "value_a", "value_b"], "filters": ["station = Utf8(\"station-a\")", "unused_tag = Utf8(\"unused-a\")", "ts >= TimestampMillisecond(0, None)", "ts < TimestampMillisecond(15000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ diff --git a/tests/cases/distributed/optimizer/range_select_projection.sql b/tests/cases/distributed/optimizer/range_select_projection.sql index 4b15a30c675..5d82804aafe 100644 --- a/tests/cases/distributed/optimizer/range_select_projection.sql +++ b/tests/cases/distributed/optimizer/range_select_projection.sql @@ -25,6 +25,7 @@ INSERT INTO range_select_projection VALUES -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE Total\s+rows:\s+\d+ Total rows: REDACTED -- SQLNESS REPLACE Dictionary\(UInt32,\s*Utf8\(([^)]*)\)\) Utf8($1) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT ts, station, diff --git a/tests/cases/distributed/optimizer/windowed_sort.result b/tests/cases/distributed/optimizer/windowed_sort.result index 0c4d1931c05..6c2944f866c 100644 --- a/tests/cases/distributed/optimizer/windowed_sort.result +++ b/tests/cases/distributed/optimizer/windowed_sort.result @@ -56,6 +56,7 @@ ORDER BY -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time_utc AS collect_time, peak_current, @@ -73,7 +74,7 @@ ORDER BY | 1_| 0_|_ProjectionExec: expr=[collect_time_utc@0 as collect_time, peak_current@1 as peak_current] REDACTED |_|_|_SortPreservingMergeExec: [collect_time_utc@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[collect_time_utc@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["collect_time_utc", "peak_current"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 8_| +-+-+-+ @@ -106,6 +107,7 @@ ORDER BY -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time_utc AS collect_time_0, peak_current, @@ -123,7 +125,7 @@ ORDER BY | 1_| 0_|_ProjectionExec: expr=[collect_time_utc@0 as collect_time_0, peak_current@1 as peak_current] REDACTED |_|_|_SortPreservingMergeExec: [collect_time_utc@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[collect_time_utc@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["collect_time_utc", "peak_current"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 8_| +-+-+-+ @@ -157,6 +159,7 @@ ORDER BY -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time AS true_collect_time, collect_time_utc AS collect_time, @@ -176,7 +179,7 @@ ORDER BY |_|_|_SortPreservingMergeExec: [collect_time@0 DESC] REDACTED |_|_|_WindowedSortExec: expr=collect_time@0 DESC num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=collect_time@0 DESC num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["collect_time", "collect_time_utc", "peak_current"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 8_| +-+-+-+ @@ -210,6 +213,7 @@ ORDER BY -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time_utc AS collect_time, collect_time AS true_collect_time, @@ -230,7 +234,7 @@ ORDER BY |_|_|_ProjectionExec: expr=[collect_time_utc@1 as collect_time_utc, collect_time@0 as collect_time, peak_current@2 as peak_current] REDACTED |_|_|_WindowedSortExec: expr=collect_time@0 DESC num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=collect_time@0 DESC num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["collect_time", "collect_time_utc", "peak_current"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 8_| +-+-+-+ diff --git a/tests/cases/distributed/optimizer/windowed_sort.sql b/tests/cases/distributed/optimizer/windowed_sort.sql index eefdacd51a1..ffdfde853e2 100644 --- a/tests/cases/distributed/optimizer/windowed_sort.sql +++ b/tests/cases/distributed/optimizer/windowed_sort.sql @@ -39,6 +39,7 @@ ORDER BY -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time_utc AS collect_time, peak_current, @@ -62,6 +63,7 @@ ORDER BY -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time_utc AS collect_time_0, peak_current, @@ -86,6 +88,7 @@ ORDER BY -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time AS true_collect_time, collect_time_utc AS collect_time, @@ -111,6 +114,7 @@ ORDER BY -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time_utc AS collect_time, collect_time AS true_collect_time, diff --git a/tests/cases/distributed/optimizer/windowed_sort_advance.result b/tests/cases/distributed/optimizer/windowed_sort_advance.result index 4476cdd5e80..618a1547a14 100644 --- a/tests/cases/distributed/optimizer/windowed_sort_advance.result +++ b/tests/cases/distributed/optimizer/windowed_sort_advance.result @@ -56,6 +56,7 @@ select ts as ts, status, value from `a` where ts >= '2026-03-12T08:00:00+08:00' -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE select ts as ts, status, value from `a` where ts >= '2026-03-12T08:00:00+08:00' and ts < '2026-03-12T08:02:01+08:00' order by ts asc; +-+-+-+ @@ -67,7 +68,7 @@ EXPLAIN ANALYZE select ts as ts, status, value from `a` where ts >= '2026-03-12T | 1_| 0_|_SortPreservingMergeExec: [ts@0 ASC NULLS LAST] REDACTED |_|_|_ProjectionExec: expr=[ts@2 as ts, status@1 as status, value@0 as value] REDACTED |_|_|_WindowedSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["value", "status", "ts"], "filters": ["ts >= TimestampNanosecond(1773273600000000000, None)", "ts < TimestampNanosecond(1773273721000000000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 10_| +-+-+-+ @@ -95,6 +96,7 @@ select to_timestamp_millis(ts) as ts, status, value from `a` where ts >= '2026-0 -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE select to_timestamp_millis(ts) as ts, status, value from `a` where ts >= '2026-03-12T08:00:00+08:00' and ts < '2026-03-12T08:02:01+08:00' order by ts asc; +-+-+-+ @@ -107,7 +109,7 @@ EXPLAIN ANALYZE select to_timestamp_millis(ts) as ts, status, value from `a` whe |_|_|_SortPreservingMergeExec: [to_timestamp_millis(a.ts)@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[to_timestamp_millis(a.ts)@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_ProjectionExec: expr=[to_timestamp_millis(ts@2) as to_timestamp_millis(a.ts), status@1 as status, value@0 as value] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["value", "status", "ts"], "filters": ["ts >= TimestampNanosecond(1773273600000000000, None)", "ts < TimestampNanosecond(1773273721000000000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 10_| +-+-+-+ diff --git a/tests/cases/distributed/optimizer/windowed_sort_advance.sql b/tests/cases/distributed/optimizer/windowed_sort_advance.sql index fa73b97933c..167a42d591c 100644 --- a/tests/cases/distributed/optimizer/windowed_sort_advance.sql +++ b/tests/cases/distributed/optimizer/windowed_sort_advance.sql @@ -22,6 +22,7 @@ select ts as ts, status, value from `a` where ts >= '2026-03-12T08:00:00+08:00' -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE select ts as ts, status, value from `a` where ts >= '2026-03-12T08:00:00+08:00' and ts < '2026-03-12T08:02:01+08:00' order by ts asc; select to_timestamp_millis(ts) as ts, status, value from `a` where ts >= '2026-03-12T08:00:00+08:00' and ts < '2026-03-12T08:02:01+08:00' order by ts asc; @@ -32,6 +33,7 @@ select to_timestamp_millis(ts) as ts, status, value from `a` where ts >= '2026-0 -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE select to_timestamp_millis(ts) as ts, status, value from `a` where ts >= '2026-03-12T08:00:00+08:00' and ts < '2026-03-12T08:02:01+08:00' order by ts asc; DROP TABLE `a`; diff --git a/tests/cases/distributed/optimizer/windowed_sort_nanos.result b/tests/cases/distributed/optimizer/windowed_sort_nanos.result index 16854798132..122c6e64a29 100644 --- a/tests/cases/distributed/optimizer/windowed_sort_nanos.result +++ b/tests/cases/distributed/optimizer/windowed_sort_nanos.result @@ -56,6 +56,7 @@ select to_timestamp_nanos(ts) as ts, status, value from `a_ms` where ts >= '2026 -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE select to_timestamp_nanos(ts) as ts, status, value from `a_ms` where ts >= '2026-03-12T08:00:00.000+08:00' and ts < '2026-03-12T08:02:01.000+08:00' order by ts asc; +-+-+-+ @@ -68,7 +69,7 @@ EXPLAIN ANALYZE select to_timestamp_nanos(ts) as ts, status, value from `a_ms` w |_|_|_SortPreservingMergeExec: [to_timestamp_nanos(a_ms.ts)@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[to_timestamp_nanos(a_ms.ts)@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_ProjectionExec: expr=[to_timestamp_nanos(ts@2) as to_timestamp_nanos(a_ms.ts), status@1 as status, value@0 as value] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["value", "status", "ts"], "filters": ["ts >= TimestampMillisecond(1773273600000, None)", "ts < TimestampMillisecond(1773273721000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 10_| +-+-+-+ diff --git a/tests/cases/distributed/optimizer/windowed_sort_nanos.sql b/tests/cases/distributed/optimizer/windowed_sort_nanos.sql index 6a6cf51c4b6..27abd0bc805 100644 --- a/tests/cases/distributed/optimizer/windowed_sort_nanos.sql +++ b/tests/cases/distributed/optimizer/windowed_sort_nanos.sql @@ -22,6 +22,7 @@ select to_timestamp_nanos(ts) as ts, status, value from `a_ms` where ts >= '2026 -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE select to_timestamp_nanos(ts) as ts, status, value from `a_ms` where ts >= '2026-03-12T08:00:00.000+08:00' and ts < '2026-03-12T08:02:01.000+08:00' order by ts asc; DROP TABLE `a_ms`; diff --git a/tests/cases/distributed/tql-explain-analyze/analyze.result b/tests/cases/distributed/tql-explain-analyze/analyze.result index 059cf55df64..695781e2c0f 100644 --- a/tests/cases/distributed/tql-explain-analyze/analyze.result +++ b/tests/cases/distributed/tql-explain-analyze/analyze.result @@ -15,6 +15,7 @@ Affected Rows: 3 -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') test; +-+-+-+ @@ -26,7 +27,7 @@ TQL ANALYZE (0, 10, '5s') test; | 1_| 0_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] REDACTED |_|_|_PromSeriesDivideExec: tags=["k"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k"], "filters": ["j >= TimestampMillisecond(-299999, None)", "j <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -40,6 +41,7 @@ TQL ANALYZE (0, 10, '5s') test; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '1s', '2s') test; +-+-+-+ @@ -51,7 +53,7 @@ TQL ANALYZE (0, 10, '1s', '2s') test; | 1_| 0_|_PromInstantManipulateExec: range=[0..10000], lookback=[2000], interval=[1000], time index=[j] REDACTED |_|_|_PromSeriesDivideExec: tags=["k"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k"], "filters": ["j >= TimestampMillisecond(-1999, None)", "j <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -64,6 +66,7 @@ TQL ANALYZE (0, 10, '1s', '2s') test; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; +-+-+-+ @@ -75,7 +78,7 @@ TQL ANALYZE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp | 1_| 0_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] REDACTED |_|_|_PromSeriesDivideExec: tags=["k"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k"], "filters": ["j >= TimestampMillisecond(-299999, None)", "j <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -91,6 +94,7 @@ TQL ANALYZE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (0, 10, '5s') test; +-+-+-+ @@ -102,7 +106,7 @@ TQL ANALYZE VERBOSE (0, 10, '5s') test; | 1_| 0_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] REDACTED |_|_|_PromSeriesDivideExec: tags=["k"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k"], "filters": ["j >= TimestampMillisecond(-299999, None)", "j <= TimestampMillisecond(10000, None)"], "REDACTED +|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k"], "filters": ["j >= TimestampMillisecond(-299999, None)", "j <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -123,6 +127,7 @@ Affected Rows: 0 -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') test; +-+-+-+ @@ -134,12 +139,12 @@ TQL ANALYZE (0, 10, '5s') test; | 1_| 0_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] REDACTED |_|_|_PromSeriesDivideExec: tags=["k", "l"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k", "l"], "filters": ["j >= TimestampMillisecond(-299999, None)", "j <= TimestampMillisecond(10000, None)", "k < Utf8(\"a\") OR k IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] REDACTED |_|_|_PromSeriesDivideExec: tags=["k", "l"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k", "l"], "filters": ["j >= TimestampMillisecond(-299999, None)", "j <= TimestampMillisecond(10000, None)", "k >= Utf8(\"a\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -151,6 +156,7 @@ TQL ANALYZE (0, 10, '5s') test; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') rate(test[10s]); +-+-+-+ @@ -166,7 +172,7 @@ TQL ANALYZE (0, 10, '5s') rate(test[10s]); |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["k", "l"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k", "l"], "filters": ["j >= TimestampMillisecond(-9999, None)", "j <= TimestampMillisecond(10000, None)", "k < Utf8(\"a\") OR k IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_ProjectionExec: expr=[j@0 as j, prom_rate(j_range,i,test.j,Int64(10000))@1 as prom_rate(j_range,i,j,Int64(10000)), k@2 as k, l@3 as l] REDACTED |_|_|_FilterExec: prom_rate(j_range,i,test.j,Int64(10000))@1 IS NOT NULL REDACTED @@ -175,7 +181,7 @@ TQL ANALYZE (0, 10, '5s') rate(test[10s]); |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["k", "l"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k", "l"], "filters": ["j >= TimestampMillisecond(-9999, None)", "j <= TimestampMillisecond(10000, None)", "k >= Utf8(\"a\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ diff --git a/tests/cases/distributed/tql-explain-analyze/analyze.sql b/tests/cases/distributed/tql-explain-analyze/analyze.sql index e8bdf92838a..7dd4ce70f1d 100644 --- a/tests/cases/distributed/tql-explain-analyze/analyze.sql +++ b/tests/cases/distributed/tql-explain-analyze/analyze.sql @@ -11,6 +11,7 @@ INSERT INTO test VALUES (1, 1, "a"), (1, 1, "b"), (2, 2, "a"); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') test; -- 'lookback' parameter is not fully supported, the test has to be updated @@ -22,6 +23,7 @@ TQL ANALYZE (0, 10, '5s') test; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '1s', '2s') test; -- analyze at 0s, 5s and 10s. No point at 0s. @@ -32,6 +34,7 @@ TQL ANALYZE (0, 10, '1s', '2s') test; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; -- analyze verbose at 0s, 5s and 10s. No point at 0s. @@ -45,6 +48,7 @@ TQL ANALYZE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (0, 10, '5s') test; DROP TABLE test; @@ -59,6 +63,7 @@ CREATE TABLE test(i DOUBLE, j TIMESTAMP TIME INDEX, k STRING, l STRING, PRIMARY -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') test; -- SQLNESS REPLACE (metrics.*) REDACTED @@ -68,6 +73,7 @@ TQL ANALYZE (0, 10, '5s') test; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') rate(test[10s]); drop table test; diff --git a/tests/cases/standalone/common/aggregate/distinct.result b/tests/cases/standalone/common/aggregate/distinct.result index 5c10ee22239..da377f8562d 100644 --- a/tests/cases/standalone/common/aggregate/distinct.result +++ b/tests/cases/standalone/common/aggregate/distinct.result @@ -85,6 +85,7 @@ SELECT DISTINCT ON (a) * FROM test ORDER BY a, t DESC; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT DISTINCT a FROM test ORDER BY a; +-+-+-+ @@ -98,7 +99,7 @@ EXPLAIN ANALYZE SELECT DISTINCT a FROM test ORDER BY a; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["a"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -110,6 +111,7 @@ EXPLAIN ANALYZE SELECT DISTINCT a FROM test ORDER BY a; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT DISTINCT a, b FROM test ORDER BY a; +-+-+-+ @@ -123,7 +125,7 @@ EXPLAIN ANALYZE SELECT DISTINCT a, b FROM test ORDER BY a; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["a", "b"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 3_| +-+-+-+ @@ -162,6 +164,7 @@ SELECT DISTINCT a, b FROM test ORDER BY a, b; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT DISTINCT a FROM test ORDER BY a; +-+-+-+ @@ -178,12 +181,12 @@ EXPLAIN ANALYZE SELECT DISTINCT a FROM test ORDER BY a; | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["a"], "filters": ["a <= Int32(10) OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["a"], "filters": ["a > Int32(10)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -195,6 +198,7 @@ EXPLAIN ANALYZE SELECT DISTINCT a FROM test ORDER BY a; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT DISTINCT a, b FROM test ORDER BY a; +-+-+-+ @@ -211,12 +215,12 @@ EXPLAIN ANALYZE SELECT DISTINCT a, b FROM test ORDER BY a; | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["a", "b"], "filters": ["a <= Int32(10) OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["a", "b"], "filters": ["a > Int32(10)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ diff --git a/tests/cases/standalone/common/aggregate/distinct.sql b/tests/cases/standalone/common/aggregate/distinct.sql index cdf9de5e6d1..adbabd694ab 100644 --- a/tests/cases/standalone/common/aggregate/distinct.sql +++ b/tests/cases/standalone/common/aggregate/distinct.sql @@ -25,6 +25,7 @@ SELECT DISTINCT ON (a) * FROM test ORDER BY a, t DESC; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT DISTINCT a FROM test ORDER BY a; -- SQLNESS REPLACE (metrics.*) REDACTED @@ -34,6 +35,7 @@ EXPLAIN ANALYZE SELECT DISTINCT a FROM test ORDER BY a; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT DISTINCT a, b FROM test ORDER BY a; DROP TABLE test; @@ -55,6 +57,7 @@ SELECT DISTINCT a, b FROM test ORDER BY a, b; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT DISTINCT a FROM test ORDER BY a; -- SQLNESS REPLACE (metrics.*) REDACTED @@ -64,6 +67,7 @@ EXPLAIN ANALYZE SELECT DISTINCT a FROM test ORDER BY a; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT DISTINCT a, b FROM test ORDER BY a; DROP TABLE test; diff --git a/tests/cases/standalone/common/aggregate/last_value_ignore_nulls.result b/tests/cases/standalone/common/aggregate/last_value_ignore_nulls.result new file mode 100644 index 00000000000..75f70c6a58f --- /dev/null +++ b/tests/cases/standalone/common/aggregate/last_value_ignore_nulls.result @@ -0,0 +1,142 @@ +create table last_value_ignore_nulls ( + ts timestamp time index, + host string primary key, + v1 double, + v2 double, +); + +Affected Rows: 0 + +insert into last_value_ignore_nulls values + (1000, 'a', 42.0, null), + (2000, 'a', null, 20.0), + (1000, 'b', null, null), + (2000, 'b', null, null); + +Affected Rows: 4 + +select + host, + last_value(v1 order by ts) ignore nulls as v1_ignore, + last_value(v2 order by ts) ignore nulls as v2_ignore, + last_value(v1 order by ts) as v1_default, + last_value(v1 order by ts) respect nulls as v1_respect +from last_value_ignore_nulls +group by host +order by host; + ++------+-----------+-----------+------------+------------+ +| host | v1_ignore | v2_ignore | v1_default | v1_respect | ++------+-----------+-----------+------------+------------+ +| a | 42.0 | 20.0 | | | +| b | | | | | ++------+-----------+-----------+------------+------------+ + +-- Default and RESPECT NULLS return the same results in both modes. +select + host, + last_value(v1 order by ts) as v1_default, + last_value(v1 order by ts) respect nulls as v1_respect +from last_value_ignore_nulls +group by host +order by host; + ++------+------------+------------+ +| host | v1_default | v1_respect | ++------+------------+------------+ +| a | | | +| b | | | ++------+------------+------------+ + +-- The IGNORE NULLS member rejects the shared selector, including when it is second. +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (partitioning.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED +explain analyze select + last_value(v1 order by ts), + last_value(v2 order by ts) ignore nulls +from last_value_ignore_nulls +group by host; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_ProjectionExec: expr=[last_value(last_value_ignore_nulls.v1) ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST]@1 as last_value(last_value_ignore_nulls.v1) ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST], last_value(last_value_ignore_nulls.v2) IGNORE NULLS ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST]@2 as last_value(last_value_ignore_nulls.v2) IGNORE NULLS ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(last_value_ignore_nulls.v1) ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST], last_value(last_value_ignore_nulls.v2) IGNORE NULLS ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST]] REDACTED +|_|_|_RepartitionExec: REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(last_value_ignore_nulls.v1) ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST], last_value(last_value_ignore_nulls.v2) IGNORE NULLS ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST]] REDACTED +|_|_|_RepartitionExec: REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_CooperativeExec REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "v1", "v2"], "time_range":REDACTED +|_|_|_| +|_|_| Total rows: 2_| ++-+-+-+ + +admin flush_table('last_value_ignore_nulls'); + ++----------------------------------------------+ +| ADMIN flush_table('last_value_ignore_nulls') | ++----------------------------------------------+ +| 0 | ++----------------------------------------------+ + +select + host, + last_value(v1 order by ts) ignore nulls as v1_ignore, + last_value(v2 order by ts) ignore nulls as v2_ignore, + last_value(v1 order by ts) as v1_default, + last_value(v1 order by ts) respect nulls as v1_respect +from last_value_ignore_nulls +group by host +order by host; + ++------+-----------+-----------+------------+------------+ +| host | v1_ignore | v2_ignore | v1_default | v1_respect | ++------+-----------+-----------+------------+------------+ +| a | 42.0 | 20.0 | | | +| b | | | | | ++------+-----------+-----------+------------+------------+ + +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (partitioning.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED +explain analyze select + last_value(v1 order by ts), + last_value(v2 order by ts) ignore nulls +from last_value_ignore_nulls +group by host; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_ProjectionExec: expr=[last_value(last_value_ignore_nulls.v1) ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST]@1 as last_value(last_value_ignore_nulls.v1) ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST], last_value(last_value_ignore_nulls.v2) IGNORE NULLS ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST]@2 as last_value(last_value_ignore_nulls.v2) IGNORE NULLS ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(last_value_ignore_nulls.v1) ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST], last_value(last_value_ignore_nulls.v2) IGNORE NULLS ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST]] REDACTED +|_|_|_RepartitionExec: REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(last_value_ignore_nulls.v1) ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST], last_value(last_value_ignore_nulls.v2) IGNORE NULLS ORDER BY [last_value_ignore_nulls.ts ASC NULLS LAST]] REDACTED +|_|_|_RepartitionExec: REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_CooperativeExec REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "v1", "v2"], "time_range":REDACTED +|_|_|_| +|_|_| Total rows: 2_| ++-+-+-+ + +drop table last_value_ignore_nulls; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/aggregate/last_value_ignore_nulls.sql b/tests/cases/standalone/common/aggregate/last_value_ignore_nulls.sql new file mode 100644 index 00000000000..659dd4fec39 --- /dev/null +++ b/tests/cases/standalone/common/aggregate/last_value_ignore_nulls.sql @@ -0,0 +1,76 @@ +create table last_value_ignore_nulls ( + ts timestamp time index, + host string primary key, + v1 double, + v2 double, +); + +insert into last_value_ignore_nulls values + (1000, 'a', 42.0, null), + (2000, 'a', null, 20.0), + (1000, 'b', null, null), + (2000, 'b', null, null); + +select + host, + last_value(v1 order by ts) ignore nulls as v1_ignore, + last_value(v2 order by ts) ignore nulls as v2_ignore, + last_value(v1 order by ts) as v1_default, + last_value(v1 order by ts) respect nulls as v1_respect +from last_value_ignore_nulls +group by host +order by host; + +-- Default and RESPECT NULLS return the same results in both modes. +select + host, + last_value(v1 order by ts) as v1_default, + last_value(v1 order by ts) respect nulls as v1_respect +from last_value_ignore_nulls +group by host +order by host; + +-- The IGNORE NULLS member rejects the shared selector, including when it is second. +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (partitioning.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED +explain analyze select + last_value(v1 order by ts), + last_value(v2 order by ts) ignore nulls +from last_value_ignore_nulls +group by host; + +admin flush_table('last_value_ignore_nulls'); + +select + host, + last_value(v1 order by ts) ignore nulls as v1_ignore, + last_value(v2 order by ts) ignore nulls as v2_ignore, + last_value(v1 order by ts) as v1_default, + last_value(v1 order by ts) respect nulls as v1_respect +from last_value_ignore_nulls +group by host +order by host; + +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (partitioning.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED +explain analyze select + last_value(v1 order by ts), + last_value(v2 order by ts) ignore nulls +from last_value_ignore_nulls +group by host; + +drop table last_value_ignore_nulls; diff --git a/tests/cases/standalone/common/aggregate/max_abs_regression.result b/tests/cases/standalone/common/aggregate/max_abs_regression.result index a0e0b394667..70f3dc06f60 100644 --- a/tests/cases/standalone/common/aggregate/max_abs_regression.result +++ b/tests/cases/standalone/common/aggregate/max_abs_regression.result @@ -67,6 +67,7 @@ SELECT MAX(v), MAX(ts) FROM t; -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] -- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT MAX(ABS(v)), MAX(ts) FROM t; +-+-+-+ @@ -78,7 +79,7 @@ EXPLAIN ANALYZE VERBOSE SELECT MAX(ABS(v)), MAX(ts) FROM t; | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[max(abs(t.v)), max(t.ts)] metrics=REDACTED_| |_|_|_CoalescePartitionsExec metrics=REDACTED_| |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[max(abs(t.v)), max(t.ts)] metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["ts", "v"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["ts", "v"], "time_range":REDACTED |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ @@ -101,6 +102,7 @@ EXPLAIN ANALYZE VERBOSE SELECT MAX(ABS(v)), MAX(ts) FROM t; -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] -- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT MAX(v), MAX(ts) FROM t; +-+-+-+ @@ -112,7 +114,7 @@ EXPLAIN ANALYZE VERBOSE SELECT MAX(v), MAX(ts) FROM t; | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[max(t.v), max(t.ts)] metrics=REDACTED_| |_|_|_CoalescePartitionsExec metrics=REDACTED_| |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[max(t.v), max(t.ts)] metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["ts", "v"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["ts", "v"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ diff --git a/tests/cases/standalone/common/aggregate/max_abs_regression.sql b/tests/cases/standalone/common/aggregate/max_abs_regression.sql index c9705f6086e..76a862eaa34 100644 --- a/tests/cases/standalone/common/aggregate/max_abs_regression.sql +++ b/tests/cases/standalone/common/aggregate/max_abs_regression.sql @@ -40,6 +40,7 @@ SELECT MAX(v), MAX(ts) FROM t; -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] -- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT MAX(ABS(v)), MAX(ts) FROM t; -- A column-only multiple-MAX query remains a dynamic-filter positive control. @@ -60,6 +61,7 @@ EXPLAIN ANALYZE VERBOSE SELECT MAX(ABS(v)), MAX(ts) FROM t; -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] -- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT MAX(v), MAX(ts) FROM t; DROP TABLE t; diff --git a/tests/cases/standalone/common/aggregate/multi_regions.result b/tests/cases/standalone/common/aggregate/multi_regions.result index bdfc5ed2dd1..7a212c5be91 100644 --- a/tests/cases/standalone/common/aggregate/multi_regions.result +++ b/tests/cases/standalone/common/aggregate/multi_regions.result @@ -21,6 +21,7 @@ Affected Rows: 0 -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select sum(val) from t group by host; @@ -34,13 +35,13 @@ select sum(val) from t group by host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[sum(t.val)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[sum(t.val)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["val", "host"], "filters": ["host < Utf8(\"1024\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_ProjectionExec: expr=[sum(t.val)@1 as sum(t.val)] REDACTED |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[sum(t.val)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[sum(t.val)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["val", "host"], "filters": ["host >= Utf8(\"1024\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -53,6 +54,7 @@ select sum(val) from t group by host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select sum(val) from t; @@ -68,12 +70,12 @@ select sum(val) from t; | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__sum_state(t.val)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__sum_state(t.val)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["val"], "filters": ["host < Utf8(\"1024\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__sum_state(t.val)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__sum_state(t.val)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["val"], "filters": ["host >= Utf8(\"1024\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -87,6 +89,7 @@ select sum(val) from t; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select sum(val) from t group by idc; @@ -103,12 +106,12 @@ select sum(val) from t group by idc; | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[idc@0 as idc], aggr=[__sum_state(t.val)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[idc@1 as idc], aggr=[__sum_state(t.val)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["val", "idc"], "filters": ["host < Utf8(\"1024\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[idc@0 as idc], aggr=[__sum_state(t.val)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_AggregateExec: mode=Partial, gby=[idc@1 as idc], aggr=[__sum_state(t.val)] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["val", "idc"], "filters": ["host >= Utf8(\"1024\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ diff --git a/tests/cases/standalone/common/aggregate/multi_regions.sql b/tests/cases/standalone/common/aggregate/multi_regions.sql index 9c8d8af4490..db6b9c02392 100644 --- a/tests/cases/standalone/common/aggregate/multi_regions.sql +++ b/tests/cases/standalone/common/aggregate/multi_regions.sql @@ -19,6 +19,7 @@ partition on columns (host) ( -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select sum(val) from t group by host; @@ -30,6 +31,7 @@ select sum(val) from t group by host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select sum(val) from t; @@ -42,6 +44,7 @@ select sum(val) from t; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select sum(val) from t group by idc; diff --git a/tests/cases/standalone/common/alter/change_col_type.result b/tests/cases/standalone/common/alter/change_col_type.result index 27926960ec3..92a986cd73b 100644 --- a/tests/cases/standalone/common/alter/change_col_type.result +++ b/tests/cases/standalone/common/alter/change_col_type.result @@ -421,3 +421,70 @@ DROP TABLE ts_overflow; Affected Rows: 0 +-- Issue 9454: persisted partition bounds keep their original unit. Widening the +-- time index unit must be rejected before any region is altered when a persisted +-- bound overflows the new unit. +CREATE TABLE ts_bound_overflow (host STRING, ts TIMESTAMP TIME INDEX) +PARTITION ON COLUMNS (ts) ( + ts < '2024-01-01 00:00:00', + ts >= '2024-01-01 00:00:00' AND ts < '3000-01-01 00:00:00', + ts >= '3000-01-01 00:00:00' +); + +Affected Rows: 0 + +INSERT INTO ts_bound_overflow VALUES ("a", "2024-06-01 12:00:00"); + +Affected Rows: 1 + +-- 3000-01-01 fits milliseconds but overflows nanoseconds, so the widening is +-- rejected before any region is altered. +-- SQLNESS REPLACE \d+\(\d+,\s+\d+\) REDACTED +ALTER TABLE ts_bound_overflow MODIFY COLUMN ts TIMESTAMP_NS; + +Error: 1004(InvalidArguments), Partition expression of region REDACTED is incompatible with the altered column 'ts': the persisted bound can no longer be compared with the new column type TimestampNanosecond + +-- The rejected ALTER leaves the schema and the stored data unchanged. +DESCRIBE ts_bound_overflow; + ++--------+----------------------+-----+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++--------+----------------------+-----+------+---------+---------------+ +| host | String | | YES | | FIELD | +| ts | TimestampMillisecond | PRI | NO | | TIMESTAMP | ++--------+----------------------+-----+------+---------+---------------+ + +SELECT * FROM ts_bound_overflow ORDER BY ts; + ++------+---------------------+ +| host | ts | ++------+---------------------+ +| a | 2024-06-01T12:00:00 | ++------+---------------------+ + +-- Widening to microseconds keeps every persisted bound representable. +ALTER TABLE ts_bound_overflow MODIFY COLUMN ts TIMESTAMP_US; + +Affected Rows: 0 + +DESCRIBE ts_bound_overflow; + ++--------+----------------------+-----+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++--------+----------------------+-----+------+---------+---------------+ +| host | String | | YES | | FIELD | +| ts | TimestampMicrosecond | PRI | NO | | TIMESTAMP | ++--------+----------------------+-----+------+---------+---------------+ + +SELECT * FROM ts_bound_overflow ORDER BY ts; + ++------+---------------------+ +| host | ts | ++------+---------------------+ +| a | 2024-06-01T12:00:00 | ++------+---------------------+ + +DROP TABLE ts_bound_overflow; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/alter/change_col_type.sql b/tests/cases/standalone/common/alter/change_col_type.sql index 27281cf5f10..2da3094a6f1 100644 --- a/tests/cases/standalone/common/alter/change_col_type.sql +++ b/tests/cases/standalone/common/alter/change_col_type.sql @@ -146,3 +146,34 @@ ALTER TABLE ts_overflow MODIFY COLUMN ts TIMESTAMP_NS; DESCRIBE ts_overflow; DROP TABLE ts_overflow; + +-- Issue 9454: persisted partition bounds keep their original unit. Widening the +-- time index unit must be rejected before any region is altered when a persisted +-- bound overflows the new unit. +CREATE TABLE ts_bound_overflow (host STRING, ts TIMESTAMP TIME INDEX) +PARTITION ON COLUMNS (ts) ( + ts < '2024-01-01 00:00:00', + ts >= '2024-01-01 00:00:00' AND ts < '3000-01-01 00:00:00', + ts >= '3000-01-01 00:00:00' +); + +INSERT INTO ts_bound_overflow VALUES ("a", "2024-06-01 12:00:00"); + +-- 3000-01-01 fits milliseconds but overflows nanoseconds, so the widening is +-- rejected before any region is altered. +-- SQLNESS REPLACE \d+\(\d+,\s+\d+\) REDACTED +ALTER TABLE ts_bound_overflow MODIFY COLUMN ts TIMESTAMP_NS; + +-- The rejected ALTER leaves the schema and the stored data unchanged. +DESCRIBE ts_bound_overflow; + +SELECT * FROM ts_bound_overflow ORDER BY ts; + +-- Widening to microseconds keeps every persisted bound representable. +ALTER TABLE ts_bound_overflow MODIFY COLUMN ts TIMESTAMP_US; + +DESCRIBE ts_bound_overflow; + +SELECT * FROM ts_bound_overflow ORDER BY ts; + +DROP TABLE ts_bound_overflow; diff --git a/tests/cases/standalone/common/filter/hash_join_dyn_filter.result b/tests/cases/standalone/common/filter/hash_join_dyn_filter.result index b1402bd4d4a..e534f55791c 100644 --- a/tests/cases/standalone/common/filter/hash_join_dyn_filter.result +++ b/tests/cases/standalone/common/filter/hash_join_dyn_filter.result @@ -78,6 +78,7 @@ WHERE c.tier = 'gold'; -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] -- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT o."id", o.amount, c."name", c.tier FROM orders o JOIN customers c ON o.customer_id = c.customer_id @@ -96,11 +97,11 @@ WHERE c.tier = 'gold'; |_|_|_| | 1_| 0_|_FilterExec: customer_id@1 IS NOT NULL metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "filters": ["customer_id IS NOT NULL"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "filters": ["customer_id IS NOT NULL"], "time_range":REDACTED |_|_|_| | 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND tier@2 = gold metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\")"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\")"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ @@ -206,6 +207,7 @@ WHERE c.tier IN ('gold', 'silver'); -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] -- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT o."id", o.amount, c."name", c.tier FROM (SELECT "id", customer_id as cid, amount, ts FROM orders) o JOIN customers c ON o.cid = c.customer_id @@ -225,11 +227,11 @@ WHERE c.tier IN ('gold', 'silver'); | 1_| 0_|_ProjectionExec: expr=[id@0 as id, customer_id@1 as cid, amount@2 as amount, ts@3 as ts] metrics=REDACTED_| |_|_|_FilterExec: customer_id@1 IS NOT NULL metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "filters": ["customer_id IS NOT NULL"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "filters": ["customer_id IS NOT NULL"], "time_range":REDACTED |_|_|_| | 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND (tier@2 = gold OR tier@2 = silver) metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ diff --git a/tests/cases/standalone/common/filter/hash_join_dyn_filter.sql b/tests/cases/standalone/common/filter/hash_join_dyn_filter.sql index d6079bc496a..405d1bc0530 100644 --- a/tests/cases/standalone/common/filter/hash_join_dyn_filter.sql +++ b/tests/cases/standalone/common/filter/hash_join_dyn_filter.sql @@ -43,6 +43,7 @@ WHERE c.tier = 'gold'; -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] -- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT o."id", o.amount, c."name", c.tier FROM orders o JOIN customers c ON o.customer_id = c.customer_id @@ -100,6 +101,7 @@ WHERE c.tier IN ('gold', 'silver'); -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] -- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT o."id", o.amount, c."name", c.tier FROM (SELECT "id", customer_id as cid, amount, ts FROM orders) o JOIN customers c ON o.cid = c.customer_id diff --git a/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.result b/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.result index 0ba0f3c4e00..1c2d05bbcbb 100644 --- a/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.result +++ b/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.result @@ -94,6 +94,7 @@ WHERE c.tier IN ('gold', 'bronze'); -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] -- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT top_orders."id", top_orders.amount, c."name", c.tier FROM ( SELECT "id", customer_id, amount, ts @@ -119,11 +120,11 @@ WHERE c.tier IN ('gold', 'bronze'); |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_SortPreservingMergeExec: [amount@2 DESC], fetch=5 metrics=REDACTED_| |_|_|_SortExec: TopK(fetch=5), expr=[amount@2 DESC], preserve_partitioning=[true] metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "dyn_filters":REDACTED |_|_|_| | 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND (tier@2 = gold OR tier@2 = bronze) metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\") OR tier = Utf8(\"bronze\")"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\") OR tier = Utf8(\"bronze\")"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ @@ -251,6 +252,7 @@ LIMIT 4; -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] -- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT "id", customer_id, "name", tier, amount FROM ( SELECT o."id", o.customer_id, c."name", c.tier, o.amount @@ -276,11 +278,11 @@ LIMIT 4; |_|_|_| | 1_| 0_|_FilterExec: customer_id@1 IS NOT NULL metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "filters": ["customer_id IS NOT NULL"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount", "ts"], "filters": ["customer_id IS NOT NULL"], "dyn_filters":REDACTED |_|_|_| | 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND (tier@2 = gold OR tier@2 = silver) metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ @@ -426,6 +428,7 @@ WHERE c.tier IN ('gold', 'silver') -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] -- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT o."id", o.amount, c."name", c.tier, p."name" as product_name, p."category" FROM orders o JOIN customers c ON o.customer_id = c.customer_id @@ -452,15 +455,15 @@ WHERE c.tier IN ('gold', 'silver') |_|_|_| | 1_| 0_|_FilterExec: customer_id@1 IS NOT NULL AND product_id@2 IS NOT NULL metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "product_id", "amount", "ts"], "filters": ["customer_id IS NOT NULL", "product_id IS NOT NULL"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "product_id", "amount", "ts"], "filters": ["customer_id IS NOT NULL", "product_id IS NOT NULL"], "time_range":REDACTED |_|_|_| | 1_| 0_|_FilterExec: customer_id@0 IS NOT NULL AND (tier@2 = gold OR tier@2 = silver) metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["customer_id", "name", "tier", "ts"], "filters": ["customer_id IS NOT NULL", "tier = Utf8(\"gold\") OR tier = Utf8(\"silver\")"], "dyn_filters":REDACTED |_|_|_| | 1_| 0_|_FilterExec: product_id@0 IS NOT NULL AND category@2 = electronics metrics=REDACTED_| |_|_|_CooperativeExec metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["product_id", "name", "category", "ts"], "filters": ["product_id IS NOT NULL", "category = Utf8(\"electronics\")"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED_| +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["product_id", "name", "category", "ts"], "filters": ["product_id IS NOT NULL", "category = Utf8(\"electronics\")"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ diff --git a/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.sql b/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.sql index 97f0c1231d7..d04c3b21bca 100644 --- a/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.sql +++ b/tests/cases/standalone/common/filter/hash_join_topk_dyn_filter.sql @@ -56,6 +56,7 @@ WHERE c.tier IN ('gold', 'bronze'); -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] -- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT top_orders."id", top_orders.amount, c."name", c.tier FROM ( SELECT "id", customer_id, amount, ts @@ -136,6 +137,7 @@ LIMIT 4; -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] -- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT "id", customer_id, "name", tier, amount FROM ( SELECT o."id", o.customer_id, c."name", c.tier, o.amount @@ -218,6 +220,7 @@ WHERE c.tier IN ('gold', 'silver') -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ,\s"dyn_filters":\s\["DynamicFilter\s\[[^"]*\]"\] , "dyn_filters": ["DynamicFilter [ REDACTED ]"] -- SQLNESS REPLACE metrics=REDACTED\s*\| metrics=REDACTED_| +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT o."id", o.amount, c."name", c.tier, p."name" as product_name, p."category" FROM orders o JOIN customers c ON o.customer_id = c.customer_id diff --git a/tests/cases/standalone/common/filter/topk_dyn_filter.result b/tests/cases/standalone/common/filter/topk_dyn_filter.result index ac905994ea3..14639053b92 100644 --- a/tests/cases/standalone/common/filter/topk_dyn_filter.result +++ b/tests/cases/standalone/common/filter/topk_dyn_filter.result @@ -87,6 +87,7 @@ LIMIT 3; -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE DynamicFilter\s\[[^\]]*\] DynamicFilter [ REDACTED ] -- SQLNESS REPLACE "files":\s\[[^\]]*\] "files": REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT "id", customer_id, amount FROM orders ORDER BY amount DESC @@ -100,7 +101,7 @@ LIMIT 3; |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [amount@2 DESC], fetch=3 metrics=REDACTED_| |_|_|_SortExec: TopK(fetch=3), expr=[amount@2 DESC], preserve_partitioning=[true] metrics=REDACTED_| -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount"], "dyn_filters": ["DynamicFilter [ REDACTED ]"], "files": REDACTED, "flat_format":REDACTED, "metrics_per_partition": REDACTED metrics=REDACTED | +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["id", "customer_id", "amount"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: REDACTED_| +-+-+-+ diff --git a/tests/cases/standalone/common/filter/topk_dyn_filter.sql b/tests/cases/standalone/common/filter/topk_dyn_filter.sql index ffce7d258e8..d77d59da4d7 100644 --- a/tests/cases/standalone/common/filter/topk_dyn_filter.sql +++ b/tests/cases/standalone/common/filter/topk_dyn_filter.sql @@ -54,6 +54,7 @@ LIMIT 3; -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE DynamicFilter\s\[[^\]]*\] DynamicFilter [ REDACTED ] -- SQLNESS REPLACE "files":\s\[[^\]]*\] "files": REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT "id", customer_id, amount FROM orders ORDER BY amount DESC diff --git a/tests/cases/standalone/common/information_schema/ssts.result b/tests/cases/standalone/common/information_schema/ssts.result index d7c7814a995..d8149d27769 100644 --- a/tests/cases/standalone/common/information_schema/ssts.result +++ b/tests/cases/standalone/common/information_schema/ssts.result @@ -243,6 +243,84 @@ FROM ( | 1 | +-----------------------+ +-- Timestamp filters must run before ordering and limiting. Sort columns must +-- remain available even when they are absent from the output projection. +SELECT min_ts, max_ts +FROM information_schema.ssts_manifest +WHERE region_id > 0 AND min_ts >= '1970-01-01T00:00:00.002Z'::TIMESTAMP_NS +ORDER BY min_ts ASC NULLS LAST, max_ts ASC +LIMIT 2; + ++-------------------------+-------------------------+ +| min_ts | max_ts | ++-------------------------+-------------------------+ +| 1970-01-01T00:00:00.002 | 1970-01-01T00:00:00.002 | +| 1970-01-01T00:00:00.003 | 1970-01-01T00:00:00.003 | ++-------------------------+-------------------------+ + +SELECT min_ts, max_ts +FROM information_schema.ssts_manifest +WHERE region_id > 0 +ORDER BY max_ts DESC NULLS FIRST, min_ts DESC +LIMIT 2 OFFSET 1; + ++-------------------------+-------------------------+ +| min_ts | max_ts | ++-------------------------+-------------------------+ +| 1970-01-01T00:00:00.100 | 1970-01-01T00:00:00.200 | +| 1970-01-01T00:00:00.003 | 1970-01-01T00:00:00.003 | ++-------------------------+-------------------------+ + +SELECT visible +FROM information_schema.ssts_manifest +WHERE region_id > 0 AND min_ts >= '1970-01-01T00:00:00.002Z'::TIMESTAMP_NS +ORDER BY min_ts DESC +LIMIT 2; + ++---------+ +| visible | ++---------+ +| true | +| true | ++---------+ + +-- Table predicates share region metadata pruning, including when table_id +-- is not part of the output projection. +SELECT min_ts, max_ts FROM information_schema.ssts_manifest +WHERE table_id > 0 ORDER BY min_ts, max_ts LIMIT 2; + ++-------------------------+-------------------------+ +| min_ts | max_ts | ++-------------------------+-------------------------+ +| 1970-01-01T00:00:00.001 | 1970-01-01T00:00:00.001 | +| 1970-01-01T00:00:00.002 | 1970-01-01T00:00:00.002 | ++-------------------------+-------------------------+ + +SELECT min_ts FROM information_schema.ssts_manifest +WHERE table_id = 0 ORDER BY min_ts LIMIT 1; + ++--------+ +| min_ts | ++--------+ ++--------+ + +-- A false region predicate and a zero limit must produce empty results. +SELECT min_ts FROM information_schema.ssts_manifest +WHERE region_id = 0 ORDER BY min_ts LIMIT 1; + ++--------+ +| min_ts | ++--------+ ++--------+ + +SELECT min_ts FROM information_schema.ssts_manifest +ORDER BY min_ts LIMIT 0; + ++--------+ +| min_ts | ++--------+ ++--------+ + DROP TABLE sst_case; Affected Rows: 0 diff --git a/tests/cases/standalone/common/information_schema/ssts.sql b/tests/cases/standalone/common/information_schema/ssts.sql index 7b40b0056bb..e630ab9a0ba 100644 --- a/tests/cases/standalone/common/information_schema/ssts.sql +++ b/tests/cases/standalone/common/information_schema/ssts.sql @@ -80,4 +80,39 @@ FROM ( LIMIT 1 ); +-- Timestamp filters must run before ordering and limiting. Sort columns must +-- remain available even when they are absent from the output projection. +SELECT min_ts, max_ts +FROM information_schema.ssts_manifest +WHERE region_id > 0 AND min_ts >= '1970-01-01T00:00:00.002Z'::TIMESTAMP_NS +ORDER BY min_ts ASC NULLS LAST, max_ts ASC +LIMIT 2; + +SELECT min_ts, max_ts +FROM information_schema.ssts_manifest +WHERE region_id > 0 +ORDER BY max_ts DESC NULLS FIRST, min_ts DESC +LIMIT 2 OFFSET 1; + +SELECT visible +FROM information_schema.ssts_manifest +WHERE region_id > 0 AND min_ts >= '1970-01-01T00:00:00.002Z'::TIMESTAMP_NS +ORDER BY min_ts DESC +LIMIT 2; + +-- Table predicates share region metadata pruning, including when table_id +-- is not part of the output projection. +SELECT min_ts, max_ts FROM information_schema.ssts_manifest +WHERE table_id > 0 ORDER BY min_ts, max_ts LIMIT 2; + +SELECT min_ts FROM information_schema.ssts_manifest +WHERE table_id = 0 ORDER BY min_ts LIMIT 1; + +-- A false region predicate and a zero limit must produce empty results. +SELECT min_ts FROM information_schema.ssts_manifest +WHERE region_id = 0 ORDER BY min_ts LIMIT 1; + +SELECT min_ts FROM information_schema.ssts_manifest +ORDER BY min_ts LIMIT 0; + DROP TABLE sst_case; diff --git a/tests/cases/standalone/common/order/order_by.result b/tests/cases/standalone/common/order/order_by.result index da47fa663c6..12f0a60f679 100644 --- a/tests/cases/standalone/common/order/order_by.result +++ b/tests/cases/standalone/common/order/order_by.result @@ -291,6 +291,7 @@ select tag from t where num > 6 order by ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE ,\sfilter=\[[^]]+\] -- SQLNESS REPLACE DynamicFilter\s\[[^\]]*\] DynamicFilter [ REDACTED ] +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select tag from t where num > 6 order by ts desc limit 2; +-+-+-+ @@ -306,14 +307,14 @@ explain analyze select tag from t where num > 6 order by ts desc limit 2; |_|_|_SortPreservingMergeExec: [ts@1 DESC], fetch=2 REDACTED |_|_|_SortExec: TopK(fetch=2), expr=[ts@1 DESC], preserve_partitioning=[true] REDACTED |_|_|_FilterExec: num@2 > 6, projection=[tag@0, ts@1] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["tag", "ts", "num"], "filters": ["num > Int64(6)", "tag <= Utf8(\"z\") OR tag IS NULL"], "dyn_filters":REDACTED |_|_|_| | 1_| 1_|_FilterExec: DynamicFilter [ REDACTED ] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_SortPreservingMergeExec: [ts@1 DESC], fetch=2 REDACTED |_|_|_SortExec: TopK(fetch=2), expr=[ts@1 DESC], preserve_partitioning=[true] REDACTED |_|_|_FilterExec: num@2 > 6, projection=[tag@0, ts@1] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["tag", "ts", "num"], "filters": ["num > Int64(6)", "tag > Utf8(\"z\")"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ diff --git a/tests/cases/standalone/common/order/order_by.sql b/tests/cases/standalone/common/order/order_by.sql index 51ffbbc4d8e..1f17be41fee 100644 --- a/tests/cases/standalone/common/order/order_by.sql +++ b/tests/cases/standalone/common/order/order_by.sql @@ -97,6 +97,7 @@ select tag from t where num > 6 order by ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE ,\sfilter=\[[^]]+\] -- SQLNESS REPLACE DynamicFilter\s\[[^\]]*\] DynamicFilter [ REDACTED ] +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select tag from t where num > 6 order by ts desc limit 2; drop table t; diff --git a/tests/cases/standalone/common/order/windowed_sort.result b/tests/cases/standalone/common/order/windowed_sort.result index f85aa8c04e6..a80ff6714ad 100644 --- a/tests/cases/standalone/common/order/windowed_sort.result +++ b/tests/cases/standalone/common/order/windowed_sort.result @@ -61,6 +61,7 @@ SELECT * FROM test ORDER BY t LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test ORDER BY t LIMIT 5; +-+-+-+ @@ -71,7 +72,7 @@ EXPLAIN ANALYZE SELECT * FROM test ORDER BY t LIMIT 5; |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [t@1 ASC NULLS LAST], fetch=5 REDACTED |_|_|_SortExec: TopK(fetch=5), expr=[t@1 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":4, "mem_ranges":1, "files":3, "file_ranges":3} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":4, "mem_ranges":1, "files":3, "file_ranges":3}, "projection": ["i", "t"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ @@ -94,6 +95,7 @@ SELECT * FROM test ORDER BY t DESC LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test ORDER BY t DESC LIMIT 5; +-+-+-+ @@ -104,7 +106,7 @@ EXPLAIN ANALYZE SELECT * FROM test ORDER BY t DESC LIMIT 5; |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [t@1 DESC], fetch=5 REDACTED |_|_|_SortExec: TopK(fetch=5), expr=[t@1 DESC], preserve_partitioning=[true] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":4, "mem_ranges":1, "files":3, "file_ranges":3} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":4, "mem_ranges":1, "files":3, "file_ranges":3}, "projection": ["i", "t"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ @@ -127,6 +129,7 @@ SELECT * FROM test where i > 2 ORDER BY t LIMIT 4; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test where i > 2 ORDER BY t LIMIT 4; +-+-+-+ @@ -138,7 +141,7 @@ EXPLAIN ANALYZE SELECT * FROM test where i > 2 ORDER BY t LIMIT 4; | 1_| 0_|_SortPreservingMergeExec: [t@1 ASC NULLS LAST], fetch=4 REDACTED |_|_|_SortExec: TopK(fetch=4), expr=[t@1 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_FilterExec: i@0 > 2 REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":4, "mem_ranges":1, "files":3, "file_ranges":3} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":4, "mem_ranges":1, "files":3, "file_ranges":3}, "projection": ["i", "t"], "filters": ["i > Int32(2)"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -161,6 +164,7 @@ SELECT * FROM test where i > 2 ORDER BY t DESC LIMIT 4; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test where i > 2 ORDER BY t DESC LIMIT 4; +-+-+-+ @@ -172,7 +176,7 @@ EXPLAIN ANALYZE SELECT * FROM test where i > 2 ORDER BY t DESC LIMIT 4; | 1_| 0_|_SortPreservingMergeExec: [t@1 DESC], fetch=4 REDACTED |_|_|_SortExec: TopK(fetch=4), expr=[t@1 DESC], preserve_partitioning=[true] REDACTED |_|_|_FilterExec: i@0 > 2 REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":4, "mem_ranges":1, "files":3, "file_ranges":3} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":4, "mem_ranges":1, "files":3, "file_ranges":3}, "projection": ["i", "t"], "filters": ["i > Int32(2)"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -195,6 +199,7 @@ SELECT * FROM test where t > 8 ORDER BY t DESC LIMIT 4; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test where t > 8 ORDER BY t DESC LIMIT 4; +-+-+-+ @@ -205,7 +210,7 @@ EXPLAIN ANALYZE SELECT * FROM test where t > 8 ORDER BY t DESC LIMIT 4; |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [t@1 DESC], fetch=4 REDACTED |_|_|_SortExec: TopK(fetch=4), expr=[t@1 DESC], preserve_partitioning=[true] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":2, "mem_ranges":1, "files":1, "file_ranges":1} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":2, "mem_ranges":1, "files":1, "file_ranges":1}, "projection": ["i", "t"], "filters": ["t > TimestampMillisecond(8, None)"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -277,6 +282,7 @@ SELECT * FROM test_pk ORDER BY t LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test_pk ORDER BY t LIMIT 5; +-+-+-+ @@ -287,7 +293,7 @@ EXPLAIN ANALYZE SELECT * FROM test_pk ORDER BY t LIMIT 5; |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [t@2 ASC NULLS LAST], fetch=5 REDACTED |_|_|_SortExec: TopK(fetch=5), expr=[t@2 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":4, "mem_ranges":1, "files":3, "file_ranges":3} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":4, "mem_ranges":1, "files":3, "file_ranges":3}, "projection": ["pk", "i", "t"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ @@ -299,6 +305,7 @@ EXPLAIN ANALYZE SELECT * FROM test_pk ORDER BY t LIMIT 5; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (files.*) REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT * FROM test_pk ORDER BY t LIMIT 5; +-+-+-+ @@ -332,6 +339,7 @@ SELECT * FROM test_pk ORDER BY t DESC LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test_pk ORDER BY t DESC LIMIT 5; +-+-+-+ @@ -342,7 +350,7 @@ EXPLAIN ANALYZE SELECT * FROM test_pk ORDER BY t DESC LIMIT 5; |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [t@2 DESC], fetch=5 REDACTED |_|_|_SortExec: TopK(fetch=5), expr=[t@2 DESC], preserve_partitioning=[true] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":4, "mem_ranges":1, "files":3, "file_ranges":3} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":4, "mem_ranges":1, "files":3, "file_ranges":3}, "projection": ["pk", "i", "t"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ @@ -366,6 +374,7 @@ SELECT * FROM test_pk where pk > 7 ORDER BY t LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test_pk where pk > 7 ORDER BY t LIMIT 5; +-+-+-+ @@ -376,7 +385,7 @@ EXPLAIN ANALYZE SELECT * FROM test_pk where pk > 7 ORDER BY t LIMIT 5; |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [t@2 ASC NULLS LAST], fetch=5 REDACTED |_|_|_SortExec: TopK(fetch=5), expr=[t@2 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":4, "mem_ranges":1, "files":3, "file_ranges":3} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":4, "mem_ranges":1, "files":3, "file_ranges":3}, "projection": ["pk", "i", "t"], "filters": ["pk > Int32(7)"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ @@ -388,6 +397,7 @@ EXPLAIN ANALYZE SELECT * FROM test_pk where pk > 7 ORDER BY t LIMIT 5; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (files.*) REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT * FROM test_pk where pk > 7 ORDER BY t LIMIT 5; +-+-+-+ diff --git a/tests/cases/standalone/common/order/windowed_sort.sql b/tests/cases/standalone/common/order/windowed_sort.sql index 1349e4d96ca..7815693e237 100644 --- a/tests/cases/standalone/common/order/windowed_sort.sql +++ b/tests/cases/standalone/common/order/windowed_sort.sql @@ -23,6 +23,7 @@ SELECT * FROM test ORDER BY t LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test ORDER BY t LIMIT 5; SELECT * FROM test ORDER BY t DESC LIMIT 5; @@ -33,6 +34,7 @@ SELECT * FROM test ORDER BY t DESC LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test ORDER BY t DESC LIMIT 5; -- Filter on a field. @@ -44,6 +46,7 @@ SELECT * FROM test where i > 2 ORDER BY t LIMIT 4; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test where i > 2 ORDER BY t LIMIT 4; -- Filter on a field. @@ -55,6 +58,7 @@ SELECT * FROM test where i > 2 ORDER BY t DESC LIMIT 4; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test where i > 2 ORDER BY t DESC LIMIT 4; -- Filter on the time index. @@ -66,6 +70,7 @@ SELECT * FROM test where t > 8 ORDER BY t DESC LIMIT 4; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test where t > 8 ORDER BY t DESC LIMIT 4; DROP TABLE test; @@ -95,6 +100,7 @@ SELECT * FROM test_pk ORDER BY t LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test_pk ORDER BY t LIMIT 5; -- SQLNESS REPLACE (-+) - @@ -104,6 +110,7 @@ EXPLAIN ANALYZE SELECT * FROM test_pk ORDER BY t LIMIT 5; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (files.*) REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT * FROM test_pk ORDER BY t LIMIT 5; SELECT * FROM test_pk ORDER BY t DESC LIMIT 5; @@ -114,6 +121,7 @@ SELECT * FROM test_pk ORDER BY t DESC LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test_pk ORDER BY t DESC LIMIT 5; -- Filter on a pk column. @@ -125,6 +133,7 @@ SELECT * FROM test_pk where pk > 7 ORDER BY t LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM test_pk where pk > 7 ORDER BY t LIMIT 5; -- SQLNESS REPLACE (-+) - @@ -134,6 +143,7 @@ EXPLAIN ANALYZE SELECT * FROM test_pk where pk > 7 ORDER BY t LIMIT 5; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (files.*) REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT * FROM test_pk where pk > 7 ORDER BY t LIMIT 5; DROP TABLE test_pk; diff --git a/tests/cases/standalone/common/partition.result b/tests/cases/standalone/common/partition.result index fe6f99df591..810899f1452 100644 --- a/tests/cases/standalone/common/partition.result +++ b/tests/cases/standalone/common/partition.result @@ -235,6 +235,7 @@ Affected Rows: 0 -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM valid_rule; +-+-+-+ @@ -244,19 +245,19 @@ EXPLAIN ANALYZE SELECT * FROM valid_rule; |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a < Int32(10) OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) AND (b < Utf8(\"a\") OR b IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) AND b >= Utf8(\"a\") AND (b < Utf8(\"o\") OR b IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 3_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) AND b >= Utf8(\"o\")"], "time_range":REDACTED |_|_|_| | 1_| 4_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a > Int32(10)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -268,6 +269,7 @@ EXPLAIN ANALYZE SELECT * FROM valid_rule; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM valid_rule WHERE a < 10; @@ -278,7 +280,7 @@ WHERE a < 10; |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a < Int32(10)", "a < Int32(10) OR a IS NULL"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -290,6 +292,7 @@ WHERE a < 10; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM valid_rule WHERE a = 10 AND b= 'z'; @@ -300,7 +303,7 @@ WHERE a = 10 AND b= 'z'; |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10)", "b = Dictionary(UInt32, Utf8(\"z\"))", "a = Int32(10) AND b >= Utf8(\"o\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -312,6 +315,7 @@ WHERE a = 10 AND b= 'z'; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM valid_rule WHERE a = 10 OR b= 'z'; @@ -323,23 +327,23 @@ WHERE a = 10 OR b= 'z'; |_|_|_| | 1_| 0_|_FilterExec: a@0 = 10 OR b@1 = z REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR b = Dictionary(UInt32, Utf8(\"z\"))", "a < Int32(10) OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_FilterExec: a@0 = 10 OR b@1 = z REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR b = Dictionary(UInt32, Utf8(\"z\"))", "a = Int32(10) AND (b < Utf8(\"a\") OR b IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_FilterExec: a@0 = 10 OR b@1 = z REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR b = Dictionary(UInt32, Utf8(\"z\"))", "a = Int32(10) AND b >= Utf8(\"a\") AND (b < Utf8(\"o\") OR b IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 3_|_FilterExec: a@0 = 10 OR b@1 = z REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR b = Dictionary(UInt32, Utf8(\"z\"))", "a = Int32(10) AND b >= Utf8(\"o\")"], "time_range":REDACTED |_|_|_| | 1_| 4_|_FilterExec: a@0 = 10 OR b@1 = z REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR b = Dictionary(UInt32, Utf8(\"z\"))", "a > Int32(10)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -351,6 +355,7 @@ WHERE a = 10 OR b= 'z'; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM valid_rule WHERE a = 10 OR ts > 1; @@ -362,23 +367,23 @@ WHERE a = 10 OR ts > 1; |_|_|_| | 1_| 0_|_FilterExec: a@0 = 10 OR ts@2 > 1 REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR ts > TimestampMillisecond(1, None)", "a < Int32(10) OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_FilterExec: a@0 = 10 OR ts@2 > 1 REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR ts > TimestampMillisecond(1, None)", "a = Int32(10) AND (b < Utf8(\"a\") OR b IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_FilterExec: a@0 = 10 OR ts@2 > 1 REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR ts > TimestampMillisecond(1, None)", "a = Int32(10) AND b >= Utf8(\"a\") AND (b < Utf8(\"o\") OR b IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 3_|_FilterExec: a@0 = 10 OR ts@2 > 1 REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR ts > TimestampMillisecond(1, None)", "a = Int32(10) AND b >= Utf8(\"o\")"], "time_range":REDACTED |_|_|_| | 1_| 4_|_FilterExec: a@0 = 10 OR ts@2 > 1 REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR ts > TimestampMillisecond(1, None)", "a > Int32(10)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -390,6 +395,7 @@ WHERE a = 10 OR ts > 1; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM valid_rule WHERE a = 10 OR (ts > 1 AND b ='h'); @@ -401,23 +407,23 @@ WHERE a = 10 OR (ts > 1 AND b ='h'); |_|_|_| | 1_| 0_|_FilterExec: a@0 = 10 OR ts@2 > 1 AND b@1 = h REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR ts > TimestampMillisecond(1, None) AND b = Dictionary(UInt32, Utf8(\"h\"))", "a < Int32(10) OR a IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_FilterExec: a@0 = 10 OR ts@2 > 1 AND b@1 = h REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR ts > TimestampMillisecond(1, None) AND b = Dictionary(UInt32, Utf8(\"h\"))", "a = Int32(10) AND (b < Utf8(\"a\") OR b IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_FilterExec: a@0 = 10 OR ts@2 > 1 AND b@1 = h REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR ts > TimestampMillisecond(1, None) AND b = Dictionary(UInt32, Utf8(\"h\"))", "a = Int32(10) AND b >= Utf8(\"a\") AND (b < Utf8(\"o\") OR b IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 3_|_FilterExec: a@0 = 10 OR ts@2 > 1 AND b@1 = h REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR ts > TimestampMillisecond(1, None) AND b = Dictionary(UInt32, Utf8(\"h\"))", "a = Int32(10) AND b >= Utf8(\"o\")"], "time_range":REDACTED |_|_|_| | 1_| 4_|_FilterExec: a@0 = 10 OR ts@2 > 1 AND b@1 = h REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "projection": ["a", "b", "ts"], "filters": ["a = Int32(10) OR ts > TimestampMillisecond(1, None) AND b = Dictionary(UInt32, Utf8(\"h\"))", "a > Int32(10)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -463,3 +469,33 @@ DROP TABLE `molestiAe`; Affected Rows: 0 +-- The minimum of a signed type is a valid bound. +CREATE TABLE int_min_bound ( + a INT PRIMARY KEY, + ts TIMESTAMP TIME INDEX, +) PARTITION ON COLUMNS (a) ( + a < -2147483648, + a >= -2147483648 AND a < 0, + a >= 0 +); + +Affected Rows: 0 + +INSERT INTO int_min_bound VALUES (-2147483648, 1), (NULL, 2), (0, 3); + +Affected Rows: 3 + +SELECT * FROM int_min_bound ORDER BY ts; + ++-------------+-------------------------+ +| a | ts | ++-------------+-------------------------+ +| -2147483648 | 1970-01-01T00:00:00.001 | +| | 1970-01-01T00:00:00.002 | +| 0 | 1970-01-01T00:00:00.003 | ++-------------+-------------------------+ + +DROP TABLE int_min_bound; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/partition.sql b/tests/cases/standalone/common/partition.sql index 7b5e8ea7750..9728a34a0c1 100644 --- a/tests/cases/standalone/common/partition.sql +++ b/tests/cases/standalone/common/partition.sql @@ -128,6 +128,7 @@ PARTITION ON COLUMNS (a, b) ( -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM valid_rule; -- SQLNESS REPLACE (metrics.*) REDACTED @@ -137,6 +138,7 @@ EXPLAIN ANALYZE SELECT * FROM valid_rule; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM valid_rule WHERE a < 10; @@ -147,6 +149,7 @@ WHERE a < 10; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM valid_rule WHERE a = 10 AND b= 'z'; @@ -157,6 +160,7 @@ WHERE a = 10 AND b= 'z'; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM valid_rule WHERE a = 10 OR b= 'z'; @@ -167,6 +171,7 @@ WHERE a = 10 OR b= 'z'; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM valid_rule WHERE a = 10 OR ts > 1; @@ -177,6 +182,7 @@ WHERE a = 10 OR ts > 1; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM valid_rule WHERE a = 10 OR (ts > 1 AND b ='h'); @@ -204,3 +210,19 @@ INSERT INTO `molestiAe` VALUES SELECT * FROM `molestiAe`; DROP TABLE `molestiAe`; + +-- The minimum of a signed type is a valid bound. +CREATE TABLE int_min_bound ( + a INT PRIMARY KEY, + ts TIMESTAMP TIME INDEX, +) PARTITION ON COLUMNS (a) ( + a < -2147483648, + a >= -2147483648 AND a < 0, + a >= 0 +); + +INSERT INTO int_min_bound VALUES (-2147483648, 1), (NULL, 2), (0, 3); + +SELECT * FROM int_min_bound ORDER BY ts; + +DROP TABLE int_min_bound; diff --git a/tests/cases/standalone/common/promql/delta_temporality.result b/tests/cases/standalone/common/promql/delta_temporality.result index ac32a907cb9..4d3b1f93f17 100644 --- a/tests/cases/standalone/common/promql/delta_temporality.result +++ b/tests/cases/standalone/common/promql/delta_temporality.result @@ -54,6 +54,7 @@ TQL EVAL (180, 180, '1m') rate(delta_temporality[3m]); -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (180, 180, '1m') rate(delta_temporality[3m]); +-+-+-+ @@ -67,7 +68,7 @@ TQL ANALYZE (180, 180, '1m') rate(delta_temporality[3m]); |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[ts], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["series", "otlp_aggregation_temporality"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "series", "otlp_aggregation_temporality"], "filters": ["ts >= TimestampMillisecond(1, None)", "ts <= TimestampMillisecond(180000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 3_| +-+-+-+ diff --git a/tests/cases/standalone/common/promql/delta_temporality.sql b/tests/cases/standalone/common/promql/delta_temporality.sql index b021f5f6b00..0ea9328e445 100644 --- a/tests/cases/standalone/common/promql/delta_temporality.sql +++ b/tests/cases/standalone/common/promql/delta_temporality.sql @@ -34,6 +34,7 @@ TQL EVAL (180, 180, '1m') rate(delta_temporality[3m]); -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (180, 180, '1m') rate(delta_temporality[3m]); -- The reserved temporality marker treats NULL as the absent cumulative state. diff --git a/tests/cases/standalone/common/promql/histogram_multi_partition.result b/tests/cases/standalone/common/promql/histogram_multi_partition.result index 64e2f1e8551..d4e8090b314 100644 --- a/tests/cases/standalone/common/promql/histogram_multi_partition.result +++ b/tests/cases/standalone/common/promql/histogram_multi_partition.result @@ -36,6 +36,7 @@ Affected Rows: 12 -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE Hash\(\[ts@1\],.* Hash([ts@1],REDACTED -- SQLNESS REPLACE Hash\(\[le@0,\sts@1\],.* Hash([le@0, ts@1],REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (0, 10, '10s') histogram_quantile(0.5, sum by (le) (histogram_gap_bucket)); +-+-+-+ @@ -56,7 +57,7 @@ tql analyze (0, 10, '10s') histogram_quantile(0.5, sum by (le) (histogram_gap_bu |_|_|_ProjectionExec: expr=[ts@0 as ts, le@1 as le, val@3 as val] REDACTED |_|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[10000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["shard", "le"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["ts", "le", "shard", "val"], "filters": ["ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(10000, None)", "shard < Utf8(\"n\") OR shard IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[le@0 as le, ts@1 as ts], aggr=[__sum_state(histogram_gap_bucket.val)] REDACTED |_|_|_RepartitionExec: partitioning=Hash([le@0, ts@1],REDACTED @@ -64,7 +65,7 @@ tql analyze (0, 10, '10s') histogram_quantile(0.5, sum by (le) (histogram_gap_bu |_|_|_ProjectionExec: expr=[ts@0 as ts, le@1 as le, val@3 as val] REDACTED |_|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[10000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["shard", "le"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["ts", "le", "shard", "val"], "filters": ["ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(10000, None)", "shard >= Utf8(\"n\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ diff --git a/tests/cases/standalone/common/promql/histogram_multi_partition.sql b/tests/cases/standalone/common/promql/histogram_multi_partition.sql index b360999fcf7..3bc11f98b99 100644 --- a/tests/cases/standalone/common/promql/histogram_multi_partition.sql +++ b/tests/cases/standalone/common/promql/histogram_multi_partition.sql @@ -32,6 +32,7 @@ insert into histogram_gap_bucket values -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE Hash\(\[ts@1\],.* Hash([ts@1],REDACTED -- SQLNESS REPLACE Hash\(\[le@0,\sts@1\],.* Hash([le@0, ts@1],REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (0, 10, '10s') histogram_quantile(0.5, sum by (le) (histogram_gap_bucket)); -- SQLNESS SORT_RESULT 2 1 diff --git a/tests/cases/standalone/common/promql/native_time_selection.result b/tests/cases/standalone/common/promql/native_time_selection.result index 945189e68f6..710f8705a85 100644 --- a/tests/cases/standalone/common/promql/native_time_selection.result +++ b/tests/cases/standalone/common/promql/native_time_selection.result @@ -65,6 +65,7 @@ TQL EXPLAIN (1, 1, '1s', '1ms') native_time_us{series="exact"}; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED -- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (1, 1, '1s', '1ms') native_time_us{series="exact"}; +-+-+-+ @@ -77,7 +78,7 @@ TQL ANALYZE VERBOSE (1, 1, '1s', '1ms') native_time_us{series="exact"}; |_|_|_PromSeriesDivideExec: tags=["series"] metrics=[output_rows: 1, REDACTED |_|_|_ProjectionExec: expr=[val@2 as val, series@1 as series, ts@0 as ts] metrics=[output_rows: 1, REDACTED |_|_|_CooperativeExec metrics=[]_| -|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "series", "val"], "filters": ["series = Dictionary(UInt32, Utf8(\"exact\"))", "ts > TimestampMicrosecond(999000, None)", "ts <= TimestampMicrosecond(1000000, None)"], "REDACTED +|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "series", "val"], "filters": ["series = Dictionary(UInt32, Utf8(\"exact\"))", "ts > TimestampMicrosecond(999000, None)", "ts <= TimestampMicrosecond(1000000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -312,6 +313,7 @@ TQL EXPLAIN (1, 1, '1s', '1ms') native_time_ns{series="exact"}; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED -- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (1, 1, '1s', '1ms') native_time_ns{series="exact"}; +-+-+-+ @@ -324,7 +326,7 @@ TQL ANALYZE VERBOSE (1, 1, '1s', '1ms') native_time_ns{series="exact"}; |_|_|_PromSeriesDivideExec: tags=["series"] metrics=[output_rows: 1, REDACTED |_|_|_ProjectionExec: expr=[val@2 as val, series@1 as series, ts@0 as ts] metrics=[output_rows: 1, REDACTED |_|_|_CooperativeExec metrics=[]_| -|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "series", "val"], "filters": ["series = Dictionary(UInt32, Utf8(\"exact\"))", "ts > TimestampNanosecond(999000000, None)", "ts <= TimestampNanosecond(1000000000, None)"], "REDACTED +|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "series", "val"], "filters": ["series = Dictionary(UInt32, Utf8(\"exact\"))", "ts > TimestampNanosecond(999000000, None)", "ts <= TimestampNanosecond(1000000000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -541,6 +543,7 @@ TQL EXPLAIN (0, 0, '1s', '2d') native_time_ns_lower_overflow{series="exact"} off -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED -- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (0, 0, '1s', '2d') native_time_ns_lower_overflow{series="exact"} offset 106750d; +-+-+-+ @@ -554,7 +557,7 @@ TQL ANALYZE VERBOSE (0, 0, '1s', '2d') native_time_ns_lower_overflow{series="exa |_|_|_PromSeriesDivideExec: tags=["series"] metrics=[output_rows: 1, REDACTED |_|_|_ProjectionExec: expr=[val@2 as val, series@1 as series, ts@0 as ts] metrics=[output_rows: 1, REDACTED |_|_|_CooperativeExec metrics=[]_| -|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "series", "val"], "filters": ["series = Dictionary(UInt32, Utf8(\"exact\"))", "ts <= TimestampNanosecond(-9223200000000000000, None)"], "REDACTED +|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "series", "val"], "filters": ["series = Dictionary(UInt32, Utf8(\"exact\"))", "ts <= TimestampNanosecond(-9223200000000000000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ diff --git a/tests/cases/standalone/common/promql/native_time_selection.sql b/tests/cases/standalone/common/promql/native_time_selection.sql index 204b047ef23..fb6e05d74b2 100644 --- a/tests/cases/standalone/common/promql/native_time_selection.sql +++ b/tests/cases/standalone/common/promql/native_time_selection.sql @@ -46,6 +46,7 @@ TQL EXPLAIN (1, 1, '1s', '1ms') native_time_us{series="exact"}; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED -- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (1, 1, '1s', '1ms') native_time_us{series="exact"}; -- The same-series future tick is in the memtable, while exact 1s remains selected. @@ -136,6 +137,7 @@ TQL EXPLAIN (1, 1, '1s', '1ms') native_time_ns{series="exact"}; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED -- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (1, 1, '1s', '1ms') native_time_ns{series="exact"}; -- The same-series future tick is in the memtable, while exact 1s remains selected. @@ -206,6 +208,7 @@ TQL EXPLAIN (0, 0, '1s', '2d') native_time_ns_lower_overflow{series="exact"} off -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED -- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (0, 0, '1s', '2d') native_time_ns_lower_overflow{series="exact"} offset 106750d; -- The representable upper bound selects only the exact row. diff --git a/tests/cases/standalone/common/promql/regex.result b/tests/cases/standalone/common/promql/regex.result index c2a10626ff8..be289cdacb8 100644 --- a/tests/cases/standalone/common/promql/regex.result +++ b/tests/cases/standalone/common/promql/regex.result @@ -66,6 +66,7 @@ TQL EVAL (0, 100, '15s') test{host=~"(10\\.0\\.160\\.237:8080|10\\.0\\.160\\.237 -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (0, 0, '1s') test{host=~".*"}; +-+-+-+ @@ -77,7 +78,7 @@ TQL ANALYZE VERBOSE (0, 0, '1s') test{host=~".*"}; | 1_| 0_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[1000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["host"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "host", "val"], "filters": ["ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(0, None)"], "REDACTED +|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "host", "val"], "filters": ["ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(0, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -90,6 +91,7 @@ TQL ANALYZE VERBOSE (0, 0, '1s') test{host=~".*"}; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (0, 0, '1s') test{host=~".+"}; +-+-+-+ @@ -101,7 +103,7 @@ TQL ANALYZE VERBOSE (0, 0, '1s') test{host=~".+"}; | 1_| 0_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[1000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["host"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "host", "val"], "filters": ["host != Dictionary(UInt32, Utf8(\"\"))", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(0, None)"], "REDACTED +|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "host", "val"], "filters": ["host != Dictionary(UInt32, Utf8(\"\"))", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(0, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -114,6 +116,7 @@ TQL ANALYZE VERBOSE (0, 0, '1s') test{host=~".+"}; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (0, 0, '1s') test{host!~".*"}; +-+-+-+ @@ -136,6 +139,7 @@ TQL ANALYZE VERBOSE (0, 0, '1s') test{host!~".*"}; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (0, 0, '1s') test{host!~".+"}; +-+-+-+ @@ -147,7 +151,7 @@ TQL ANALYZE VERBOSE (0, 0, '1s') test{host!~".+"}; | 1_| 0_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[1000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["host"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "host", "val"], "filters": ["host IS NULL OR host = Dictionary(UInt32, Utf8(\"\"))", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(0, None)"], "REDACTED +|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "host", "val"], "filters": ["host IS NULL OR host = Dictionary(UInt32, Utf8(\"\"))", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(0, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ diff --git a/tests/cases/standalone/common/promql/regex.sql b/tests/cases/standalone/common/promql/regex.sql index 80a8f3ed381..3ad88c6ea8c 100644 --- a/tests/cases/standalone/common/promql/regex.sql +++ b/tests/cases/standalone/common/promql/regex.sql @@ -27,6 +27,7 @@ TQL EVAL (0, 100, '15s') test{host=~"(10\\.0\\.160\\.237:8080|10\\.0\\.160\\.237 -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (0, 0, '1s') test{host=~".*"}; -- SQLNESS REPLACE (metrics.*) REDACTED @@ -37,6 +38,7 @@ TQL ANALYZE VERBOSE (0, 0, '1s') test{host=~".*"}; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (0, 0, '1s') test{host=~".+"}; -- SQLNESS REPLACE (metrics.*) REDACTED @@ -47,6 +49,7 @@ TQL ANALYZE VERBOSE (0, 0, '1s') test{host=~".+"}; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (0, 0, '1s') test{host!~".*"}; -- SQLNESS REPLACE (metrics.*) REDACTED @@ -57,6 +60,7 @@ TQL ANALYZE VERBOSE (0, 0, '1s') test{host!~".*"}; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (0, 0, '1s') test{host!~".+"}; INSERT INTO test VALUES (0, NULL, 2), (0, '', 3), (0, 'ext4', 4), (0, 'tmpfs', 5); diff --git a/tests/cases/standalone/common/promql/subquery_at_modifier.result b/tests/cases/standalone/common/promql/subquery_at_modifier.result new file mode 100644 index 00000000000..02dc4111c8d --- /dev/null +++ b/tests/cases/standalone/common/promql/subquery_at_modifier.result @@ -0,0 +1,229 @@ +-- Tests for the PromQL `@` modifier on subqueries. +-- +-- Prometheus folds the `@` of a subquery into its offset (`setOffsetForAtModifier`) and treats +-- the subquery as step-invariant: every evaluation step reports the window +-- `(anchor - offset - range, anchor - offset]`, sampled on the absolute multiples of the subquery +-- step, exactly like `@` on a range selector. `@ start()` / `@ end()` anchor at the start/end of +-- the statement's evaluation range. The output timestamps still follow the evaluation grid. +-- +-- Two series with different values, so that a dropped or merged series shows up as a wrong row. +-- Sample timestamps are in milliseconds, while `@` and `TQL EVAL` timestamps are in seconds. +-- The samples lie on the 10s subquery step grid. +CREATE TABLE subquery_at_total ( + ts TIMESTAMP TIME INDEX, + host STRING PRIMARY KEY, + val DOUBLE, +); + +Affected Rows: 0 + +INSERT INTO subquery_at_total VALUES + (0, 'a', 1), + (10000, 'a', 2), + (20000, 'a', 3), + (30000, 'a', 4), + (40000, 'a', 5), + (50000, 'a', 6), + (60000, 'a', 7), + (0, 'b', 10), + (10000, 'b', 20), + (20000, 'b', 30), + (30000, 'b', 40), + (40000, 'b', 50), + (50000, 'b', 60), + (60000, 'b', 70); + +Affected Rows: 14 + +-- The range selector form folds (40s, 60s] at every step: 6 + 7 = 13 (and 130 for 'b'). +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time(subquery_at_total[20s] @ 60); + ++---------------------+----------------------------------+------+ +| ts | prom_sum_over_time(ts_range,val) | host | ++---------------------+----------------------------------+------+ +| 1970-01-01T00:00:30 | 13.0 | a | +| 1970-01-01T00:00:30 | 130.0 | b | +| 1970-01-01T00:01:00 | 13.0 | a | +| 1970-01-01T00:01:00 | 130.0 | b | ++---------------------+----------------------------------+------+ + +-- The subquery with `@ 60` must report the same window at every step. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time(subquery_at_total[20s:10s] @ 60); + ++---------------------+----------------------------------+------+ +| ts | prom_sum_over_time(ts_range,val) | host | ++---------------------+----------------------------------+------+ +| 1970-01-01T00:00:30 | 13.0 | a | +| 1970-01-01T00:00:30 | 130.0 | b | +| 1970-01-01T00:01:00 | 13.0 | a | +| 1970-01-01T00:01:00 | 130.0 | b | ++---------------------+----------------------------------+------+ + +-- Baseline without `@`: the window follows the grid, 3 + 4 at 30s and 6 + 7 at 60s. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time(subquery_at_total[20s:10s]); + ++---------------------+----------------------------------+------+ +| ts | prom_sum_over_time(ts_range,val) | host | ++---------------------+----------------------------------+------+ +| 1970-01-01T00:00:30 | 7.0 | a | +| 1970-01-01T00:00:30 | 70.0 | b | +| 1970-01-01T00:01:00 | 13.0 | a | +| 1970-01-01T00:01:00 | 130.0 | b | ++---------------------+----------------------------------+------+ + +-- `@ end()` anchors at 60s: 13 at both steps. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time(subquery_at_total[20s:10s] @ end()); + ++---------------------+----------------------------------+------+ +| ts | prom_sum_over_time(ts_range,val) | host | ++---------------------+----------------------------------+------+ +| 1970-01-01T00:00:30 | 13.0 | a | +| 1970-01-01T00:00:30 | 130.0 | b | +| 1970-01-01T00:01:00 | 13.0 | a | +| 1970-01-01T00:01:00 | 130.0 | b | ++---------------------+----------------------------------+------+ + +-- `@ start()` anchors at 30s: (10s, 30s] holds 3 + 4 = 7 at both steps. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time(subquery_at_total[20s:10s] @ start()); + ++---------------------+----------------------------------+------+ +| ts | prom_sum_over_time(ts_range,val) | host | ++---------------------+----------------------------------+------+ +| 1970-01-01T00:00:30 | 7.0 | a | +| 1970-01-01T00:00:30 | 70.0 | b | +| 1970-01-01T00:01:00 | 7.0 | a | +| 1970-01-01T00:01:00 | 70.0 | b | ++---------------------+----------------------------------+------+ + +-- `offset` applies relative to the anchor, in either modifier order: `@ 60 offset 30s` folds +-- (10s, 30s] = 7 at both steps. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time(subquery_at_total[20s:10s] @ 60 offset 30s); + ++---------------------+----------------------------------+------+ +| ts | prom_sum_over_time(ts_range,val) | host | ++---------------------+----------------------------------+------+ +| 1970-01-01T00:00:30 | 7.0 | a | +| 1970-01-01T00:00:30 | 70.0 | b | +| 1970-01-01T00:01:00 | 7.0 | a | +| 1970-01-01T00:01:00 | 70.0 | b | ++---------------------+----------------------------------+------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time(subquery_at_total[20s:10s] offset 30s @ 60); + ++---------------------+----------------------------------+------+ +| ts | prom_sum_over_time(ts_range,val) | host | ++---------------------+----------------------------------+------+ +| 1970-01-01T00:00:30 | 7.0 | a | +| 1970-01-01T00:00:30 | 70.0 | b | +| 1970-01-01T00:01:00 | 7.0 | a | +| 1970-01-01T00:01:00 | 70.0 | b | ++---------------------+----------------------------------+------+ + +-- An instant query away from the anchor still folds the anchored window: 13. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 0, '1s') sum_over_time(subquery_at_total[20s:10s] @ 60); + ++---------------------+----------------------------------+------+ +| ts | prom_sum_over_time(ts_range,val) | host | ++---------------------+----------------------------------+------+ +| 1970-01-01T00:00:00 | 13.0 | a | +| 1970-01-01T00:00:00 | 130.0 | b | ++---------------------+----------------------------------+------+ + +-- Other `*_over_time` functions over the anchored window (40s, 60s]: points 50s and 60s. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') max_over_time(subquery_at_total[20s:10s] @ 60); + ++---------------------+----------------------------------+------+ +| ts | prom_max_over_time(ts_range,val) | host | ++---------------------+----------------------------------+------+ +| 1970-01-01T00:00:30 | 7.0 | a | +| 1970-01-01T00:00:30 | 70.0 | b | +| 1970-01-01T00:01:00 | 7.0 | a | +| 1970-01-01T00:01:00 | 70.0 | b | ++---------------------+----------------------------------+------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') count_over_time(subquery_at_total[20s:10s] @ 60); + ++---------------------+------------------------------------+------+ +| ts | prom_count_over_time(ts_range,val) | host | ++---------------------+------------------------------------+------+ +| 1970-01-01T00:00:30 | 2.0 | a | +| 1970-01-01T00:00:30 | 2.0 | b | +| 1970-01-01T00:01:00 | 2.0 | a | +| 1970-01-01T00:01:00 | 2.0 | b | ++---------------------+------------------------------------+------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') avg_over_time(subquery_at_total[20s:10s] @ 60); + ++---------------------+----------------------------------+------+ +| ts | prom_avg_over_time(ts_range,val) | host | ++---------------------+----------------------------------+------+ +| 1970-01-01T00:00:30 | 6.5 | a | +| 1970-01-01T00:00:30 | 65.0 | b | +| 1970-01-01T00:01:00 | 6.5 | a | +| 1970-01-01T00:01:00 | 65.0 | b | ++---------------------+----------------------------------+------+ + +-- `rate` over (30s, 60s] -- points 40s, 50s, 60s -- extrapolates from the anchored window, not +-- from the evaluation step, so the subquery form matches the range selector form at every step. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') rate(subquery_at_total[30s] @ 60); + ++---------------------+-----------------------------------------+------+ +| ts | prom_rate(ts_range,val,ts,Int64(30000)) | host | ++---------------------+-----------------------------------------+------+ +| 1970-01-01T00:00:30 | 0.1 | a | +| 1970-01-01T00:00:30 | 1.0 | b | +| 1970-01-01T00:01:00 | 0.1 | a | +| 1970-01-01T00:01:00 | 1.0 | b | ++---------------------+-----------------------------------------+------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') rate(subquery_at_total[30s:10s] @ 60); + ++---------------------+-----------------------------------------+------+ +| ts | prom_rate(ts_range,val,ts,Int64(30000)) | host | ++---------------------+-----------------------------------------+------+ +| 1970-01-01T00:00:30 | 0.1 | a | +| 1970-01-01T00:00:30 | 1.0 | b | +| 1970-01-01T00:01:00 | 0.1 | a | +| 1970-01-01T00:01:00 | 1.0 | b | ++---------------------+-----------------------------------------+------+ + +-- A computed inner expression and an aggregation above the anchored call. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time((subquery_at_total * 2)[20s:10s] @ 60); + ++---------------------+-----------------------------------------------+------+ +| ts | prom_sum_over_time(ts_range,val * Float64(2)) | host | ++---------------------+-----------------------------------------------+------+ +| 1970-01-01T00:00:30 | 26.0 | a | +| 1970-01-01T00:00:30 | 260.0 | b | +| 1970-01-01T00:01:00 | 26.0 | a | +| 1970-01-01T00:01:00 | 260.0 | b | ++---------------------+-----------------------------------------------+------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum(sum_over_time(subquery_at_total[20s:10s] @ 60)); + ++---------------------+---------------------------------------+ +| ts | sum(prom_sum_over_time(ts_range,val)) | ++---------------------+---------------------------------------+ +| 1970-01-01T00:00:30 | 143.0 | +| 1970-01-01T00:01:00 | 143.0 | ++---------------------+---------------------------------------+ + +DROP TABLE subquery_at_total; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/promql/subquery_at_modifier.sql b/tests/cases/standalone/common/promql/subquery_at_modifier.sql new file mode 100644 index 00000000000..5ae70098664 --- /dev/null +++ b/tests/cases/standalone/common/promql/subquery_at_modifier.sql @@ -0,0 +1,92 @@ +-- Tests for the PromQL `@` modifier on subqueries. +-- +-- Prometheus folds the `@` of a subquery into its offset (`setOffsetForAtModifier`) and treats +-- the subquery as step-invariant: every evaluation step reports the window +-- `(anchor - offset - range, anchor - offset]`, sampled on the absolute multiples of the subquery +-- step, exactly like `@` on a range selector. `@ start()` / `@ end()` anchor at the start/end of +-- the statement's evaluation range. The output timestamps still follow the evaluation grid. +-- +-- Two series with different values, so that a dropped or merged series shows up as a wrong row. +-- Sample timestamps are in milliseconds, while `@` and `TQL EVAL` timestamps are in seconds. +-- The samples lie on the 10s subquery step grid. + +CREATE TABLE subquery_at_total ( + ts TIMESTAMP TIME INDEX, + host STRING PRIMARY KEY, + val DOUBLE, +); + +INSERT INTO subquery_at_total VALUES + (0, 'a', 1), + (10000, 'a', 2), + (20000, 'a', 3), + (30000, 'a', 4), + (40000, 'a', 5), + (50000, 'a', 6), + (60000, 'a', 7), + (0, 'b', 10), + (10000, 'b', 20), + (20000, 'b', 30), + (30000, 'b', 40), + (40000, 'b', 50), + (50000, 'b', 60), + (60000, 'b', 70); + +-- The range selector form folds (40s, 60s] at every step: 6 + 7 = 13 (and 130 for 'b'). +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time(subquery_at_total[20s] @ 60); + +-- The subquery with `@ 60` must report the same window at every step. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time(subquery_at_total[20s:10s] @ 60); + +-- Baseline without `@`: the window follows the grid, 3 + 4 at 30s and 6 + 7 at 60s. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time(subquery_at_total[20s:10s]); + +-- `@ end()` anchors at 60s: 13 at both steps. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time(subquery_at_total[20s:10s] @ end()); + +-- `@ start()` anchors at 30s: (10s, 30s] holds 3 + 4 = 7 at both steps. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time(subquery_at_total[20s:10s] @ start()); + +-- `offset` applies relative to the anchor, in either modifier order: `@ 60 offset 30s` folds +-- (10s, 30s] = 7 at both steps. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time(subquery_at_total[20s:10s] @ 60 offset 30s); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time(subquery_at_total[20s:10s] offset 30s @ 60); + +-- An instant query away from the anchor still folds the anchored window: 13. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 0, '1s') sum_over_time(subquery_at_total[20s:10s] @ 60); + +-- Other `*_over_time` functions over the anchored window (40s, 60s]: points 50s and 60s. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') max_over_time(subquery_at_total[20s:10s] @ 60); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') count_over_time(subquery_at_total[20s:10s] @ 60); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') avg_over_time(subquery_at_total[20s:10s] @ 60); + +-- `rate` over (30s, 60s] -- points 40s, 50s, 60s -- extrapolates from the anchored window, not +-- from the evaluation step, so the subquery form matches the range selector form at every step. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') rate(subquery_at_total[30s] @ 60); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') rate(subquery_at_total[30s:10s] @ 60); + +-- A computed inner expression and an aggregation above the anchored call. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum_over_time((subquery_at_total * 2)[20s:10s] @ 60); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (30, 60, '30s') sum(sum_over_time(subquery_at_total[20s:10s] @ 60)); + +DROP TABLE subquery_at_total; diff --git a/tests/cases/standalone/common/promql/tsid_binary_join_regression.result b/tests/cases/standalone/common/promql/tsid_binary_join_regression.result index 0288a045390..144f9defc35 100644 --- a/tests/cases/standalone/common/promql/tsid_binary_join_regression.result +++ b/tests/cases/standalone/common/promql/tsid_binary_join_regression.result @@ -110,6 +110,8 @@ Affected Rows: 4 -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / tsid_binary_join_right; +-+-+-+ @@ -127,13 +129,13 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / tsid_binary_join_right; |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -150,6 +152,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / tsid_binary_join_right; -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left + tsid_binary_join_right) / tsid_binary_join_left; +-+-+-+ @@ -167,13 +171,13 @@ TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left + tsid_binary_join_right) / tsid |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -189,6 +193,8 @@ TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left + tsid_binary_join_right) / tsid -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left + tsid_binary_join_right) * (tsid_binary_join_left - tsid_binary_join_third)) / (tsid_binary_join_left + 2); +-+-+-+ @@ -210,19 +216,19 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left + tsid_binary_join_right) * (ts |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -238,6 +244,8 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left + tsid_binary_join_right) * (ts -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / ignoring(host) tsid_binary_join_right; +-+-+-+ @@ -262,13 +270,13 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / ignoring(host) tsid_binary_join |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -284,6 +292,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / ignoring(host) tsid_binary_join -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / on(job) tsid_binary_join_right_by_job; +-+-+-+ @@ -308,13 +318,13 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / on(job) tsid_binary_join_right_ |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, job@3 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -331,6 +341,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / on(job) tsid_binary_join_right_ -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > tsid_binary_join_right; +-+-+-+ @@ -347,13 +359,13 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > tsid_binary_join_right; |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -369,6 +381,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > tsid_binary_join_right; -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > bool tsid_binary_join_right; +-+-+-+ @@ -386,13 +400,13 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > bool tsid_binary_join_right; |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -408,6 +422,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > bool tsid_binary_join_right; -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > tsid_binary_join_right) / tsid_binary_join_left) * 100; +-+-+-+ @@ -430,19 +446,19 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > tsid_binary_join_right) / tsi |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -457,6 +473,8 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > tsid_binary_join_right) / tsi -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > bool tsid_binary_join_right) + tsid_binary_join_left) / tsid_binary_join_left; +-+-+-+ @@ -485,25 +503,25 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > bool tsid_binary_join_right) |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -519,6 +537,8 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > bool tsid_binary_join_right) -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left or tsid_binary_join_right) / tsid_binary_join_left; +-+-+-+ @@ -541,7 +561,7 @@ TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left or tsid_binary_join_right) / tsi |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_ProjectionExec: expr=[ts@2 as ts, __tsid@3 as __tsid, greptime_value@4 as greptime_value, host@5 as host, job@6 as job, CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 ELSE_END as __promql_or_match_0, CASE WHEN __common_expr_2@1 IS NOT NULL THEN __common_expr_2@1 ELSE_END as __promql_or_match_1] REDACTED |_|_|_ProjectionExec: expr=[CAST(host@1 AS Utf8) as __common_expr_1, CAST(job@2 AS Utf8) as __common_expr_2, ts@4 as ts, __tsid@3 as __tsid, greptime_value@0 as greptime_value, host@1 as host, job@2 as job] REDACTED @@ -549,13 +569,13 @@ TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left or tsid_binary_join_right) / tsi |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -571,6 +591,8 @@ TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left or tsid_binary_join_right) / tsi -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left / ignoring(host) group_left tsid_binary_join_right) / tsid_binary_join_left; +-+-+-+ @@ -597,19 +619,19 @@ TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left / ignoring(host) group_left tsid |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries", "projection": ["ts", "greptime_value", "__tsid", "host", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(5000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ diff --git a/tests/cases/standalone/common/promql/tsid_binary_join_regression.sql b/tests/cases/standalone/common/promql/tsid_binary_join_regression.sql index 627680330aa..07c0d5bb606 100644 --- a/tests/cases/standalone/common/promql/tsid_binary_join_regression.sql +++ b/tests/cases/standalone/common/promql/tsid_binary_join_regression.sql @@ -93,6 +93,8 @@ INSERT INTO tsid_binary_join_third (host, job, ts, greptime_value) VALUES -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / tsid_binary_join_right; -- Repeated operands in a safe arithmetic island should be planned once and reused @@ -107,6 +109,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / tsid_binary_join_right; -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left + tsid_binary_join_right) / tsid_binary_join_left; -- A larger arithmetic island should still plan each distinct vector selector only once @@ -120,6 +124,8 @@ TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left + tsid_binary_join_right) / tsid -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left + tsid_binary_join_right) * (tsid_binary_join_left - tsid_binary_join_third)) / (tsid_binary_join_left + 2); -- Label modifiers must disable the TSID shortcut and keep matching on the remaining labels. @@ -133,6 +139,8 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left + tsid_binary_join_right) * (ts -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / ignoring(host) tsid_binary_join_right; -- `on(job)` must stay label-based when only the left side has extra row-key labels. @@ -146,6 +154,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / ignoring(host) tsid_binary_join -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / on(job) tsid_binary_join_right_by_job; -- Comparison filters can join on `__tsid`, but the filtered result must still behave like @@ -160,6 +170,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / on(job) tsid_binary_join_right_ -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > tsid_binary_join_right; -- `bool` comparison should follow the same TSID-backed matching path. @@ -173,6 +185,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > tsid_binary_join_right; -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > bool tsid_binary_join_right; -- Comparison filters are a barrier for binary island coalescing because they filter the @@ -186,6 +200,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > bool tsid_binary_join_right; -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > tsid_binary_join_right) / tsid_binary_join_left) * 100; -- Bool comparisons are intentionally outside the first island optimization version. @@ -198,6 +214,8 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > tsid_binary_join_right) / tsi -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > bool tsid_binary_join_right) + tsid_binary_join_left) / tsid_binary_join_left; -- Set operators are a barrier because they have distinct matching and output-domain @@ -211,6 +229,8 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > bool tsid_binary_join_right) -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left or tsid_binary_join_right) / tsid_binary_join_left; -- Group modifiers are many-to-one/one-to-many matching barriers and must stay on the @@ -224,6 +244,8 @@ TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left or tsid_binary_join_right) / tsi -- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left / ignoring(host) group_left tsid_binary_join_right) / tsid_binary_join_left; -- A group modifier that keeps the many side's whole tag set keeps its `__tsid` too, so the diff --git a/tests/cases/standalone/common/promql/tsid_histogram_quantile_regression.result b/tests/cases/standalone/common/promql/tsid_histogram_quantile_regression.result index 10b31cac1e1..e128850b241 100644 --- a/tests/cases/standalone/common/promql/tsid_histogram_quantile_regression.result +++ b/tests/cases/standalone/common/promql/tsid_histogram_quantile_regression.result @@ -103,6 +103,8 @@ TQL EVAL (0, 10, '5s') avg_over_time(tsid_no_aggr_metric[5s]); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') histogram_quantile(0.5, tsid_no_aggr_histogram_bucket); +-+-+-+ @@ -118,7 +120,7 @@ TQL ANALYZE (0, 10, '5s') histogram_quantile(0.5, tsid_no_aggr_histogram_bucket) |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[val@1 as val, job@3 as job, le@4 as le, __tsid@2 as __tsid, ts@0 as ts] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["ts", "val", "__tsid", "job", "le"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 3_| +-+-+-+ diff --git a/tests/cases/standalone/common/promql/tsid_histogram_quantile_regression.sql b/tests/cases/standalone/common/promql/tsid_histogram_quantile_regression.sql index 2c450d3bc23..7faea98007e 100644 --- a/tests/cases/standalone/common/promql/tsid_histogram_quantile_regression.sql +++ b/tests/cases/standalone/common/promql/tsid_histogram_quantile_regression.sql @@ -70,6 +70,8 @@ TQL EVAL (0, 10, '5s') avg_over_time(tsid_no_aggr_metric[5s]); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') histogram_quantile(0.5, tsid_no_aggr_histogram_bucket); -- SQLNESS SORT_RESULT 3 1 diff --git a/tests/cases/standalone/common/range/nest.result b/tests/cases/standalone/common/range/nest.result index e283c457526..2409155486c 100644 --- a/tests/cases/standalone/common/range/nest.result +++ b/tests/cases/standalone/common/range/nest.result @@ -147,6 +147,7 @@ EXPLAIN SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s'; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s'; +-+-+-+ @@ -157,7 +158,7 @@ EXPLAIN ANALYZE SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s'; |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "val"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 10_| +-+-+-+ diff --git a/tests/cases/standalone/common/range/nest.sql b/tests/cases/standalone/common/range/nest.sql index d92f9f1345f..49be120b887 100644 --- a/tests/cases/standalone/common/range/nest.sql +++ b/tests/cases/standalone/common/range/nest.sql @@ -84,6 +84,7 @@ EXPLAIN SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s'; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s'; DROP TABLE host; diff --git a/tests/cases/standalone/common/select/prune.result b/tests/cases/standalone/common/select/prune.result index 5e219a195db..36b636a514b 100644 --- a/tests/cases/standalone/common/select/prune.result +++ b/tests/cases/standalone/common/select/prune.result @@ -84,6 +84,7 @@ select * from demo where collector='disk' order by ts; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select * from demo where idc='idc1'; +-+-+-+ @@ -93,7 +94,7 @@ explain analyze select * from demo where idc='idc1'; |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "value", "host", "idc", "collector"], "filters": ["idc = Dictionary(UInt32, Utf8(\"idc1\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -112,6 +113,7 @@ SELECT * FROM demo where host in ('test1'); -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze SELECT * FROM demo where host in ('test1'); +-+-+-+ @@ -121,7 +123,7 @@ explain analyze SELECT * FROM demo where host in ('test1'); |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_CooperativeExec REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "value", "host", "idc", "collector"], "filters": ["host = Dictionary(UInt32, Utf8(\"test1\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ diff --git a/tests/cases/standalone/common/select/prune.sql b/tests/cases/standalone/common/select/prune.sql index 226c1cef69c..7a938e9c43e 100644 --- a/tests/cases/standalone/common/select/prune.sql +++ b/tests/cases/standalone/common/select/prune.sql @@ -25,6 +25,7 @@ select * from demo where collector='disk' order by ts; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select * from demo where idc='idc1'; SELECT * FROM demo where host in ('test1'); @@ -35,6 +36,7 @@ SELECT * FROM demo where host in ('test1'); -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze SELECT * FROM demo where host in ('test1'); drop table demo; diff --git a/tests/cases/standalone/common/select/skipping_index.result b/tests/cases/standalone/common/select/skipping_index.result index 3802e1d7df5..3131e68c4c7 100644 --- a/tests/cases/standalone/common/select/skipping_index.result +++ b/tests/cases/standalone/common/select/skipping_index.result @@ -51,6 +51,7 @@ ADMIN FLUSH_TABLE('skipping_table'); -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM skipping_table WHERE id = 'id2' ORDER BY `name`; +-+-+-+ @@ -62,7 +63,7 @@ EXPLAIN ANALYZE SELECT * FROM skipping_table WHERE id = 'id2' ORDER BY `name`; | 1_| 0_|_SortPreservingMergeExec: [name@2 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[name@2 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_FilterExec: id@1 = id2 REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":3, "mem_ranges":0, "files":3, "file_ranges":3} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":3, "mem_ranges":0, "files":3, "file_ranges":3}, "projection": ["ts", "id", "name"], "filters": ["id = Utf8(\"id2\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -73,6 +74,7 @@ EXPLAIN ANALYZE SELECT * FROM skipping_table WHERE id = 'id2' ORDER BY `name`; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM skipping_table WHERE id = 'id5' ORDER BY `name`; +-+-+-+ @@ -84,7 +86,7 @@ EXPLAIN ANALYZE SELECT * FROM skipping_table WHERE id = 'id5' ORDER BY `name`; | 1_| 0_|_SortPreservingMergeExec: [name@2 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[name@2 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_FilterExec: id@1 = id5 REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":3, "mem_ranges":0, "files":3, "file_ranges":3} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":3, "mem_ranges":0, "files":3, "file_ranges":3}, "projection": ["ts", "id", "name"], "filters": ["id = Utf8(\"id5\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ diff --git a/tests/cases/standalone/common/select/skipping_index.sql b/tests/cases/standalone/common/select/skipping_index.sql index b17d70f598f..a2ac365a525 100644 --- a/tests/cases/standalone/common/select/skipping_index.sql +++ b/tests/cases/standalone/common/select/skipping_index.sql @@ -25,6 +25,7 @@ ADMIN FLUSH_TABLE('skipping_table'); -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM skipping_table WHERE id = 'id2' ORDER BY `name`; -- SQLNESS REPLACE (metrics.*) REDACTED @@ -33,6 +34,7 @@ EXPLAIN ANALYZE SELECT * FROM skipping_table WHERE id = 'id2' ORDER BY `name`; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT * FROM skipping_table WHERE id = 'id5' ORDER BY `name`; DROP TABLE IF EXISTS skipping_table; diff --git a/tests/cases/standalone/common/select/tql_filter.result b/tests/cases/standalone/common/select/tql_filter.result index 45f85ead00d..d15fb075755 100644 --- a/tests/cases/standalone/common/select/tql_filter.result +++ b/tests/cases/standalone/common/select/tql_filter.result @@ -13,6 +13,7 @@ Affected Rows: 3 -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1, 3, '1s') t1{ a = "a" }; +-+-+-+ @@ -24,7 +25,7 @@ tql analyze (1, 3, '1s') t1{ a = "a" }; | 1_| 0_|_PromInstantManipulateExec: range=[1000..3000], lookback=[300000], interval=[1000], time index=[b] REDACTED |_|_|_PromSeriesDivideExec: tags=["a"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c"], "filters": ["a = Dictionary(UInt32, Utf8(\"a\"))", "b >= TimestampMillisecond(-298999, None)", "b <= TimestampMillisecond(3000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 3_| +-+-+-+ @@ -36,6 +37,7 @@ tql analyze (1, 3, '1s') t1{ a = "a" }; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1, 3, '1s') t1{ a =~ ".*" }; +-+-+-+ @@ -47,7 +49,7 @@ tql analyze (1, 3, '1s') t1{ a =~ ".*" }; | 1_| 0_|_PromInstantManipulateExec: range=[1000..3000], lookback=[300000], interval=[1000], time index=[b] REDACTED |_|_|_PromSeriesDivideExec: tags=["a"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c"], "filters": ["b >= TimestampMillisecond(-298999, None)", "b <= TimestampMillisecond(3000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 6_| +-+-+-+ @@ -59,6 +61,7 @@ tql analyze (1, 3, '1s') t1{ a =~ ".*" }; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1, 3, '1s') t1{ a =~ "a.*" }; +-+-+-+ @@ -70,7 +73,7 @@ tql analyze (1, 3, '1s') t1{ a =~ "a.*" }; | 1_| 0_|_PromInstantManipulateExec: range=[1000..3000], lookback=[300000], interval=[1000], time index=[b] REDACTED |_|_|_PromSeriesDivideExec: tags=["a"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["a", "b", "c"], "filters": ["b >= TimestampMillisecond(-298999, None)", "b <= TimestampMillisecond(3000, None)", "a ~ Utf8(\"^(?:a.*)$\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 3_| +-+-+-+ diff --git a/tests/cases/standalone/common/select/tql_filter.sql b/tests/cases/standalone/common/select/tql_filter.sql index 442efbdd145..325fdd06d9a 100644 --- a/tests/cases/standalone/common/select/tql_filter.sql +++ b/tests/cases/standalone/common/select/tql_filter.sql @@ -9,6 +9,7 @@ insert into t1 values ("a", 1000, 1.0), ("b", 2000, 2.0), ("c", 3000, 3.0); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1, 3, '1s') t1{ a = "a" }; -- SQLNESS REPLACE (metrics.*) REDACTED @@ -18,6 +19,7 @@ tql analyze (1, 3, '1s') t1{ a = "a" }; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1, 3, '1s') t1{ a =~ ".*" }; -- SQLNESS REPLACE (metrics.*) REDACTED @@ -27,6 +29,7 @@ tql analyze (1, 3, '1s') t1{ a =~ ".*" }; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (1, 3, '1s') t1{ a =~ "a.*" }; drop table t1; diff --git a/tests/cases/standalone/common/tql/partition.result b/tests/cases/standalone/common/tql/partition.result index ac3072342d7..04efb194222 100644 --- a/tests/cases/standalone/common/tql/partition.result +++ b/tests/cases/standalone/common/tql/partition.result @@ -14,6 +14,7 @@ Affected Rows: 0 -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (0, 10, '1s') 100 - (avg by (k) (irate(t[1m])) * 100); +-+-+-+ @@ -32,7 +33,7 @@ tql analyze (0, 10, '1s') 100 - (avg by (k) (irate(t[1m])) * 100); |_|_|_PromRangeManipulateExec: req range=[0..10000], interval=[1000], eval range=[60000], time index=[j] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["k"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k"], "filters": ["j >= TimestampMillisecond(-59999, None)", "j <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -59,6 +60,7 @@ Affected Rows: 0 -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (0, 10, '1s') 100 - (avg by (k) (irate(t[1m])) * 100); +-+-+-+ @@ -82,7 +84,7 @@ tql analyze (0, 10, '1s') 100 - (avg by (k) (irate(t[1m])) * 100); |_|_|_PromRangeManipulateExec: req range=[0..10000], interval=[1000], eval range=[60000], time index=[j] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["k", "l"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k", "l"], "filters": ["j >= TimestampMillisecond(-59999, None)", "j <= TimestampMillisecond(10000, None)", "k < Utf8(\"a\") OR k IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[k@0 as k, j@1 as j], aggr=[__avg_state(prom_irate(j_range,i))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED @@ -92,7 +94,7 @@ tql analyze (0, 10, '1s') 100 - (avg by (k) (irate(t[1m])) * 100); |_|_|_PromRangeManipulateExec: req range=[0..10000], interval=[1000], eval range=[60000], time index=[j] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["k", "l"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k", "l"], "filters": ["j >= TimestampMillisecond(-59999, None)", "j <= TimestampMillisecond(10000, None)", "k >= Utf8(\"a\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -119,6 +121,7 @@ Affected Rows: 0 -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (0, 10, '1s') 100 - (avg by (k) (irate(t[1m])) * 100); +-+-+-+ @@ -141,10 +144,10 @@ tql analyze (0, 10, '1s') 100 - (avg by (k) (irate(t[1m])) * 100); |_|_|_MergeScanExec: REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [k@2 ASC, l@3 ASC, j@1 ASC] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k", "l"], "filters": ["j >= TimestampMillisecond(-59999, None)", "j <= TimestampMillisecond(10000, None)", "i < Float64(1) OR i IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_SortPreservingMergeExec: [k@2 ASC, l@3 ASC, j@1 ASC] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k", "l"], "filters": ["j >= TimestampMillisecond(-59999, None)", "j <= TimestampMillisecond(10000, None)", "i >= Float64(1)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ diff --git a/tests/cases/standalone/common/tql/partition.sql b/tests/cases/standalone/common/tql/partition.sql index 332caa4252e..d68ce313e87 100644 --- a/tests/cases/standalone/common/tql/partition.sql +++ b/tests/cases/standalone/common/tql/partition.sql @@ -12,6 +12,7 @@ create table t ( -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (0, 10, '1s') 100 - (avg by (k) (irate(t[1m])) * 100); drop table t; @@ -32,6 +33,7 @@ create table t ( -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (0, 10, '1s') 100 - (avg by (k) (irate(t[1m])) * 100); drop table t; @@ -52,6 +54,7 @@ create table t ( -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED tql analyze (0, 10, '1s') 100 - (avg by (k) (irate(t[1m])) * 100); drop table t; diff --git a/tests/cases/standalone/optimizer/count.result b/tests/cases/standalone/optimizer/count.result index 925e89eeb23..1ffeca4b69e 100644 --- a/tests/cases/standalone/optimizer/count.result +++ b/tests/cases/standalone/optimizer/count.result @@ -89,6 +89,7 @@ Affected Rows: 5 -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (\{count\[count\]:)\d+(\}) {count[count]:REDACTED} -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug; @@ -126,6 +127,7 @@ select count(1) from count_where_bug; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where `tag` = 'b'; @@ -138,7 +140,7 @@ select count(1) from count_where_bug where `tag` = 'b'; | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(count_where_bug.ts) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(count_where_bug.ts) as count(Int64(1))] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts"], "filters": ["tag = Dictionary(UInt32, Utf8(\"b\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -167,6 +169,7 @@ select count(1) from count_where_bug where `tag` = 'b'; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (\{count\[count\]:)\d+(\}) {count[count]:REDACTED} -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; @@ -179,7 +182,7 @@ select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[count(count_where_bug.ts) as count(Int64(1))] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(count_where_bug.ts) as count(Int64(1))] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["ts > TimestampMillisecond(1725602404000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -206,6 +209,7 @@ select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where num != 3; @@ -219,7 +223,7 @@ select count(1) from count_where_bug where num != 3; |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(count_where_bug.ts) as count(Int64(1))] REDACTED |_|_|_FilterExec: num@1 != 3, projection=[ts@0] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "num"], "filters": ["num != Int64(3)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -269,6 +273,7 @@ Affected Rows: 5 -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (\{count\[count\]:)\d+(\}) {count[count]:REDACTED} -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug; @@ -305,6 +310,7 @@ select count(1) from count_where_bug; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where `tag` = 'b'; @@ -320,7 +326,7 @@ select count(1) from count_where_bug where `tag` = 'b'; | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts"], "filters": ["tag = Dictionary(UInt32, Utf8(\"b\"))", "tag > Utf8(\"a\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -332,6 +338,7 @@ select count(1) from count_where_bug where `tag` = 'b'; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where `tag` = 'b'; @@ -347,7 +354,7 @@ select count(1) from count_where_bug where `tag` = 'b'; | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts"], "filters": ["tag = Dictionary(UInt32, Utf8(\"b\"))", "tag > Utf8(\"a\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -369,6 +376,7 @@ select count(1) from count_where_bug where `tag` = 'b'; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (\{count\[count]:)\d+(\}) {count[count]:REDACTED} -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; @@ -384,12 +392,12 @@ select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["ts > TimestampMillisecond(1725602404000, None)", "tag <= Utf8(\"a\") OR tag IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["ts > TimestampMillisecond(1725602404000, None)", "tag > Utf8(\"a\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -409,6 +417,7 @@ select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where num != 3; @@ -425,13 +434,13 @@ select count(1) from count_where_bug where num != 3; |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED |_|_|_FilterExec: num@1 != 3, projection=[ts@0] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "num"], "filters": ["num != Int64(3)", "tag <= Utf8(\"a\") OR tag IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(count_where_bug.ts)] REDACTED |_|_|_FilterExec: num@1 != 3, projection=[ts@0] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "num"], "filters": ["num != Int64(3)", "tag > Utf8(\"a\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ diff --git a/tests/cases/standalone/optimizer/count.sql b/tests/cases/standalone/optimizer/count.sql index caa961b1f01..fc2494589f8 100644 --- a/tests/cases/standalone/optimizer/count.sql +++ b/tests/cases/standalone/optimizer/count.sql @@ -50,6 +50,7 @@ values ('a', '2024-09-06T06:00:01Z', 1), -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (\{count\[count\]:)\d+(\}) {count[count]:REDACTED} -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug; @@ -69,6 +70,7 @@ select count(1) from count_where_bug; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where `tag` = 'b'; @@ -90,6 +92,7 @@ select count(1) from count_where_bug where `tag` = 'b'; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (\{count\[count\]:)\d+(\}) {count[count]:REDACTED} -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; @@ -109,6 +112,7 @@ select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where num != 3; @@ -145,6 +149,7 @@ values ('a', '2024-09-06T06:00:01Z', 1), -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (\{count\[count\]:)\d+(\}) {count[count]:REDACTED} -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug; @@ -157,6 +162,7 @@ select count(1) from count_where_bug; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where `tag` = 'b'; @@ -167,6 +173,7 @@ select count(1) from count_where_bug where `tag` = 'b'; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where `tag` = 'b'; @@ -181,6 +188,7 @@ select count(1) from count_where_bug where `tag` = 'b'; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (\{count\[count]:)\d+(\}) {count[count]:REDACTED} -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; @@ -193,6 +201,7 @@ select count(1) from count_where_bug where ts > '2024-09-06T06:00:04Z'; -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select count(1) from count_where_bug where num != 3; diff --git a/tests/cases/standalone/optimizer/filter_column_pruning.result b/tests/cases/standalone/optimizer/filter_column_pruning.result index 3a16bef50df..f966b0504e4 100644 --- a/tests/cases/standalone/optimizer/filter_column_pruning.result +++ b/tests/cases/standalone/optimizer/filter_column_pruning.result @@ -44,6 +44,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' ORDER BY ts; +-+-+-+ @@ -57,7 +58,7 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'ho |_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED |_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))"], "flat_format": REDACTED, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 3_| +-+-+-+ @@ -80,6 +81,7 @@ SELECT mem_usage, cpu_usage FROM filter_prune_test WHERE host = 'host1' AND `reg -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT mem_usage, cpu_usage FROM filter_prune_test WHERE host = 'host1' AND `region` = 'us-east' ORDER BY ts; +-+-+-+ @@ -93,7 +95,7 @@ EXPLAIN ANALYZE VERBOSE SELECT mem_usage, cpu_usage FROM filter_prune_test WHERE |_|_|_ProjectionExec: expr=[mem_usage@2 as mem_usage, cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED |_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage", "mem_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], "flat_format": REDACTED, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage", "mem_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -118,6 +120,7 @@ SELECT host, `region` FROM filter_prune_test WHERE cpu_usage > 20.0 ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT host, `region` FROM filter_prune_test WHERE cpu_usage > 20.0 ORDER BY ts; +-+-+-+ @@ -131,7 +134,7 @@ EXPLAIN ANALYZE VERBOSE SELECT host, `region` FROM filter_prune_test WHERE cpu_u |_|_|_WindowedSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_FilterExec: cpu_usage@3 > 20, projection=[host@1, region@2, ts@0] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["cpu_usage > Float64(20)"], "flat_format": REDACTED, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["cpu_usage > Float64(20)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -156,6 +159,7 @@ SELECT host, cpu_usage FROM filter_prune_test WHERE ts > 2000 ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT host, cpu_usage FROM filter_prune_test WHERE ts > 2000 ORDER BY ts; +-+-+-+ @@ -169,7 +173,7 @@ EXPLAIN ANALYZE VERBOSE SELECT host, cpu_usage FROM filter_prune_test WHERE ts > |_|_|_ProjectionExec: expr=[host@1 as host, cpu_usage@2 as cpu_usage, ts@0 as ts] REDACTED |_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "cpu_usage"], "filters": ["ts > TimestampMillisecond(2000, None)"], "flat_format": REDACTED, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "cpu_usage"], "filters": ["ts > TimestampMillisecond(2000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -192,6 +196,7 @@ SELECT mem_usage FROM filter_prune_test WHERE host = 'host1' AND cpu_usage > 10. -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT mem_usage FROM filter_prune_test WHERE host = 'host1' AND cpu_usage > 10.0 AND `region` = 'us-east' ORDER BY ts; +-+-+-+ @@ -205,7 +210,7 @@ EXPLAIN ANALYZE VERBOSE SELECT mem_usage FROM filter_prune_test WHERE host = 'ho |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_FilterExec: cpu_usage@1 > 10, projection=[mem_usage@2, ts@0] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage", "mem_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "cpu_usage > Float64(10)", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], "flat_format": REDACTED, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage", "mem_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "cpu_usage > Float64(10)", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -227,6 +232,7 @@ SELECT SUM(cpu_usage) as total_cpu FROM filter_prune_test WHERE host = 'host1'; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT SUM(cpu_usage) as total_cpu FROM filter_prune_test WHERE host = 'host1'; +-+-+-+ @@ -239,7 +245,7 @@ EXPLAIN ANALYZE VERBOSE SELECT SUM(cpu_usage) as total_cpu FROM filter_prune_tes |_|_|_AggregateExec: mode=Final, gby=[], aggr=[sum(filter_prune_test.cpu_usage)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[sum(filter_prune_test.cpu_usage)] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))"], "flat_format": REDACTED, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -263,6 +269,7 @@ SELECT `region`, AVG(cpu_usage) as avg_cpu FROM filter_prune_test WHERE mem_usag -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT `region`, AVG(cpu_usage) as avg_cpu FROM filter_prune_test WHERE mem_usage > 25.0 GROUP BY `region` ORDER BY `region`; +-+-+-+ @@ -278,7 +285,7 @@ EXPLAIN ANALYZE VERBOSE SELECT `region`, AVG(cpu_usage) as avg_cpu FROM filter_p |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[region@0 as region], aggr=[avg(filter_prune_test.cpu_usage)] REDACTED |_|_|_FilterExec: mem_usage@2 > 25, projection=[region@0, cpu_usage@1] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["region", "cpu_usage", "mem_usage"], "filters": ["mem_usage > Float64(25)"], "flat_format": REDACTED, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["region", "cpu_usage", "mem_usage"], "filters": ["mem_usage > Float64(25)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 3_| +-+-+-+ @@ -304,6 +311,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host IN ('host1', 'host2') ORDER B -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host IN ('host1', 'host2') ORDER BY ts; +-+-+-+ @@ -317,7 +325,7 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host IN (' |_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED |_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\")) OR host = Dictionary(UInt32, Utf8(\"host2\"))"], "flat_format": REDACTED, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\")) OR host = Dictionary(UInt32, Utf8(\"host2\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ @@ -341,6 +349,7 @@ SELECT host FROM filter_prune_test WHERE cpu_usage BETWEEN 15.0 AND 30.0 ORDER B -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT host FROM filter_prune_test WHERE cpu_usage BETWEEN 15.0 AND 30.0 ORDER BY ts; +-+-+-+ @@ -354,7 +363,7 @@ EXPLAIN ANALYZE VERBOSE SELECT host FROM filter_prune_test WHERE cpu_usage BETWE |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_FilterExec: cpu_usage@2 >= 15 AND cpu_usage@2 <= 30, projection=[host@1, ts@0] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "cpu_usage"], "filters": ["cpu_usage >= Float64(15)", "cpu_usage <= Float64(30)"], "flat_format": REDACTED, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "cpu_usage"], "filters": ["cpu_usage >= Float64(15)", "cpu_usage <= Float64(30)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 3_| +-+-+-+ @@ -380,6 +389,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host LIKE 'host%' AND `region` LIK -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host LIKE 'host%' AND `region` LIKE 'us-%' ORDER BY ts; +-+-+-+ @@ -393,7 +403,7 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host LIKE |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_FilterExec: host@1 LIKE host% AND region@2 LIKE us-%, projection=[cpu_usage@3, ts@0] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["host LIKE Dictionary(UInt32, Utf8(\"host%\"))", "region LIKE Dictionary(UInt32, Utf8(\"us-%\"))"], "flat_format": REDACTED, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["host LIKE Dictionary(UInt32, Utf8(\"host%\"))", "region LIKE Dictionary(UInt32, Utf8(\"us-%\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ @@ -418,6 +428,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' OR `region` = 'eu-w -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' OR `region` = 'eu-west' ORDER BY ts; +-+-+-+ @@ -431,7 +442,7 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'ho |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_FilterExec: host@1 = host1 OR region@2 = eu-west, projection=[cpu_usage@3, ts@0] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\")) OR region = Dictionary(UInt32, Utf8(\"eu-west\"))"], "flat_format": REDACTED, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\")) OR region = Dictionary(UInt32, Utf8(\"eu-west\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -454,6 +465,7 @@ SELECT cpu_usage FROM (SELECT * FROM filter_prune_test WHERE host = 'host1') sub -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM (SELECT * FROM filter_prune_test WHERE host = 'host1') sub WHERE `region` = 'us-east' ORDER BY ts; +-+-+-+ @@ -467,7 +479,7 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM (SELECT * FROM filter_prune_test W |_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED |_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], "flat_format": REDACTED, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -501,6 +513,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' ORDER BY ts; +-+-+-+ @@ -514,7 +527,7 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'ho |_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED |_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 3_| +-+-+-+ @@ -538,6 +551,7 @@ SELECT mem_usage, cpu_usage FROM filter_prune_test WHERE host = 'host1' AND `reg -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT mem_usage, cpu_usage FROM filter_prune_test WHERE host = 'host1' AND `region` = 'us-east' ORDER BY ts; +-+-+-+ @@ -551,7 +565,7 @@ EXPLAIN ANALYZE VERBOSE SELECT mem_usage, cpu_usage FROM filter_prune_test WHERE |_|_|_ProjectionExec: expr=[mem_usage@2 as mem_usage, cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED |_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage", "mem_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage", "mem_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -577,6 +591,7 @@ SELECT host, `region` FROM filter_prune_test WHERE cpu_usage > 20.0 ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT host, `region` FROM filter_prune_test WHERE cpu_usage > 20.0 ORDER BY ts; +-+-+-+ @@ -590,7 +605,7 @@ EXPLAIN ANALYZE VERBOSE SELECT host, `region` FROM filter_prune_test WHERE cpu_u |_|_|_WindowedSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_FilterExec: cpu_usage@3 > 20, projection=[host@1, region@2, ts@0] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["cpu_usage > Float64(20)"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["cpu_usage > Float64(20)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -616,6 +631,7 @@ SELECT host, cpu_usage FROM filter_prune_test WHERE ts > 2000 ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT host, cpu_usage FROM filter_prune_test WHERE ts > 2000 ORDER BY ts; +-+-+-+ @@ -629,7 +645,7 @@ EXPLAIN ANALYZE VERBOSE SELECT host, cpu_usage FROM filter_prune_test WHERE ts > |_|_|_ProjectionExec: expr=[host@1 as host, cpu_usage@2 as cpu_usage, ts@0 as ts] REDACTED |_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "cpu_usage"], "filters": ["ts > TimestampMillisecond(2000, None)"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "cpu_usage"], "filters": ["ts > TimestampMillisecond(2000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -653,6 +669,7 @@ SELECT mem_usage FROM filter_prune_test WHERE host = 'host1' AND cpu_usage > 10. -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT mem_usage FROM filter_prune_test WHERE host = 'host1' AND cpu_usage > 10.0 AND `region` = 'us-east' ORDER BY ts; +-+-+-+ @@ -666,7 +683,7 @@ EXPLAIN ANALYZE VERBOSE SELECT mem_usage FROM filter_prune_test WHERE host = 'ho |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_FilterExec: cpu_usage@1 > 10, projection=[mem_usage@2, ts@0] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage", "mem_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "cpu_usage > Float64(10)", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage", "mem_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "cpu_usage > Float64(10)", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -689,6 +706,7 @@ SELECT SUM(cpu_usage) as total_cpu FROM filter_prune_test WHERE host = 'host1'; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT SUM(cpu_usage) as total_cpu FROM filter_prune_test WHERE host = 'host1'; +-+-+-+ @@ -701,7 +719,7 @@ EXPLAIN ANALYZE VERBOSE SELECT SUM(cpu_usage) as total_cpu FROM filter_prune_tes |_|_|_AggregateExec: mode=Final, gby=[], aggr=[sum(filter_prune_test.cpu_usage)] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[sum(filter_prune_test.cpu_usage)] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -726,6 +744,7 @@ SELECT `region`, AVG(cpu_usage) as avg_cpu FROM filter_prune_test WHERE mem_usag -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT `region`, AVG(cpu_usage) as avg_cpu FROM filter_prune_test WHERE mem_usage > 25.0 GROUP BY `region` ORDER BY `region`; +-+-+-+ @@ -741,7 +760,7 @@ EXPLAIN ANALYZE VERBOSE SELECT `region`, AVG(cpu_usage) as avg_cpu FROM filter_p |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[region@0 as region], aggr=[avg(filter_prune_test.cpu_usage)] REDACTED |_|_|_FilterExec: mem_usage@2 > 25, projection=[region@0, cpu_usage@1] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["region", "cpu_usage", "mem_usage"], "filters": ["mem_usage > Float64(25)"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["region", "cpu_usage", "mem_usage"], "filters": ["mem_usage > Float64(25)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 3_| +-+-+-+ @@ -768,6 +787,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host IN ('host1', 'host2') ORDER B -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host IN ('host1', 'host2') ORDER BY ts; +-+-+-+ @@ -781,7 +801,7 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host IN (' |_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED |_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\")) OR host = Dictionary(UInt32, Utf8(\"host2\"))"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\")) OR host = Dictionary(UInt32, Utf8(\"host2\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ @@ -806,6 +826,7 @@ SELECT host FROM filter_prune_test WHERE cpu_usage BETWEEN 15.0 AND 30.0 ORDER B -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT host FROM filter_prune_test WHERE cpu_usage BETWEEN 15.0 AND 30.0 ORDER BY ts; +-+-+-+ @@ -819,7 +840,7 @@ EXPLAIN ANALYZE VERBOSE SELECT host FROM filter_prune_test WHERE cpu_usage BETWE |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_FilterExec: cpu_usage@2 >= 15 AND cpu_usage@2 <= 30, projection=[host@1, ts@0] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "cpu_usage"], "filters": ["cpu_usage >= Float64(15)", "cpu_usage <= Float64(30)"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "cpu_usage"], "filters": ["cpu_usage >= Float64(15)", "cpu_usage <= Float64(30)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 3_| +-+-+-+ @@ -846,6 +867,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host LIKE 'host%' AND `region` LIK -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host LIKE 'host%' AND `region` LIKE 'us-%' ORDER BY ts; +-+-+-+ @@ -859,7 +881,7 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host LIKE |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_FilterExec: host@1 LIKE host% AND region@2 LIKE us-%, projection=[cpu_usage@3, ts@0] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["host LIKE Dictionary(UInt32, Utf8(\"host%\"))", "region LIKE Dictionary(UInt32, Utf8(\"us-%\"))"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["host LIKE Dictionary(UInt32, Utf8(\"host%\"))", "region LIKE Dictionary(UInt32, Utf8(\"us-%\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ @@ -885,6 +907,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' OR `region` = 'eu-w -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' OR `region` = 'eu-west' ORDER BY ts; +-+-+-+ @@ -898,7 +921,7 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'ho |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_FilterExec: host@1 = host1 OR region@2 = eu-west, projection=[cpu_usage@3, ts@0] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\")) OR region = Dictionary(UInt32, Utf8(\"eu-west\"))"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "region", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\")) OR region = Dictionary(UInt32, Utf8(\"eu-west\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -922,6 +945,7 @@ SELECT cpu_usage FROM (SELECT * FROM filter_prune_test WHERE host = 'host1') sub -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM (SELECT * FROM filter_prune_test WHERE host = 'host1') sub WHERE `region` = 'us-east' ORDER BY ts; +-+-+-+ @@ -935,7 +959,7 @@ EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM (SELECT * FROM filter_prune_test W |_|_|_ProjectionExec: expr=[cpu_usage@1 as cpu_usage, ts@0 as ts] REDACTED |_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "cpu_usage"], "filters": ["host = Dictionary(UInt32, Utf8(\"host1\"))", "region = Dictionary(UInt32, Utf8(\"us-east\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -1011,6 +1035,7 @@ SELECT field1 FROM filter_prune_files WHERE tag_key = 'a' ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT field1 FROM filter_prune_files WHERE tag_key = 'a' ORDER BY ts; +-+-+-+ @@ -1024,7 +1049,7 @@ EXPLAIN ANALYZE VERBOSE SELECT field1 FROM filter_prune_files WHERE tag_key = 'a |_|_|_ProjectionExec: expr=[field1@1 as field1, ts@0 as ts] REDACTED |_|_|_WindowedSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@0 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":3, "mem_ranges":1, "files":2, "file_ranges":2}, "projection": ["ts", "field1"], "filters": ["tag_key = Dictionary(UInt32, Utf8(\"a\"))"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":3, "mem_ranges":1, "files":2, "file_ranges":2}, "projection": ["ts", "field1"], "filters": ["tag_key = Dictionary(UInt32, Utf8(\"a\"))"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -1051,6 +1076,7 @@ SELECT field1 FROM filter_prune_files WHERE field2 > 5.0 ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT field1 FROM filter_prune_files WHERE field2 > 5.0 ORDER BY ts; +-+-+-+ @@ -1064,7 +1090,7 @@ EXPLAIN ANALYZE VERBOSE SELECT field1 FROM filter_prune_files WHERE field2 > 5.0 |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_FilterExec: field2@2 > 5, projection=[field1@1, ts@0] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":3, "mem_ranges":1, "files":2, "file_ranges":2}, "projection": ["ts", "field1", "field2"], "filters": ["field2 > Float64(5)"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":3, "mem_ranges":1, "files":2, "file_ranges":2}, "projection": ["ts", "field1", "field2"], "filters": ["field2 > Float64(5)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ @@ -1088,6 +1114,7 @@ SELECT field3 FROM filter_prune_files WHERE tag_key = 'b' AND field2 > 7.0 ORDER -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT field3 FROM filter_prune_files WHERE tag_key = 'b' AND field2 > 7.0 ORDER BY ts; +-+-+-+ @@ -1101,7 +1128,7 @@ EXPLAIN ANALYZE VERBOSE SELECT field3 FROM filter_prune_files WHERE tag_key = 'b |_|_|_WindowedSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=ts@1 ASC NULLS LAST num_ranges=REDACTED REDACTED |_|_|_FilterExec: field2@1 > 7, projection=[field3@2, ts@0] REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":3, "mem_ranges":1, "files":2, "file_ranges":2}, "projection": ["ts", "field2", "field3"], "filters": ["tag_key = Dictionary(UInt32, Utf8(\"b\"))", "field2 > Float64(7)"], \"file\":REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":{"count":3, "mem_ranges":1, "files":2, "file_ranges":2}, "projection": ["ts", "field2", "field3"], "filters": ["tag_key = Dictionary(UInt32, Utf8(\"b\"))", "field2 > Float64(7)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ diff --git a/tests/cases/standalone/optimizer/filter_column_pruning.sql b/tests/cases/standalone/optimizer/filter_column_pruning.sql index a71db303631..5c8f73c0fff 100644 --- a/tests/cases/standalone/optimizer/filter_column_pruning.sql +++ b/tests/cases/standalone/optimizer/filter_column_pruning.sql @@ -33,6 +33,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' ORDER BY ts; -- Filter on multiple columns (host, region) but only select value columns @@ -46,6 +47,7 @@ SELECT mem_usage, cpu_usage FROM filter_prune_test WHERE host = 'host1' AND `reg -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT mem_usage, cpu_usage FROM filter_prune_test WHERE host = 'host1' AND `region` = 'us-east' ORDER BY ts; -- Filter on value column (cpu_usage) not in projection @@ -59,6 +61,7 @@ SELECT host, `region` FROM filter_prune_test WHERE cpu_usage > 20.0 ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT host, `region` FROM filter_prune_test WHERE cpu_usage > 20.0 ORDER BY ts; -- Filter on time index but time index not in projection @@ -72,6 +75,7 @@ SELECT host, cpu_usage FROM filter_prune_test WHERE ts > 2000 ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT host, cpu_usage FROM filter_prune_test WHERE ts > 2000 ORDER BY ts; -- Complex filter: multiple columns in WHERE, only one in SELECT @@ -85,6 +89,7 @@ SELECT mem_usage FROM filter_prune_test WHERE host = 'host1' AND cpu_usage > 10. -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT mem_usage FROM filter_prune_test WHERE host = 'host1' AND cpu_usage > 10.0 AND `region` = 'us-east' ORDER BY ts; -- Aggregation with filter on non-projected column @@ -98,6 +103,7 @@ SELECT SUM(cpu_usage) as total_cpu FROM filter_prune_test WHERE host = 'host1'; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT SUM(cpu_usage) as total_cpu FROM filter_prune_test WHERE host = 'host1'; -- GROUP BY with filter on column not in projection or grouping @@ -111,6 +117,7 @@ SELECT `region`, AVG(cpu_usage) as avg_cpu FROM filter_prune_test WHERE mem_usag -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT `region`, AVG(cpu_usage) as avg_cpu FROM filter_prune_test WHERE mem_usage > 25.0 GROUP BY `region` ORDER BY `region`; -- Filter with IN clause on non-projected column @@ -124,6 +131,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host IN ('host1', 'host2') ORDER B -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host IN ('host1', 'host2') ORDER BY ts; -- Filter with BETWEEN on non-projected column @@ -137,6 +145,7 @@ SELECT host FROM filter_prune_test WHERE cpu_usage BETWEEN 15.0 AND 30.0 ORDER B -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT host FROM filter_prune_test WHERE cpu_usage BETWEEN 15.0 AND 30.0 ORDER BY ts; -- Filter with LIKE on non-projected column @@ -150,6 +159,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host LIKE 'host%' AND `region` LIK -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host LIKE 'host%' AND `region` LIKE 'us-%' ORDER BY ts; -- Filter with OR conditions on non-projected columns @@ -163,6 +173,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' OR `region` = 'eu-w -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' OR `region` = 'eu-west' ORDER BY ts; -- Subquery with filter on non-projected column @@ -176,6 +187,7 @@ SELECT cpu_usage FROM (SELECT * FROM filter_prune_test WHERE host = 'host1') sub -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM (SELECT * FROM filter_prune_test WHERE host = 'host1') sub WHERE `region` = 'us-east' ORDER BY ts; -- Repeat with data flushed to verify column pruning works with on-disk data @@ -194,6 +206,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' ORDER BY ts; -- Filter on multiple columns (host, region) but only select value columns @@ -208,6 +221,7 @@ SELECT mem_usage, cpu_usage FROM filter_prune_test WHERE host = 'host1' AND `reg -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT mem_usage, cpu_usage FROM filter_prune_test WHERE host = 'host1' AND `region` = 'us-east' ORDER BY ts; -- Filter on value column (cpu_usage) not in projection @@ -222,6 +236,7 @@ SELECT host, `region` FROM filter_prune_test WHERE cpu_usage > 20.0 ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT host, `region` FROM filter_prune_test WHERE cpu_usage > 20.0 ORDER BY ts; -- Filter on time index but time index not in projection @@ -236,6 +251,7 @@ SELECT host, cpu_usage FROM filter_prune_test WHERE ts > 2000 ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT host, cpu_usage FROM filter_prune_test WHERE ts > 2000 ORDER BY ts; -- Complex filter: multiple columns in WHERE, only one in SELECT @@ -250,6 +266,7 @@ SELECT mem_usage FROM filter_prune_test WHERE host = 'host1' AND cpu_usage > 10. -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT mem_usage FROM filter_prune_test WHERE host = 'host1' AND cpu_usage > 10.0 AND `region` = 'us-east' ORDER BY ts; -- Aggregation with filter on non-projected column @@ -264,6 +281,7 @@ SELECT SUM(cpu_usage) as total_cpu FROM filter_prune_test WHERE host = 'host1'; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT SUM(cpu_usage) as total_cpu FROM filter_prune_test WHERE host = 'host1'; -- GROUP BY with filter on column not in projection or grouping @@ -278,6 +296,7 @@ SELECT `region`, AVG(cpu_usage) as avg_cpu FROM filter_prune_test WHERE mem_usag -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT `region`, AVG(cpu_usage) as avg_cpu FROM filter_prune_test WHERE mem_usage > 25.0 GROUP BY `region` ORDER BY `region`; -- Filter with IN clause on non-projected column @@ -292,6 +311,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host IN ('host1', 'host2') ORDER B -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host IN ('host1', 'host2') ORDER BY ts; -- Filter with BETWEEN on non-projected column @@ -306,6 +326,7 @@ SELECT host FROM filter_prune_test WHERE cpu_usage BETWEEN 15.0 AND 30.0 ORDER B -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT host FROM filter_prune_test WHERE cpu_usage BETWEEN 15.0 AND 30.0 ORDER BY ts; -- Filter with LIKE on non-projected column @@ -320,6 +341,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host LIKE 'host%' AND `region` LIK -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host LIKE 'host%' AND `region` LIKE 'us-%' ORDER BY ts; -- Filter with OR conditions on non-projected columns @@ -334,6 +356,7 @@ SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' OR `region` = 'eu-w -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM filter_prune_test WHERE host = 'host1' OR `region` = 'eu-west' ORDER BY ts; -- Subquery with filter on non-projected column @@ -348,6 +371,7 @@ SELECT cpu_usage FROM (SELECT * FROM filter_prune_test WHERE host = 'host1') sub -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT cpu_usage FROM (SELECT * FROM filter_prune_test WHERE host = 'host1') sub WHERE `region` = 'us-east' ORDER BY ts; DROP TABLE filter_prune_test; @@ -392,6 +416,7 @@ SELECT field1 FROM filter_prune_files WHERE tag_key = 'a' ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT field1 FROM filter_prune_files WHERE tag_key = 'a' ORDER BY ts; -- Query: filter on field2 (not in projection), select only field1 @@ -406,6 +431,7 @@ SELECT field1 FROM filter_prune_files WHERE field2 > 5.0 ORDER BY ts; -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT field1 FROM filter_prune_files WHERE field2 > 5.0 ORDER BY ts; -- Complex: filter on tag and field2, select only field3 @@ -420,6 +446,7 @@ SELECT field3 FROM filter_prune_files WHERE tag_key = 'b' AND field2 > 7.0 ORDER -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED -- SQLNESS REPLACE (RepartitionExec:.*) RepartitionExec: REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT field3 FROM filter_prune_files WHERE tag_key = 'b' AND field2 > 7.0 ORDER BY ts; DROP TABLE filter_prune_files; diff --git a/tests/cases/standalone/optimizer/first_value_advance.result b/tests/cases/standalone/optimizer/first_value_advance.result index f3b24db6d45..4ee877d14b7 100644 --- a/tests/cases/standalone/optimizer/first_value_advance.result +++ b/tests/cases/standalone/optimizer/first_value_advance.result @@ -95,6 +95,8 @@ explain select -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts), @@ -113,7 +115,7 @@ explain analyze |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts", "host", "not_pk", "val"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -153,6 +155,8 @@ explain select first_value(ts order by ts) from t; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(ts order by ts) from t; @@ -165,7 +169,7 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[first_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1} REDACTED +|_|_|_UnorderedScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "projection": ["ts"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -255,6 +259,8 @@ order by ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts) as ordered_host, @@ -276,7 +282,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "not_pk", "val"], "filters": ["host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ordered_host@0 ASC NULLS LAST], preserve_REDACTED @@ -284,7 +290,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "not_pk", "val"], "filters": ["host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ordered_host@0 ASC NULLS LAST], preserve_REDACTED @@ -292,7 +298,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[first_value(t.host) ORDER BY [t.ts ASC NULLS LAST], first_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], first_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "not_pk", "val"], "filters": ["host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -337,6 +343,8 @@ explain select first_value(ts order by ts) from t; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(ts order by ts) from t; @@ -352,17 +360,17 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -428,6 +436,8 @@ order by time_window, ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts) as ordered_host, @@ -453,17 +463,17 @@ order by time_window, ordered_host; | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@0) as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "val"], "filters": ["host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@0) as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "val"], "filters": ["host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@0) as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__first_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __first_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "val"], "filters": ["host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -541,6 +551,8 @@ explain -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(ts order by ts) from t1; @@ -556,17 +568,17 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["__table_id=UInt32(REDACTED)", "host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -624,6 +636,8 @@ order by ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts) as ordered_host, @@ -644,7 +658,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ordered_host@0 ASC NULLS LAST], preserve_REDACTED @@ -652,7 +666,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ordered_host@0 ASC NULLS LAST], preserve_REDACTED @@ -660,7 +674,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[first_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], first_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -726,6 +740,8 @@ order by time_window, ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts) as ordered_host, @@ -751,17 +767,17 @@ order by time_window, ordered_host; | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@1) as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@1) as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@1) as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__first_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __first_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ diff --git a/tests/cases/standalone/optimizer/first_value_advance.sql b/tests/cases/standalone/optimizer/first_value_advance.sql index 2f8da34b99a..b683761221a 100644 --- a/tests/cases/standalone/optimizer/first_value_advance.sql +++ b/tests/cases/standalone/optimizer/first_value_advance.sql @@ -55,6 +55,8 @@ explain select -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts), @@ -79,6 +81,8 @@ explain select first_value(ts order by ts) from t; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(ts order by ts) from t; @@ -138,6 +142,8 @@ order by ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts) as ordered_host, @@ -165,6 +171,8 @@ explain select first_value(ts order by ts) from t; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(ts order by ts) from t; @@ -202,6 +210,8 @@ order by time_window, ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts) as ordered_host, @@ -256,6 +266,8 @@ explain -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(ts order by ts) from t1; @@ -289,6 +301,8 @@ order by ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts) as ordered_host, @@ -330,6 +344,8 @@ order by time_window, ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select first_value(host order by ts) as ordered_host, diff --git a/tests/cases/standalone/optimizer/last_value.result b/tests/cases/standalone/optimizer/last_value.result index 55ecfc8bb56..4131a2240d2 100644 --- a/tests/cases/standalone/optimizer/last_value.result +++ b/tests/cases/standalone/optimizer/last_value.result @@ -69,6 +69,7 @@ explain select -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts), @@ -87,7 +88,7 @@ explain analyze |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: false }", "projection": ["ts", "host", "not_pk", "val"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -127,6 +128,7 @@ explain select last_value(ts order by ts) from t; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(ts order by ts) from t; @@ -139,11 +141,41 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (partitioning.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED +explain analyze select + last_value(val order by ts), + last_value(val order by ts) respect nulls +from t +group by host; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_CooperativeExec REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_ProjectionExec: expr=[last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@1 as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) RESPECT NULLS ORDER BY [t.ts ASC NULLS LAST]@2 as last_value(t.val) RESPECT NULLS ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) RESPECT NULLS ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_RepartitionExec: REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.val) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) RESPECT NULLS ORDER BY [t.ts ASC NULLS LAST]] REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: false }", "projection": ["ts", "host", "val"], "time_range":REDACTED +|_|_|_| +|_|_| Total rows: 4_| ++-+-+-+ + drop table t; Affected Rows: 0 diff --git a/tests/cases/standalone/optimizer/last_value.sql b/tests/cases/standalone/optimizer/last_value.sql index 760ee799d48..7618276217b 100644 --- a/tests/cases/standalone/optimizer/last_value.sql +++ b/tests/cases/standalone/optimizer/last_value.sql @@ -43,6 +43,7 @@ explain select -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts), @@ -67,7 +68,23 @@ explain select last_value(ts order by ts) from t; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(ts order by ts) from t; +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (elapsed_compute.*) REDACTED +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (partitioning.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED +explain analyze select + last_value(val order by ts), + last_value(val order by ts) respect nulls +from t +group by host; + drop table t; \ No newline at end of file diff --git a/tests/cases/standalone/optimizer/last_value_advance.result b/tests/cases/standalone/optimizer/last_value_advance.result index 6b4960be6a1..1b85741eb89 100644 --- a/tests/cases/standalone/optimizer/last_value_advance.result +++ b/tests/cases/standalone/optimizer/last_value_advance.result @@ -95,6 +95,8 @@ explain select -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts), @@ -113,7 +115,7 @@ explain analyze |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow { after_merge: false }", "projection": ["ts", "host", "not_pk", "val"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -153,6 +155,8 @@ explain select last_value(ts order by ts) from t; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(ts order by ts) from t; @@ -165,7 +169,7 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -255,6 +259,8 @@ order by ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts) as ordered_host, @@ -276,7 +282,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts", "host", "not_pk", "val"], "filters": ["host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ordered_host@0 ASC NULLS LAST], preserve_REDACTED @@ -284,7 +290,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts", "host", "not_pk", "val"], "filters": ["host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ordered_host@0 ASC NULLS LAST], preserve_REDACTED @@ -292,7 +298,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts", "host", "not_pk", "val"], "filters": ["host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -337,6 +343,8 @@ explain select last_value(ts order by ts) from t; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(ts order by ts) from t; @@ -352,17 +360,17 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "filters": ["host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "filters": ["host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "filters": ["host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -428,6 +436,8 @@ order by time_window, ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts) as ordered_host, @@ -453,17 +463,17 @@ order by time_window, ordered_host; | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@0) as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "val"], "filters": ["host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@0) as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "val"], "filters": ["host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t.ts)@0 as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@0) as date_bin(Utf8("5 milliseconds"),t.ts)], aggr=[__last_value_state(t.host) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.val) ORDER BY [t.ts ASC NULLS LAST], __last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["ts", "host", "val"], "filters": ["host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -541,6 +551,8 @@ explain -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(ts order by ts) from t1; @@ -556,17 +568,17 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "filters": ["__table_id=UInt32(REDACTED)", "host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["ts"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -624,6 +636,8 @@ order by ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts) as ordered_host, @@ -644,7 +658,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ordered_host@0 ASC NULLS LAST], preserve_REDACTED @@ -652,7 +666,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ordered_host@0 ASC NULLS LAST], preserve_REDACTED @@ -660,7 +674,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }", "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -726,6 +740,8 @@ order by time_window, ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts) as ordered_host, @@ -751,17 +767,17 @@ order by time_window, ordered_host; | 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@1) as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host < Utf8(\"b\") OR host IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@1) as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"b\") AND (host < Utf8(\"d\") OR host IS NULL)"], "time_range":REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=FinalPartitioned, gby=[date_bin(Utf8("5 milliseconds"),t1.ts)@0 as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 5000000 }, ts@1) as date_bin(Utf8("5 milliseconds"),t1.ts)], aggr=[__last_value_state(t1.host) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.val) ORDER BY [t1.ts ASC NULLS LAST], __last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "projection": ["host", "ts", "val"], "filters": ["__table_id=UInt32(REDACTED)", "host >= Utf8(\"d\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ diff --git a/tests/cases/standalone/optimizer/last_value_advance.sql b/tests/cases/standalone/optimizer/last_value_advance.sql index 9090b399b07..6e83b8751c5 100644 --- a/tests/cases/standalone/optimizer/last_value_advance.sql +++ b/tests/cases/standalone/optimizer/last_value_advance.sql @@ -55,6 +55,8 @@ explain select -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts), @@ -79,6 +81,8 @@ explain select last_value(ts order by ts) from t; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (partitioning.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(ts order by ts) from t; @@ -138,6 +142,8 @@ order by ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts) as ordered_host, @@ -165,6 +171,8 @@ explain select last_value(ts order by ts) from t; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(ts order by ts) from t; @@ -202,6 +210,8 @@ order by time_window, ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts) as ordered_host, @@ -256,6 +266,8 @@ explain -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(ts order by ts) from t1; @@ -289,6 +301,8 @@ order by ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts) as ordered_host, @@ -330,6 +344,8 @@ order by time_window, ordered_host; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- might write to different partitions -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED explain analyze select last_value(host order by ts) as ordered_host, diff --git a/tests/cases/standalone/optimizer/order_by.result b/tests/cases/standalone/optimizer/order_by.result index 020e1dc28ac..ba5119d8a9c 100644 --- a/tests/cases/standalone/optimizer/order_by.result +++ b/tests/cases/standalone/optimizer/order_by.result @@ -132,6 +132,7 @@ SELECT i, t AS alias_ts FROM test_pk ORDER BY alias_ts DESC LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY t DESC LIMIT 5; +-+-+-+ @@ -143,7 +144,7 @@ EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY t DESC LIMIT 5; | 1_| 0_|_SortPreservingMergeExec: [alias_ts@1 DESC], fetch=5 REDACTED |_|_|_ProjectionExec: expr=[i@0 as i, t@1 as alias_ts] REDACTED |_|_|_SortExec: TopK(fetch=5), expr=[t@1 DESC], preserve_partitioning=[true], filter=[t@1 IS NULL OR t@1 > 2] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["i", "t"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ @@ -154,6 +155,7 @@ EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY t DESC LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY alias_ts DESC LIMIT 5; +-+-+-+ @@ -165,7 +167,7 @@ EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY alias_ts DESC LIMI | 1_| 0_|_SortPreservingMergeExec: [alias_ts@1 DESC], fetch=5 REDACTED |_|_|_ProjectionExec: expr=[i@0 as i, t@1 as alias_ts] REDACTED |_|_|_SortExec: TopK(fetch=5), expr=[t@1 DESC], preserve_partitioning=[true], filter=[t@1 IS NULL OR t@1 > 2] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["i", "t"], "dyn_filters":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ diff --git a/tests/cases/standalone/optimizer/order_by.sql b/tests/cases/standalone/optimizer/order_by.sql index ee6b6bc0bdd..b785af2fd4b 100644 --- a/tests/cases/standalone/optimizer/order_by.sql +++ b/tests/cases/standalone/optimizer/order_by.sql @@ -29,6 +29,7 @@ SELECT i, t AS alias_ts FROM test_pk ORDER BY alias_ts DESC LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY t DESC LIMIT 5; -- SQLNESS REPLACE (-+) - @@ -37,6 +38,7 @@ EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY t DESC LIMIT 5; -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT i, t AS alias_ts FROM test_pk ORDER BY alias_ts DESC LIMIT 5; DROP TABLE test_pk; diff --git a/tests/cases/standalone/optimizer/time_index_filter_pushdown.result b/tests/cases/standalone/optimizer/time_index_filter_pushdown.result index c39df6c8652..106a78c49d2 100644 --- a/tests/cases/standalone/optimizer/time_index_filter_pushdown.result +++ b/tests/cases/standalone/optimizer/time_index_filter_pushdown.result @@ -123,6 +123,7 @@ Affected Rows: 3 -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT rack FROM @@ -138,7 +139,7 @@ WHERE |_|_|_| | 1_| 0_|_FilterExec: usage_small@1 >= 10, projection=[rack@0] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_UnorderedScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["rack", "usage_small"], "filters": ["usage_small >= Int16(10)"], "flat_format": REDACTED, "REDACTED +|_|_|_UnorderedScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["rack", "usage_small"], "filters": ["usage_small >= Int16(10)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 3_| +-+-+-+ diff --git a/tests/cases/standalone/optimizer/time_index_filter_pushdown.sql b/tests/cases/standalone/optimizer/time_index_filter_pushdown.sql index 71a22330d88..64f0fbc0e26 100644 --- a/tests/cases/standalone/optimizer/time_index_filter_pushdown.sql +++ b/tests/cases/standalone/optimizer/time_index_filter_pushdown.sql @@ -81,6 +81,7 @@ VALUES -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE VERBOSE SELECT rack FROM diff --git a/tests/cases/standalone/optimizer/windowed_sort.result b/tests/cases/standalone/optimizer/windowed_sort.result index 9bffb3d58d3..0636e19ba34 100644 --- a/tests/cases/standalone/optimizer/windowed_sort.result +++ b/tests/cases/standalone/optimizer/windowed_sort.result @@ -56,6 +56,7 @@ ORDER BY -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time_utc AS collect_time, peak_current, @@ -73,7 +74,7 @@ ORDER BY | 1_| 0_|_SortPreservingMergeExec: [collect_time@0 ASC NULLS LAST] REDACTED |_|_|_ProjectionExec: expr=[collect_time_utc@0 as collect_time, peak_current@1 as peak_current] REDACTED |_|_|_SortExec: expr=[collect_time_utc@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["collect_time_utc", "peak_current"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 8_| +-+-+-+ @@ -105,6 +106,7 @@ ORDER BY -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time_utc AS collect_time_0, peak_current, @@ -122,7 +124,7 @@ ORDER BY | 1_| 0_|_SortPreservingMergeExec: [collect_time_0@0 ASC NULLS LAST] REDACTED |_|_|_ProjectionExec: expr=[collect_time_utc@0 as collect_time_0, peak_current@1 as peak_current] REDACTED |_|_|_SortExec: expr=[collect_time_utc@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["collect_time_utc", "peak_current"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 8_| +-+-+-+ @@ -156,6 +158,7 @@ ORDER BY -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time AS true_collect_time, collect_time_utc AS collect_time, @@ -175,7 +178,7 @@ ORDER BY |_|_|_ProjectionExec: expr=[collect_time@0 as true_collect_time, collect_time_utc@1 as collect_time, peak_current@2 as peak_current] REDACTED |_|_|_WindowedSortExec: expr=collect_time@0 DESC num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=collect_time@0 DESC num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["collect_time", "collect_time_utc", "peak_current"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 8_| +-+-+-+ @@ -209,6 +212,7 @@ ORDER BY -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time_utc AS collect_time, collect_time AS true_collect_time, @@ -228,7 +232,7 @@ ORDER BY |_|_|_ProjectionExec: expr=[collect_time_utc@1 as collect_time, collect_time@0 as true_collect_time, peak_current@2 as peak_current] REDACTED |_|_|_WindowedSortExec: expr=collect_time@0 DESC num_ranges=REDACTED REDACTED |_|_|_PartSortExec: expr=collect_time@0 DESC num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["collect_time", "collect_time_utc", "peak_current"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 8_| +-+-+-+ @@ -274,6 +278,7 @@ TQL EVAL('2023-10-01 00:00:00.000'::TIMESTAMP, '2023-10-01 00:00:05.000'::TIMEST -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE('2023-10-01 00:00:00.000'::TIMESTAMP, '2023-10-01 00:00:05.000'::TIMESTAMP, '1s') sum(instance_job_metrics); +-+-+-+ @@ -290,7 +295,7 @@ TQL ANALYZE('2023-10-01 00:00:00.000'::TIMESTAMP, '2023-10-01 00:00:05.000'::TIM |_|_|_ProjectionExec: expr=[greptime_timestamp@0 as greptime_timestamp, greptime_value@1 as greptime_value] REDACTED |_|_|_PromInstantManipulateExec: range=[1696118400000..1696118405000], lookback=[300000], interval=[1000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesDivideExec: tags=["instance", "job"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["greptime_timestamp", "greptime_value", "instance", "job"], "filters": ["greptime_timestamp >= TimestampMillisecond(1696118100001, None)", "greptime_timestamp <= TimestampMillisecond(1696118405000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 5_| +-+-+-+ diff --git a/tests/cases/standalone/optimizer/windowed_sort.sql b/tests/cases/standalone/optimizer/windowed_sort.sql index b74db2b6b72..b19f06ec9ab 100644 --- a/tests/cases/standalone/optimizer/windowed_sort.sql +++ b/tests/cases/standalone/optimizer/windowed_sort.sql @@ -39,6 +39,7 @@ ORDER BY -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time_utc AS collect_time, peak_current, @@ -61,6 +62,7 @@ ORDER BY -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time_utc AS collect_time_0, peak_current, @@ -85,6 +87,7 @@ ORDER BY -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time AS true_collect_time, collect_time_utc AS collect_time, @@ -110,6 +113,7 @@ ORDER BY -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE SELECT collect_time_utc AS collect_time, collect_time AS true_collect_time, @@ -144,6 +148,7 @@ TQL EVAL('2023-10-01 00:00:00.000'::TIMESTAMP, '2023-10-01 00:00:05.000'::TIMEST -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE('2023-10-01 00:00:00.000'::TIMESTAMP, '2023-10-01 00:00:05.000'::TIMESTAMP, '1s') sum(instance_job_metrics); DROP TABLE IF EXISTS `instance_job_metrics`; diff --git a/tests/cases/standalone/optimizer/windowed_sort_advance.result b/tests/cases/standalone/optimizer/windowed_sort_advance.result index 44af88635cb..123635de8c2 100644 --- a/tests/cases/standalone/optimizer/windowed_sort_advance.result +++ b/tests/cases/standalone/optimizer/windowed_sort_advance.result @@ -56,6 +56,7 @@ select ts as ts, status, value from `a` where ts >= '2026-03-12T08:00:00+08:00' -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE select ts as ts, status, value from `a` where ts >= '2026-03-12T08:00:00+08:00' and ts < '2026-03-12T08:02:01+08:00' order by ts asc; +-+-+-+ @@ -67,7 +68,7 @@ EXPLAIN ANALYZE select ts as ts, status, value from `a` where ts >= '2026-03-12T | 1_| 0_|_SortPreservingMergeExec: [ts@0 ASC NULLS LAST] REDACTED |_|_|_ProjectionExec: expr=[ts@2 as ts, status@1 as status, value@0 as value] REDACTED |_|_|_WindowedSortExec: expr=ts@2 ASC NULLS LAST num_ranges=REDACTED REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["value", "status", "ts"], "filters": ["ts >= TimestampNanosecond(1773273600000000000, None)", "ts < TimestampNanosecond(1773273721000000000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 10_| +-+-+-+ @@ -95,6 +96,7 @@ select to_timestamp_millis(ts) as ts, status, value from `a` where ts >= '2026-0 -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE select to_timestamp_millis(ts) as ts, status, value from `a` where ts >= '2026-03-12T08:00:00+08:00' and ts < '2026-03-12T08:02:01+08:00' order by ts asc; +-+-+-+ @@ -106,7 +108,7 @@ EXPLAIN ANALYZE select to_timestamp_millis(ts) as ts, status, value from `a` whe | 1_| 0_|_SortPreservingMergeExec: [ts@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_ProjectionExec: expr=[to_timestamp_millis(ts@2) as ts, status@1 as status, value@0 as value] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["value", "status", "ts"], "filters": ["ts >= TimestampNanosecond(1773273600000000000, None)", "ts < TimestampNanosecond(1773273721000000000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 10_| +-+-+-+ diff --git a/tests/cases/standalone/optimizer/windowed_sort_advance.sql b/tests/cases/standalone/optimizer/windowed_sort_advance.sql index fa73b97933c..167a42d591c 100644 --- a/tests/cases/standalone/optimizer/windowed_sort_advance.sql +++ b/tests/cases/standalone/optimizer/windowed_sort_advance.sql @@ -22,6 +22,7 @@ select ts as ts, status, value from `a` where ts >= '2026-03-12T08:00:00+08:00' -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE select ts as ts, status, value from `a` where ts >= '2026-03-12T08:00:00+08:00' and ts < '2026-03-12T08:02:01+08:00' order by ts asc; select to_timestamp_millis(ts) as ts, status, value from `a` where ts >= '2026-03-12T08:00:00+08:00' and ts < '2026-03-12T08:02:01+08:00' order by ts asc; @@ -32,6 +33,7 @@ select to_timestamp_millis(ts) as ts, status, value from `a` where ts >= '2026-0 -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE select to_timestamp_millis(ts) as ts, status, value from `a` where ts >= '2026-03-12T08:00:00+08:00' and ts < '2026-03-12T08:02:01+08:00' order by ts asc; DROP TABLE `a`; diff --git a/tests/cases/standalone/optimizer/windowed_sort_nanos.result b/tests/cases/standalone/optimizer/windowed_sort_nanos.result index ddcac3ba26f..3c808912b67 100644 --- a/tests/cases/standalone/optimizer/windowed_sort_nanos.result +++ b/tests/cases/standalone/optimizer/windowed_sort_nanos.result @@ -56,6 +56,7 @@ select to_timestamp_nanos(ts) as ts, status, value from `a_ms` where ts >= '2026 -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE select to_timestamp_nanos(ts) as ts, status, value from `a_ms` where ts >= '2026-03-12T08:00:00.000+08:00' and ts < '2026-03-12T08:02:01.000+08:00' order by ts asc; +-+-+-+ @@ -67,7 +68,7 @@ EXPLAIN ANALYZE select to_timestamp_nanos(ts) as ts, status, value from `a_ms` w | 1_| 0_|_SortPreservingMergeExec: [ts@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED |_|_|_ProjectionExec: expr=[to_timestamp_nanos(ts@2) as ts, status@1 as status, value@0 as value] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0} REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "projection": ["value", "status", "ts"], "filters": ["ts >= TimestampMillisecond(1773273600000, None)", "ts < TimestampMillisecond(1773273721000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 10_| +-+-+-+ diff --git a/tests/cases/standalone/optimizer/windowed_sort_nanos.sql b/tests/cases/standalone/optimizer/windowed_sort_nanos.sql index 6a6cf51c4b6..27abd0bc805 100644 --- a/tests/cases/standalone/optimizer/windowed_sort_nanos.sql +++ b/tests/cases/standalone/optimizer/windowed_sort_nanos.sql @@ -22,6 +22,7 @@ select to_timestamp_nanos(ts) as ts, status, value from `a_ms` where ts >= '2026 -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE num_ranges=\d+ num_ranges=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED EXPLAIN ANALYZE select to_timestamp_nanos(ts) as ts, status, value from `a_ms` where ts >= '2026-03-12T08:00:00.000+08:00' and ts < '2026-03-12T08:02:01.000+08:00' order by ts asc; DROP TABLE `a_ms`; diff --git a/tests/cases/standalone/tql-explain-analyze/analyze.result b/tests/cases/standalone/tql-explain-analyze/analyze.result index 7c0b9b16e6b..45942403722 100644 --- a/tests/cases/standalone/tql-explain-analyze/analyze.result +++ b/tests/cases/standalone/tql-explain-analyze/analyze.result @@ -15,6 +15,7 @@ Affected Rows: 3 -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') test; +-+-+-+ @@ -26,7 +27,7 @@ TQL ANALYZE (0, 10, '5s') test; | 1_| 0_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] REDACTED |_|_|_PromSeriesDivideExec: tags=["k"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k"], "filters": ["j >= TimestampMillisecond(-299999, None)", "j <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -40,6 +41,7 @@ TQL ANALYZE (0, 10, '5s') test; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '1s', '2s') test; +-+-+-+ @@ -51,7 +53,7 @@ TQL ANALYZE (0, 10, '1s', '2s') test; | 1_| 0_|_PromInstantManipulateExec: range=[0..10000], lookback=[2000], interval=[1000], time index=[j] REDACTED |_|_|_PromSeriesDivideExec: tags=["k"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k"], "filters": ["j >= TimestampMillisecond(-1999, None)", "j <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -64,6 +66,7 @@ TQL ANALYZE (0, 10, '1s', '2s') test; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; +-+-+-+ @@ -75,7 +78,7 @@ TQL ANALYZE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp | 1_| 0_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] REDACTED |_|_|_PromSeriesDivideExec: tags=["k"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k"], "filters": ["j >= TimestampMillisecond(-299999, None)", "j <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -91,6 +94,7 @@ TQL ANALYZE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (0, 10, '5s') test; +-+-+-+ @@ -102,7 +106,7 @@ TQL ANALYZE VERBOSE (0, 10, '5s') test; | 1_| 0_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] REDACTED |_|_|_PromSeriesDivideExec: tags=["k"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k"], "filters": ["j >= TimestampMillisecond(-299999, None)", "j <= TimestampMillisecond(10000, None)"], "REDACTED +|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k"], "filters": ["j >= TimestampMillisecond(-299999, None)", "j <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -123,6 +127,7 @@ Affected Rows: 0 -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') test; +-+-+-+ @@ -134,12 +139,12 @@ TQL ANALYZE (0, 10, '5s') test; | 1_| 0_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] REDACTED |_|_|_PromSeriesDivideExec: tags=["k", "l"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k", "l"], "filters": ["j >= TimestampMillisecond(-299999, None)", "j <= TimestampMillisecond(10000, None)", "k < Utf8(\"a\") OR k IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] REDACTED |_|_|_PromSeriesDivideExec: tags=["k", "l"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k", "l"], "filters": ["j >= TimestampMillisecond(-299999, None)", "j <= TimestampMillisecond(10000, None)", "k >= Utf8(\"a\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -151,6 +156,7 @@ TQL ANALYZE (0, 10, '5s') test; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') rate(test[10s]); +-+-+-+ @@ -165,7 +171,7 @@ TQL ANALYZE (0, 10, '5s') rate(test[10s]); |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["k", "l"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k", "l"], "filters": ["j >= TimestampMillisecond(-9999, None)", "j <= TimestampMillisecond(10000, None)", "k < Utf8(\"a\") OR k IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_FilterExec: prom_rate(j_range,i,j,Int64(10000))@1 IS NOT NULL REDACTED |_|_|_ProjectionExec: expr=[j@1 as j, prom_rate(j_range@4, i@0, j@1, 10000) as prom_rate(j_range,i,j,Int64(10000)), k@2 as k, l@3 as l] REDACTED @@ -173,7 +179,7 @@ TQL ANALYZE (0, 10, '5s') rate(test[10s]); |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["k", "l"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k", "l"], "filters": ["j >= TimestampMillisecond(-9999, None)", "j <= TimestampMillisecond(10000, None)", "k >= Utf8(\"a\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -187,6 +193,7 @@ TQL ANALYZE (0, 10, '5s') rate(test[10s]); -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE FORMAT JSON (0, 10, '5s') test; +-+-+-+ @@ -208,6 +215,7 @@ TQL ANALYZE FORMAT JSON (0, 10, '5s') test; -- SQLNESS REPLACE (Duration.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE FORMAT JSON (0, 10, '5s') test; +-+-+-+ @@ -227,6 +235,7 @@ TQL ANALYZE VERBOSE FORMAT JSON (0, 10, '5s') test; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE FORMAT TEXT (0, 10, '5s') test; +-+-+-+ @@ -238,12 +247,12 @@ TQL ANALYZE FORMAT TEXT (0, 10, '5s') test; | 1_| 0_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] REDACTED |_|_|_PromSeriesDivideExec: tags=["k", "l"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k", "l"], "filters": ["j >= TimestampMillisecond(-299999, None)", "j <= TimestampMillisecond(10000, None)", "k < Utf8(\"a\") OR k IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[j] REDACTED |_|_|_PromSeriesDivideExec: tags=["k", "l"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["i", "j", "k", "l"], "filters": ["j >= TimestampMillisecond(-299999, None)", "j <= TimestampMillisecond(10000, None)", "k >= Utf8(\"a\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ @@ -280,6 +289,7 @@ TQL EVAL sum(test2); -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE sum(test2); +-+-+-+ @@ -299,7 +309,7 @@ TQL ANALYZE sum(test2); |_|_|_ProjectionExec: expr=[greptime_timestamp@0 as greptime_timestamp, greptime_value@1 as greptime_value] REDACTED |_|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesDivideExec: tags=["shard"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["greptime_timestamp", "greptime_value", "shard"], "filters": ["greptime_timestamp >= TimestampMillisecond(-299999, None)", "greptime_timestamp <= TimestampMillisecond(0, None)", "shard <= Utf8(\"2\") OR shard IS NULL"], "time_range":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[__sum_state(test2.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED @@ -307,7 +317,7 @@ TQL ANALYZE sum(test2); |_|_|_ProjectionExec: expr=[greptime_timestamp@0 as greptime_timestamp, greptime_value@1 as greptime_value] REDACTED |_|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesDivideExec: tags=["shard"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["greptime_timestamp", "greptime_value", "shard"], "filters": ["greptime_timestamp >= TimestampMillisecond(-299999, None)", "greptime_timestamp <= TimestampMillisecond(0, None)", "shard > Utf8(\"2\")"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ diff --git a/tests/cases/standalone/tql-explain-analyze/analyze.sql b/tests/cases/standalone/tql-explain-analyze/analyze.sql index 598cdf0aa98..cfcb615c289 100644 --- a/tests/cases/standalone/tql-explain-analyze/analyze.sql +++ b/tests/cases/standalone/tql-explain-analyze/analyze.sql @@ -11,6 +11,7 @@ INSERT INTO test VALUES (1, 1, "a"), (1, 1, "b"), (2, 2, "a"); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') test; -- 'lookback' parameter is not fully supported, the test has to be updated @@ -22,6 +23,7 @@ TQL ANALYZE (0, 10, '5s') test; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '1s', '2s') test; -- analyze at 0s, 5s and 10s. No point at 0s. @@ -32,6 +34,7 @@ TQL ANALYZE (0, 10, '1s', '2s') test; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '10 seconds'::interval, '5s') test; -- analyze verbose at 0s, 5s and 10s. No point at 0s. @@ -45,6 +48,7 @@ TQL ANALYZE ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE (0, 10, '5s') test; DROP TABLE test; @@ -59,6 +63,7 @@ CREATE TABLE test(i DOUBLE, j TIMESTAMP TIME INDEX, k STRING, l STRING, PRIMARY -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') test; -- SQLNESS REPLACE (metrics.*) REDACTED @@ -68,6 +73,7 @@ TQL ANALYZE (0, 10, '5s') test; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') rate(test[10s]); -- Test new FORMAT functionality for ANALYZE @@ -79,6 +85,7 @@ TQL ANALYZE (0, 10, '5s') rate(test[10s]); -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE FORMAT JSON (0, 10, '5s') test; -- analyze verbose with JSON format @@ -91,6 +98,7 @@ TQL ANALYZE FORMAT JSON (0, 10, '5s') test; -- SQLNESS REPLACE (Duration.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (flat_format.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE VERBOSE FORMAT JSON (0, 10, '5s') test; -- analyze with TEXT format (should be same as default) @@ -101,6 +109,7 @@ TQL ANALYZE VERBOSE FORMAT JSON (0, 10, '5s') test; -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE FORMAT TEXT (0, 10, '5s') test; drop table test; @@ -126,6 +135,7 @@ TQL EVAL sum(test2); -- SQLNESS REPLACE (\s\s+) _ -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE sum(test2); DROP TABLE test2; diff --git a/tests/cases/standalone/tql-explain-analyze/column_pruning.result b/tests/cases/standalone/tql-explain-analyze/column_pruning.result index cecb72fcee1..24ac84a7bc0 100644 --- a/tests/cases/standalone/tql-explain-analyze/column_pruning.result +++ b/tests/cases/standalone/tql-explain-analyze/column_pruning.result @@ -29,6 +29,7 @@ Affected Rows: 9 -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') sum(promql_column_pruning); +-+-+-+ @@ -45,7 +46,7 @@ TQL ANALYZE (0, 10, '5s') sum(promql_column_pruning); |_|_|_ProjectionExec: expr=[ts@0 as ts, greptime_value@4 as greptime_value] REDACTED |_|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["job", "instance", "region"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["ts", "job", "instance", "region", "greptime_value"], "filters": ["ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 3_| +-+-+-+ @@ -57,6 +58,7 @@ TQL ANALYZE (0, 10, '5s') sum(promql_column_pruning); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') sum(rate(promql_column_pruning[5s])); +-+-+-+ @@ -75,7 +77,7 @@ TQL ANALYZE (0, 10, '5s') sum(rate(promql_column_pruning[5s])); |_|_|_PromRangeManipulateExec: req range=[0..10000], interval=[5000], eval range=[5000], time index=[ts] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[ts], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["job", "instance", "region"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["ts", "job", "instance", "region", "greptime_value"], "filters": ["ts >= TimestampMillisecond(-4999, None)", "ts <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ diff --git a/tests/cases/standalone/tql-explain-analyze/column_pruning.sql b/tests/cases/standalone/tql-explain-analyze/column_pruning.sql index 88ab7484ab7..74684136fb5 100644 --- a/tests/cases/standalone/tql-explain-analyze/column_pruning.sql +++ b/tests/cases/standalone/tql-explain-analyze/column_pruning.sql @@ -25,6 +25,7 @@ INSERT INTO promql_column_pruning VALUES -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') sum(promql_column_pruning); -- SQLNESS REPLACE (metrics.*) REDACTED @@ -34,6 +35,7 @@ TQL ANALYZE (0, 10, '5s') sum(promql_column_pruning); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') sum(rate(promql_column_pruning[5s])); DROP TABLE promql_column_pruning; diff --git a/tests/cases/standalone/tql-explain-analyze/tsid_column.result b/tests/cases/standalone/tql-explain-analyze/tsid_column.result index 3b509cdd20b..723b1803237 100644 --- a/tests/cases/standalone/tql-explain-analyze/tsid_column.result +++ b/tests/cases/standalone/tql-explain-analyze/tsid_column.result @@ -37,6 +37,8 @@ Affected Rows: 6 -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') sum(tsid_metric); +-+-+-+ @@ -54,7 +56,7 @@ TQL ANALYZE (0, 10, '5s') sum(tsid_metric); |_|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[val@1 as val, __tsid@2 as __tsid, ts@0 as ts] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["ts", "val", "__tsid"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 3_| +-+-+-+ @@ -66,6 +68,8 @@ TQL ANALYZE (0, 10, '5s') sum(tsid_metric); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') sum by (job, instance) (tsid_metric); +-+-+-+ @@ -82,7 +86,7 @@ TQL ANALYZE (0, 10, '5s') sum by (job, instance) (tsid_metric); |_|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[val@1 as val, instance@3 as instance, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["ts", "val", "__tsid", "instance", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 6_| +-+-+-+ @@ -94,6 +98,8 @@ TQL ANALYZE (0, 10, '5s') sum by (job, instance) (tsid_metric); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') sum(irate(tsid_metric[1h])) / scalar(count(count(tsid_metric) by (job))); +-+-+-+ @@ -120,7 +126,7 @@ TQL ANALYZE (0, 10, '5s') sum(irate(tsid_metric[1h])) / scalar(count(count(tsid |_|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[val@1 as val, job@3 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["ts", "val", "__tsid", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [ts@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED @@ -133,7 +139,7 @@ TQL ANALYZE (0, 10, '5s') sum(irate(tsid_metric[1h])) / scalar(count(count(tsid |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[ts], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[val@1 as val, __tsid@2 as __tsid, ts@0 as ts] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["ts", "val", "__tsid"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-3599999, None)", "ts <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -145,6 +151,8 @@ TQL ANALYZE (0, 10, '5s') sum(irate(tsid_metric[1h])) / scalar(count(count(tsid -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') sum(irate(tsid_metric[1h])) / scalar(count(sum(tsid_metric) by (job))); +-+-+-+ @@ -172,7 +180,7 @@ TQL ANALYZE (0, 10, '5s') sum(irate(tsid_metric[1h])) / scalar(count(sum(tsid_m |_|_|_PromInstantManipulateExec: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[val@1 as val, job@3 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["ts", "val", "__tsid", "job"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| | 1_| 0_|_SortPreservingMergeExec: [ts@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[true] REDACTED @@ -185,7 +193,7 @@ TQL ANALYZE (0, 10, '5s') sum(irate(tsid_metric[1h])) / scalar(count(sum(tsid_m |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[ts], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED |_|_|_ProjectionExec: expr=[val@1 as val, __tsid@2 as __tsid, ts@0 as ts] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["ts", "val", "__tsid"], "filters": ["__table_id=UInt32(REDACTED)", "ts >= TimestampMillisecond(-3599999, None)", "ts <= TimestampMillisecond(10000, None)"], "time_range":REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ diff --git a/tests/cases/standalone/tql-explain-analyze/tsid_column.sql b/tests/cases/standalone/tql-explain-analyze/tsid_column.sql index dedce2dfb19..51cb584b6f4 100644 --- a/tests/cases/standalone/tql-explain-analyze/tsid_column.sql +++ b/tests/cases/standalone/tql-explain-analyze/tsid_column.sql @@ -31,6 +31,8 @@ INSERT INTO tsid_metric VALUES -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') sum(tsid_metric); -- SQLNESS REPLACE (metrics.*) REDACTED @@ -40,6 +42,8 @@ TQL ANALYZE (0, 10, '5s') sum(tsid_metric); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') sum by (job, instance) (tsid_metric); -- SQLNESS REPLACE (metrics.*) REDACTED @@ -49,6 +53,8 @@ TQL ANALYZE (0, 10, '5s') sum by (job, instance) (tsid_metric); -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') sum(irate(tsid_metric[1h])) / scalar(count(count(tsid_metric) by (job))); -- SQLNESS REPLACE (metrics.*) REDACTED @@ -58,6 +64,8 @@ TQL ANALYZE (0, 10, '5s') sum(irate(tsid_metric[1h])) / scalar(count(count(tsid -- SQLNESS REPLACE (peers.*) REDACTED -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED -- SQLNESS REPLACE (Hash.*) REDACTED +-- SQLNESS REPLACE __table_id\s*=\s*UInt32\(\d+\) __table_id=UInt32(REDACTED) +-- SQLNESS REPLACE \\?"(dyn_filters|time_range|file_summary|max_part_range)\\?":.* "$1":REDACTED TQL ANALYZE (0, 10, '5s') sum(irate(tsid_metric[1h])) / scalar(count(sum(tsid_metric) by (job))); DROP TABLE tsid_metric; diff --git a/tests/compatibility/cases/analyze_verbose_remote_metrics_extension/verify.result b/tests/compatibility/cases/analyze_verbose_remote_metrics_extension/verify.result index 7394ca8faee..a3ed4dbba0e 100644 --- a/tests/compatibility/cases/analyze_verbose_remote_metrics_extension/verify.result +++ b/tests/compatibility/cases/analyze_verbose_remote_metrics_extension/verify.result @@ -8,6 +8,7 @@ -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ("files":\s\[.*\],) "files": REDACTED, -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE "(time_range|file_summary|max_part_range)":.* "scan_details":REDACTED EXPLAIN ANALYZE VERBOSE SELECT count(*) FROM t_analyze_verbose_remote_metrics_extension; +-+-+-+ @@ -23,12 +24,12 @@ EXPLAIN ANALYZE VERBOSE SELECT count(*) FROM t_analyze_verbose_remote_metrics_ex | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(t_analyze_verbose_remote_REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(t_analyze_verbose_remote_REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["ts"], "filters": ["host < Utf8(\"m\") OR host IS NULL"], "files": REDACTED, "flat_format": REDACTED, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["ts"], "filters": ["host < Utf8(\"m\") OR host IS NULL"], "scan_details":REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__count_state(t_analyze_verbose_remote_REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__count_state(t_analyze_verbose_remote_REDACTED -|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["ts"], "filters": ["host >= Utf8(\"m\") AND host IS NOT NULL"], "flat_format": REDACTED, "REDACTED +|_|_|_SeqScan: region=REDACTED, {"partition_count":REDACTED, "projection": ["ts"], "filters": ["host >= Utf8(\"m\")"], "scan_details":REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ diff --git a/tests/compatibility/cases/analyze_verbose_remote_metrics_extension/verify.sql b/tests/compatibility/cases/analyze_verbose_remote_metrics_extension/verify.sql index 81eb489e663..f50b4a29116 100644 --- a/tests/compatibility/cases/analyze_verbose_remote_metrics_extension/verify.sql +++ b/tests/compatibility/cases/analyze_verbose_remote_metrics_extension/verify.sql @@ -8,4 +8,5 @@ -- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED -- SQLNESS REPLACE ("files":\s\[.*\],) "files": REDACTED, -- SQLNESS REPLACE "flat_format":\s\w+, "flat_format": REDACTED, +-- SQLNESS REPLACE "(time_range|file_summary|max_part_range)":.* "scan_details":REDACTED EXPLAIN ANALYZE VERBOSE SELECT count(*) FROM t_analyze_verbose_remote_metrics_extension;