From d99b0df374eec0329ac3f430e40b80a1e6c6963c Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 17 Aug 2026 20:06:33 +0800 Subject: [PATCH] fix(query): restore columnar group-by for dictionary-encoded tags Bump the DataFusion fork to be93ffd85 (feat/dict-group-column-53), which backports apache/datafusion #23187: DictionaryGroupValuesColumn lets dictionary-encoded group keys use the columnar GroupValuesColumn fast path (hash distinct dictionary values once per batch, resolve rows by key index) instead of falling back to row-based GroupValuesRows. This fixes the TSBS double-groupby regression introduced by #8541 (preserve dictionary-encoded query labels): v1.2.0-beta.1 scan output changed tag columns to Dictionary(UInt32, Utf8), which DataFusion 53.1.0 did not support in GroupValuesColumn's supported_type allow-list, so GROUP BY queries silently dropped to the ~60% slower row path (time_calculating_group_ids +57%, peak_mem +50%, end-to-end +38%). Adds an end-to-end integration test (dict_groupby_sst) that flushes a flat-format SST with dictionary-encoded hostname, runs the tsbs-style double-groupby query, and asserts correct results with no CastExec inserted before the aggregate. Signed-off-by: discord9 --- Cargo.lock | 68 +++++----- Cargo.toml | 30 ++--- tests-integration/tests/dict_groupby_sst.rs | 139 ++++++++++++++++++++ tests-integration/tests/main.rs | 1 + 4 files changed, 189 insertions(+), 49 deletions(-) create mode 100644 tests-integration/tests/dict_groupby_sst.rs diff --git a/Cargo.lock b/Cargo.lock index 0f473d4e03..7837f6140d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3744,7 +3744,7 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "datafusion" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "arrow-schema 58.3.0", @@ -3798,7 +3798,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3822,7 +3822,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3844,7 +3844,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -3868,7 +3868,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "futures", "log", @@ -3878,7 +3878,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "async-compression", @@ -3912,7 +3912,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "arrow-ipc 58.3.0", @@ -3935,7 +3935,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3957,7 +3957,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "async-trait", @@ -3980,7 +3980,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "async-trait", @@ -4009,12 +4009,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" [[package]] name = "datafusion-execution" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "arrow-buffer 58.3.0", @@ -4036,7 +4036,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "async-trait", @@ -4058,7 +4058,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4070,7 +4070,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "arrow-buffer 58.3.0", @@ -4101,7 +4101,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4122,7 +4122,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4134,7 +4134,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "arrow-ord 58.3.0", @@ -4158,7 +4158,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "async-trait", @@ -4173,7 +4173,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4190,7 +4190,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -4199,7 +4199,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "datafusion-doc", "quote", @@ -4209,7 +4209,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "chrono", @@ -4258,7 +4258,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4281,7 +4281,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4295,7 +4295,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4311,7 +4311,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4329,7 +4329,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "ahash 0.8.12", "arrow 58.3.0", @@ -4360,7 +4360,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "chrono", @@ -4387,7 +4387,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4397,7 +4397,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "datafusion-common", @@ -4413,7 +4413,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "async-trait", "datafusion-common", @@ -4426,7 +4426,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "arrow 58.3.0", "bigdecimal 0.4.8", @@ -4444,7 +4444,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "53.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=d705d4433d2fcbe2bead54e3f1d6cc59727857e1#d705d4433d2fcbe2bead54e3f1d6cc59727857e1" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=be93ffd85cdd8c05912df65c0e37630abd5c91ec#be93ffd85cdd8c05912df65c0e37630abd5c91ec" dependencies = [ "async-recursion", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index daee1bd409..7b60695170 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -348,21 +348,21 @@ git = "https://github.com/GreptimeTeam/greptime-meter.git" rev = "5618e779cf2bb4755b499c630fba4c35e91898cb" [patch.crates-io] -datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } -datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } -datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } -datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } -datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } -datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } -datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } -datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } -datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } -datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } -datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } -datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } -datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } -datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } -datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "d705d4433d2fcbe2bead54e3f1d6cc59727857e1" } +datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } +datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } +datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } +datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } +datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } +datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } +datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } +datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } +datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } +datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } +datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } +datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } +datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } +datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } +datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "be93ffd85cdd8c05912df65c0e37630abd5c91ec" } sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "2aefa08a8d69c96eec2d6d6703598a009bba6e4c" } # on branch v0.61.x [profile.release] diff --git a/tests-integration/tests/dict_groupby_sst.rs b/tests-integration/tests/dict_groupby_sst.rs new file mode 100644 index 0000000000..862b77ef10 --- /dev/null +++ b/tests-integration/tests/dict_groupby_sst.rs @@ -0,0 +1,139 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use common_query::Output; +use frontend::instance::Instance; +use servers::query_handler::sql::SqlQueryHandler; +use session::context::QueryContext; +use tests_integration::standalone::GreptimeDbStandaloneBuilder; + +async fn execute_sql(instance: &Instance, sql: &str) -> Output { + SqlQueryHandler::do_query(instance, sql, QueryContext::arc()) + .await + .remove(0) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_dict_groupby_sst() { + common_telemetry::init_default_ut_logging(); + + let standalone = GreptimeDbStandaloneBuilder::new("dict_groupby_sst") + .build() + .await; + let frontend = standalone.fe_instance(); + + execute_sql( + frontend, + r#" + CREATE TABLE cpu ( + hostname STRING PRIMARY KEY, + usage_user DOUBLE, + usage_system DOUBLE, + usage_idle DOUBLE, + greptime_timestamp TIMESTAMP TIME INDEX + ) WITH (append_mode = 'true', sst_format = 'flat') + "#, + ) + .await; + + let mut rows = String::new(); + for index in 0..1200 { + if index != 0 { + rows.push(','); + } + let hostname = match index % 3 { + 0 => "host_a", + 1 => "host_b", + _ => "host_c", + }; + let usage_user = match index % 3 { + 0 => 10.0, + 1 => 20.0, + _ => 30.0, + }; + let hour = 11 + index / 600; + let minute = (index % 600) / 60; + let second = index % 60; + rows.push_str(&format!( + "('{hostname}', {usage_user}, 1.0, 0.0, '2023-06-12T{hour:02}:{minute:02}:{second:02}Z')" + )); + } + execute_sql(frontend, &format!("INSERT INTO cpu VALUES {rows}")).await; + + execute_sql(frontend, "ADMIN FLUSH_TABLE('cpu')").await; + + let count = execute_sql(frontend, "SELECT count(*) FROM cpu") + .await + .data + .pretty_print() + .await; + assert!( + count.contains("| 1200 |"), + "unexpected row count:\n{count}" + ); + + let query = r#" + SELECT hostname, avg(usage_user), date_trunc('hour', greptime_timestamp) + FROM cpu + WHERE greptime_timestamp >= '2023-06-12T00:00:00Z' + AND greptime_timestamp < '2023-06-14T00:00:00Z' + GROUP BY date_trunc('hour', greptime_timestamp), hostname + ORDER BY hostname, date_trunc('hour', greptime_timestamp) + "#; + let groups = execute_sql(frontend, query).await.data.pretty_print().await; + assert_eq!( + 6, + groups + .lines() + .filter(|line| line.starts_with("| host_")) + .count(), + "expected one group per hostname and hour:\n{groups}" + ); + for (hostname, average) in [("host_a", "10.0"), ("host_b", "20.0"), ("host_c", "30.0")] { + assert_eq!( + 2, + groups.matches(&format!("| {hostname} ")).count(), + "expected two hourly groups for {hostname}:\n{groups}" + ); + assert!( + groups.contains(average), + "expected average {average} for {hostname}:\n{groups}" + ); + } + + let explain = execute_sql( + frontend, + r#" + EXPLAIN ANALYZE VERBOSE + SELECT avg(usage_user), date_trunc('hour', greptime_timestamp) + FROM cpu + WHERE greptime_timestamp >= '2023-06-12T00:00:00Z' + AND greptime_timestamp < '2023-06-14T00:00:00Z' + GROUP BY date_trunc('hour', greptime_timestamp), hostname + "#, + ) + .await + .data + .pretty_print() + .await; + assert!( + explain.contains("AggregateExec"), + "expected aggregate in physical plan:\n{explain}" + ); + assert!( + !explain.contains("CastExec") && !explain.contains("CAST(hostname"), + "dictionary hostname must not be cast before aggregation:\n{explain}" + ); +} diff --git a/tests-integration/tests/main.rs b/tests-integration/tests/main.rs index 3112cb72c3..965a3a6746 100644 --- a/tests-integration/tests/main.rs +++ b/tests-integration/tests/main.rs @@ -16,6 +16,7 @@ mod admin_function_event; mod database_ddl_event; +mod dict_groupby_sst; mod event_recorder_test_util; #[macro_use] mod grpc;