rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-23 12:22:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: backport scan statistics and partitioning fixes to release/5e02efa9 (#9783)

Browse Source 
* fix: preserve count correctness after repartition (#9154)

* fix: preserve count correctness after repartition

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix: restore full predicate guard for count statistics

Signed-off-by: WenyXu <wenymedia@gmail.com>

* test: update count plans for full predicate guard

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix: preserve count statistics for safe partition scans

Signed-off-by: WenyXu <wenymedia@gmail.com>

* test: retain repartition home guard and cover staging flush

Signed-off-by: WenyXu <wenymedia@gmail.com>

---------

Signed-off-by: WenyXu <wenymedia@gmail.com>
(cherry picked from commit d9b97796a6)
Signed-off-by: Lei, HUANG <ratuthomm@gmail.com>

* fix: only push down aggregates grouped by the partition columns themselves (#9337)

* fix: only push down aggregates grouped by the partition columns themselves

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* fix: keep grouping sets on the frontend for partitioned tables

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

---------

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
(cherry picked from commit def0c2ec5c)
Signed-off-by: Lei, HUANG <ratuthomm@gmail.com>

* fix(table): require complete primary keys for hash partitioning (#9507)

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
(cherry picked from commit 28ec3ff5aa)
Signed-off-by: Lei, HUANG <ratuthomm@gmail.com>

* fix(table): stop claiming exact null and distinct counts for append scans (#9593)

RegionScanExec reported null_count = 0 and distinct_count = total_rows for
every column of an append-only table. DataFusion answers count(col) and
count(DISTINCT col) from these statistics, so both returned the row count.
Keep the exact row count and an exact zero null count for non-nullable
columns, so count(*) is still answered from statistics.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
(cherry picked from commit 5cfdc30a69)
Signed-off-by: Lei, HUANG <ratuthomm@gmail.com>

* fix(table): stop claiming tag order for logical metric scans (#9522)

* test: cover metric logical scan tag ordering

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(table): only claim projected primary key prefix ordering

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(table): keep physical key ordering for metric projections

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(table): compare unknown partition counts explicitly

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(table): model scoped internal TSID scan properties

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

---------

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
(cherry picked from commit 75bc4ce46d)
Signed-off-by: Lei, HUANG <ratuthomm@gmail.com>

* test: regenerate append_count_nulls result for release plan rendering

Signed-off-by: Lei, HUANG <ratuthomm@gmail.com>

---------

Signed-off-by: WenyXu <wenymedia@gmail.com>
Signed-off-by: Lei, HUANG <ratuthomm@gmail.com>
Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
Co-authored-by: Weny Xu <wenymedia@gmail.com>
Co-authored-by: dennis zhuang <killme2008@gmail.com>
Co-authored-by: discord9 <discord9@163.com>
This commit is contained in:
Lei, HUANG
2026-10-20 08:32:32 -07:00
committed by GitHub
co-authored by Weny Xu dennis zhuang discord9
parent 1e5fef5809
commit fd3faeb7ed
16 changed files with 1442 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/cases/standalone/common/aggregate/append_count_nulls.result
+165
View File
@@ -0,0 +1,165 @@
-- count(col) and count(DISTINCT col) on append-only tables must not be answered
-- from row-count statistics, while count(*) still is.
CREATE TABLE n1 (ts TIMESTAMP TIME INDEX, v BIGINT, s STRING) WITH (append_mode='true');
Affected Rows: 0
INSERT INTO n1 VALUES (1000, 1, 'a'), (2000, NULL, 'a'), (3000, NULL, NULL), (4000, 4, 'b');
Affected Rows: 4
SELECT count(*), count(v), count(s), count(DISTINCT s) FROM n1;
+----------+-------------+-------------+----------------------+
| count(*) | count(n1.v) | count(n1.s) | count(DISTINCT n1.s) |
+----------+-------------+-------------+----------------------+
| 4 | 2 | 3 | 2 |
+----------+-------------+-------------+----------------------+
SELECT count(v) FROM n1;
+-------------+
| count(n1.v) |
+-------------+
| 2 |
+-------------+
-- SQLNESS REPLACE (metrics.*) REDACTED
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
-- SQLNESS REPLACE (-+) -
-- SQLNESS REPLACE (\s\s+) _
-- SQLNESS REPLACE (peers.*) REDACTED
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
EXPLAIN ANALYZE SELECT count(*) FROM n1;
+-+-+-+
| stage | node | plan_|
+-+-+-+
| 0_| 0_|_CooperativeExec REDACTED
|_|_|_MergeScanExec: REDACTED
|_|_|_|
| 1_| 0_|_ProjectionExec: expr=[4 as count(*)] REDACTED
|_|_|_PlaceholderRowExec REDACTED
|_|_|_|
|_|_| Total rows: 1_|
+-+-+-+
ADMIN FLUSH_TABLE('n1');
+-------------------------+
| ADMIN FLUSH_TABLE('n1') |
+-------------------------+
| 0 |
+-------------------------+
SELECT count(*), count(v), count(s), count(DISTINCT s) FROM n1;
+----------+-------------+-------------+----------------------+
| count(*) | count(n1.v) | count(n1.s) | count(DISTINCT n1.s) |
+----------+-------------+-------------+----------------------+
| 4 | 2 | 3 | 2 |
+----------+-------------+-------------+----------------------+
-- SQLNESS REPLACE (metrics.*) REDACTED
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
-- SQLNESS REPLACE (-+) -
-- SQLNESS REPLACE (\s\s+) _
-- SQLNESS REPLACE (peers.*) REDACTED
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
EXPLAIN ANALYZE SELECT count(*) FROM n1;
+-+-+-+
| stage | node | plan_|
+-+-+-+
| 0_| 0_|_CooperativeExec REDACTED
|_|_|_MergeScanExec: REDACTED
|_|_|_|
| 1_| 0_|_ProjectionExec: expr=[4 as count(*)] REDACTED
|_|_|_PlaceholderRowExec REDACTED
|_|_|_|
|_|_| Total rows: 1_|
+-+-+-+
DROP TABLE n1;
Affected Rows: 0
CREATE TABLE n2 (
host STRING,
ts TIMESTAMP TIME INDEX,
v BIGINT,
s STRING,
PRIMARY KEY (host)
)
PARTITION ON COLUMNS (host) (
host < 'm',
host >= 'm'
)
WITH (append_mode='true');
Affected Rows: 0
INSERT INTO n2 VALUES
('a', 1000, 1, 'a'),
('b', 2000, NULL, 'a'),
('x', 3000, NULL, NULL),
('y', 4000, 4, 'b');
Affected Rows: 4
-- count is pushed down to each region as a state aggregate.
SELECT count(v), count(s) FROM n2;
+-------------+-------------+
| count(n2.v) | count(n2.s) |
+-------------+-------------+
| 2 | 3 |
+-------------+-------------+
ADMIN FLUSH_TABLE('n2');
+-------------------------+
| ADMIN FLUSH_TABLE('n2') |
+-------------------------+
| 0 |
+-------------------------+
SELECT count(v), count(s) FROM n2;
+-------------+-------------+
| count(n2.v) | count(n2.s) |
+-------------+-------------+
| 2 | 3 |
+-------------+-------------+
-- SQLNESS REPLACE (metrics.*) REDACTED
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
-- SQLNESS REPLACE (-+) -
-- SQLNESS REPLACE (\s\s+) _
-- SQLNESS REPLACE (peers.*) REDACTED
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
EXPLAIN ANALYZE SELECT count(*) FROM n2;
+-+-+-+
| stage | node | plan_|
+-+-+-+
| 0_| 0_|_ProjectionExec: expr=[count(Int64(1))@0 as count(*)] REDACTED
|_|_|_AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] REDACTED
|_|_|_CoalescePartitionsExec REDACTED
|_|_|_AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] REDACTED
|_|_|_RepartitionExec: partitioning=REDACTED
|_|_|_MergeScanExec: REDACTED
|_|_|_|
| 1_| 0_|_ProjectionExec: expr=[{count[count]:2} as __count_state(n2.ts)] REDACTED
|_|_|_PlaceholderRowExec REDACTED
|_|_|_|
| 1_| 1_|_ProjectionExec: expr=[{count[count]:2} as __count_state(n2.ts)] REDACTED
|_|_|_PlaceholderRowExec REDACTED
|_|_|_|
|_|_| Total rows: 1_|
+-+-+-+
DROP TABLE n2;
Affected Rows: 0
tests/cases/standalone/common/aggregate/append_count_nulls.sql
+67
View File
@@ -0,0 +1,67 @@
-- count(col) and count(DISTINCT col) on append-only tables must not be answered
-- from row-count statistics, while count(*) still is.
CREATE TABLE n1 (ts TIMESTAMP TIME INDEX, v BIGINT, s STRING) WITH (append_mode='true');
INSERT INTO n1 VALUES (1000, 1, 'a'), (2000, NULL, 'a'), (3000, NULL, NULL), (4000, 4, 'b');
SELECT count(*), count(v), count(s), count(DISTINCT s) FROM n1;
SELECT count(v) FROM n1;
-- SQLNESS REPLACE (metrics.*) REDACTED
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
-- SQLNESS REPLACE (-+) -
-- SQLNESS REPLACE (\s\s+) _
-- SQLNESS REPLACE (peers.*) REDACTED
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
EXPLAIN ANALYZE SELECT count(*) FROM n1;
ADMIN FLUSH_TABLE('n1');
SELECT count(*), count(v), count(s), count(DISTINCT s) FROM n1;
-- SQLNESS REPLACE (metrics.*) REDACTED
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
-- SQLNESS REPLACE (-+) -
-- SQLNESS REPLACE (\s\s+) _
-- SQLNESS REPLACE (peers.*) REDACTED
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
EXPLAIN ANALYZE SELECT count(*) FROM n1;
DROP TABLE n1;
CREATE TABLE n2 (
host STRING,
ts TIMESTAMP TIME INDEX,
v BIGINT,
s STRING,
PRIMARY KEY (host)
)
PARTITION ON COLUMNS (host) (
host < 'm',
host >= 'm'
)
WITH (append_mode='true');
INSERT INTO n2 VALUES
('a', 1000, 1, 'a'),
('b', 2000, NULL, 'a'),
('x', 3000, NULL, NULL),
('y', 4000, 4, 'b');
-- count is pushed down to each region as a state aggregate.
SELECT count(v), count(s) FROM n2;
ADMIN FLUSH_TABLE('n2');
SELECT count(v), count(s) FROM n2;
-- SQLNESS REPLACE (metrics.*) REDACTED
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
-- SQLNESS REPLACE (-+) -
-- SQLNESS REPLACE (\s\s+) _
-- SQLNESS REPLACE (peers.*) REDACTED
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
EXPLAIN ANALYZE SELECT count(*) FROM n2;
DROP TABLE n2;
tests/cases/standalone/common/aggregate/multi_regions.result
+48
View File
@@ -113,6 +113,54 @@ select sum(val) from t group by idc;
|_|_| Total rows: 0_|
+-+-+-+
insert into t values
(1000, 1, '1000-x', 'a'),
(1000, 2, '1000-y', 'a'),
(1000, 3, '2000-x', 'a'),
(1000, 4, '2000-y', 'a');
Affected Rows: 4
-- group keys derived from the partition column span regions
select substr(host, 6, 1) as g, count(*), sum(val) from t group by g order by g;
+---+----------+------------+
| g | count(*) | sum(t.val) |
+---+----------+------------+
| x | 2 | 4.0 |
| y | 2 | 6.0 |
+---+----------+------------+
-- grouping sets are computed on the frontend, only the PostgreSQL dialect parses them
-- SQLNESS PROTOCOL POSTGRES
select host, idc, sum(val) from t group by grouping sets ((host, idc), (host)) order by host, idc;
+--------+-----+------------+
| host | idc | sum(t.val) |
+--------+-----+------------+
| 1000-x | a | 1.0 |
| 1000-x | | 1.0 |
| 1000-y | a | 2.0 |
| 1000-y | | 2.0 |
| 2000-x | a | 3.0 |
| 2000-x | | 3.0 |
| 2000-y | a | 4.0 |
| 2000-y | | 4.0 |
+--------+-----+------------+
-- SQLNESS PROTOCOL POSTGRES
select host, sum(val) from t group by rollup(host) order by host;
+--------+------------+
| host | sum(t.val) |
+--------+------------+
| 1000-x | 1.0 |
| 1000-y | 2.0 |
| 2000-x | 3.0 |
| 2000-y | 4.0 |
| | 10.0 |
+--------+------------+
drop table t;
Affected Rows: 0
tests/cases/standalone/common/aggregate/multi_regions.sql
+16
View File
@@ -45,4 +45,20 @@ select sum(val) from t;
explain analyze
select sum(val) from t group by idc;
insert into t values
(1000, 1, '1000-x', 'a'),
(1000, 2, '1000-y', 'a'),
(1000, 3, '2000-x', 'a'),
(1000, 4, '2000-y', 'a');
-- group keys derived from the partition column span regions
select substr(host, 6, 1) as g, count(*), sum(val) from t group by g order by g;
-- grouping sets are computed on the frontend, only the PostgreSQL dialect parses them
-- SQLNESS PROTOCOL POSTGRES
select host, idc, sum(val) from t group by grouping sets ((host, idc), (host)) order by host, idc;
-- SQLNESS PROTOCOL POSTGRES
select host, sum(val) from t group by rollup(host) order by host;
drop table t;
tests/cases/standalone/common/aggregate/projected_pk_group_by.result
+69
View File
@@ -0,0 +1,69 @@
CREATE TABLE projected_pk_group_by (
marker TINYINT UNSIGNED,
host STRING,
ts TIMESTAMP TIME INDEX,
PRIMARY KEY (marker, host)
);
Affected Rows: 0
INSERT INTO projected_pk_group_by VALUES
(0, 'a', 1000),
(0, 'a', 2000),
(0, 'b', 1000),
(0, 'b', 2000),
(0, 'c', 1000),
(1, NULL, 1000);
Affected Rows: 6
SELECT marker, count(*) AS row_count
FROM projected_pk_group_by
GROUP BY marker
ORDER BY marker;
+--------+-----------+
| marker | row_count |
+--------+-----------+
| 0 | 5 |
| 1 | 1 |
+--------+-----------+
SELECT marker, host, count(*) AS row_count
FROM projected_pk_group_by
GROUP BY marker, host
ORDER BY marker, host;
+--------+------+-----------+
| marker | host | row_count |
+--------+------+-----------+
| 0 | a | 2 |
| 0 | b | 2 |
| 0 | c | 1 |
| 1 | | 1 |
+--------+------+-----------+
ADMIN FLUSH_TABLE('projected_pk_group_by');
+--------------------------------------------+
| ADMIN FLUSH_TABLE('projected_pk_group_by') |
+--------------------------------------------+
| 0 |
+--------------------------------------------+
SELECT marker, count(*) AS row_count
FROM projected_pk_group_by
GROUP BY marker
ORDER BY marker;
+--------+-----------+
| marker | row_count |
+--------+-----------+
| 0 | 5 |
| 1 | 1 |
+--------+-----------+
DROP TABLE projected_pk_group_by;
Affected Rows: 0
tests/cases/standalone/common/aggregate/projected_pk_group_by.sql
+33
View File
@@ -0,0 +1,33 @@
CREATE TABLE projected_pk_group_by (
marker TINYINT UNSIGNED,
host STRING,
ts TIMESTAMP TIME INDEX,
PRIMARY KEY (marker, host)
);
INSERT INTO projected_pk_group_by VALUES
(0, 'a', 1000),
(0, 'a', 2000),
(0, 'b', 1000),
(0, 'b', 2000),
(0, 'c', 1000),
(1, NULL, 1000);
SELECT marker, count(*) AS row_count
FROM projected_pk_group_by
GROUP BY marker
ORDER BY marker;
SELECT marker, host, count(*) AS row_count
FROM projected_pk_group_by
GROUP BY marker, host
ORDER BY marker, host;
ADMIN FLUSH_TABLE('projected_pk_group_by');
SELECT marker, count(*) AS row_count
FROM projected_pk_group_by
GROUP BY marker
ORDER BY marker;
DROP TABLE projected_pk_group_by;