mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-01-04 20:32:56 +00:00
* feat: add metrics for request wait time and adjust stall metrics (#6540) * feat: add metric greptime_mito_request_wait_time to observe wait time Signed-off-by: evenyag <realevenyag@gmail.com> * feat: add worker to wait time metric Signed-off-by: evenyag <realevenyag@gmail.com> * refactor: rename stall gauge to greptime_mito_write_stalling_count Signed-off-by: evenyag <realevenyag@gmail.com> * feat: change greptime_mito_write_stall_total to total stalled requests Signed-off-by: evenyag <realevenyag@gmail.com> * refactor: merge lazy static blocks Signed-off-by: evenyag <realevenyag@gmail.com> --------- Signed-off-by: evenyag <realevenyag@gmail.com> * fix: estimate mem size for bulk ingester (#6550) Signed-off-by: evenyag <realevenyag@gmail.com> * fix: flow mirror cache (#6551) * fix: invalid cache when flownode change address Signed-off-by: discord9 <discord9@163.com> * update comments Signed-off-by: discord9 <discord9@163.com> * fix Signed-off-by: discord9 <discord9@163.com> * refactor: add log&rename Signed-off-by: discord9 <discord9@163.com> * stuff Signed-off-by: discord9 <discord9@163.com> --------- Signed-off-by: discord9 <discord9@163.com> Signed-off-by: evenyag <realevenyag@gmail.com> * feat: impl timestamp function for promql (#6556) * feat: impl timestamp function for promql Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * chore: style and typo Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * fix: test Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * docs: update comments Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * chore: comment Signed-off-by: Dennis Zhuang <killme2008@gmail.com> --------- Signed-off-by: Dennis Zhuang <killme2008@gmail.com> Signed-off-by: evenyag <realevenyag@gmail.com> * feat: MergeScan print input (#6563) * feat: MergeScan print input Signed-off-by: discord9 <discord9@163.com> * test: fix ut Signed-off-by: discord9 <discord9@163.com> --------- Signed-off-by: discord9 <discord9@163.com> Signed-off-by: evenyag <realevenyag@gmail.com> * fix: aggr group by all partition cols use partial commutative (#6534) * fix: aggr group by all partition cols use partial commutative Signed-off-by: discord9 <discord9@163.com> * test: bugged case Signed-off-by: discord9 <discord9@163.com> * test: sqlness fix Signed-off-by: discord9 <discord9@163.com> * test: more redacted Signed-off-by: discord9 <discord9@163.com> * more cases Signed-off-by: discord9 <discord9@163.com> * even more test cases Signed-off-by: discord9 <discord9@163.com> * join testcase Signed-off-by: discord9 <discord9@163.com> * fix: column requirement added in correct location Signed-off-by: discord9 <discord9@163.com> * fix test Signed-off-by: discord9 <discord9@163.com> * chore: clippy Signed-off-by: discord9 <discord9@163.com> * track col reqs per stack Signed-off-by: discord9 <discord9@163.com> * fix: continue Signed-off-by: discord9 <discord9@163.com> * chore: clippy Signed-off-by: discord9 <discord9@163.com> * refactor: test mod Signed-off-by: discord9 <discord9@163.com> * test utils Signed-off-by: discord9 <discord9@163.com> * test: better test Signed-off-by: discord9 <discord9@163.com> * more testcases Signed-off-by: discord9 <discord9@163.com> * test limit push down Signed-off-by: discord9 <discord9@163.com> * more testcases Signed-off-by: discord9 <discord9@163.com> * more testcase Signed-off-by: discord9 <discord9@163.com> * more test Signed-off-by: discord9 <discord9@163.com> * chore: update sqlness Signed-off-by: discord9 <discord9@163.com> * chore: update commnets Signed-off-by: discord9 <discord9@163.com> * fix: check col reqs from bottom to upper Signed-off-by: discord9 <discord9@163.com> * chore: more comment Signed-off-by: discord9 <discord9@163.com> * docs: more todo Signed-off-by: discord9 <discord9@163.com> * chore: comments Signed-off-by: discord9 <discord9@163.com> * test: a new failing test that should be fixed Signed-off-by: discord9 <discord9@163.com> * fix: part col alias tracking Signed-off-by: discord9 <discord9@163.com> * chore: unused Signed-off-by: discord9 <discord9@163.com> * chore: clippy Signed-off-by: discord9 <discord9@163.com> * docs: comment Signed-off-by: discord9 <discord9@163.com> * mroe testcase Signed-off-by: discord9 <discord9@163.com> * more testcase for step/part aggr combine Signed-off-by: discord9 <discord9@163.com> * FIXME: a new bug Signed-off-by: discord9 <discord9@163.com> * literally unfixable Signed-off-by: discord9 <discord9@163.com> * chore: remove some debug print Signed-off-by: discord9 <discord9@163.com> --------- Signed-off-by: discord9 <discord9@163.com> Signed-off-by: evenyag <realevenyag@gmail.com> --------- Signed-off-by: evenyag <realevenyag@gmail.com> Signed-off-by: discord9 <discord9@163.com> Signed-off-by: Dennis Zhuang <killme2008@gmail.com> Co-authored-by: fys <40801205+fengys1996@users.noreply.github.com> Co-authored-by: discord9 <55937128+discord9@users.noreply.github.com> Co-authored-by: dennis zhuang <killme2008@gmail.com>
308 lines
9.0 KiB
SQL
308 lines
9.0 KiB
SQL
CREATE TABLE IF NOT EXISTS aggr_optimize_not (
|
|
a STRING NULL,
|
|
b STRING NULL,
|
|
c STRING NULL,
|
|
d STRING NULL,
|
|
greptime_timestamp TIMESTAMP(3) NOT NULL,
|
|
greptime_value DOUBLE NULL,
|
|
TIME INDEX (greptime_timestamp),
|
|
PRIMARY KEY (a, b, c, d)
|
|
) PARTITION ON COLUMNS (a, b, c) (a < 'b', a >= 'b',);
|
|
|
|
-- Case 0: group by columns are the same as partition columns.
|
|
-- This query shouldn't push down aggregation even if group by columns are partitioned.
|
|
-- because sort is already pushed down.
|
|
-- If it does, it will cause a wrong result.
|
|
-- explain at 0s, 5s and 10s. No point at 0s.
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
tql explain (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr_optimize_not [2m]));
|
|
|
|
-- SQLNESS REPLACE (metrics.*) REDACTED
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
-- SQLNESS REPLACE (-+) -
|
|
-- SQLNESS REPLACE (\s\s+) _
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
|
tql analyze (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr_optimize_not [2m]));
|
|
|
|
-- Case 1: group by columns are prefix of partition columns.
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
tql explain (1752591864, 1752592164, '30s') sum by (a, b) (max_over_time(aggr_optimize_not [2m]));
|
|
|
|
-- SQLNESS REPLACE (metrics.*) REDACTED
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
-- SQLNESS REPLACE (-+) -
|
|
-- SQLNESS REPLACE (\s\s+) _
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
|
tql analyze (1752591864, 1752592164, '30s') sum by (a, b) (max_over_time(aggr_optimize_not [2m]));
|
|
|
|
-- Case 2: group by columns are prefix of partition columns.
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
tql explain (1752591864, 1752592164, '30s') avg by (a) (max_over_time(aggr_optimize_not [2m]));
|
|
|
|
-- SQLNESS REPLACE (metrics.*) REDACTED
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
-- SQLNESS REPLACE (-+) -
|
|
-- SQLNESS REPLACE (\s\s+) _
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
|
tql analyze (1752591864, 1752592164, '30s') avg by (a) (max_over_time(aggr_optimize_not [2m]));
|
|
|
|
-- Case 3: group by columns are superset of partition columns.
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
tql explain (1752591864, 1752592164, '30s') count by (a, b, c, d) (max_over_time(aggr_optimize_not [2m]));
|
|
|
|
-- SQLNESS REPLACE (metrics.*) REDACTED
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
-- SQLNESS REPLACE (-+) -
|
|
-- SQLNESS REPLACE (\s\s+) _
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
|
tql analyze (1752591864, 1752592164, '30s') count by (a, b, c, d) (max_over_time(aggr_optimize_not [2m]));
|
|
|
|
-- Case 4: group by columns are not prefix of partition columns.
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
tql explain (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr_optimize_not [2m]));
|
|
|
|
-- SQLNESS REPLACE (metrics.*) REDACTED
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
-- SQLNESS REPLACE (-+) -
|
|
-- SQLNESS REPLACE (\s\s+) _
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
|
tql analyze (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr_optimize_not [2m]));
|
|
|
|
-- Case 5: a simple sum
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
tql explain sum(aggr_optimize_not);
|
|
|
|
-- SQLNESS REPLACE (metrics.*) REDACTED
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
-- SQLNESS REPLACE (-+) -
|
|
-- SQLNESS REPLACE (\s\s+) _
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
|
tql analyze sum(aggr_optimize_not);
|
|
|
|
-- TODO(discord9): more cases for aggr push down interacting with partitioning&tql
|
|
CREATE TABLE IF NOT EXISTS aggr_optimize_not_count (
|
|
a STRING NULL,
|
|
b STRING NULL,
|
|
c STRING NULL,
|
|
d STRING NULL,
|
|
greptime_timestamp TIMESTAMP(3) NOT NULL,
|
|
greptime_value DOUBLE NULL,
|
|
TIME INDEX (greptime_timestamp),
|
|
PRIMARY KEY (a, b, c, d)
|
|
) PARTITION ON COLUMNS (a, b, c) (a < 'b', a >= 'b',);
|
|
|
|
-- Case 6: Test average rate (sum/count like)
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
tql explain (1752591864, 1752592164, '30s') sum by (a, b, c) (rate(aggr_optimize_not [2m])) / sum by (a, b, c) (rate(aggr_optimize_not_count [2m]));
|
|
|
|
-- SQLNESS REPLACE (metrics.*) REDACTED
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
-- SQLNESS REPLACE (-+) -
|
|
-- SQLNESS REPLACE (\s\s+) _
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
|
tql analyze (1752591864, 1752592164, '30s') sum by (a, b, c) (rate(aggr_optimize_not [2m])) / sum by (a, b, c) (rate(aggr_optimize_not_count [2m]));
|
|
|
|
-- Case 7: aggregate without sort should be pushed down. This one push down for include all partition columns.
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
EXPLAIN
|
|
SELECT
|
|
min(greptime_value)
|
|
FROM
|
|
aggr_optimize_not
|
|
GROUP BY
|
|
a,
|
|
b,
|
|
c;
|
|
|
|
-- SQLNESS REPLACE (metrics.*) REDACTED
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
-- SQLNESS REPLACE (-+) -
|
|
-- SQLNESS REPLACE (\s\s+) _
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
|
EXPLAIN ANALYZE
|
|
SELECT
|
|
min(greptime_value)
|
|
FROM
|
|
aggr_optimize_not
|
|
GROUP BY
|
|
a,
|
|
b,
|
|
c;
|
|
|
|
-- Case 8: aggregate without sort should be pushed down. This one push down for include all partition columns then some
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
EXPLAIN
|
|
SELECT
|
|
min(greptime_value)
|
|
FROM
|
|
aggr_optimize_not
|
|
GROUP BY
|
|
a,
|
|
b,
|
|
c,
|
|
d;
|
|
|
|
-- SQLNESS REPLACE (metrics.*) REDACTED
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
-- SQLNESS REPLACE (-+) -
|
|
-- SQLNESS REPLACE (\s\s+) _
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
|
EXPLAIN ANALYZE
|
|
SELECT
|
|
min(greptime_value)
|
|
FROM
|
|
aggr_optimize_not
|
|
GROUP BY
|
|
a,
|
|
b,
|
|
c,
|
|
d;
|
|
|
|
-- Case 9: aggregate without sort should be pushed down. This one push down for step aggr push down.
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
EXPLAIN
|
|
SELECT
|
|
min(greptime_value)
|
|
FROM
|
|
aggr_optimize_not
|
|
GROUP BY
|
|
a,
|
|
b;
|
|
|
|
-- SQLNESS REPLACE (metrics.*) REDACTED
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
-- SQLNESS REPLACE (-+) -
|
|
-- SQLNESS REPLACE (\s\s+) _
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
|
EXPLAIN ANALYZE
|
|
SELECT
|
|
min(greptime_value)
|
|
FROM
|
|
aggr_optimize_not
|
|
GROUP BY
|
|
a,
|
|
b;
|
|
|
|
-- Case 10: aggregate without sort should be pushed down. This one push down for step aggr push down with complex aggr
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
EXPLAIN
|
|
SELECT
|
|
min(greptime_value) + max(greptime_value)
|
|
FROM
|
|
aggr_optimize_not
|
|
GROUP BY
|
|
a,
|
|
b;
|
|
|
|
-- SQLNESS REPLACE (metrics.*) REDACTED
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
-- SQLNESS REPLACE (-+) -
|
|
-- SQLNESS REPLACE (\s\s+) _
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
|
EXPLAIN ANALYZE
|
|
SELECT
|
|
min(greptime_value) + max(greptime_value)
|
|
FROM
|
|
aggr_optimize_not
|
|
GROUP BY
|
|
a,
|
|
b;
|
|
|
|
|
|
-- Case 11: aggregate with subquery
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
EXPLAIN
|
|
SELECT
|
|
a,
|
|
min(greptime_value)
|
|
FROM
|
|
(
|
|
SELECT
|
|
a,
|
|
b,
|
|
greptime_value
|
|
FROM
|
|
aggr_optimize_not
|
|
ORDER BY
|
|
a,
|
|
b
|
|
)
|
|
GROUP BY
|
|
a;
|
|
|
|
-- SQLNESS REPLACE (metrics.*) REDACTED
|
|
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
|
|
-- SQLNESS REPLACE (Hash.*) REDACTED
|
|
-- SQLNESS REPLACE (-+) -
|
|
-- SQLNESS REPLACE (\s\s+) _
|
|
-- SQLNESS REPLACE (peers.*) REDACTED
|
|
-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
|
|
EXPLAIN ANALYZE
|
|
SELECT
|
|
a,
|
|
min(greptime_value)
|
|
FROM
|
|
(
|
|
SELECT
|
|
a,
|
|
b,
|
|
greptime_value
|
|
FROM
|
|
aggr_optimize_not
|
|
ORDER BY
|
|
a,
|
|
b
|
|
)
|
|
GROUP BY
|
|
a;
|
|
|
|
drop table aggr_optimize_not_count;
|
|
|
|
drop table aggr_optimize_not;
|