rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-01-09 23:02:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(flow): support prom ql(in tql) in flow (#6063)

Browse Source 
* feat: support parse prom ql in create flow

* refactor

* fix: just run tql unmodified

* refactor: determine type faster

* fix: pass original query

* tests: sqlness

* test: fix format&chore

* fix: get raw query

* test: fix sqlness randomness

* chore: what's the box for?

* test: location_to_index

* test: make sqlness more determinstic

* fix: tmp add sleep 1s after flush_flow

* undo test sleep 1s&rm done todo

* chore: more tests
This commit is contained in:
discord9
2025-05-22 11:06:09 +08:00
committed by GitHub
parent f55af5838c
commit fc6300a2ba
18 changed files with 704 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

172
tests/cases/standalone/common/flow/flow_tql.result Normal file
View File

@@ -0,0 +1,172 @@
CREATE TABLE http_requests (
ts timestamp(3) time index,
host STRING,
idc STRING,
val BIGINT,
PRIMARY KEY(host, idc),
);
Affected Rows: 0
CREATE FLOW calc_reqs SINK TO cnt_reqs AS
TQL EVAL (now() - '1m'::interval, now(), '5s') count_values("status_code", http_requests);
Affected Rows: 0
SHOW CREATE TABLE cnt_reqs;
+----------+-------------------------------------------+
| Table | Create Table |
+----------+-------------------------------------------+
| cnt_reqs | CREATE TABLE IF NOT EXISTS "cnt_reqs" ( |
| | "count(http_requests.val)" BIGINT NULL, |
| | "ts" TIMESTAMP(3) NOT NULL, |
| | "status_code" BIGINT NULL, |
| | "update_at" TIMESTAMP(3) NULL, |
| | TIME INDEX ("ts"), |
| | PRIMARY KEY ("status_code") |
| | ) |
| | |
| | ENGINE=mito |
| | |
+----------+-------------------------------------------+
INSERT INTO TABLE http_requests VALUES
(now() - '17s'::interval, 'host1', 'idc1', 200),
(now() - '17s'::interval, 'host2', 'idc1', 200),
(now() - '17s'::interval, 'host3', 'idc2', 200),
(now() - '17s'::interval, 'host4', 'idc2', 401),
(now() - '13s'::interval, 'host1', 'idc1', 404),
(now() - '13s'::interval, 'host2', 'idc1', 401),
(now() - '13s'::interval, 'host3', 'idc2', 404),
(now() - '13s'::interval, 'host4', 'idc2', 500),
(now() - '7s'::interval, 'host1', 'idc1', 200),
(now() - '7s'::interval, 'host2', 'idc1', 200),
(now() - '7s'::interval, 'host3', 'idc2', 201),
(now() - '7s'::interval, 'host4', 'idc2', 201),
(now() - '3s'::interval, 'host1', 'idc1', 500),
(now() - '3s'::interval, 'host2', 'idc1', 500),
(now() - '3s'::interval, 'host3', 'idc2', 500),
(now() - '3s'::interval, 'host4', 'idc2', 500);
Affected Rows: 16
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
ADMIN FLUSH_FLOW('calc_reqs');
+-------------------------------+
| ADMIN FLUSH_FLOW('calc_reqs') |
+-------------------------------+
| FLOW_FLUSHED |
+-------------------------------+
-- too much indeterminsticity in the test, so just check that the flow is running
SELECT count(*) > 0 FROM cnt_reqs;
+---------------------+
| count(*) > Int64(0) |
+---------------------+
| true |
+---------------------+
DROP FLOW calc_reqs;
Affected Rows: 0
DROP TABLE http_requests;
Affected Rows: 0
DROP TABLE cnt_reqs;
Affected Rows: 0
CREATE TABLE http_requests (
ts timestamp(3) time index,
host STRING,
idc STRING,
val BIGINT,
PRIMARY KEY(host, idc),
);
Affected Rows: 0
CREATE FLOW calc_reqs SINK TO cnt_reqs AS
TQL EVAL (0, 15, '5s') count_values("status_code", http_requests);
Affected Rows: 0
SHOW CREATE TABLE cnt_reqs;
+----------+-------------------------------------------+
| Table | Create Table |
+----------+-------------------------------------------+
| cnt_reqs | CREATE TABLE IF NOT EXISTS "cnt_reqs" ( |
| | "count(http_requests.val)" BIGINT NULL, |
| | "ts" TIMESTAMP(3) NOT NULL, |
| | "status_code" BIGINT NULL, |
| | "update_at" TIMESTAMP(3) NULL, |
| | TIME INDEX ("ts"), |
| | PRIMARY KEY ("status_code") |
| | ) |
| | |
| | ENGINE=mito |
| | |
+----------+-------------------------------------------+
INSERT INTO TABLE http_requests VALUES
(0::Timestamp, 'host1', 'idc1', 200),
(0::Timestamp, 'host2', 'idc1', 200),
(0::Timestamp, 'host3', 'idc2', 200),
(0::Timestamp, 'host4', 'idc2', 401),
(5000::Timestamp, 'host1', 'idc1', 404),
(5000::Timestamp, 'host2', 'idc1', 401),
(5000::Timestamp, 'host3', 'idc2', 404),
(5000::Timestamp, 'host4', 'idc2', 500),
(10000::Timestamp, 'host1', 'idc1', 200),
(10000::Timestamp, 'host2', 'idc1', 200),
(10000::Timestamp, 'host3', 'idc2', 201),
(10000::Timestamp, 'host4', 'idc2', 201),
(15000::Timestamp, 'host1', 'idc1', 500),
(15000::Timestamp, 'host2', 'idc1', 500),
(15000::Timestamp, 'host3', 'idc2', 500),
(15000::Timestamp, 'host4', 'idc2', 500);
Affected Rows: 16
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
ADMIN FLUSH_FLOW('calc_reqs');
+-------------------------------+
| ADMIN FLUSH_FLOW('calc_reqs') |
+-------------------------------+
| FLOW_FLUSHED |
+-------------------------------+
SELECT "count(http_requests.val)", ts, status_code FROM cnt_reqs ORDER BY ts, status_code;
+--------------------------+---------------------+-------------+
| count(http_requests.val) | ts | status_code |
+--------------------------+---------------------+-------------+
| 3 | 1970-01-01T00:00:00 | 200 |
| 1 | 1970-01-01T00:00:00 | 401 |
| 1 | 1970-01-01T00:00:05 | 401 |
| 2 | 1970-01-01T00:00:05 | 404 |
| 1 | 1970-01-01T00:00:05 | 500 |
| 2 | 1970-01-01T00:00:10 | 200 |
| 2 | 1970-01-01T00:00:10 | 201 |
| 4 | 1970-01-01T00:00:15 | 500 |
+--------------------------+---------------------+-------------+
DROP FLOW calc_reqs;
Affected Rows: 0
DROP TABLE http_requests;
Affected Rows: 0
DROP TABLE cnt_reqs;
Affected Rows: 0

80
tests/cases/standalone/common/flow/flow_tql.sql Normal file
View File

@@ -0,0 +1,80 @@
CREATE TABLE http_requests (
ts timestamp(3) time index,
host STRING,
idc STRING,
val BIGINT,
PRIMARY KEY(host, idc),
);
CREATE FLOW calc_reqs SINK TO cnt_reqs AS
TQL EVAL (now() - '1m'::interval, now(), '5s') count_values("status_code", http_requests);
SHOW CREATE TABLE cnt_reqs;
INSERT INTO TABLE http_requests VALUES
(now() - '17s'::interval, 'host1', 'idc1', 200),
(now() - '17s'::interval, 'host2', 'idc1', 200),
(now() - '17s'::interval, 'host3', 'idc2', 200),
(now() - '17s'::interval, 'host4', 'idc2', 401),
(now() - '13s'::interval, 'host1', 'idc1', 404),
(now() - '13s'::interval, 'host2', 'idc1', 401),
(now() - '13s'::interval, 'host3', 'idc2', 404),
(now() - '13s'::interval, 'host4', 'idc2', 500),
(now() - '7s'::interval, 'host1', 'idc1', 200),
(now() - '7s'::interval, 'host2', 'idc1', 200),
(now() - '7s'::interval, 'host3', 'idc2', 201),
(now() - '7s'::interval, 'host4', 'idc2', 201),
(now() - '3s'::interval, 'host1', 'idc1', 500),
(now() - '3s'::interval, 'host2', 'idc1', 500),
(now() - '3s'::interval, 'host3', 'idc2', 500),
(now() - '3s'::interval, 'host4', 'idc2', 500);
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
ADMIN FLUSH_FLOW('calc_reqs');
-- too much indeterminsticity in the test, so just check that the flow is running
SELECT count(*) > 0 FROM cnt_reqs;
DROP FLOW calc_reqs;
DROP TABLE http_requests;
DROP TABLE cnt_reqs;
CREATE TABLE http_requests (
ts timestamp(3) time index,
host STRING,
idc STRING,
val BIGINT,
PRIMARY KEY(host, idc),
);
CREATE FLOW calc_reqs SINK TO cnt_reqs AS
TQL EVAL (0, 15, '5s') count_values("status_code", http_requests);
SHOW CREATE TABLE cnt_reqs;
INSERT INTO TABLE http_requests VALUES
(0::Timestamp, 'host1', 'idc1', 200),
(0::Timestamp, 'host2', 'idc1', 200),
(0::Timestamp, 'host3', 'idc2', 200),
(0::Timestamp, 'host4', 'idc2', 401),
(5000::Timestamp, 'host1', 'idc1', 404),
(5000::Timestamp, 'host2', 'idc1', 401),
(5000::Timestamp, 'host3', 'idc2', 404),
(5000::Timestamp, 'host4', 'idc2', 500),
(10000::Timestamp, 'host1', 'idc1', 200),
(10000::Timestamp, 'host2', 'idc1', 200),
(10000::Timestamp, 'host3', 'idc2', 201),
(10000::Timestamp, 'host4', 'idc2', 201),
(15000::Timestamp, 'host1', 'idc1', 500),
(15000::Timestamp, 'host2', 'idc1', 500),
(15000::Timestamp, 'host3', 'idc2', 500),
(15000::Timestamp, 'host4', 'idc2', 500);
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
ADMIN FLUSH_FLOW('calc_reqs');
SELECT "count(http_requests.val)", ts, status_code FROM cnt_reqs ORDER BY ts, status_code;
DROP FLOW calc_reqs;
DROP TABLE http_requests;
DROP TABLE cnt_reqs;

20
tests/cases/standalone/common/flow/show_create_flow.result
View File

@@ -44,7 +44,7 @@ SELECT flow_name, table_catalog, flow_definition, source_table_names FROM INFORM
+---------------------+---------------+---------------------------------------------------------+------------------------------------+
| flow_name | table_catalog | flow_definition | source_table_names |
+---------------------+---------------+---------------------------------------------------------+------------------------------------+
| filter_numbers_show | greptime | SELECT number FROM numbers_input_show WHERE number > 10 | greptime.public.numbers_input_show |
| filter_numbers_show | greptime | SELECT number FROM numbers_input_show where number > 10 | greptime.public.numbers_input_show |
+---------------------+---------------+---------------------------------------------------------+------------------------------------+
SHOW FLOWS LIKE 'filter_numbers_show';
@@ -80,7 +80,7 @@ SELECT flow_name, table_catalog, flow_definition, source_table_names FROM INFORM
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
| flow_name | table_catalog | flow_definition | source_table_names |
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > 10 | greptime.public.numbers_input_show |
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show where number > 10 | greptime.public.numbers_input_show |
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
-- this one should error out
@@ -94,7 +94,7 @@ SELECT flow_name, table_catalog, flow_definition, source_table_names FROM INFORM
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
| flow_name | table_catalog | flow_definition | source_table_names |
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > 10 | greptime.public.numbers_input_show |
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show where number > 10 | greptime.public.numbers_input_show |
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
-- makesure it's not replaced in flownode
@@ -131,7 +131,7 @@ SELECT flow_name, table_catalog, flow_definition, source_table_names FROM INFORM
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
| flow_name | table_catalog | flow_definition | source_table_names |
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > 10 | greptime.public.numbers_input_show |
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show where number > 10 | greptime.public.numbers_input_show |
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
-- makesure it's not replaced in flownode
@@ -169,7 +169,7 @@ SELECT flow_name, table_catalog, flow_definition, source_table_names FROM INFORM
+---------------------+---------------+------------------------------------------------------------+------------------------------------+
| flow_name | table_catalog | flow_definition | source_table_names |
+---------------------+---------------+------------------------------------------------------------+------------------------------------+
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > 3 | greptime.public.numbers_input_show |
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show where number > 3 | greptime.public.numbers_input_show |
+---------------------+---------------+------------------------------------------------------------+------------------------------------+
-- makesure it's replaced in flownode
@@ -209,7 +209,7 @@ SELECT flow_name, table_catalog, flow_definition, source_table_names FROM INFORM
+---------------------+---------------+------------------------------------------------------------+------------------------------------+
| flow_name | table_catalog | flow_definition | source_table_names |
+---------------------+---------------+------------------------------------------------------------+------------------------------------+
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > 3 | greptime.public.numbers_input_show |
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show where number > 3 | greptime.public.numbers_input_show |
+---------------------+---------------+------------------------------------------------------------+------------------------------------+
DROP FLOW filter_numbers_show;
@@ -241,7 +241,7 @@ SELECT flow_name, table_catalog, flow_definition, source_table_names FROM INFORM
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
| flow_name | table_catalog | flow_definition | source_table_names |
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > -2 | greptime.public.numbers_input_show |
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show where number > -2 | greptime.public.numbers_input_show |
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
DROP FLOW filter_numbers_show;
@@ -258,7 +258,7 @@ SELECT flow_name, table_catalog, flow_definition, source_table_names FROM INFORM
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
| flow_name | table_catalog | flow_definition | source_table_names |
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > -3 | greptime.public.numbers_input_show |
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show where number > -3 | greptime.public.numbers_input_show |
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
-- makesure after recover should be the same
@@ -277,7 +277,7 @@ SELECT flow_name, table_catalog, flow_definition, source_table_names FROM INFORM
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
| flow_name | table_catalog | flow_definition | source_table_names |
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > -3 | greptime.public.numbers_input_show |
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show where number > -3 | greptime.public.numbers_input_show |
+---------------------+---------------+-------------------------------------------------------------+------------------------------------+
SELECT * FROM out_num_cnt_show;
@@ -389,7 +389,7 @@ SELECT flow_definition, source_table_names FROM INFORMATION_SCHEMA.FLOWS WHERE f
+---------------------------------------------------------------+------------------------------------+
| flow_definition | source_table_names |
+---------------------------------------------------------------+------------------------------------+
| SELECT number AS n1 FROM numbers_input_show WHERE number > 10 | greptime.public.numbers_input_show |
| SELECT number as n1 FROM numbers_input_show where number > 10 | greptime.public.numbers_input_show |
+---------------------------------------------------------------+------------------------------------+
INSERT INTO numbers_input_show VALUES (10, 6),(11, 8),(15, 7),(18, 3);