mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-05-28 02:40:38 +00:00
feat: CREATE OR REPLACE FLOW (#5001)
* feat: Replace flow * refactor: better show create flow&tests: better check * tests: sqlness result update * tests: unit test for update * refactor: cmp with raw bytes * refactor: rename * refactor: per review
This commit is contained in:
discord9
GitHub
@@ -259,7 +259,7 @@ SHOW CREATE FLOW filter_numbers_basic;
|
||||
+----------------------+----------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| Flow | Create Flow |
|
||||
+----------------------+----------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| filter_numbers_basic | CREATE OR REPLACE FLOW IF NOT EXISTS filter_numbers_basic |
|
||||
| filter_numbers_basic | CREATE FLOW IF NOT EXISTS filter_numbers_basic |
|
||||
| | SINK TO out_num_cnt_basic |
|
||||
| | AS SELECT INTERVAL '1 day 1 second', INTERVAL '1 month 1 day 1 second', INTERVAL '1 year 1 month' FROM numbers_input_basic WHERE number > 10 |
|
||||
+----------------------+----------------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
@@ -9,7 +9,9 @@ Affected Rows: 0
|
||||
|
||||
create table out_num_cnt_show (
|
||||
number INT,
|
||||
ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP TIME INDEX);
|
||||
ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP TIME INDEX,
|
||||
PRIMARY KEY(number),
|
||||
);
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
@@ -32,7 +34,7 @@ SHOW CREATE FLOW filter_numbers_show;
|
||||
+---------------------+------------------------------------------------------------+
|
||||
| Flow | Create Flow |
|
||||
+---------------------+------------------------------------------------------------+
|
||||
| filter_numbers_show | CREATE OR REPLACE FLOW IF NOT EXISTS filter_numbers_show |
|
||||
| filter_numbers_show | CREATE FLOW IF NOT EXISTS filter_numbers_show |
|
||||
| | SINK TO out_num_cnt_show |
|
||||
| | AS SELECT number FROM numbers_input_show WHERE number > 10 |
|
||||
+---------------------+------------------------------------------------------------+
|
||||
@@ -67,6 +69,251 @@ SHOW FLOWS LIKE 'filter_numbers_show';
|
||||
++
|
||||
++
|
||||
|
||||
-- also test `CREATE OR REPLACE` and `IF NOT EXISTS`
|
||||
-- (flow exists, replace, if not exists)=(false, false, false)
|
||||
CREATE FLOW filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > 10;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
| flow_name | table_catalog | flow_definition |
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > 10 |
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
|
||||
-- this one should error out
|
||||
-- (flow exists, replace, if not exists)=(true, false, false)
|
||||
CREATE FLOW filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > 15;
|
||||
|
||||
Error: 8000(FlowAlreadyExists), Flow already exists: greptime.filter_numbers_show
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
| flow_name | table_catalog | flow_definition |
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > 10 |
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
|
||||
-- makesure it's not replaced in flownode
|
||||
INSERT INTO numbers_input_show VALUES (10, 0),(15, 1),(16, 2);
|
||||
|
||||
Affected Rows: 3
|
||||
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('filter_numbers_show');
|
||||
|
||||
+-----------------------------------------+
|
||||
| ADMIN FLUSH_FLOW('filter_numbers_show') |
|
||||
+-----------------------------------------+
|
||||
| FLOW_FLUSHED |
|
||||
+-----------------------------------------+
|
||||
|
||||
SELECT number, ts FROM out_num_cnt_show;
|
||||
|
||||
+--------+-------------------------+
|
||||
| number | ts |
|
||||
+--------+-------------------------+
|
||||
| 15 | 1970-01-01T00:00:00.001 |
|
||||
| 16 | 1970-01-01T00:00:00.002 |
|
||||
+--------+-------------------------+
|
||||
|
||||
-- after this one, the flow SHOULD NOT be replaced
|
||||
-- (flow exists, replace, if not exists)=(true, false, true)
|
||||
CREATE FLOW IF NOT EXISTS filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > 5;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
| flow_name | table_catalog | flow_definition |
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > 10 |
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
|
||||
-- makesure it's not replaced in flownode
|
||||
INSERT INTO numbers_input_show VALUES (4,4),(5,4),(10, 3),(11, 4);
|
||||
|
||||
Affected Rows: 4
|
||||
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('filter_numbers_show');
|
||||
|
||||
+-----------------------------------------+
|
||||
| ADMIN FLUSH_FLOW('filter_numbers_show') |
|
||||
+-----------------------------------------+
|
||||
| FLOW_FLUSHED |
|
||||
+-----------------------------------------+
|
||||
|
||||
SELECT number, ts FROM out_num_cnt_show;
|
||||
|
||||
+--------+-------------------------+
|
||||
| number | ts |
|
||||
+--------+-------------------------+
|
||||
| 11 | 1970-01-01T00:00:00.004 |
|
||||
| 15 | 1970-01-01T00:00:00.001 |
|
||||
| 16 | 1970-01-01T00:00:00.002 |
|
||||
+--------+-------------------------+
|
||||
|
||||
-- after this, the flow SHOULD be replaced
|
||||
-- (flow exists, replace, if not exists)=(true, true, false)
|
||||
CREATE OR REPLACE FLOW filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > 3;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
+---------------------+---------------+------------------------------------------------------------+
|
||||
| flow_name | table_catalog | flow_definition |
|
||||
+---------------------+---------------+------------------------------------------------------------+
|
||||
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > 3 |
|
||||
+---------------------+---------------+------------------------------------------------------------+
|
||||
|
||||
-- makesure it's replaced in flownode
|
||||
INSERT INTO numbers_input_show VALUES (3, 1),(4, 2),(10, 3),(11, 4);
|
||||
|
||||
Affected Rows: 4
|
||||
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('filter_numbers_show');
|
||||
|
||||
+-----------------------------------------+
|
||||
| ADMIN FLUSH_FLOW('filter_numbers_show') |
|
||||
+-----------------------------------------+
|
||||
| FLOW_FLUSHED |
|
||||
+-----------------------------------------+
|
||||
|
||||
SELECT number, ts FROM out_num_cnt_show;
|
||||
|
||||
+--------+-------------------------+
|
||||
| number | ts |
|
||||
+--------+-------------------------+
|
||||
| 4 | 1970-01-01T00:00:00.002 |
|
||||
| 10 | 1970-01-01T00:00:00.003 |
|
||||
| 11 | 1970-01-01T00:00:00.004 |
|
||||
| 15 | 1970-01-01T00:00:00.001 |
|
||||
| 16 | 1970-01-01T00:00:00.002 |
|
||||
+--------+-------------------------+
|
||||
|
||||
-- after this, the flow SHOULD error out since having both `replace` and `if not exists`
|
||||
-- (flow exists, replace, if not exists)=(true, true, true)
|
||||
CREATE OR REPLACE FLOW IF NOT EXISTS filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > 0;
|
||||
|
||||
Error: 1001(Unsupported), Unsupported operation Create flow with both `IF NOT EXISTS` and `OR REPLACE`
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
+---------------------+---------------+------------------------------------------------------------+
|
||||
| flow_name | table_catalog | flow_definition |
|
||||
+---------------------+---------------+------------------------------------------------------------+
|
||||
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > 3 |
|
||||
+---------------------+---------------+------------------------------------------------------------+
|
||||
|
||||
DROP FLOW filter_numbers_show;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
-- (flow exists, replace, if not exists)=(false, true, true)
|
||||
CREATE OR REPLACE FLOW IF NOT EXISTS filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > -1;
|
||||
|
||||
Error: 1001(Unsupported), Unsupported operation Create flow with both `IF NOT EXISTS` and `OR REPLACE`
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
++
|
||||
++
|
||||
|
||||
DROP FLOW filter_numbers_show;
|
||||
|
||||
Error: 8001(FlowNotFound), Flow not found: greptime.filter_numbers_show
|
||||
|
||||
-- following always create since didn't exist
|
||||
-- (flow exists, replace, if not exists)=(false, true, false)
|
||||
CREATE OR REPLACE FLOW filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > -2;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
| flow_name | table_catalog | flow_definition |
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > -2 |
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
|
||||
DROP FLOW filter_numbers_show;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
-- (flow exists, replace, if not exists)=(false, false, true)
|
||||
CREATE OR REPLACE FLOW filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > -3;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
| flow_name | table_catalog | flow_definition |
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > -3 |
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
|
||||
-- makesure after recover should be the same
|
||||
-- SQLNESS ARG restart=true
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
| flow_name | table_catalog | flow_definition |
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
| filter_numbers_show | greptime | SELECT number, ts FROM numbers_input_show WHERE number > -3 |
|
||||
+---------------------+---------------+-------------------------------------------------------------+
|
||||
|
||||
SELECT * FROM out_num_cnt_show;
|
||||
|
||||
+--------+-------------------------+
|
||||
| number | ts |
|
||||
+--------+-------------------------+
|
||||
| 4 | 1970-01-01T00:00:00.002 |
|
||||
| 10 | 1970-01-01T00:00:00.003 |
|
||||
| 11 | 1970-01-01T00:00:00.004 |
|
||||
| 15 | 1970-01-01T00:00:00.001 |
|
||||
| 16 | 1970-01-01T00:00:00.002 |
|
||||
+--------+-------------------------+
|
||||
|
||||
INSERT INTO numbers_input_show VALUES(-4,0), (-3,1), (-2,2), (-1,3);
|
||||
|
||||
Affected Rows: 4
|
||||
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('filter_numbers_show');
|
||||
|
||||
+-----------------------------------------+
|
||||
| ADMIN FLUSH_FLOW('filter_numbers_show') |
|
||||
+-----------------------------------------+
|
||||
| FLOW_FLUSHED |
|
||||
+-----------------------------------------+
|
||||
|
||||
SELECT * FROM out_num_cnt_show;
|
||||
|
||||
+--------+-------------------------+
|
||||
| number | ts |
|
||||
+--------+-------------------------+
|
||||
| -2 | 1970-01-01T00:00:00.002 |
|
||||
| -1 | 1970-01-01T00:00:00.003 |
|
||||
| 4 | 1970-01-01T00:00:00.002 |
|
||||
| 10 | 1970-01-01T00:00:00.003 |
|
||||
| 11 | 1970-01-01T00:00:00.004 |
|
||||
| 15 | 1970-01-01T00:00:00.001 |
|
||||
| 16 | 1970-01-01T00:00:00.002 |
|
||||
+--------+-------------------------+
|
||||
|
||||
DROP FLOW filter_numbers_show;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
drop table out_num_cnt_show;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
@@ -6,7 +6,9 @@ CREATE TABLE numbers_input_show (
|
||||
);
|
||||
create table out_num_cnt_show (
|
||||
number INT,
|
||||
ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP TIME INDEX);
|
||||
ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP TIME INDEX,
|
||||
PRIMARY KEY(number),
|
||||
);
|
||||
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
@@ -27,6 +29,99 @@ SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS W
|
||||
|
||||
SHOW FLOWS LIKE 'filter_numbers_show';
|
||||
|
||||
-- also test `CREATE OR REPLACE` and `IF NOT EXISTS`
|
||||
|
||||
-- (flow exists, replace, if not exists)=(false, false, false)
|
||||
CREATE FLOW filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > 10;
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
-- this one should error out
|
||||
-- (flow exists, replace, if not exists)=(true, false, false)
|
||||
CREATE FLOW filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > 15;
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
-- makesure it's not replaced in flownode
|
||||
INSERT INTO numbers_input_show VALUES (10, 0),(15, 1),(16, 2);
|
||||
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('filter_numbers_show');
|
||||
|
||||
SELECT number, ts FROM out_num_cnt_show;
|
||||
|
||||
-- after this one, the flow SHOULD NOT be replaced
|
||||
-- (flow exists, replace, if not exists)=(true, false, true)
|
||||
CREATE FLOW IF NOT EXISTS filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > 5;
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
-- makesure it's not replaced in flownode
|
||||
INSERT INTO numbers_input_show VALUES (4,4),(5,4),(10, 3),(11, 4);
|
||||
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('filter_numbers_show');
|
||||
|
||||
SELECT number, ts FROM out_num_cnt_show;
|
||||
|
||||
-- after this, the flow SHOULD be replaced
|
||||
-- (flow exists, replace, if not exists)=(true, true, false)
|
||||
CREATE OR REPLACE FLOW filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > 3;
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
-- makesure it's replaced in flownode
|
||||
INSERT INTO numbers_input_show VALUES (3, 1),(4, 2),(10, 3),(11, 4);
|
||||
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('filter_numbers_show');
|
||||
|
||||
SELECT number, ts FROM out_num_cnt_show;
|
||||
|
||||
-- after this, the flow SHOULD error out since having both `replace` and `if not exists`
|
||||
-- (flow exists, replace, if not exists)=(true, true, true)
|
||||
CREATE OR REPLACE FLOW IF NOT EXISTS filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > 0;
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
DROP FLOW filter_numbers_show;
|
||||
|
||||
-- (flow exists, replace, if not exists)=(false, true, true)
|
||||
CREATE OR REPLACE FLOW IF NOT EXISTS filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > -1;
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
DROP FLOW filter_numbers_show;
|
||||
|
||||
-- following always create since didn't exist
|
||||
-- (flow exists, replace, if not exists)=(false, true, false)
|
||||
CREATE OR REPLACE FLOW filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > -2;
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
DROP FLOW filter_numbers_show;
|
||||
|
||||
-- (flow exists, replace, if not exists)=(false, false, true)
|
||||
CREATE OR REPLACE FLOW filter_numbers_show SINK TO out_num_cnt_show AS SELECT number, ts FROM numbers_input_show where number > -3;
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
-- makesure after recover should be the same
|
||||
-- SQLNESS ARG restart=true
|
||||
|
||||
SELECT flow_name, table_catalog, flow_definition FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name='filter_numbers_show';
|
||||
|
||||
SELECT * FROM out_num_cnt_show;
|
||||
|
||||
INSERT INTO numbers_input_show VALUES(-4,0), (-3,1), (-2,2), (-1,3);
|
||||
|
||||
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
||||
ADMIN FLUSH_FLOW('filter_numbers_show');
|
||||
|
||||
SELECT * FROM out_num_cnt_show;
|
||||
|
||||
DROP FLOW filter_numbers_show;
|
||||
|
||||
drop table out_num_cnt_show;
|
||||
|
||||
drop table numbers_input_show;
|
||||
|
||||
Reference in New Issue
Block a user