mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-10-03 10:35:35 +00:00
dennis zhuang
d59725b04a
* test(sqlness): run environments concurrently with external backends The runner forced both environment and instance parallelism to 1 whenever etcd/PG/MySQL or an external Kafka was set up, so the standalone and distributed environments ran one after the other. Only distributed uses the kv backend, and the two environments use different Kafka topic prefixes, so they can run at the same time. Keep one instance per environment, since instances would share the metadata table and Kafka topics. Runs with a runner-managed Kafka, an external server, a test filter, or `-j 1` stay fully serial. Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * test(sqlness): drop redundant restarts and sleeps - flow_rebuild: drop the R2 block, which repeats R1 from the same state, and the restart in R5, whose result R1 already asserts. - flow_advance_ttl, flow_view, ttl_instant: drop sleeps that no assertion depends on. - flow_basic, flow_null, flow_call_df_func: drop the bytes_log section (covered by flow_insert), a duplicate state_size query, an unchecked insert, and flushes that run with no new data. - alter_table_options, skip_wal: share restarts between independent tables. - session_skip_wal, copy_skip_wal: check every case after a single restart. Tables copied or inserted with skip_wal = true first get a WAL row that is truncated, so the check that truncated WAL entries are not replayed stays. - region_statistics: wait once for the statistics of all three tables. - build_index_table: fold its index_size checks into build_index_table_restart, which runs the same fixture and waits. Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * test(sqlness): restore session_skip_wal and copy_skip_wal Checking every case after a single restart dropped two state transitions the original cases cover: recovering a region whose memtable only held skipped rows, and writing to the recovered region before restarting again. Restore the original cases. Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * test(sqlness): restore the one-restart check in flow_auto_sink_table #5112 checked the auto-created sink before a restart and the flow and sink after it. #5987 moved the restart in front of the first SHOW and added a second one. Flow recovery only creates the sink when it is missing, so the second restart recovers from the same persisted state as the first. Restore the original before/after check with one restart. Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * test(sqlness): keep one instance per environment with external store addresses Instances of the distributed environment share the etcd behind --store-addrs, so run one instance per environment when it is set, same as --setup-etcd. Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * test(sqlness): restore the false-branch batch in flow_basic and assert it The kept batch (20, 22) is all above the threshold. The second batch (10, 23) is the only input that hits the false branch of the CASE and makes the flow compute again. Restore it and check the result, which the original case never did. Signed-off-by: Dennis Zhuang <killme2008@gmail.com> --------- Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
376 lines
9.1 KiB
SQL
376 lines
9.1 KiB
SQL
CREATE TABLE input_basic (
|
|
"number" INT,
|
|
ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY(number),
|
|
TIME INDEX(ts)
|
|
)WITH(
|
|
append_mode = 'true'
|
|
);
|
|
|
|
CREATE FLOW test_wildcard_basic sink TO out_basic EVAL INTERVAL '1m' AS
|
|
SELECT
|
|
COUNT(*) as wildcard
|
|
FROM
|
|
input_basic;
|
|
|
|
INSERT INTO
|
|
input_basic
|
|
VALUES
|
|
(23, "2021-07-01 00:00:01.000"),
|
|
(24, "2021-07-01 00:00:01.500");
|
|
|
|
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
|
ADMIN FLUSH_FLOW('test_wildcard_basic');
|
|
|
|
SELECT wildcard FROM out_basic;
|
|
|
|
DROP TABLE input_basic;
|
|
|
|
DROP TABLE out_basic;
|
|
|
|
DROP FLOW test_wildcard_basic;
|
|
|
|
-- combination of different order of rebuild input table/flow
|
|
|
|
CREATE TABLE input_basic (
|
|
"number" INT,
|
|
ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY(number),
|
|
TIME INDEX(ts)
|
|
);
|
|
|
|
CREATE FLOW test_wildcard_basic sink TO out_basic EVAL INTERVAL '1m' AS
|
|
SELECT
|
|
COUNT(*) as wildcard
|
|
FROM
|
|
input_basic;
|
|
|
|
INSERT INTO
|
|
input_basic
|
|
VALUES
|
|
(23, "2021-07-01 00:00:01.000"),
|
|
(24, "2021-07-01 00:00:01.500");
|
|
|
|
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
|
ADMIN FLUSH_FLOW('test_wildcard_basic');
|
|
|
|
SELECT wildcard FROM out_basic;
|
|
|
|
DROP TABLE input_basic;
|
|
|
|
CREATE TABLE input_basic (
|
|
"number" INT,
|
|
ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY(number),
|
|
TIME INDEX(ts)
|
|
);
|
|
|
|
INSERT INTO
|
|
input_basic
|
|
VALUES
|
|
(23, "2021-07-01 00:00:01.000"),
|
|
(24, "2021-07-01 00:00:01.500");
|
|
|
|
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
|
ADMIN FLUSH_FLOW('test_wildcard_basic');
|
|
|
|
-- this is expected to be the same as above("2") since the new `input_basic` table
|
|
-- have different table id, so is a different table
|
|
SELECT wildcard FROM out_basic;
|
|
|
|
DROP FLOW test_wildcard_basic;
|
|
|
|
-- recreate flow so that it use new table id
|
|
CREATE FLOW test_wildcard_basic sink TO out_basic EVAL INTERVAL '1m' AS
|
|
SELECT
|
|
COUNT(*) as wildcard
|
|
FROM
|
|
input_basic;
|
|
|
|
INSERT INTO
|
|
input_basic
|
|
VALUES
|
|
(23, "2021-07-01 00:00:01.000"),
|
|
(24, "2021-07-01 00:00:01.500"),
|
|
(25, "2021-07-01 00:00:01.700");
|
|
|
|
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
|
ADMIN FLUSH_FLOW('test_wildcard_basic');
|
|
|
|
-- flow batching mode
|
|
SELECT wildcard FROM out_basic;
|
|
|
|
SELECT count(*) FROM input_basic;
|
|
|
|
DROP TABLE input_basic;
|
|
DROP FLOW test_wildcard_basic;
|
|
DROP TABLE out_basic;
|
|
|
|
CREATE TABLE input_basic (
|
|
"number" INT,
|
|
ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY(number),
|
|
TIME INDEX(ts)
|
|
);
|
|
|
|
CREATE FLOW test_wildcard_basic sink TO out_basic EVAL INTERVAL '1m' AS
|
|
SELECT
|
|
COUNT(*) as wildcard
|
|
FROM
|
|
input_basic;
|
|
|
|
INSERT INTO
|
|
input_basic
|
|
VALUES
|
|
(23, "2021-07-01 00:00:01.000"),
|
|
(24, "2021-07-01 00:00:01.500");
|
|
|
|
|
|
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
|
ADMIN FLUSH_FLOW('test_wildcard_basic');
|
|
|
|
SELECT wildcard FROM out_basic;
|
|
|
|
DROP FLOW test_wildcard_basic;
|
|
|
|
DROP TABLE out_basic;
|
|
|
|
CREATE FLOW test_wildcard_basic sink TO out_basic EVAL INTERVAL '1m' AS
|
|
SELECT
|
|
COUNT(*) as wildcard
|
|
FROM
|
|
input_basic;
|
|
|
|
INSERT INTO
|
|
input_basic
|
|
VALUES
|
|
(23, "2021-07-01 00:00:01.000"),
|
|
(24, "2021-07-01 00:00:01.500"),
|
|
(25, "2021-07-01 00:00:01.700");
|
|
|
|
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
|
ADMIN FLUSH_FLOW('test_wildcard_basic');
|
|
|
|
-- SQLNESS SLEEP 3s
|
|
SELECT wildcard FROM out_basic;
|
|
|
|
-- test again, this time with db restart
|
|
DROP TABLE input_basic;
|
|
DROP TABLE out_basic;
|
|
DROP FLOW test_wildcard_basic;
|
|
|
|
CREATE TABLE input_basic (
|
|
"number" INT,
|
|
ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY(number),
|
|
TIME INDEX(ts)
|
|
);
|
|
|
|
CREATE FLOW test_wildcard_basic sink TO out_basic EVAL INTERVAL '1m' AS
|
|
SELECT
|
|
COUNT(*) as wildcard
|
|
FROM
|
|
input_basic;
|
|
|
|
-- SQLNESS ARG restart=true
|
|
SELECT 1;
|
|
|
|
-- SQLNESS SLEEP 3s
|
|
INSERT INTO
|
|
input_basic
|
|
VALUES
|
|
(23, "2021-07-01 00:00:01.000"),
|
|
(24, "2021-07-01 00:00:01.500");
|
|
|
|
-- give flownode a second to rebuild flow
|
|
-- SQLNESS SLEEP 3s
|
|
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
|
ADMIN FLUSH_FLOW('test_wildcard_basic');
|
|
|
|
SELECT wildcard FROM out_basic;
|
|
|
|
-- combination of different order of rebuild input table/flow
|
|
|
|
DROP TABLE input_basic;
|
|
|
|
CREATE TABLE input_basic (
|
|
"number" INT,
|
|
ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY(number),
|
|
TIME INDEX(ts)
|
|
);
|
|
|
|
-- SQLNESS ARG restart=true
|
|
SELECT 1;
|
|
|
|
-- SQLNESS SLEEP 3s
|
|
INSERT INTO
|
|
input_basic
|
|
VALUES
|
|
(23, "2021-07-01 00:00:01.000"),
|
|
(24, "2021-07-01 00:00:01.500"),
|
|
(26, "2021-07-01 00:00:02.000");
|
|
|
|
-- give flownode a second to rebuild flow
|
|
-- SQLNESS SLEEP 3s
|
|
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
|
ADMIN FLUSH_FLOW('test_wildcard_basic');
|
|
|
|
-- this is expected to be the same as above("2") since the new `input_basic` table
|
|
-- have different table id, so is a different table
|
|
SELECT wildcard FROM out_basic;
|
|
|
|
DROP FLOW test_wildcard_basic;
|
|
|
|
-- recreate flow so that it use new table id
|
|
CREATE FLOW test_wildcard_basic sink TO out_basic EVAL INTERVAL '1m' AS
|
|
SELECT
|
|
COUNT(*) as wildcard
|
|
FROM
|
|
input_basic;
|
|
|
|
-- give flownode a second to rebuild flow
|
|
-- SQLNESS ARG restart=true
|
|
SELECT 1;
|
|
|
|
-- SQLNESS SLEEP 3s
|
|
INSERT INTO
|
|
input_basic
|
|
VALUES
|
|
(23, "2021-07-01 00:00:01.000"),
|
|
(24, "2021-07-01 00:00:01.500"),
|
|
(25, "2021-07-01 00:00:01.700");
|
|
|
|
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
|
ADMIN FLUSH_FLOW('test_wildcard_basic');
|
|
|
|
-- 4 is also expected, since flow batching mode
|
|
SELECT wildcard FROM out_basic;
|
|
|
|
SELECT count(*) FROM input_basic;
|
|
|
|
DROP TABLE input_basic;
|
|
DROP FLOW test_wildcard_basic;
|
|
DROP TABLE out_basic;
|
|
|
|
CREATE TABLE input_basic (
|
|
"number" INT,
|
|
ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY(number),
|
|
TIME INDEX(ts)
|
|
);
|
|
|
|
CREATE FLOW test_wildcard_basic sink TO out_basic EVAL INTERVAL '1m' AS
|
|
SELECT
|
|
COUNT(*) as wildcard
|
|
FROM
|
|
input_basic;
|
|
|
|
INSERT INTO
|
|
input_basic
|
|
VALUES
|
|
(23, "2021-07-01 00:00:01.000"),
|
|
(24, "2021-07-01 00:00:01.500");
|
|
|
|
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
|
ADMIN FLUSH_FLOW('test_wildcard_basic');
|
|
|
|
SELECT wildcard FROM out_basic;
|
|
|
|
DROP FLOW test_wildcard_basic;
|
|
|
|
DROP TABLE out_basic;
|
|
|
|
CREATE FLOW test_wildcard_basic sink TO out_basic EVAL INTERVAL '1m' AS
|
|
SELECT
|
|
COUNT(*) as wildcard
|
|
FROM
|
|
input_basic;
|
|
|
|
-- SQLNESS ARG restart=true
|
|
SELECT 1;
|
|
|
|
-- SQLNESS SLEEP 3s
|
|
INSERT INTO
|
|
input_basic
|
|
VALUES
|
|
(23, "2021-07-01 00:00:01.000"),
|
|
(24, "2021-07-01 00:00:01.500"),
|
|
(25, "2021-07-01 00:00:01.700");
|
|
|
|
-- give flownode a second to rebuild flow
|
|
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
|
ADMIN FLUSH_FLOW('test_wildcard_basic');
|
|
|
|
SELECT wildcard FROM out_basic;
|
|
|
|
DROP FLOW test_wildcard_basic;
|
|
|
|
DROP TABLE input_basic;
|
|
|
|
DROP TABLE out_basic;
|
|
|
|
-- check if different schema is working as expected
|
|
|
|
CREATE DATABASE jsdp_log;
|
|
USE jsdp_log;
|
|
|
|
CREATE TABLE IF NOT EXISTS `api_log` (
|
|
`time` TIMESTAMP(9) NOT NULL,
|
|
`key` STRING NULL SKIPPING INDEX WITH(granularity = '1024', type = 'BLOOM'),
|
|
`status_code` TINYINT NULL,
|
|
`method` STRING NULL,
|
|
`path` STRING NULL,
|
|
`raw_query` STRING NULL,
|
|
`user_agent` STRING NULL,
|
|
`client_ip` STRING NULL,
|
|
`duration` INT NULL,
|
|
`count` INT NULL,
|
|
TIME INDEX (`time`)
|
|
) ENGINE=mito WITH(
|
|
append_mode = 'true'
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS `api_stats` (
|
|
`time` TIMESTAMP(0) NOT NULL,
|
|
`key` STRING NULL,
|
|
`qpm` BIGINT NULL,
|
|
`rpm` BIGINT NULL,
|
|
TIME INDEX (`time`),
|
|
PRIMARY KEY (`key`)
|
|
) ENGINE=mito;
|
|
|
|
CREATE FLOW IF NOT EXISTS api_stats_flow
|
|
SINK TO api_stats AS
|
|
SELECT date_trunc('minute', `time`::TimestampSecond) AS `time1`, `key`, count(*), sum(`count`)
|
|
FROM api_log
|
|
GROUP BY `time1`, `key`;
|
|
|
|
INSERT INTO `api_log` (`time`, `key`, `status_code`, `method`, `path`, `raw_query`, `user_agent`, `client_ip`, `duration`, `count`) VALUES (0::TimestampSecond, '1', 0, 'GET', '/lightning/v1/query', 'key=1&since=600', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', '1', 21, 1);
|
|
|
|
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
|
ADMIN FLUSH_FLOW('api_stats_flow');
|
|
|
|
SELECT * FROM api_stats;
|
|
|
|
-- SQLNESS ARG restart=true
|
|
SELECT 1;
|
|
|
|
-- SQLNESS SLEEP 5s
|
|
INSERT INTO `api_log` (`time`, `key`, `status_code`, `method`, `path`, `raw_query`, `user_agent`, `client_ip`, `duration`, `count`) VALUES (0::TimestampSecond, '2', 0, 'GET', '/lightning/v1/query', 'key=1&since=600', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', '1', 21, 1);
|
|
|
|
-- wait more time so flownode have time to recover flows
|
|
-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED |
|
|
ADMIN FLUSH_FLOW('api_stats_flow');
|
|
|
|
-- SQLNESS SLEEP 5s
|
|
SELECT * FROM api_stats;
|
|
|
|
DROP FLOW api_stats_flow;
|
|
|
|
DROP TABLE api_log;
|
|
DROP TABLE api_stats;
|
|
|
|
USE public;
|
|
DROP DATABASE jsdp_log;
|