rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-29 15:06:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(pipeline): introduce pipeline doc version 2 for combine-transform (#6360)

Browse Source 
* chore: init commit of pipeline doc version v2

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: remove unused code

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: remove unused code

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: add test

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: add test

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: compatible with v1 to remain field in the map during transform

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* refactor: pipeline.exec_mut

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* fix: typo

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: change from v2 to 2 in version setting

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

---------

Signed-off-by: shuiyisong <xixing.sys@gmail.com>
This commit is contained in:
shuiyisong
2025-06-22 00:58:36 +00:00
committed by GitHub
parent 15616d0c43
commit 7cd6be41ce
17 changed files with 612 additions and 1090 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests-integration/tests/http.rs
+82
View File
@@ -108,6 +108,7 @@ macro_rules! http_tests {
test_pipeline_context,
test_pipeline_with_vrl,
test_pipeline_with_hint_vrl,
test_pipeline_2,
test_pipeline_skip_error,
test_otlp_metrics,
@@ -2497,6 +2498,87 @@ transform:
guard.remove_all().await;
}
pub async fn test_pipeline_2(storage_type: StorageType) {
common_telemetry::init_default_ut_logging();
let (app, mut guard) = setup_test_http_app_with_frontend(storage_type, "test_pipeline_2").await;
// handshake
let client = TestClient::new(app).await;
let pipeline = r#"
version: 2
processors:
- date:
field: time
formats:
- "%Y-%m-%d %H:%M:%S%.3f"
transform:
- field: id1
type: int32
index: inverted
- field: time
type: time
index: timestamp
"#;
// 1. create pipeline
let res = client
.post("/v1/events/pipelines/root")
.header("Content-Type", "application/x-yaml")
.body(pipeline)
.send()
.await;
assert_eq!(res.status(), StatusCode::OK);
// 2. write data
let data_body = r#"
[
{
"id1": "123",
"id2": "2436",
"time": "2024-05-25 20:16:37.217"
}
]
"#;
let res = client
.post("/v1/events/logs?db=public&table=d_table&pipeline_name=root")
.header("Content-Type", "application/json")
.body(data_body)
.send()
.await;
assert_eq!(res.status(), StatusCode::OK);
// CREATE TABLE IF NOT EXISTS "d_table" (
// "id1" INT NULL INVERTED INDEX,
// "time" TIMESTAMP(9) NOT NULL,
// "id2" STRING NULL,
// TIME INDEX ("time")
// )
// ENGINE=mito
// WITH(
// append_mode = 'true'
// )
validate_data(
"test_pipeline_2_schema",
&client,
"show create table d_table",
"[[\"d_table\",\"CREATE TABLE IF NOT EXISTS \\\"d_table\\\" (\\n \\\"id1\\\" INT NULL INVERTED INDEX,\\n \\\"time\\\" TIMESTAMP(9) NOT NULL,\\n \\\"id2\\\" STRING NULL,\\n TIME INDEX (\\\"time\\\")\\n)\\n\\nENGINE=mito\\nWITH(\\n append_mode = 'true'\\n)\"]]",
)
.await;
validate_data(
"test_pipeline_2_data",
&client,
"select * from d_table",
"[[123,1716668197217000000,\"2436\"]]",
)
.await;
guard.remove_all().await;
}
pub async fn test_identity_pipeline_with_flatten(store_type: StorageType) {
common_telemetry::init_default_ut_logging();
let (app, mut guard) =