rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-07-10 16:00:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(flow): harden incremental read correctness (#8196)

Browse Source 
* fix(flow): harden incremental read correctness

Signed-off-by: discord9 <discord9@163.com>

* fix(flow): propagate dirty window options

Signed-off-by: discord9 <discord9@163.com>

* test: more

Signed-off-by: discord9 <discord9@163.com>

* chore: test config api

Signed-off-by: discord9 <discord9@163.com>

* refactor: split gen

Signed-off-by: discord9 <discord9@163.com>

* chore: per review

Signed-off-by: discord9 <discord9@163.com>

* fix: allowlist key

Signed-off-by: discord9 <discord9@163.com>

---------

Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
discord9
2026-06-01 10:48:00 +08:00
committed by GitHub
parent ed9312f8e3
commit 28fd796f4e
21 changed files with 873 additions and 437 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/common/meta/src/ddl/create_flow.rs
View File

@@ -437,11 +437,13 @@ pub fn defer_on_missing_source(flow_task: &CreateFlowTask) -> Result<bool> {
pub fn validate_flow_options(flow_task: &CreateFlowTask) -> Result<()> {
for key in flow_task.flow_options.keys() {
match key.as_str() {
DEFER_ON_MISSING_SOURCE_KEY | FlowType::FLOW_TYPE_KEY => {}
DEFER_ON_MISSING_SOURCE_KEY
| FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY
| FlowType::FLOW_TYPE_KEY => {}
unknown => {
return UnexpectedSnafu {
err_msg: format!(
"Unknown flow option '{unknown}', supported user options: {DEFER_ON_MISSING_SOURCE_KEY}"
"Unknown flow option '{unknown}', supported user options: {DEFER_ON_MISSING_SOURCE_KEY}, {FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY}"
),
}
.fail();
@@ -487,6 +489,9 @@ pub enum FlowType {
Streaming,
}
pub const FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY: &str =
"experimental_enable_incremental_read";
impl FlowType {
pub const BATCHING: &str = "batching";
pub const STREAMING: &str = "streaming";

21
src/common/meta/src/ddl/tests/create_flow.rs
View File

@@ -24,8 +24,9 @@ use table::table_name::TableName;
use crate::ddl::DdlContext;
use crate::ddl::create_flow::{
CreateFlowData, CreateFlowProcedure, CreateFlowState, DEFER_ON_MISSING_SOURCE_KEY, FlowType,
defer_on_missing_source,
CreateFlowData, CreateFlowProcedure, CreateFlowState, DEFER_ON_MISSING_SOURCE_KEY,
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType, defer_on_missing_source,
validate_flow_options,
};
use crate::ddl::test_util::create_table::test_create_table_task;
use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler;
@@ -275,6 +276,22 @@ fn test_defer_on_missing_source_invalid_value() {
);
}
#[test]
fn test_validate_flow_options_allows_incremental_read_option() {
let mut task = test_create_flow_task(
"my_flow",
vec![],
TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
false,
);
task.flow_options.insert(
FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(),
"true".to_string(),
);
validate_flow_options(&task).unwrap();
}
#[tokio::test]
async fn test_create_flow_rejects_unknown_option_in_meta_task() {
let mut task = test_create_flow_task(