rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-14 12:00:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(flow): parse defer on miss src table (#7980)

Browse Source 
* feat: parse create flow with

Signed-off-by: discord9 <discord9@163.com>

* feat: validate after parse

Signed-off-by: discord9 <discord9@163.com>

* pcr

Signed-off-by: discord9 <discord9@163.com>

* chore: sqlness

Signed-off-by: discord9 <discord9@163.com>

---------

Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
discord9
2026-04-27 11:02:13 +08:00
committed by GitHub
parent 793545d8e6
commit d2d256909f
9 changed files with 427 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
src/sql/src/parsers/create_parser.rs
View File

@@ -68,6 +68,17 @@ pub const VECTOR: &str = "VECTOR";
pub type RawIntervalExpr = String;
// Preserve raw CREATE FLOW option entries until operator-side validation.
// Do not use `OptionMap::new()` here: it can drop non-string values for
// redacted keys before the flow option allowlist rejects them.
fn flow_option_map(options: HashMap<String, OptionValue>) -> OptionMap {
let mut flow_options = OptionMap::default();
for (key, value) in options {
flow_options.insert_options(&key, value);
}
flow_options
}
/// Parses create [table] statement
impl<'a> ParserContext<'a> {
pub(crate) fn parse_create(&mut self) -> Result<Statement> {
@@ -339,6 +350,14 @@ impl<'a> ParserContext<'a> {
None
};
let flow_options = self
.parser
.parse_options(Keyword::WITH)
.context(SyntaxSnafu)?
.into_iter()
.map(parse_option_string)
.collect::<Result<HashMap<String, OptionValue>>>()?;
self.parser
.expect_keyword(Keyword::AS)
.context(SyntaxSnafu)?;
@@ -353,6 +372,7 @@ impl<'a> ParserContext<'a> {
expire_after,
eval_interval,
comment,
flow_options: flow_option_map(flow_options),
query,
}))
}
@@ -1256,6 +1276,20 @@ mod tests {
use crate::dialect::GreptimeDbDialect;
use crate::parser::ParseOptions;
fn string_option_map(
entries: impl IntoIterator<Item = (&'static str, &'static str)>,
) -> OptionMap {
OptionMap::new(entries.into_iter().map(|(key, value)| {
(
key.to_string(),
OptionValue::try_new(Expr::Value(
Value::SingleQuotedString(value.to_string()).into(),
))
.unwrap(),
)
}))
}
#[test]
fn test_parse_create_table_like() {
let sql = "CREATE TABLE t1 LIKE t2";
@@ -1498,6 +1532,8 @@ mod tests {
pub expire_after: Option<i64>,
/// Comment string
pub comment: Option<String>,
/// Flow creation options
pub flow_options: OptionMap,
}
let testcases = vec![
(
@@ -1518,6 +1554,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
if_not_exists: true,
expire_after: Some(300),
comment: Some("test comment".to_string()),
flow_options: OptionMap::default(),
},
),
(
@@ -1538,6 +1575,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
if_not_exists: true,
expire_after: Some(300),
comment: Some("test comment".to_string()),
flow_options: OptionMap::default(),
},
),
(
@@ -1558,6 +1596,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
if_not_exists: true,
expire_after: Some(300),
comment: Some("test comment".to_string()),
flow_options: OptionMap::default(),
},
),
(
@@ -1578,6 +1617,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
if_not_exists: true,
expire_after: Some(300),
comment: Some("test comment".to_string()),
flow_options: OptionMap::default(),
},
),
(
@@ -1597,6 +1637,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
if_not_exists: false,
expire_after: Some(2 * 86400 + 3600 + 2 * 60),
comment: None,
flow_options: OptionMap::default(),
},
),
(
@@ -1616,6 +1657,7 @@ select max(c1), min(c2) from schema_2.table_2;",
if_not_exists: false,
expire_after: Some(600), // 10 minutes in seconds
comment: None,
flow_options: OptionMap::default(),
},
),
(
@@ -1636,6 +1678,27 @@ select max(c1), min(c2) from schema_2.table_2;",
if_not_exists: true,
expire_after: Some(7200), // 2 hours in seconds
comment: Some("lowercase test".to_string()),
flow_options: OptionMap::default(),
},
),
(
r"
CREATE FLOW task_5
SINK TO schema_1.table_1
WITH (defer_on_missing_source = 'true')
AS
SELECT max(c1), min(c2) FROM schema_2.table_2;",
CreateFlowWoutQuery {
flow_name: ObjectName::from(vec![Ident::new("task_5")]),
sink_table_name: ObjectName::from(vec![
Ident::new("schema_1"),
Ident::new("table_1"),
]),
or_replace: false,
if_not_exists: false,
expire_after: None,
comment: None,
flow_options: string_option_map([("defer_on_missing_source", "true")]),
},
),
];
@@ -1651,6 +1714,7 @@ select max(c1), min(c2) from schema_2.table_2;",
expire_after: expected.expire_after,
eval_interval: None,
comment: expected.comment,
flow_options: expected.flow_options,
// ignore query parse result
query: create_task.query.clone(),
};
@@ -1696,6 +1760,8 @@ select max(c1), min(c2) from schema_2.table_2;",
pub eval_interval: Option<i64>,
/// Comment string
pub comment: Option<String>,
/// Flow creation options
pub flow_options: OptionMap,
}
// create flow without `OR REPLACE`, `IF NOT EXISTS`, `EXPIRE AFTER` and `COMMENT`
@@ -1719,6 +1785,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
expire_after: Some(300),
eval_interval: None,
comment: Some("test comment".to_string()),
flow_options: OptionMap::default(),
},
),
(
@@ -1740,6 +1807,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
expire_after: Some(300),
eval_interval: None,
comment: Some("test comment".to_string()),
flow_options: OptionMap::default(),
},
),
(
@@ -1762,6 +1830,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
expire_after: Some(300),
eval_interval: Some(10),
comment: Some("test comment".to_string()),
flow_options: OptionMap::default(),
},
),
(
@@ -1784,6 +1853,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
expire_after: Some(300),
eval_interval: Some(10),
comment: Some("test comment".to_string()),
flow_options: OptionMap::default(),
},
),
(
@@ -1806,6 +1876,32 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
expire_after: Some(2 * 86400 + 3600 + 2 * 60),
eval_interval: None,
comment: None,
flow_options: OptionMap::default(),
},
),
(
r"
CREATE FLOW task_3
SINK TO schema_1.table_1
EVAL INTERVAL '10 seconds'
WITH (defer_on_missing_source = 'true', foo = 'bar')
AS
SELECT max(c1), min(c2) FROM schema_2.table_2;",
CreateFlowWoutQuery {
flow_name: ObjectName(vec![ObjectNamePart::Identifier(Ident::new("task_3"))]),
sink_table_name: ObjectName(vec![
ObjectNamePart::Identifier(Ident::new("schema_1")),
ObjectNamePart::Identifier(Ident::new("table_1")),
]),
or_replace: false,
if_not_exists: false,
expire_after: None,
eval_interval: Some(10),
comment: None,
flow_options: string_option_map([
("defer_on_missing_source", "true"),
("foo", "bar"),
]),
},
),
];
@@ -1821,6 +1917,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;",
expire_after: expected.expire_after,
eval_interval: expected.eval_interval,
comment: expected.comment,
flow_options: expected.flow_options,
// ignore query parse result
query: create_task.query.clone(),
};

6
src/sql/src/statements/create.rs
View File

@@ -615,6 +615,8 @@ pub struct CreateFlow {
pub eval_interval: Option<i64>,
/// Comment string
pub comment: Option<String>,
/// Flow creation options from `WITH (...)`
pub flow_options: OptionMap,
/// SQL statement
pub query: Box<SqlOrTql>,
}
@@ -672,6 +674,10 @@ impl Display for CreateFlow {
if let Some(comment) = &self.comment {
writeln!(f, "COMMENT '{}'", comment)?;
}
if !self.flow_options.is_empty() {
let options = self.flow_options.kv_pairs();
writeln!(f, "WITH ({})", format_list_comma!(options))?;
}
write!(f, "AS {}", &self.query)
}
}

28
src/sql/src/statements/option_map.rs
View File

@@ -49,6 +49,18 @@ impl OptionMap {
}
}
pub fn from_filtered_string_map(
options: &HashMap<String, String>,
hidden_keys: &[&str],
) -> Self {
Self::from(
options
.iter()
.filter(|(key, _)| !hidden_keys.contains(&key.as_str()))
.map(|(key, value)| (key.clone(), value.clone())),
)
}
pub fn insert(&mut self, k: String, v: String) {
if REDACTED_OPTIONS.contains(&k.as_str()) {
self.secrets.insert(k, SecretString::new(Box::new(v)));
@@ -221,6 +233,8 @@ impl VisitMut for OptionMap {
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use crate::statements::OptionMap;
#[test]
@@ -237,4 +251,18 @@ mod tests {
map.insert("a.b".to_string(), "中文comment\n".to_string());
assert_eq!("'a.b' = '中文comment\\n'", map.kv_pairs()[0]);
}
#[test]
fn test_from_filtered_string_map() {
let map = OptionMap::from_filtered_string_map(
&HashMap::from([
("visible".to_string(), "1".to_string()),
("hidden".to_string(), "2".to_string()),
]),
&["hidden"],
);
assert_eq!(map.get("visible"), Some("1"));
assert_eq!(map.get("hidden"), None);
}
}