feat(pipeline): support table-aware JSON2 transforms (#8964)

* feat(pipeline): support table-aware JSON2 transforms

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* feat(pipeline): support JSON2 type hints in transforms

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* fix(pipeline): default failed JSON2 transforms to null

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* refactor(json2): distinguish invalid settings from layout errors

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

---------

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

src/sql/src/lib.rs

@@ -23,6 +23,6 @@ pub mod partition;
 pub mod statements;
 pub mod util;
 
-pub use parsers::create_parser::{ENGINE, MAXVALUE};
+pub use parsers::create_parser::{ENGINE, MAXVALUE, parse_json2_type_hint_path};
 pub use parsers::tql_parser::TQL;
 pub use parsers::with_tql_parser::{CteContent, HybridCteWith};

src/sql/src/parsers/create_parser.rs

@@ -24,6 +24,7 @@ use datafusion_common::ScalarValue;
 use datatypes::arrow::datatypes::{DataType as ArrowDataType, IntervalUnit};
 use datatypes::data_type::ConcreteDataType;
 use itertools::Itertools;
+pub use json::parse_json2_type_hint_path;
 use snafu::{OptionExt, ResultExt, ensure};
 use sqlparser::ast::{
     ColumnOption, ColumnOptionDef, DataType, Expr, KeyOrIndexDisplay, NullsDistinctOption,

src/sql/src/parsers/create_parser/json.rs

@@ -21,6 +21,7 @@ use sqlparser::parser::Parser;
 use sqlparser::tokenizer::Token;
 
 use crate::ast::Ident;
+use crate::dialect::GreptimeDbDialect;
 use crate::error::{InvalidSqlSnafu, Result, SyntaxSnafu};
 use crate::parsers::create_parser::{INVERTED, SKIPPING};
 use crate::statements::create::{Json2Options, JsonTypeHint};
@@ -29,6 +30,25 @@ use crate::statements::transform::type_alias::get_type_by_alias;
 const JSON2_TYPE_NAME: &str = "JSON2";
 const MAX_AUTO_EXPANDED_PATHS: &str = "max_auto_expanded_paths";
 
+/// Parses a JSON2 type hint path with the same grammar used by `CREATE TABLE`.
+pub fn parse_json2_type_hint_path(path: &str) -> Result<Vec<String>> {
+    let dialect = GreptimeDbDialect {};
+    let mut parser = Parser::new(&dialect)
+        .try_with_sql(path)
+        .context(SyntaxSnafu)?;
+    let path = parse_json2_path(&mut parser)?;
+    ensure!(
+        parser.peek_token().token == Token::EOF,
+        InvalidSqlSnafu {
+            msg: format!(
+                "unexpected token '{}' in JSON2 type hint path",
+                parser.peek_token()
+            )
+        }
+    );
+    Ok(path)
+}
+
 pub(super) fn parse_json2_type_and_options(
     parser: &mut Parser<'_>,
 ) -> Result<Option<(DataType, Option<Json2Options>)>> {
@@ -322,6 +342,7 @@ fn ensure_no_path_conflict(hints: &[JsonTypeHint], path: &[String]) -> Result<()
 mod tests {
     use sqlparser::ast::{DataType, ExactNumberInfo};
 
+    use super::parse_json2_type_hint_path;
     use crate::dialect::GreptimeDbDialect;
     use crate::parser::{ParseOptions, ParserContext};
     use crate::statements::create::Column;
@@ -339,6 +360,15 @@ mod tests {
         create_table.columns.remove(0)
     }
 
+    #[test]
+    fn test_parse_json2_type_hint_path() {
+        assert_eq!(
+            parse_json2_type_hint_path(r#"attrs."http.status_code""#).unwrap(),
+            vec!["attrs", "http.status_code"]
+        );
+        assert!(parse_json2_type_hint_path("user.id trailing").is_err());
+    }
+
     #[test]
     fn test_parse_json2_type_hints() {
         let column = parse_json2_column(