fix(json2): restrict JSON2 type hints (#9316)

* fix(sql): restrict JSON2 type hints Signed-off-by: fys <fengys1996@gmail.com> * fix(sql): allow equivalent 64-bit aliases in JSON2 type hints Signed-off-by: fys <fengys1996@gmail.com> * fix(sql): support UInt64 conversion and preserve JSON2 hint aliases Signed-off-by: fys <fengys1996@gmail.com> * refactor(sql): remove redundant JSON2 type hint normalization Signed-off-by: fys <fengys1996@gmail.com> * fix(datatypes): restrict JSON2 hint types in JsonSettings::try_new Signed-off-by: fys <fengys1996@gmail.com> --------- Signed-off-by: fys <fengys1996@gmail.com>
2026-10-04 19:15:34 +00:00 · 2026-09-23 09:19:18 +00:00
parent bc80071409
commit 5a78e07be6
14 changed files with 300 additions and 132 deletions
@@ -25,7 +25,6 @@ use crate::dialect::GreptimeDbDialect;
 use crate::error::{InvalidSqlSnafu, Result, SyntaxSnafu};
 use crate::parsers::create_parser::{INVERTED, SKIPPING};
 use crate::statements::create::{Json2Options, JsonTypeHint};
-use crate::statements::transform::type_alias::get_type_by_alias;

 const JSON2_TYPE_NAME: &str = "JSON2";
 const MAX_AUTO_EXPANDED_PATHS: &str = "max_auto_expanded_paths";
@@ -155,7 +154,7 @@ fn parse_json2_type_hint(parser: &mut Parser<'_>) -> Result<JsonTypeHint> {
        }
    );
    let data_type = parser.parse_data_type().context(SyntaxSnafu)?;
-    let data_type = normalize_json2_type_hint_type(data_type)?;
+    let data_type = validate_json2_type_hint_type(data_type)?;

    let mut inverted_index = false;

@@ -238,35 +237,23 @@ fn parse_json2_path(parser: &mut Parser<'_>) -> Result<Vec<String>> {
    Ok(path)
 }

-fn normalize_json2_type_hint_type(data_type: DataType) -> Result<DataType> {
-    let data_type = get_type_by_alias(&data_type).unwrap_or(data_type);
-    let normalized = match data_type {
-        DataType::String(_) | DataType::Text | DataType::Varchar(_) | DataType::Char(_) => {
-            DataType::String(None)
+fn validate_json2_type_hint_type(data_type: DataType) -> Result<DataType> {
+    match data_type {
+        DataType::Int64
+        | DataType::UInt64
+        | DataType::Float64
+        | DataType::String(None)
+        | DataType::BigInt(None)
+        | DataType::BigIntUnsigned(None)
+        | DataType::Double(ExactNumberInfo::None)
+        | DataType::Boolean => Ok(data_type),
+        _ => InvalidSqlSnafu {
+            msg: format!(
+                "unsupported JSON2 type hint data type: {data_type}; supported types: STRING, BIGINT, BIGINT UNSIGNED, DOUBLE, BOOLEAN; supported aliases: INT64, UINT64, FLOAT64"
+            ),
        }
-        DataType::TinyInt(_)
-        | DataType::SmallInt(_)
-        | DataType::Int(_)
-        | DataType::Integer(_)
-        | DataType::BigInt(_) => DataType::BigInt(None),
-        DataType::TinyIntUnsigned(_)
-        | DataType::SmallIntUnsigned(_)
-        | DataType::IntUnsigned(_)
-        | DataType::UnsignedInteger
-        | DataType::BigIntUnsigned(_) => DataType::BigIntUnsigned(None),
-        DataType::Float(_) | DataType::Real | DataType::Double(_) => {
-            DataType::Double(ExactNumberInfo::None)
-        }
-        DataType::Boolean => DataType::Boolean,
-        _ => {
-            return InvalidSqlSnafu {
-                msg: format!("unsupported JSON2 type hint data type: {data_type}"),
-            }
-            .fail();
-        }
-    };
-
-    Ok(normalized)
+        .fail(),
+    }
 }

 fn ensure_no_path_conflict(hints: &[JsonTypeHint], path: &[String]) -> Result<()> {
@@ -330,7 +317,7 @@ CREATE TABLE traces (
    log_json_data JSON2 (
        "service.name" STRING INVERTED INDEX,
        http.method STRING,
-        status_code INT64,
+        status_code BIGINT,
        comment STRING,
    ),
    ts TIMESTAMP TIME INDEX,
@@ -463,40 +450,81 @@ CREATE TABLE traces (
    }

    #[test]
-    fn test_parse_json2_type_hint_normalizes_numeric_types() {
-        let column = parse_json2_column(
-            r#"
-CREATE TABLE traces (
-    log_json_data JSON2 (
-        tinyint_value TINYINT,
-        smallint_value SMALLINT,
-        int_value INT,
-        integer_value INTEGER,
-        bigint_value BIGINT,
-        int64_value INT64,
-        tinyuint_value TINYINT UNSIGNED,
-        smalluint_value SMALLINT UNSIGNED,
-        uint_value INT UNSIGNED,
-        uint64_value UINT64,
-        float_value FLOAT,
-        real_value REAL,
-        double_value DOUBLE,
-        float64_value FLOAT64
-    ),
-    ts TIMESTAMP TIME INDEX,
-)"#,
-        );
+    fn test_parse_json2_type_hint_supported_types() {
+        for (sql_type, expected) in [
+            ("STRING", DataType::String(None)),
+            ("BIGINT", DataType::BigInt(None)),
+            ("BIGINT UNSIGNED", DataType::BigIntUnsigned(None)),
+            ("DOUBLE", DataType::Double(ExactNumberInfo::None)),
+            ("BOOLEAN", DataType::Boolean),
+            ("INT64", DataType::Int64),
+            ("UINT64", DataType::UInt64),
+            ("FLOAT64", DataType::Float64),
+        ] {
+            for sql_type in [sql_type.to_string(), sql_type.to_lowercase()] {
+                let column = parse_json2_column(&format!(
+                    "CREATE TABLE traces (j JSON2 (value {sql_type}), ts TIMESTAMP TIME INDEX)"
+                ));
+                let options = column.extensions.json2_options.unwrap();
+                let settings = options.build_json_settings().unwrap();
+                assert_eq!(settings.type_hints().len(), 1);
+                let hints = options.type_hints;
+                assert_eq!(hints[0].data_type, expected);
+            }
+        }
+    }

-        let hints = column.extensions.json2_options.unwrap().type_hints;
-        assert_eq!(hints.len(), 14);
-        for hint in hints.iter().take(6) {
-            assert_eq!(hint.data_type, DataType::BigInt(None));
-        }
-        for hint in hints.iter().skip(6).take(4) {
-            assert_eq!(hint.data_type, DataType::BigIntUnsigned(None));
-        }
-        for hint in hints.iter().skip(10) {
-            assert_eq!(hint.data_type, DataType::Double(ExactNumberInfo::None));
+    #[test]
+    fn test_parse_json2_type_hint_rejects_unsupported_types() {
+        for sql_type in [
+            "INT2",
+            "INT4",
+            "INT8",
+            "INT16",
+            "INT32",
+            "TINYINT",
+            "SMALLINT",
+            "INT",
+            "INTEGER",
+            "UINT8",
+            "UINT16",
+            "UINT32",
+            "TINYINT UNSIGNED",
+            "SMALLINT UNSIGNED",
+            "INT UNSIGNED",
+            "FLOAT",
+            "REAL",
+            "FLOAT4",
+            "FLOAT8",
+            "FLOAT32",
+            "BOOL",
+            "TEXT",
+            "VARCHAR(10)",
+            "CHAR(10)",
+            "TIMESTAMP",
+            "DECIMAL(10, 2)",
+            "STRING(10)",
+            "BIGINT(10)",
+        ] {
+            for sql in [
+                format!(
+                    "CREATE TABLE traces (j JSON2 (value {sql_type}), ts TIMESTAMP TIME INDEX)"
+                ),
+                format!("ALTER TABLE traces MODIFY COLUMN j JSON2 (value {sql_type})"),
+            ] {
+                let err = ParserContext::create_with_dialect(
+                    &sql,
+                    &GreptimeDbDialect {},
+                    ParseOptions::default(),
+                )
+                .unwrap_err();
+                assert!(
+                    err.to_string().contains(
+                        "supported types: STRING, BIGINT, BIGINT UNSIGNED, DOUBLE, BOOLEAN; supported aliases: INT64, UINT64, FLOAT64"
+                    ),
+                    "{sql}: {err}"
+                );
+            }
        }
    }

@@ -504,7 +532,7 @@ CREATE TABLE traces (
    fn test_parse_json2_type_hint_rejects_default() {
        for default in ["-5", "abs(-1)"] {
            let sql = format!(
-                "CREATE TABLE traces (log_json_data JSON2 (status_code INT64 DEFAULT {default}), ts TIMESTAMP TIME INDEX)"
+                "CREATE TABLE traces (log_json_data JSON2 (status_code BIGINT DEFAULT {default}), ts TIMESTAMP TIME INDEX)"
            );
            let err = ParserContext::create_with_dialect(
                &sql,
@@ -521,7 +549,7 @@ CREATE TABLE traces (
        let result = ParserContext::create_with_dialect(
            r#"
 CREATE TABLE traces (
-    log_json_data JSON2 (a.b STRING, a.b INT64),
+    log_json_data JSON2 (a.b STRING, a.b BIGINT),
    ts TIMESTAMP TIME INDEX,
 )"#,
            &GreptimeDbDialect {},
@@ -537,7 +565,7 @@ CREATE TABLE traces (
        let result = ParserContext::create_with_dialect(
            r#"
 CREATE TABLE traces (
-    log_json_data JSON2 (a STRING, a.b INT64),
+    log_json_data JSON2 (a STRING, a.b BIGINT),
    ts TIMESTAMP TIME INDEX,
 )"#,
            &GreptimeDbDialect {},
@@ -225,7 +225,9 @@ pub fn sql_column_def_to_grpc_column_def(
 pub fn sql_data_type_to_concrete_data_type(data_type: &SqlDataType) -> Result<ConcreteDataType> {
    match data_type {
        SqlDataType::BigInt(_) | SqlDataType::Int64 => Ok(ConcreteDataType::int64_datatype()),
-        SqlDataType::BigIntUnsigned(_) => Ok(ConcreteDataType::uint64_datatype()),
+        SqlDataType::BigIntUnsigned(_) | SqlDataType::UInt64 => {
+            Ok(ConcreteDataType::uint64_datatype())
+        }
        SqlDataType::Int(_) | SqlDataType::Integer(_) => Ok(ConcreteDataType::int32_datatype()),
        SqlDataType::IntUnsigned(_) | SqlDataType::UnsignedInteger => {
            Ok(ConcreteDataType::uint32_datatype())
@@ -422,6 +424,7 @@ mod tests {
            SqlDataType::BigIntUnsigned(None),
            ConcreteDataType::uint64_datatype(),
        );
+        check_type(SqlDataType::UInt64, ConcreteDataType::uint64_datatype());
        check_type(
            SqlDataType::IntUnsigned(None),
            ConcreteDataType::uint32_datatype(),
@@ -165,7 +165,7 @@ impl Json2Options {
            .map(|hint| {
                Ok(datatypes::json::JsonTypeHint {
                    path: hint.path.clone(),
-                    data_type: json_type_hint_concrete_data_type(&hint.data_type)?,
+                    data_type: sql_data_type_to_concrete_data_type(&hint.data_type)?,
                    inverted_index: hint.inverted_index,
                })
            })
@@ -396,47 +396,19 @@ impl ColumnExtensions {
    }
 }

-fn json_type_hint_concrete_data_type(data_type: &DataType) -> Result<ConcreteDataType> {
-    let data_type = sql_data_type_to_concrete_data_type(data_type)?;
-    normalize_json_type_hint_concrete_data_type(&data_type)
-}
-
-fn normalize_json_type_hint_concrete_data_type(
-    data_type: &ConcreteDataType,
-) -> Result<ConcreteDataType> {
-    let normalized = match data_type {
-        ConcreteDataType::String(_) => ConcreteDataType::string_datatype(),
-        ConcreteDataType::Int8(_)
-        | ConcreteDataType::Int16(_)
-        | ConcreteDataType::Int32(_)
-        | ConcreteDataType::Int64(_) => ConcreteDataType::int64_datatype(),
-        ConcreteDataType::UInt8(_)
-        | ConcreteDataType::UInt16(_)
-        | ConcreteDataType::UInt32(_)
-        | ConcreteDataType::UInt64(_) => ConcreteDataType::uint64_datatype(),
-        ConcreteDataType::Float32(_) | ConcreteDataType::Float64(_) => {
-            ConcreteDataType::float64_datatype()
-        }
-        ConcreteDataType::Boolean(_) => ConcreteDataType::boolean_datatype(),
-        _ => {
-            return InvalidSqlSnafu {
-                msg: format!("unsupported JSON2 type hint data type: {data_type}"),
-            }
-            .fail();
-        }
-    };
-    Ok(normalized)
-}
-
 fn json_type_hint_sql_data_type(data_type: &ConcreteDataType) -> Result<DataType> {
-    let data_type = normalize_json_type_hint_concrete_data_type(data_type)?;
    let sql_type = match data_type {
        ConcreteDataType::String(_) => DataType::String(None),
        ConcreteDataType::Int64(_) => DataType::BigInt(None),
        ConcreteDataType::UInt64(_) => DataType::BigIntUnsigned(None),
        ConcreteDataType::Float64(_) => DataType::Double(sqlparser::ast::ExactNumberInfo::None),
        ConcreteDataType::Boolean(_) => DataType::Boolean,
-        _ => unreachable!("JSON2 type hint data type should have been normalized"),
+        _ => {
+            return InvalidSqlSnafu {
+                msg: format!("unsupported JSON2 type hint data type: {data_type}"),
+            }
+            .fail();
+        }
    };
    Ok(sql_type)
 }
@@ -933,7 +905,7 @@ ENGINE=mito
        let sql = r#"CREATE TABLE traces (
            log_json_data JSON2 (
                "service.name" STRING,
-                "a.b"."c" INT64,
+                "a.b"."c" BIGINT,
                a."b.c" STRING
            ),
            ts TIMESTAMP TIME INDEX
@@ -977,7 +949,7 @@ ENGINE=mito
    fn test_parse_json2_max_auto_expanded_paths_option() -> Result<()> {
        let sql = r#"CREATE TABLE traces (
            log_json_data JSON2 (
-                status_code INT64,
+                status_code BIGINT,
                max_auto_expanded_paths = 1
            ),
            ts TIMESTAMP TIME INDEX
@@ -1003,7 +975,7 @@ ENGINE=mito
        let sql = r#"CREATE TABLE traces (
            log_json_data JSON2 (
                "1abc" STRING,
-                a."2b" INT64
+                a."2b" BIGINT
            ),
            ts TIMESTAMP TIME INDEX
        )"#;
@@ -1045,8 +1017,8 @@ ENGINE=mito
    fn test_json2_type_hint_rejects_default() {
        let sql = r#"CREATE TABLE traces (
            log_json_data JSON2 (
-                status_code INT64 DEFAULT -5,
-                duration FLOAT64 DEFAULT +1.5,
+                status_code BIGINT DEFAULT -5,
+                duration DOUBLE DEFAULT +1.5,
                error BOOLEAN DEFAULT false,
                message STRING DEFAULT 'unknown'
            ),
@@ -1062,7 +1034,7 @@ ENGINE=mito
    fn test_json2_type_hint_rejects_not_null() {
        let sql = r#"CREATE TABLE traces (
            log_json_data JSON2 (
-                status_code INT64 NOT NULL DEFAULT NULL
+                status_code BIGINT NOT NULL DEFAULT NULL
            ),
            ts TIMESTAMP TIME INDEX
        )"#;
@@ -1073,23 +1045,23 @@ ENGINE=mito
    }

    #[test]
-    fn test_set_json_settings_normalizes_type_hint_sql_types() -> Result<()> {
+    fn test_set_json_settings_preserves_type_hint_sql_types() -> Result<()> {
        let mut extensions = super::ColumnExtensions::default();
        let settings = JsonSettings::try_new(
            vec![
                DatatypeJsonTypeHint {
                    path: vec!["i".to_string()],
-                    data_type: ConcreteDataType::int32_datatype(),
+                    data_type: ConcreteDataType::int64_datatype(),
                    inverted_index: false,
                },
                DatatypeJsonTypeHint {
                    path: vec!["f".to_string()],
-                    data_type: ConcreteDataType::float32_datatype(),
+                    data_type: ConcreteDataType::float64_datatype(),
                    inverted_index: false,
                },
                DatatypeJsonTypeHint {
                    path: vec!["u".to_string()],
-                    data_type: ConcreteDataType::uint32_datatype(),
+                    data_type: ConcreteDataType::uint64_datatype(),
                    inverted_index: false,
                },
                DatatypeJsonTypeHint {