sql/parsers/create_parser/
json.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use datatypes::extension::json::JSON2_REMAINDER_FIELD_NAME;
16use datatypes::json::JSON2_MAX_STRUCTURED_DEPTH;
17use snafu::{ResultExt, ensure};
18use sqlparser::ast::{DataType, ExactNumberInfo, Expr, ObjectName, UnaryOperator};
19use sqlparser::dialect::keywords::Keyword;
20use sqlparser::parser::Parser;
21use sqlparser::tokenizer::Token;
22
23use crate::ast::Ident;
24use crate::error::{InvalidSqlSnafu, Result, SyntaxSnafu};
25use crate::parsers::create_parser::{INVERTED, SKIPPING};
26use crate::statements::create::{Json2Options, JsonTypeHint};
27use crate::statements::transform::type_alias::get_type_by_alias;
28
29const JSON2_TYPE_NAME: &str = "JSON2";
30const MAX_AUTO_EXPANDED_PATHS: &str = "max_auto_expanded_paths";
31
32pub(super) fn parse_json2_type_and_options(
33    parser: &mut Parser<'_>,
34) -> Result<Option<(DataType, Option<Json2Options>)>> {
35    let token = parser.peek_token();
36    let Token::Word(word) = &token.token else {
37        return Ok(None);
38    };
39
40    if !word.value.eq_ignore_ascii_case(JSON2_TYPE_NAME) || word.quote_style.is_some() {
41        return Ok(None);
42    }
43
44    parser.next_token();
45    let data_type = DataType::Custom(ObjectName::from(vec![Ident::new(JSON2_TYPE_NAME)]), vec![]);
46    let options = if parser.consume_token(&Token::LParen) {
47        parse_json2_options(parser)?
48    } else {
49        None
50    };
51
52    Ok(Some((data_type, options)))
53}
54
55fn parse_json2_options(parser: &mut Parser<'_>) -> Result<Option<Json2Options>> {
56    if parser.consume_token(&Token::RParen) {
57        return Ok(None);
58    }
59
60    let mut max_auto_expanded_paths = None;
61    let mut type_hints = Vec::new();
62    loop {
63        let token = parser.peek_token();
64        let is_max_auto_expanded_paths = matches!(
65            &token.token,
66            Token::Word(word)
67                if word.quote_style.is_none()
68                    && word.value.eq_ignore_ascii_case(MAX_AUTO_EXPANDED_PATHS)
69        );
70        if is_max_auto_expanded_paths {
71            parser.next_token();
72            ensure!(
73                max_auto_expanded_paths.is_none(),
74                InvalidSqlSnafu {
75                    msg: format!("duplicated JSON2 option '{MAX_AUTO_EXPANDED_PATHS}'")
76                }
77            );
78            parser.expect_token(&Token::Eq).context(SyntaxSnafu)?;
79
80            let token = parser.next_token();
81            let Token::Number(value, _) = token.token else {
82                return InvalidSqlSnafu {
83                    msg: format!(
84                        "JSON2 option '{MAX_AUTO_EXPANDED_PATHS}' expects a non-negative integer"
85                    ),
86                }
87                .fail();
88            };
89            max_auto_expanded_paths = Some(value.parse::<u32>().map_err(|_| {
90                InvalidSqlSnafu {
91                    msg: format!(
92                        "JSON2 option '{MAX_AUTO_EXPANDED_PATHS}' expects a non-negative integer"
93                    ),
94                }
95                .build()
96            })?);
97        } else {
98            let hint = parse_json2_type_hint(parser)?;
99            ensure_no_path_conflict(&type_hints, &hint.path)?;
100            type_hints.push(hint);
101        }
102
103        if parser.consume_token(&Token::Comma) {
104            if parser.consume_token(&Token::RParen) {
105                break;
106            }
107        } else {
108            parser.expect_token(&Token::RParen).context(SyntaxSnafu)?;
109            break;
110        }
111    }
112
113    Ok(Some(Json2Options {
114        max_auto_expanded_paths,
115        type_hints,
116    }))
117}
118
119fn parse_json2_type_hint(parser: &mut Parser<'_>) -> Result<JsonTypeHint> {
120    let path = parse_json2_path(parser)?;
121    ensure!(
122        path.first().is_none_or(|x| x != JSON2_REMAINDER_FIELD_NAME),
123        InvalidSqlSnafu {
124            msg: format!(
125                "JSON2 type hint path cannot be rooted at reserved field '{JSON2_REMAINDER_FIELD_NAME}'"
126            )
127        }
128    );
129    ensure!(
130        path.len() <= JSON2_MAX_STRUCTURED_DEPTH,
131        InvalidSqlSnafu {
132            msg: format!(
133                "JSON2 type hint path cannot exceed {JSON2_MAX_STRUCTURED_DEPTH} segments"
134            ),
135        }
136    );
137    let data_type = parser.parse_data_type().context(SyntaxSnafu)?;
138    let data_type = normalize_json2_type_hint_type(data_type)?;
139
140    let mut nullable = true;
141    let mut nullable_set = false;
142    let mut default = None;
143    let mut inverted_index = false;
144
145    loop {
146        if parser.parse_keywords(&[Keyword::NOT, Keyword::NULL]) {
147            ensure!(
148                !nullable_set,
149                InvalidSqlSnafu {
150                    msg: format!(
151                        "NULL/NOT NULL option already specified for JSON2 type hint '{}'",
152                        path.join(".")
153                    )
154                }
155            );
156            nullable = false;
157            nullable_set = true;
158        } else if parser.parse_keyword(Keyword::NULL) {
159            ensure!(
160                !nullable_set,
161                InvalidSqlSnafu {
162                    msg: format!(
163                        "NULL/NOT NULL option already specified for JSON2 type hint '{}'",
164                        path.join(".")
165                    )
166                }
167            );
168            nullable = true;
169            nullable_set = true;
170        } else if parser.parse_keyword(Keyword::DEFAULT) {
171            ensure!(
172                default.is_none(),
173                InvalidSqlSnafu {
174                    msg: format!(
175                        "duplicated DEFAULT option for JSON2 type hint '{}'",
176                        path.join(".")
177                    )
178                }
179            );
180            let expr = parser.parse_expr().context(SyntaxSnafu)?;
181            ensure_json2_default_expr_is_literal(&expr)?;
182            default = Some(expr);
183        } else if let Token::Word(word) = parser.peek_token().token
184            && word.value.eq_ignore_ascii_case(INVERTED)
185        {
186            parser.next_token();
187            ensure!(
188                parser.parse_keyword(Keyword::INDEX),
189                InvalidSqlSnafu {
190                    msg: format!(
191                        "expect INDEX after INVERTED keyword for JSON2 type hint '{}'",
192                        path.join(".")
193                    )
194                }
195            );
196            ensure!(
197                !inverted_index,
198                InvalidSqlSnafu {
199                    msg: format!(
200                        "duplicated INVERTED INDEX option for JSON2 type hint '{}'",
201                        path.join(".")
202                    )
203                }
204            );
205            inverted_index = true;
206        } else if let Token::Word(word) = parser.peek_token().token
207            && word.value.eq_ignore_ascii_case(SKIPPING)
208        {
209            return InvalidSqlSnafu {
210                msg: "JSON2 type hint SKIPPING INDEX is not supported yet".to_string(),
211            }
212            .fail();
213        } else if matches!(parser.peek_token().token, Token::Comma | Token::RParen) {
214            break;
215        } else {
216            return parser
217                .expected("JSON2 type hint option", parser.peek_token())
218                .context(SyntaxSnafu);
219        }
220    }
221
222    Ok(JsonTypeHint {
223        path,
224        data_type,
225        nullable,
226        default,
227        inverted_index,
228    })
229}
230
231fn parse_json2_path(parser: &mut Parser<'_>) -> Result<Vec<String>> {
232    let first = parser.parse_identifier().context(SyntaxSnafu)?;
233    let mut path = vec![first.value];
234
235    while parser.consume_token(&Token::Period) {
236        let segment = parser.parse_identifier().context(SyntaxSnafu)?;
237        path.push(segment.value);
238    }
239
240    ensure!(
241        !path.iter().any(|segment| segment.is_empty()),
242        InvalidSqlSnafu {
243            msg: "JSON2 type hint path segment cannot be empty".to_string(),
244        }
245    );
246
247    Ok(path)
248}
249
250fn normalize_json2_type_hint_type(data_type: DataType) -> Result<DataType> {
251    let data_type = get_type_by_alias(&data_type).unwrap_or(data_type);
252    let normalized = match data_type {
253        DataType::String(_) | DataType::Text | DataType::Varchar(_) | DataType::Char(_) => {
254            DataType::String(None)
255        }
256        DataType::TinyInt(_)
257        | DataType::SmallInt(_)
258        | DataType::Int(_)
259        | DataType::Integer(_)
260        | DataType::BigInt(_) => DataType::BigInt(None),
261        DataType::TinyIntUnsigned(_)
262        | DataType::SmallIntUnsigned(_)
263        | DataType::IntUnsigned(_)
264        | DataType::UnsignedInteger
265        | DataType::BigIntUnsigned(_) => DataType::BigIntUnsigned(None),
266        DataType::Float(_) | DataType::Real | DataType::Double(_) => {
267            DataType::Double(ExactNumberInfo::None)
268        }
269        DataType::Boolean => DataType::Boolean,
270        _ => {
271            return InvalidSqlSnafu {
272                msg: format!("unsupported JSON2 type hint data type: {data_type}"),
273            }
274            .fail();
275        }
276    };
277
278    Ok(normalized)
279}
280
281fn ensure_json2_default_expr_is_literal(expr: &Expr) -> Result<()> {
282    let is_literal = match expr {
283        Expr::Value(_) => true,
284        Expr::UnaryOp { op, expr } => {
285            matches!(op, UnaryOperator::Plus | UnaryOperator::Minus)
286                && matches!(expr.as_ref(), Expr::Value(_))
287        }
288        _ => false,
289    };
290    ensure!(
291        is_literal,
292        InvalidSqlSnafu {
293            msg: "JSON2 type hint DEFAULT only supports literal values",
294        }
295    );
296    Ok(())
297}
298
299fn ensure_no_path_conflict(hints: &[JsonTypeHint], path: &[String]) -> Result<()> {
300    for hint in hints {
301        ensure!(
302            hint.path != path,
303            InvalidSqlSnafu {
304                msg: format!("duplicated JSON2 type hint path '{}'", path.join("."))
305            }
306        );
307        ensure!(
308            !hint.path.starts_with(path) && !path.starts_with(&hint.path),
309            InvalidSqlSnafu {
310                msg: format!(
311                    "JSON2 type hint path '{}' conflicts with '{}'",
312                    path.join("."),
313                    hint.path.join(".")
314                )
315            }
316        );
317    }
318    Ok(())
319}
320
321#[cfg(test)]
322mod tests {
323    use sqlparser::ast::{DataType, ExactNumberInfo};
324
325    use crate::dialect::GreptimeDbDialect;
326    use crate::parser::{ParseOptions, ParserContext};
327    use crate::statements::create::Column;
328    use crate::statements::statement::Statement;
329
330    fn parse_json2_column(sql: &str) -> Column {
331        let Statement::CreateTable(mut create_table) =
332            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
333                .unwrap()
334                .remove(0)
335        else {
336            unreachable!()
337        };
338
339        create_table.columns.remove(0)
340    }
341
342    #[test]
343    fn test_parse_json2_type_hints() {
344        let column = parse_json2_column(
345            r#"
346CREATE TABLE traces (
347    log_json_data JSON2 (
348        "service.name" STRING NOT NULL DEFAULT 'null' INVERTED INDEX,
349        http.method STRING NOT NULL,
350        status_code INT64 NOT NULL,
351        comment STRING NULL,
352    ),
353    ts TIMESTAMP TIME INDEX,
354)"#,
355        );
356
357        assert!(matches!(
358            column.column_def.data_type,
359            DataType::Custom(_, _)
360        ));
361        let hints = column.extensions.json2_options.unwrap().type_hints;
362        assert_eq!(hints.len(), 4);
363
364        assert_eq!(hints[0].path, vec!["service.name"]);
365        assert_eq!(hints[0].data_type, DataType::String(None));
366        assert!(!hints[0].nullable);
367        assert_eq!(
368            hints[0]
369                .default
370                .as_ref()
371                .map(|expr| expr.to_string())
372                .as_deref(),
373            Some("'null'")
374        );
375        assert!(hints[0].inverted_index);
376
377        assert_eq!(hints[1].path, vec!["http", "method"]);
378        assert_eq!(hints[1].data_type, DataType::String(None));
379        assert!(!hints[1].nullable);
380        assert!(!hints[1].inverted_index);
381
382        assert_eq!(hints[2].path, vec!["status_code"]);
383        assert_eq!(hints[2].data_type, DataType::BigInt(None));
384        assert!(!hints[2].nullable);
385
386        assert_eq!(hints[3].path, vec!["comment"]);
387        assert_eq!(hints[3].data_type, DataType::String(None));
388        assert!(hints[3].nullable);
389    }
390
391    #[test]
392    fn test_parse_json2_max_auto_expanded_paths() {
393        let column = parse_json2_column(
394            r#"
395CREATE TABLE traces (
396    log_json_data JSON2 (
397        http.method STRING,
398        max_auto_expanded_paths = 0
399    ),
400    ts TIMESTAMP TIME INDEX,
401)"#,
402        );
403
404        let options = column.extensions.json2_options.unwrap();
405        assert_eq!(options.max_auto_expanded_paths, Some(0));
406        assert_eq!(options.type_hints.len(), 1);
407
408        let empty = parse_json2_column(
409            r#"
410CREATE TABLE traces (
411    log_json_data JSON2 (),
412    ts TIMESTAMP TIME INDEX,
413)"#,
414        );
415        assert!(empty.extensions.json2_options.is_none());
416
417        let quoted = parse_json2_column(
418            r#"
419CREATE TABLE traces (
420    log_json_data JSON2 (
421        "max_auto_expanded_paths" STRING,
422        nested."!__remainder__!" STRING
423    ),
424    ts TIMESTAMP TIME INDEX,
425)"#,
426        );
427        let options = quoted.extensions.json2_options.unwrap();
428        assert_eq!(options.max_auto_expanded_paths, None);
429        assert_eq!(options.type_hints.len(), 2);
430    }
431
432    #[test]
433    fn test_parse_json2_max_auto_expanded_paths_rejects_invalid_options() {
434        for options in [
435            "max_auto_expanded_paths = 0, max_auto_expanded_paths = 1",
436            "max_auto_expanded_paths = -1",
437            "max_auto_expanded_paths = 1.5",
438            "max_auto_expanded_paths = 4294967296",
439            r#""!__remainder__!".value STRING"#,
440        ] {
441            let sql = format!(
442                "CREATE TABLE traces (log_json_data JSON2 ({options}), ts TIMESTAMP TIME INDEX)"
443            );
444            assert!(
445                ParserContext::create_with_dialect(
446                    &sql,
447                    &GreptimeDbDialect {},
448                    ParseOptions::default()
449                )
450                .is_err(),
451                "{options}"
452            );
453        }
454    }
455
456    #[test]
457    fn test_parse_json2_type_hint_default_nullable() {
458        let column = parse_json2_column(
459            r#"
460CREATE TABLE traces (
461    log_json_data JSON2 (http.method STRING),
462    ts TIMESTAMP TIME INDEX,
463)"#,
464        );
465
466        let hints = column.extensions.json2_options.unwrap().type_hints;
467        assert_eq!(hints.len(), 1);
468        assert!(hints[0].nullable);
469    }
470
471    #[test]
472    fn test_parse_json2_type_hint_quoted_path_segments() {
473        let column = parse_json2_column(
474            r#"
475CREATE TABLE traces (
476    log_json_data JSON2 (
477        "a".b STRING,
478        "x"."y" STRING,
479        "a.b"."c" STRING,
480        a."b.c" STRING
481    ),
482    ts TIMESTAMP TIME INDEX,
483)"#,
484        );
485
486        let hints = column.extensions.json2_options.unwrap().type_hints;
487        assert_eq!(hints.len(), 4);
488        assert_eq!(hints[0].path, vec!["a", "b"]);
489        assert_eq!(hints[1].path, vec!["x", "y"]);
490        assert_eq!(hints[2].path, vec!["a.b", "c"]);
491        assert_eq!(hints[3].path, vec!["a", "b.c"]);
492    }
493
494    #[test]
495    fn test_parse_json2_type_hint_normalizes_numeric_types() {
496        let column = parse_json2_column(
497            r#"
498CREATE TABLE traces (
499    log_json_data JSON2 (
500        tinyint_value TINYINT,
501        smallint_value SMALLINT,
502        int_value INT,
503        integer_value INTEGER,
504        bigint_value BIGINT,
505        int64_value INT64,
506        tinyuint_value TINYINT UNSIGNED,
507        smalluint_value SMALLINT UNSIGNED,
508        uint_value INT UNSIGNED,
509        uint64_value UINT64,
510        float_value FLOAT,
511        real_value REAL,
512        double_value DOUBLE,
513        float64_value FLOAT64
514    ),
515    ts TIMESTAMP TIME INDEX,
516)"#,
517        );
518
519        let hints = column.extensions.json2_options.unwrap().type_hints;
520        assert_eq!(hints.len(), 14);
521        for hint in hints.iter().take(6) {
522            assert_eq!(hint.data_type, DataType::BigInt(None));
523        }
524        for hint in hints.iter().skip(6).take(4) {
525            assert_eq!(hint.data_type, DataType::BigIntUnsigned(None));
526        }
527        for hint in hints.iter().skip(10) {
528            assert_eq!(hint.data_type, DataType::Double(ExactNumberInfo::None));
529        }
530    }
531
532    #[test]
533    fn test_parse_json2_type_hint_default_accepts_signed_literals() {
534        let column = parse_json2_column(
535            r#"
536CREATE TABLE traces (
537    log_json_data JSON2 (
538        negative_int INT64 DEFAULT -5,
539        positive_float FLOAT64 DEFAULT +1.5
540    ),
541    ts TIMESTAMP TIME INDEX,
542)"#,
543        );
544
545        let hints = column.extensions.json2_options.unwrap().type_hints;
546        assert_eq!(hints.len(), 2);
547        assert_eq!(
548            hints[0]
549                .default
550                .as_ref()
551                .map(|expr| expr.to_string())
552                .as_deref(),
553            Some("-5")
554        );
555        assert_eq!(
556            hints[1]
557                .default
558                .as_ref()
559                .map(|expr| expr.to_string())
560                .as_deref(),
561            Some("+1.5")
562        );
563    }
564
565    #[test]
566    fn test_parse_json2_type_hint_default_rejects_function() {
567        let result = ParserContext::create_with_dialect(
568            r#"
569CREATE TABLE traces (
570    log_json_data JSON2 (status_code INT64 DEFAULT abs(-1)),
571    ts TIMESTAMP TIME INDEX,
572)"#,
573            &GreptimeDbDialect {},
574            ParseOptions::default(),
575        );
576
577        assert!(result.is_err());
578        assert!(
579            result
580                .unwrap_err()
581                .to_string()
582                .contains("DEFAULT only supports literal values")
583        );
584    }
585
586    #[test]
587    fn test_parse_json2_type_hint_rejects_duplicate_path() {
588        let result = ParserContext::create_with_dialect(
589            r#"
590CREATE TABLE traces (
591    log_json_data JSON2 (a.b STRING, a.b INT64),
592    ts TIMESTAMP TIME INDEX,
593)"#,
594            &GreptimeDbDialect {},
595            ParseOptions::default(),
596        );
597
598        assert!(result.is_err());
599        assert!(result.unwrap_err().to_string().contains("duplicated"));
600    }
601
602    #[test]
603    fn test_parse_json2_type_hint_rejects_parent_child_path() {
604        let result = ParserContext::create_with_dialect(
605            r#"
606CREATE TABLE traces (
607    log_json_data JSON2 (a STRING, a.b INT64),
608    ts TIMESTAMP TIME INDEX,
609)"#,
610            &GreptimeDbDialect {},
611            ParseOptions::default(),
612        );
613
614        assert!(result.is_err());
615        assert!(result.unwrap_err().to_string().contains("conflicts"));
616    }
617
618    #[test]
619    fn test_parse_json2_type_hint_rejects_duplicated_nullability() {
620        for sql in [
621            r#"
622CREATE TABLE traces (
623    log_json_data JSON2 (a STRING NULL NULL),
624    ts TIMESTAMP TIME INDEX,
625)"#,
626            r#"
627CREATE TABLE traces (
628    log_json_data JSON2 (a STRING NOT NULL NOT NULL),
629    ts TIMESTAMP TIME INDEX,
630)"#,
631            r#"
632CREATE TABLE traces (
633    log_json_data JSON2 (a STRING NOT NULL NULL),
634    ts TIMESTAMP TIME INDEX,
635)"#,
636            r#"
637CREATE TABLE traces (
638    log_json_data JSON2 (a STRING NULL NOT NULL),
639    ts TIMESTAMP TIME INDEX,
640)"#,
641        ] {
642            let result = ParserContext::create_with_dialect(
643                sql,
644                &GreptimeDbDialect {},
645                ParseOptions::default(),
646            );
647
648            assert!(result.is_err());
649            assert!(
650                result
651                    .unwrap_err()
652                    .to_string()
653                    .contains("NULL/NOT NULL option already specified")
654            );
655        }
656    }
657}
sql/parsers/create_parser/json.rs

sql/parsers/create_parser/
json.rs
