sql/parsers/create_parser/
json.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use datatypes::extension::json::JSON2_REMAINDER_FIELD_NAME;
16use datatypes::json::JSON2_MAX_STRUCTURED_DEPTH;
17use snafu::{ResultExt, ensure};
18use sqlparser::ast::{DataType, ExactNumberInfo, ObjectName};
19use sqlparser::dialect::keywords::Keyword;
20use sqlparser::parser::Parser;
21use sqlparser::tokenizer::Token;
22
23use crate::ast::Ident;
24use crate::dialect::GreptimeDbDialect;
25use crate::error::{InvalidSqlSnafu, Result, SyntaxSnafu};
26use crate::parsers::create_parser::{INVERTED, SKIPPING};
27use crate::statements::create::{Json2Options, JsonTypeHint};
28
29const JSON2_TYPE_NAME: &str = "JSON2";
30const MAX_AUTO_EXPANDED_PATHS: &str = "max_auto_expanded_paths";
31
32/// Parses a JSON2 type hint path with the same grammar used by `CREATE TABLE`.
33pub fn parse_json2_type_hint_path(path: &str) -> Result<Vec<String>> {
34    let dialect = GreptimeDbDialect {};
35    let mut parser = Parser::new(&dialect)
36        .try_with_sql(path)
37        .context(SyntaxSnafu)?;
38    let path = parse_json2_path(&mut parser)?;
39    ensure!(
40        parser.peek_token().token == Token::EOF,
41        InvalidSqlSnafu {
42            msg: format!(
43                "unexpected token '{}' in JSON2 type hint path",
44                parser.peek_token()
45            )
46        }
47    );
48    Ok(path)
49}
50
51pub(crate) fn parse_json2_type_and_options(
52    parser: &mut Parser<'_>,
53) -> Result<Option<(DataType, Option<Json2Options>)>> {
54    let token = parser.peek_token();
55    let Token::Word(word) = &token.token else {
56        return Ok(None);
57    };
58
59    if !word.value.eq_ignore_ascii_case(JSON2_TYPE_NAME) || word.quote_style.is_some() {
60        return Ok(None);
61    }
62
63    parser.next_token();
64    let data_type = DataType::Custom(ObjectName::from(vec![Ident::new(JSON2_TYPE_NAME)]), vec![]);
65    let options = if parser.consume_token(&Token::LParen) {
66        parse_json2_options(parser)?
67    } else {
68        None
69    };
70
71    Ok(Some((data_type, options)))
72}
73
74fn parse_json2_options(parser: &mut Parser<'_>) -> Result<Option<Json2Options>> {
75    if parser.consume_token(&Token::RParen) {
76        return Ok(None);
77    }
78
79    let mut max_auto_expanded_paths = None;
80    let mut type_hints = Vec::new();
81    loop {
82        let token = parser.peek_token();
83        let is_max_auto_expanded_paths = matches!(
84            &token.token,
85            Token::Word(word)
86                if word.quote_style.is_none()
87                    && word.value.eq_ignore_ascii_case(MAX_AUTO_EXPANDED_PATHS)
88        );
89        if is_max_auto_expanded_paths {
90            parser.next_token();
91            ensure!(
92                max_auto_expanded_paths.is_none(),
93                InvalidSqlSnafu {
94                    msg: format!("duplicated JSON2 option '{MAX_AUTO_EXPANDED_PATHS}'")
95                }
96            );
97            parser.expect_token(&Token::Eq).context(SyntaxSnafu)?;
98
99            let token = parser.next_token();
100            let Token::Number(value, _) = token.token else {
101                return InvalidSqlSnafu {
102                    msg: format!(
103                        "JSON2 option '{MAX_AUTO_EXPANDED_PATHS}' expects a non-negative integer"
104                    ),
105                }
106                .fail();
107            };
108            max_auto_expanded_paths = Some(value.parse::<u32>().map_err(|_| {
109                InvalidSqlSnafu {
110                    msg: format!(
111                        "JSON2 option '{MAX_AUTO_EXPANDED_PATHS}' expects a non-negative integer"
112                    ),
113                }
114                .build()
115            })?);
116        } else {
117            let hint = parse_json2_type_hint(parser)?;
118            ensure_no_path_conflict(&type_hints, &hint.path)?;
119            type_hints.push(hint);
120        }
121
122        if parser.consume_token(&Token::Comma) {
123            if parser.consume_token(&Token::RParen) {
124                break;
125            }
126        } else {
127            parser.expect_token(&Token::RParen).context(SyntaxSnafu)?;
128            break;
129        }
130    }
131
132    Ok(Some(Json2Options {
133        max_auto_expanded_paths,
134        type_hints,
135    }))
136}
137
138fn parse_json2_type_hint(parser: &mut Parser<'_>) -> Result<JsonTypeHint> {
139    let path = parse_json2_path(parser)?;
140    ensure!(
141        path.first().is_none_or(|x| x != JSON2_REMAINDER_FIELD_NAME),
142        InvalidSqlSnafu {
143            msg: format!(
144                "JSON2 type hint path cannot be rooted at reserved field '{JSON2_REMAINDER_FIELD_NAME}'"
145            )
146        }
147    );
148    ensure!(
149        path.len() <= JSON2_MAX_STRUCTURED_DEPTH,
150        InvalidSqlSnafu {
151            msg: format!(
152                "JSON2 type hint path cannot exceed {JSON2_MAX_STRUCTURED_DEPTH} segments"
153            ),
154        }
155    );
156    let data_type = parser.parse_data_type().context(SyntaxSnafu)?;
157    let data_type = validate_json2_type_hint_type(data_type)?;
158
159    let mut inverted_index = false;
160
161    loop {
162        if parser.parse_keywords(&[Keyword::NOT, Keyword::NULL])
163            || parser.parse_keyword(Keyword::NULL)
164        {
165            return InvalidSqlSnafu {
166                msg: "JSON2 type hint NULL/NOT NULL is not supported; hinted fields are always nullable"
167                    .to_string(),
168            }
169            .fail();
170        } else if parser.parse_keyword(Keyword::DEFAULT) {
171            return InvalidSqlSnafu {
172                msg: "JSON2 type hint DEFAULT is not supported".to_string(),
173            }
174            .fail();
175        } else if let Token::Word(word) = parser.peek_token().token
176            && word.value.eq_ignore_ascii_case(INVERTED)
177        {
178            parser.next_token();
179            ensure!(
180                parser.parse_keyword(Keyword::INDEX),
181                InvalidSqlSnafu {
182                    msg: format!(
183                        "expect INDEX after INVERTED keyword for JSON2 type hint '{}'",
184                        path.join(".")
185                    )
186                }
187            );
188            ensure!(
189                !inverted_index,
190                InvalidSqlSnafu {
191                    msg: format!(
192                        "duplicated INVERTED INDEX option for JSON2 type hint '{}'",
193                        path.join(".")
194                    )
195                }
196            );
197            inverted_index = true;
198        } else if let Token::Word(word) = parser.peek_token().token
199            && word.value.eq_ignore_ascii_case(SKIPPING)
200        {
201            return InvalidSqlSnafu {
202                msg: "JSON2 type hint SKIPPING INDEX is not supported yet".to_string(),
203            }
204            .fail();
205        } else if matches!(parser.peek_token().token, Token::Comma | Token::RParen) {
206            break;
207        } else {
208            return parser
209                .expected("JSON2 type hint option", parser.peek_token())
210                .context(SyntaxSnafu);
211        }
212    }
213
214    Ok(JsonTypeHint {
215        path,
216        data_type,
217        inverted_index,
218    })
219}
220
221fn parse_json2_path(parser: &mut Parser<'_>) -> Result<Vec<String>> {
222    let first = parser.parse_identifier().context(SyntaxSnafu)?;
223    let mut path = vec![first.value];
224
225    while parser.consume_token(&Token::Period) {
226        let segment = parser.parse_identifier().context(SyntaxSnafu)?;
227        path.push(segment.value);
228    }
229
230    ensure!(
231        !path.iter().any(|segment| segment.is_empty()),
232        InvalidSqlSnafu {
233            msg: "JSON2 type hint path segment cannot be empty".to_string(),
234        }
235    );
236
237    Ok(path)
238}
239
240fn validate_json2_type_hint_type(data_type: DataType) -> Result<DataType> {
241    match data_type {
242        DataType::Int64
243        | DataType::UInt64
244        | DataType::Float64
245        | DataType::String(None)
246        | DataType::BigInt(None)
247        | DataType::BigIntUnsigned(None)
248        | DataType::Double(ExactNumberInfo::None)
249        | DataType::Boolean => Ok(data_type),
250        _ => InvalidSqlSnafu {
251            msg: format!(
252                "unsupported JSON2 type hint data type: {data_type}; supported types: STRING, BIGINT, BIGINT UNSIGNED, DOUBLE, BOOLEAN; supported aliases: INT64, UINT64, FLOAT64"
253            ),
254        }
255        .fail(),
256    }
257}
258
259fn ensure_no_path_conflict(hints: &[JsonTypeHint], path: &[String]) -> Result<()> {
260    for hint in hints {
261        ensure!(
262            hint.path != path,
263            InvalidSqlSnafu {
264                msg: format!("duplicated JSON2 type hint path '{}'", path.join("."))
265            }
266        );
267        ensure!(
268            !hint.path.starts_with(path) && !path.starts_with(&hint.path),
269            InvalidSqlSnafu {
270                msg: format!(
271                    "JSON2 type hint path '{}' conflicts with '{}'",
272                    path.join("."),
273                    hint.path.join(".")
274                )
275            }
276        );
277    }
278    Ok(())
279}
280
281#[cfg(test)]
282mod tests {
283    use sqlparser::ast::{DataType, ExactNumberInfo};
284
285    use super::parse_json2_type_hint_path;
286    use crate::dialect::GreptimeDbDialect;
287    use crate::parser::{ParseOptions, ParserContext};
288    use crate::statements::create::Column;
289    use crate::statements::statement::Statement;
290
291    fn parse_json2_column(sql: &str) -> Column {
292        let Statement::CreateTable(mut create_table) =
293            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
294                .unwrap()
295                .remove(0)
296        else {
297            unreachable!()
298        };
299
300        create_table.columns.remove(0)
301    }
302
303    #[test]
304    fn test_parse_json2_type_hint_path() {
305        assert_eq!(
306            parse_json2_type_hint_path(r#"attrs."http.status_code""#).unwrap(),
307            vec!["attrs", "http.status_code"]
308        );
309        assert!(parse_json2_type_hint_path("user.id trailing").is_err());
310    }
311
312    #[test]
313    fn test_parse_json2_type_hints() {
314        let column = parse_json2_column(
315            r#"
316CREATE TABLE traces (
317    log_json_data JSON2 (
318        "service.name" STRING INVERTED INDEX,
319        http.method STRING,
320        status_code BIGINT,
321        comment STRING,
322    ),
323    ts TIMESTAMP TIME INDEX,
324)"#,
325        );
326
327        assert!(matches!(
328            column.column_def.data_type,
329            DataType::Custom(_, _)
330        ));
331        let hints = column.extensions.json2_options.unwrap().type_hints;
332        assert_eq!(hints.len(), 4);
333
334        assert_eq!(hints[0].path, vec!["service.name"]);
335        assert_eq!(hints[0].data_type, DataType::String(None));
336        assert!(hints[0].inverted_index);
337
338        assert_eq!(hints[1].path, vec!["http", "method"]);
339        assert_eq!(hints[1].data_type, DataType::String(None));
340        assert!(!hints[1].inverted_index);
341
342        assert_eq!(hints[2].path, vec!["status_code"]);
343        assert_eq!(hints[2].data_type, DataType::BigInt(None));
344
345        assert_eq!(hints[3].path, vec!["comment"]);
346        assert_eq!(hints[3].data_type, DataType::String(None));
347    }
348
349    #[test]
350    fn test_parse_json2_max_auto_expanded_paths() {
351        let column = parse_json2_column(
352            r#"
353CREATE TABLE traces (
354    log_json_data JSON2 (
355        http.method STRING,
356        max_auto_expanded_paths = 0
357    ),
358    ts TIMESTAMP TIME INDEX,
359)"#,
360        );
361
362        let options = column.extensions.json2_options.unwrap();
363        assert_eq!(options.max_auto_expanded_paths, Some(0));
364        assert_eq!(options.type_hints.len(), 1);
365
366        let empty = parse_json2_column(
367            r#"
368CREATE TABLE traces (
369    log_json_data JSON2 (),
370    ts TIMESTAMP TIME INDEX,
371)"#,
372        );
373        assert!(empty.extensions.json2_options.is_none());
374
375        let quoted = parse_json2_column(
376            r#"
377CREATE TABLE traces (
378    log_json_data JSON2 (
379        "max_auto_expanded_paths" STRING,
380        nested."!__remainder__!" STRING
381    ),
382    ts TIMESTAMP TIME INDEX,
383)"#,
384        );
385        let options = quoted.extensions.json2_options.unwrap();
386        assert_eq!(options.max_auto_expanded_paths, None);
387        assert_eq!(options.type_hints.len(), 2);
388    }
389
390    #[test]
391    fn test_parse_json2_max_auto_expanded_paths_rejects_invalid_options() {
392        for options in [
393            "max_auto_expanded_paths = 0, max_auto_expanded_paths = 1",
394            "max_auto_expanded_paths = -1",
395            "max_auto_expanded_paths = 1.5",
396            "max_auto_expanded_paths = 4294967296",
397            r#""!__remainder__!".value STRING"#,
398        ] {
399            let sql = format!(
400                "CREATE TABLE traces (log_json_data JSON2 ({options}), ts TIMESTAMP TIME INDEX)"
401            );
402            assert!(
403                ParserContext::create_with_dialect(
404                    &sql,
405                    &GreptimeDbDialect {},
406                    ParseOptions::default()
407                )
408                .is_err(),
409                "{options}"
410            );
411        }
412    }
413
414    #[test]
415    fn test_parse_json2_type_hint_defaults_to_nullable() {
416        let column = parse_json2_column(
417            r#"
418CREATE TABLE traces (
419    log_json_data JSON2 (http.method STRING),
420    ts TIMESTAMP TIME INDEX,
421)"#,
422        );
423
424        let hints = column.extensions.json2_options.unwrap().type_hints;
425        assert_eq!(hints.len(), 1);
426        assert_eq!(hints[0].data_type, DataType::String(None));
427    }
428
429    #[test]
430    fn test_parse_json2_type_hint_quoted_path_segments() {
431        let column = parse_json2_column(
432            r#"
433CREATE TABLE traces (
434    log_json_data JSON2 (
435        "a".b STRING,
436        "x"."y" STRING,
437        "a.b"."c" STRING,
438        a."b.c" STRING
439    ),
440    ts TIMESTAMP TIME INDEX,
441)"#,
442        );
443
444        let hints = column.extensions.json2_options.unwrap().type_hints;
445        assert_eq!(hints.len(), 4);
446        assert_eq!(hints[0].path, vec!["a", "b"]);
447        assert_eq!(hints[1].path, vec!["x", "y"]);
448        assert_eq!(hints[2].path, vec!["a.b", "c"]);
449        assert_eq!(hints[3].path, vec!["a", "b.c"]);
450    }
451
452    #[test]
453    fn test_parse_json2_type_hint_supported_types() {
454        for (sql_type, expected) in [
455            ("STRING", DataType::String(None)),
456            ("BIGINT", DataType::BigInt(None)),
457            ("BIGINT UNSIGNED", DataType::BigIntUnsigned(None)),
458            ("DOUBLE", DataType::Double(ExactNumberInfo::None)),
459            ("BOOLEAN", DataType::Boolean),
460            ("INT64", DataType::Int64),
461            ("UINT64", DataType::UInt64),
462            ("FLOAT64", DataType::Float64),
463        ] {
464            for sql_type in [sql_type.to_string(), sql_type.to_lowercase()] {
465                let column = parse_json2_column(&format!(
466                    "CREATE TABLE traces (j JSON2 (value {sql_type}), ts TIMESTAMP TIME INDEX)"
467                ));
468                let options = column.extensions.json2_options.unwrap();
469                let settings = options.build_json_settings().unwrap();
470                assert_eq!(settings.type_hints().len(), 1);
471                let hints = options.type_hints;
472                assert_eq!(hints[0].data_type, expected);
473            }
474        }
475    }
476
477    #[test]
478    fn test_parse_json2_type_hint_rejects_unsupported_types() {
479        for sql_type in [
480            "INT2",
481            "INT4",
482            "INT8",
483            "INT16",
484            "INT32",
485            "TINYINT",
486            "SMALLINT",
487            "INT",
488            "INTEGER",
489            "UINT8",
490            "UINT16",
491            "UINT32",
492            "TINYINT UNSIGNED",
493            "SMALLINT UNSIGNED",
494            "INT UNSIGNED",
495            "FLOAT",
496            "REAL",
497            "FLOAT4",
498            "FLOAT8",
499            "FLOAT32",
500            "BOOL",
501            "TEXT",
502            "VARCHAR(10)",
503            "CHAR(10)",
504            "TIMESTAMP",
505            "DECIMAL(10, 2)",
506            "STRING(10)",
507            "BIGINT(10)",
508        ] {
509            for sql in [
510                format!(
511                    "CREATE TABLE traces (j JSON2 (value {sql_type}), ts TIMESTAMP TIME INDEX)"
512                ),
513                format!("ALTER TABLE traces MODIFY COLUMN j JSON2 (value {sql_type})"),
514            ] {
515                let err = ParserContext::create_with_dialect(
516                    &sql,
517                    &GreptimeDbDialect {},
518                    ParseOptions::default(),
519                )
520                .unwrap_err();
521                assert!(
522                    err.to_string().contains(
523                        "supported types: STRING, BIGINT, BIGINT UNSIGNED, DOUBLE, BOOLEAN; supported aliases: INT64, UINT64, FLOAT64"
524                    ),
525                    "{sql}: {err}"
526                );
527            }
528        }
529    }
530
531    #[test]
532    fn test_parse_json2_type_hint_rejects_default() {
533        for default in ["-5", "abs(-1)"] {
534            let sql = format!(
535                "CREATE TABLE traces (log_json_data JSON2 (status_code BIGINT DEFAULT {default}), ts TIMESTAMP TIME INDEX)"
536            );
537            let err = ParserContext::create_with_dialect(
538                &sql,
539                &GreptimeDbDialect {},
540                ParseOptions::default(),
541            )
542            .unwrap_err();
543            assert!(err.to_string().contains("DEFAULT is not supported"));
544        }
545    }
546
547    #[test]
548    fn test_parse_json2_type_hint_rejects_duplicate_path() {
549        let result = ParserContext::create_with_dialect(
550            r#"
551CREATE TABLE traces (
552    log_json_data JSON2 (a.b STRING, a.b BIGINT),
553    ts TIMESTAMP TIME INDEX,
554)"#,
555            &GreptimeDbDialect {},
556            ParseOptions::default(),
557        );
558
559        assert!(result.is_err());
560        assert!(result.unwrap_err().to_string().contains("duplicated"));
561    }
562
563    #[test]
564    fn test_parse_json2_type_hint_rejects_parent_child_path() {
565        let result = ParserContext::create_with_dialect(
566            r#"
567CREATE TABLE traces (
568    log_json_data JSON2 (a STRING, a.b BIGINT),
569    ts TIMESTAMP TIME INDEX,
570)"#,
571            &GreptimeDbDialect {},
572            ParseOptions::default(),
573        );
574
575        assert!(result.is_err());
576        assert!(result.unwrap_err().to_string().contains("conflicts"));
577    }
578
579    #[test]
580    fn test_parse_json2_type_hint_rejects_nullability() {
581        for sql in [
582            r#"
583CREATE TABLE traces (
584    log_json_data JSON2 (a STRING NULL NULL),
585    ts TIMESTAMP TIME INDEX,
586)"#,
587            r#"
588CREATE TABLE traces (
589    log_json_data JSON2 (a STRING NOT NULL NOT NULL),
590    ts TIMESTAMP TIME INDEX,
591)"#,
592            r#"
593CREATE TABLE traces (
594    log_json_data JSON2 (a STRING NOT NULL NULL),
595    ts TIMESTAMP TIME INDEX,
596)"#,
597            r#"
598CREATE TABLE traces (
599    log_json_data JSON2 (a STRING NULL NOT NULL),
600    ts TIMESTAMP TIME INDEX,
601)"#,
602        ] {
603            let result = ParserContext::create_with_dialect(
604                sql,
605                &GreptimeDbDialect {},
606                ParseOptions::default(),
607            );
608
609            assert!(result.is_err());
610            assert!(
611                result
612                    .unwrap_err()
613                    .to_string()
614                    .contains("NULL/NOT NULL is not supported")
615            );
616        }
617    }
618}
sql/parsers/create_parser/json.rs

sql/parsers/create_parser/
json.rs
