sql/parsers/create_parser/
json.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use datatypes::extension::json::JSON2_REMAINDER_FIELD_NAME;
16use datatypes::json::JSON2_MAX_STRUCTURED_DEPTH;
17use snafu::{ResultExt, ensure};
18use sqlparser::ast::{DataType, ExactNumberInfo, ObjectName};
19use sqlparser::dialect::keywords::Keyword;
20use sqlparser::parser::Parser;
21use sqlparser::tokenizer::Token;
22
23use crate::ast::Ident;
24use crate::dialect::GreptimeDbDialect;
25use crate::error::{InvalidSqlSnafu, Result, SyntaxSnafu};
26use crate::parsers::create_parser::{INVERTED, SKIPPING};
27use crate::statements::create::{Json2Options, JsonTypeHint};
28
29const JSON2_TYPE_NAME: &str = "JSON2";
30const MAX_AUTO_EXPANDED_PATHS: &str = "max_auto_expanded_paths";
31
32/// Parses a JSON2 type hint path with the same grammar used by `CREATE TABLE`.
33pub fn parse_json2_type_hint_path(path: &str) -> Result<Vec<String>> {
34    let dialect = GreptimeDbDialect {};
35    let mut parser = Parser::new(&dialect)
36        .try_with_sql(path)
37        .context(SyntaxSnafu)?;
38    let path = parse_json2_path(&mut parser)?;
39    ensure!(
40        parser.peek_token().token == Token::EOF,
41        InvalidSqlSnafu {
42            msg: format!(
43                "unexpected token '{}' in JSON2 type hint path",
44                parser.peek_token()
45            )
46        }
47    );
48    Ok(path)
49}
50
51pub(crate) fn parse_json2_type_and_options(
52    parser: &mut Parser<'_>,
53) -> Result<Option<(DataType, Option<Json2Options>)>> {
54    let token = parser.peek_token();
55    let Token::Word(word) = &token.token else {
56        return Ok(None);
57    };
58
59    if !word.value.eq_ignore_ascii_case(JSON2_TYPE_NAME) || word.quote_style.is_some() {
60        return Ok(None);
61    }
62
63    parser.next_token();
64    let data_type = DataType::Custom(ObjectName::from(vec![Ident::new(JSON2_TYPE_NAME)]), vec![]);
65    let options = if parser.consume_token(&Token::LParen) {
66        parse_json2_options(parser)?
67    } else {
68        None
69    };
70
71    Ok(Some((data_type, options)))
72}
73
74fn parse_json2_options(parser: &mut Parser<'_>) -> Result<Option<Json2Options>> {
75    if parser.consume_token(&Token::RParen) {
76        return Ok(None);
77    }
78
79    let mut max_auto_expanded_paths = None;
80    let mut type_hints = Vec::new();
81    loop {
82        let token = parser.peek_token();
83        let is_max_auto_expanded_paths = matches!(
84            &token.token,
85            Token::Word(word)
86                if word.quote_style.is_none()
87                    && word.value.eq_ignore_ascii_case(MAX_AUTO_EXPANDED_PATHS)
88        );
89        if is_max_auto_expanded_paths {
90            parser.next_token();
91            ensure!(
92                max_auto_expanded_paths.is_none(),
93                InvalidSqlSnafu {
94                    msg: format!("duplicated JSON2 option '{MAX_AUTO_EXPANDED_PATHS}'")
95                }
96            );
97            parser.expect_token(&Token::Eq).context(SyntaxSnafu)?;
98
99            let token = parser.next_token();
100            let Token::Number(value, _) = token.token else {
101                return InvalidSqlSnafu {
102                    msg: format!(
103                        "JSON2 option '{MAX_AUTO_EXPANDED_PATHS}' expects a non-negative integer"
104                    ),
105                }
106                .fail();
107            };
108            max_auto_expanded_paths = Some(value.parse::<u32>().map_err(|_| {
109                InvalidSqlSnafu {
110                    msg: format!(
111                        "JSON2 option '{MAX_AUTO_EXPANDED_PATHS}' expects a non-negative integer"
112                    ),
113                }
114                .build()
115            })?);
116        } else {
117            let hint = parse_json2_type_hint(parser)?;
118            ensure_no_path_conflict(&type_hints, &hint.path)?;
119            type_hints.push(hint);
120        }
121
122        if parser.consume_token(&Token::Comma) {
123            if parser.consume_token(&Token::RParen) {
124                break;
125            }
126        } else {
127            parser.expect_token(&Token::RParen).context(SyntaxSnafu)?;
128            break;
129        }
130    }
131
132    Ok(Some(Json2Options {
133        max_auto_expanded_paths,
134        type_hints,
135    }))
136}
137
138fn parse_json2_type_hint(parser: &mut Parser<'_>) -> Result<JsonTypeHint> {
139    let path = parse_json2_path(parser)?;
140    ensure!(
141        path.first().is_none_or(|x| x != JSON2_REMAINDER_FIELD_NAME),
142        InvalidSqlSnafu {
143            msg: format!(
144                "JSON2 type hint path cannot be rooted at reserved field '{JSON2_REMAINDER_FIELD_NAME}'"
145            )
146        }
147    );
148    ensure!(
149        path.len() <= JSON2_MAX_STRUCTURED_DEPTH,
150        InvalidSqlSnafu {
151            msg: format!(
152                "JSON2 type hint path cannot exceed {JSON2_MAX_STRUCTURED_DEPTH} segments"
153            ),
154        }
155    );
156    let data_type = parser.parse_data_type().context(SyntaxSnafu)?;
157    let data_type = validate_json2_type_hint_type(data_type)?;
158
159    let mut inverted_index = false;
160    let mut skipping_index = None;
161
162    loop {
163        if parser.parse_keywords(&[Keyword::NOT, Keyword::NULL])
164            || parser.parse_keyword(Keyword::NULL)
165        {
166            return InvalidSqlSnafu {
167                msg: "JSON2 type hint NULL/NOT NULL is not supported; hinted fields are always nullable"
168                    .to_string(),
169            }
170            .fail();
171        } else if parser.parse_keyword(Keyword::DEFAULT) {
172            return InvalidSqlSnafu {
173                msg: "JSON2 type hint DEFAULT is not supported".to_string(),
174            }
175            .fail();
176        } else if let Token::Word(word) = parser.peek_token().token
177            && word.value.eq_ignore_ascii_case(INVERTED)
178        {
179            parser.next_token();
180            ensure!(
181                parser.parse_keyword(Keyword::INDEX),
182                InvalidSqlSnafu {
183                    msg: format!(
184                        "expect INDEX after INVERTED keyword for JSON2 type hint '{}'",
185                        path.join(".")
186                    )
187                }
188            );
189            ensure!(
190                !inverted_index,
191                InvalidSqlSnafu {
192                    msg: format!(
193                        "duplicated INVERTED INDEX option for JSON2 type hint '{}'",
194                        path.join(".")
195                    )
196                }
197            );
198            ensure!(
199                !path
200                    .iter()
201                    .any(|segment| segment == JSON2_REMAINDER_FIELD_NAME),
202                InvalidSqlSnafu {
203                    msg: format!(
204                        "JSON2 indexed type hint path cannot contain reserved field '{JSON2_REMAINDER_FIELD_NAME}'"
205                    )
206                }
207            );
208            inverted_index = true;
209        } else if let Token::Word(word) = parser.peek_token().token
210            && word.value.eq_ignore_ascii_case(SKIPPING)
211        {
212            parser.next_token();
213            ensure!(
214                parser.parse_keyword(Keyword::INDEX),
215                InvalidSqlSnafu {
216                    msg: "expect INDEX after SKIPPING keyword",
217                }
218            );
219            ensure!(
220                skipping_index.is_none(),
221                InvalidSqlSnafu {
222                    msg: "duplicated SKIPPING INDEX option for JSON2 type hint",
223                }
224            );
225            let options = parser
226                .parse_options(Keyword::WITH)
227                .context(SyntaxSnafu)?
228                .into_iter()
229                .map(crate::util::parse_option_string)
230                .collect::<Result<Vec<_>>>()?;
231            for (key, _) in &options {
232                ensure!(
233                    crate::parsers::utils::validate_column_skipping_index_create_option(key),
234                    InvalidSqlSnafu {
235                        msg: format!("invalid SKIPPING INDEX option: {key}"),
236                    }
237                );
238            }
239            skipping_index = Some(
240                datatypes::schema::SkippingIndexOptions::try_from(
241                    crate::statements::OptionMap::new(options).into_map(),
242                )
243                .context(crate::error::SetSkippingIndexOptionSnafu)?,
244            );
245        } else if matches!(parser.peek_token().token, Token::Comma | Token::RParen) {
246            break;
247        } else {
248            return parser
249                .expected("JSON2 type hint option", parser.peek_token())
250                .context(SyntaxSnafu);
251        }
252    }
253
254    Ok(JsonTypeHint {
255        path,
256        data_type,
257        inverted_index,
258        skipping_index,
259    })
260}
261
262fn parse_json2_path(parser: &mut Parser<'_>) -> Result<Vec<String>> {
263    let first = parser.parse_identifier().context(SyntaxSnafu)?;
264    let mut path = vec![first.value];
265
266    while parser.consume_token(&Token::Period) {
267        let segment = parser.parse_identifier().context(SyntaxSnafu)?;
268        path.push(segment.value);
269    }
270
271    ensure!(
272        !path.iter().any(|segment| segment.is_empty()),
273        InvalidSqlSnafu {
274            msg: "JSON2 type hint path segment cannot be empty".to_string(),
275        }
276    );
277
278    Ok(path)
279}
280
281fn validate_json2_type_hint_type(data_type: DataType) -> Result<DataType> {
282    match data_type {
283        DataType::Int64
284        | DataType::UInt64
285        | DataType::Float64
286        | DataType::String(None)
287        | DataType::BigInt(None)
288        | DataType::BigIntUnsigned(None)
289        | DataType::Double(ExactNumberInfo::None)
290        | DataType::Boolean => Ok(data_type),
291        _ => InvalidSqlSnafu {
292            msg: format!(
293                "unsupported JSON2 type hint data type: {data_type}; supported types: STRING, BIGINT, BIGINT UNSIGNED, DOUBLE, BOOLEAN; supported aliases: INT64, UINT64, FLOAT64"
294            ),
295        }
296        .fail(),
297    }
298}
299
300fn ensure_no_path_conflict(hints: &[JsonTypeHint], path: &[String]) -> Result<()> {
301    for hint in hints {
302        ensure!(
303            hint.path != path,
304            InvalidSqlSnafu {
305                msg: format!("duplicated JSON2 type hint path '{}'", path.join("."))
306            }
307        );
308        ensure!(
309            !hint.path.starts_with(path) && !path.starts_with(&hint.path),
310            InvalidSqlSnafu {
311                msg: format!(
312                    "JSON2 type hint path '{}' conflicts with '{}'",
313                    path.join("."),
314                    hint.path.join(".")
315                )
316            }
317        );
318    }
319    Ok(())
320}
321
322#[cfg(test)]
323mod tests {
324    use sqlparser::ast::{DataType, ExactNumberInfo};
325
326    use super::parse_json2_type_hint_path;
327    use crate::dialect::GreptimeDbDialect;
328    use crate::parser::{ParseOptions, ParserContext};
329    use crate::statements::create::Column;
330    use crate::statements::statement::Statement;
331
332    fn parse_json2_column(sql: &str) -> Column {
333        let Statement::CreateTable(mut create_table) =
334            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
335                .unwrap()
336                .remove(0)
337        else {
338            unreachable!()
339        };
340
341        create_table.columns.remove(0)
342    }
343
344    #[test]
345    fn test_json2_skipping_index() {
346        let column = parse_json2_column(
347            "CREATE TABLE logs (attrs JSON2 (service.name STRING SKIPPING INDEX WITH (granularity='8', false_positive_rate='0.05') INVERTED INDEX, status BIGINT SKIPPING INDEX), ts TIMESTAMP TIME INDEX)",
348        );
349        let options = column.extensions.json2_options.as_ref().unwrap();
350        let settings = options.build_json_settings().unwrap();
351        let hint = &settings.type_hints()[0];
352        assert!(hint.inverted_index);
353        assert_eq!(hint.skipping_index.as_ref().unwrap().granularity, 8);
354        assert_eq!(
355            hint.skipping_index.as_ref().unwrap().false_positive_rate(),
356            0.05
357        );
358        assert_eq!(
359            settings.type_hints()[1].skipping_index,
360            Some(datatypes::schema::SkippingIndexOptions::default())
361        );
362        let rebuilt = parse_json2_column(&format!(
363            "CREATE TABLE logs (attrs JSON2 {options}, ts TIMESTAMP TIME INDEX)"
364        ));
365        assert_eq!(
366            settings,
367            rebuilt
368                .extensions
369                .json2_options
370                .unwrap()
371                .build_json_settings()
372                .unwrap()
373        );
374        for option in [
375            "SKIPPING INDEX SKIPPING INDEX",
376            "SKIPPING",
377            "SKIPPING INDEX WITH (granularity='0')",
378            "SKIPPING INDEX WITH (false_positive_rate='0')",
379            "SKIPPING INDEX WITH (type='unknown')",
380            "SKIPPING INDEX WITH (unknown='1')",
381        ] {
382            let sql = format!(
383                "CREATE TABLE logs (attrs JSON2 (a STRING {option}), ts TIMESTAMP TIME INDEX)"
384            );
385            assert!(
386                ParserContext::create_with_dialect(
387                    &sql,
388                    &GreptimeDbDialect {},
389                    ParseOptions::default()
390                )
391                .is_err(),
392                "{sql}"
393            );
394        }
395    }
396
397    #[test]
398    fn test_parse_json2_type_hint_path() {
399        assert_eq!(
400            parse_json2_type_hint_path(r#"attrs."http.status_code""#).unwrap(),
401            vec!["attrs", "http.status_code"]
402        );
403        assert!(parse_json2_type_hint_path("user.id trailing").is_err());
404    }
405
406    #[test]
407    fn test_parse_json2_type_hints() {
408        let column = parse_json2_column(
409            r#"
410CREATE TABLE traces (
411    log_json_data JSON2 (
412        "service.name" STRING INVERTED INDEX,
413        http.method STRING,
414        status_code BIGINT,
415        comment STRING,
416    ),
417    ts TIMESTAMP TIME INDEX,
418)"#,
419        );
420
421        assert!(matches!(
422            column.column_def.data_type,
423            DataType::Custom(_, _)
424        ));
425        let hints = column.extensions.json2_options.unwrap().type_hints;
426        assert_eq!(hints.len(), 4);
427
428        assert_eq!(hints[0].path, vec!["service.name"]);
429        assert_eq!(hints[0].data_type, DataType::String(None));
430        assert!(hints[0].inverted_index);
431
432        assert_eq!(hints[1].path, vec!["http", "method"]);
433        assert_eq!(hints[1].data_type, DataType::String(None));
434        assert!(!hints[1].inverted_index);
435
436        assert_eq!(hints[2].path, vec!["status_code"]);
437        assert_eq!(hints[2].data_type, DataType::BigInt(None));
438
439        assert_eq!(hints[3].path, vec!["comment"]);
440        assert_eq!(hints[3].data_type, DataType::String(None));
441    }
442
443    #[test]
444    fn test_parse_json2_max_auto_expanded_paths() {
445        let column = parse_json2_column(
446            r#"
447CREATE TABLE traces (
448    log_json_data JSON2 (
449        http.method STRING,
450        max_auto_expanded_paths = 0
451    ),
452    ts TIMESTAMP TIME INDEX,
453)"#,
454        );
455
456        let options = column.extensions.json2_options.unwrap();
457        assert_eq!(options.max_auto_expanded_paths, Some(0));
458        assert_eq!(options.type_hints.len(), 1);
459
460        let empty = parse_json2_column(
461            r#"
462CREATE TABLE traces (
463    log_json_data JSON2 (),
464    ts TIMESTAMP TIME INDEX,
465)"#,
466        );
467        assert!(empty.extensions.json2_options.is_none());
468
469        let quoted = parse_json2_column(
470            r#"
471CREATE TABLE traces (
472    log_json_data JSON2 (
473        "max_auto_expanded_paths" STRING,
474        nested."!__remainder__!" STRING
475    ),
476    ts TIMESTAMP TIME INDEX,
477)"#,
478        );
479        let options = quoted.extensions.json2_options.unwrap();
480        assert_eq!(options.max_auto_expanded_paths, None);
481        assert_eq!(options.type_hints.len(), 2);
482    }
483
484    #[test]
485    fn test_parse_json2_max_auto_expanded_paths_rejects_invalid_options() {
486        for options in [
487            "max_auto_expanded_paths = 0, max_auto_expanded_paths = 1",
488            "max_auto_expanded_paths = -1",
489            "max_auto_expanded_paths = 1.5",
490            "max_auto_expanded_paths = 4294967296",
491            r#""!__remainder__!".value STRING"#,
492        ] {
493            let sql = format!(
494                "CREATE TABLE traces (log_json_data JSON2 ({options}), ts TIMESTAMP TIME INDEX)"
495            );
496            assert!(
497                ParserContext::create_with_dialect(
498                    &sql,
499                    &GreptimeDbDialect {},
500                    ParseOptions::default()
501                )
502                .is_err(),
503                "{options}"
504            );
505        }
506    }
507
508    #[test]
509    fn test_parse_json2_type_hint_defaults_to_nullable() {
510        let column = parse_json2_column(
511            r#"
512CREATE TABLE traces (
513    log_json_data JSON2 (http.method STRING),
514    ts TIMESTAMP TIME INDEX,
515)"#,
516        );
517
518        let hints = column.extensions.json2_options.unwrap().type_hints;
519        assert_eq!(hints.len(), 1);
520        assert_eq!(hints[0].data_type, DataType::String(None));
521    }
522
523    #[test]
524    fn test_parse_json2_type_hint_quoted_path_segments() {
525        let column = parse_json2_column(
526            r#"
527CREATE TABLE traces (
528    log_json_data JSON2 (
529        "a".b STRING,
530        "x"."y" STRING,
531        "a.b"."c" STRING,
532        a."b.c" STRING
533    ),
534    ts TIMESTAMP TIME INDEX,
535)"#,
536        );
537
538        let hints = column.extensions.json2_options.unwrap().type_hints;
539        assert_eq!(hints.len(), 4);
540        assert_eq!(hints[0].path, vec!["a", "b"]);
541        assert_eq!(hints[1].path, vec!["x", "y"]);
542        assert_eq!(hints[2].path, vec!["a.b", "c"]);
543        assert_eq!(hints[3].path, vec!["a", "b.c"]);
544    }
545
546    #[test]
547    fn test_parse_json2_type_hint_supported_types() {
548        for (sql_type, expected) in [
549            ("STRING", DataType::String(None)),
550            ("BIGINT", DataType::BigInt(None)),
551            ("BIGINT UNSIGNED", DataType::BigIntUnsigned(None)),
552            ("DOUBLE", DataType::Double(ExactNumberInfo::None)),
553            ("BOOLEAN", DataType::Boolean),
554            ("INT64", DataType::Int64),
555            ("UINT64", DataType::UInt64),
556            ("FLOAT64", DataType::Float64),
557        ] {
558            for sql_type in [sql_type.to_string(), sql_type.to_lowercase()] {
559                let column = parse_json2_column(&format!(
560                    "CREATE TABLE traces (j JSON2 (value {sql_type}), ts TIMESTAMP TIME INDEX)"
561                ));
562                let options = column.extensions.json2_options.unwrap();
563                let settings = options.build_json_settings().unwrap();
564                assert_eq!(settings.type_hints().len(), 1);
565                let hints = options.type_hints;
566                assert_eq!(hints[0].data_type, expected);
567            }
568        }
569    }
570
571    #[test]
572    fn test_parse_json2_type_hint_rejects_unsupported_types() {
573        for sql_type in [
574            "INT2",
575            "INT4",
576            "INT8",
577            "INT16",
578            "INT32",
579            "TINYINT",
580            "SMALLINT",
581            "INT",
582            "INTEGER",
583            "UINT8",
584            "UINT16",
585            "UINT32",
586            "TINYINT UNSIGNED",
587            "SMALLINT UNSIGNED",
588            "INT UNSIGNED",
589            "FLOAT",
590            "REAL",
591            "FLOAT4",
592            "FLOAT8",
593            "FLOAT32",
594            "BOOL",
595            "TEXT",
596            "VARCHAR(10)",
597            "CHAR(10)",
598            "TIMESTAMP",
599            "DECIMAL(10, 2)",
600            "STRING(10)",
601            "BIGINT(10)",
602        ] {
603            for sql in [
604                format!(
605                    "CREATE TABLE traces (j JSON2 (value {sql_type}), ts TIMESTAMP TIME INDEX)"
606                ),
607                format!("ALTER TABLE traces MODIFY COLUMN j JSON2 (value {sql_type})"),
608            ] {
609                let err = ParserContext::create_with_dialect(
610                    &sql,
611                    &GreptimeDbDialect {},
612                    ParseOptions::default(),
613                )
614                .unwrap_err();
615                assert!(
616                    err.to_string().contains(
617                        "supported types: STRING, BIGINT, BIGINT UNSIGNED, DOUBLE, BOOLEAN; supported aliases: INT64, UINT64, FLOAT64"
618                    ),
619                    "{sql}: {err}"
620                );
621            }
622        }
623    }
624
625    #[test]
626    fn test_parse_json2_type_hint_rejects_default() {
627        for default in ["-5", "abs(-1)"] {
628            let sql = format!(
629                "CREATE TABLE traces (log_json_data JSON2 (status_code BIGINT DEFAULT {default}), ts TIMESTAMP TIME INDEX)"
630            );
631            let err = ParserContext::create_with_dialect(
632                &sql,
633                &GreptimeDbDialect {},
634                ParseOptions::default(),
635            )
636            .unwrap_err();
637            assert!(err.to_string().contains("DEFAULT is not supported"));
638        }
639    }
640
641    #[test]
642    fn test_parse_json2_type_hint_rejects_duplicate_path() {
643        let result = ParserContext::create_with_dialect(
644            r#"
645CREATE TABLE traces (
646    log_json_data JSON2 (a.b STRING, a.b BIGINT),
647    ts TIMESTAMP TIME INDEX,
648)"#,
649            &GreptimeDbDialect {},
650            ParseOptions::default(),
651        );
652
653        assert!(result.is_err());
654        assert!(result.unwrap_err().to_string().contains("duplicated"));
655    }
656
657    #[test]
658    fn test_parse_json2_type_hint_rejects_parent_child_path() {
659        let result = ParserContext::create_with_dialect(
660            r#"
661CREATE TABLE traces (
662    log_json_data JSON2 (a STRING, a.b BIGINT),
663    ts TIMESTAMP TIME INDEX,
664)"#,
665            &GreptimeDbDialect {},
666            ParseOptions::default(),
667        );
668
669        assert!(result.is_err());
670        assert!(result.unwrap_err().to_string().contains("conflicts"));
671    }
672
673    #[test]
674    fn test_parse_json2_type_hint_rejects_nullability() {
675        for sql in [
676            r#"
677CREATE TABLE traces (
678    log_json_data JSON2 (a STRING NULL NULL),
679    ts TIMESTAMP TIME INDEX,
680)"#,
681            r#"
682CREATE TABLE traces (
683    log_json_data JSON2 (a STRING NOT NULL NOT NULL),
684    ts TIMESTAMP TIME INDEX,
685)"#,
686            r#"
687CREATE TABLE traces (
688    log_json_data JSON2 (a STRING NOT NULL NULL),
689    ts TIMESTAMP TIME INDEX,
690)"#,
691            r#"
692CREATE TABLE traces (
693    log_json_data JSON2 (a STRING NULL NOT NULL),
694    ts TIMESTAMP TIME INDEX,
695)"#,
696        ] {
697            let result = ParserContext::create_with_dialect(
698                sql,
699                &GreptimeDbDialect {},
700                ParseOptions::default(),
701            );
702
703            assert!(result.is_err());
704            assert!(
705                result
706                    .unwrap_err()
707                    .to_string()
708                    .contains("NULL/NOT NULL is not supported")
709            );
710        }
711    }
712}
sql/parsers/create_parser/json.rs

sql/parsers/create_parser/
json.rs
