sql/parsers/create_parser/
json.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use datatypes::extension::json::JSON2_REMAINDER_FIELD_NAME;
16use datatypes::json::JSON2_MAX_STRUCTURED_DEPTH;
17use snafu::{ResultExt, ensure};
18use sqlparser::ast::{DataType, ExactNumberInfo, ObjectName};
19use sqlparser::dialect::keywords::Keyword;
20use sqlparser::parser::Parser;
21use sqlparser::tokenizer::Token;
22
23use crate::ast::Ident;
24use crate::dialect::GreptimeDbDialect;
25use crate::error::{InvalidSqlSnafu, Result, SyntaxSnafu};
26use crate::parsers::create_parser::{INVERTED, SKIPPING};
27use crate::statements::create::{Json2Options, JsonTypeHint};
28
29const JSON2_TYPE_NAME: &str = "JSON2";
30const MAX_AUTO_EXPANDED_PATHS: &str = "max_auto_expanded_paths";
31
32/// Parses a JSON2 type hint path with the same grammar used by `CREATE TABLE`.
33pub fn parse_json2_type_hint_path(path: &str) -> Result<Vec<String>> {
34    let dialect = GreptimeDbDialect {};
35    let mut parser = Parser::new(&dialect)
36        .try_with_sql(path)
37        .context(SyntaxSnafu)?;
38    let path = parse_json2_path(&mut parser)?;
39    ensure!(
40        parser.peek_token().token == Token::EOF,
41        InvalidSqlSnafu {
42            msg: format!(
43                "unexpected token '{}' in JSON2 type hint path",
44                parser.peek_token()
45            )
46        }
47    );
48    Ok(path)
49}
50
51pub(crate) fn parse_json2_type_and_options(
52    parser: &mut Parser<'_>,
53) -> Result<Option<(DataType, Option<Json2Options>)>> {
54    let token = parser.peek_token();
55    let Token::Word(word) = &token.token else {
56        return Ok(None);
57    };
58
59    if !word.value.eq_ignore_ascii_case(JSON2_TYPE_NAME) || word.quote_style.is_some() {
60        return Ok(None);
61    }
62
63    parser.next_token();
64    let data_type = DataType::Custom(ObjectName::from(vec![Ident::new(JSON2_TYPE_NAME)]), vec![]);
65    let options = if parser.consume_token(&Token::LParen) {
66        parse_json2_options(parser)?
67    } else {
68        None
69    };
70
71    Ok(Some((data_type, options)))
72}
73
74fn parse_json2_options(parser: &mut Parser<'_>) -> Result<Option<Json2Options>> {
75    if parser.consume_token(&Token::RParen) {
76        return Ok(None);
77    }
78
79    let mut max_auto_expanded_paths = None;
80    let mut type_hints = Vec::new();
81    loop {
82        let token = parser.peek_token();
83        let is_max_auto_expanded_paths = matches!(
84            &token.token,
85            Token::Word(word)
86                if word.quote_style.is_none()
87                    && word.value.eq_ignore_ascii_case(MAX_AUTO_EXPANDED_PATHS)
88        );
89        if is_max_auto_expanded_paths {
90            parser.next_token();
91            ensure!(
92                max_auto_expanded_paths.is_none(),
93                InvalidSqlSnafu {
94                    msg: format!("duplicated JSON2 option '{MAX_AUTO_EXPANDED_PATHS}'")
95                }
96            );
97            parser.expect_token(&Token::Eq).context(SyntaxSnafu)?;
98
99            let token = parser.next_token();
100            let Token::Number(value, _) = token.token else {
101                return InvalidSqlSnafu {
102                    msg: format!(
103                        "JSON2 option '{MAX_AUTO_EXPANDED_PATHS}' expects a non-negative integer"
104                    ),
105                }
106                .fail();
107            };
108            max_auto_expanded_paths = Some(value.parse::<u32>().map_err(|_| {
109                InvalidSqlSnafu {
110                    msg: format!(
111                        "JSON2 option '{MAX_AUTO_EXPANDED_PATHS}' expects a non-negative integer"
112                    ),
113                }
114                .build()
115            })?);
116        } else {
117            let hint = parse_json2_type_hint(parser)?;
118            ensure_no_path_conflict(&type_hints, &hint.path)?;
119            type_hints.push(hint);
120        }
121
122        if parser.consume_token(&Token::Comma) {
123            if parser.consume_token(&Token::RParen) {
124                break;
125            }
126        } else {
127            parser.expect_token(&Token::RParen).context(SyntaxSnafu)?;
128            break;
129        }
130    }
131
132    Ok(Some(Json2Options {
133        max_auto_expanded_paths,
134        type_hints,
135    }))
136}
137
138fn parse_json2_type_hint(parser: &mut Parser<'_>) -> Result<JsonTypeHint> {
139    let path = parse_json2_path(parser)?;
140    ensure!(
141        path.first().is_none_or(|x| x != JSON2_REMAINDER_FIELD_NAME),
142        InvalidSqlSnafu {
143            msg: format!(
144                "JSON2 type hint path cannot be rooted at reserved field '{JSON2_REMAINDER_FIELD_NAME}'"
145            )
146        }
147    );
148    ensure!(
149        path.len() <= JSON2_MAX_STRUCTURED_DEPTH,
150        InvalidSqlSnafu {
151            msg: format!(
152                "JSON2 type hint path cannot exceed {JSON2_MAX_STRUCTURED_DEPTH} segments"
153            ),
154        }
155    );
156    let data_type = parser.parse_data_type().context(SyntaxSnafu)?;
157    let data_type = validate_json2_type_hint_type(data_type)?;
158
159    let mut inverted_index = false;
160
161    loop {
162        if parser.parse_keywords(&[Keyword::NOT, Keyword::NULL])
163            || parser.parse_keyword(Keyword::NULL)
164        {
165            return InvalidSqlSnafu {
166                msg: "JSON2 type hint NULL/NOT NULL is not supported; hinted fields are always nullable"
167                    .to_string(),
168            }
169            .fail();
170        } else if parser.parse_keyword(Keyword::DEFAULT) {
171            return InvalidSqlSnafu {
172                msg: "JSON2 type hint DEFAULT is not supported".to_string(),
173            }
174            .fail();
175        } else if let Token::Word(word) = parser.peek_token().token
176            && word.value.eq_ignore_ascii_case(INVERTED)
177        {
178            parser.next_token();
179            ensure!(
180                parser.parse_keyword(Keyword::INDEX),
181                InvalidSqlSnafu {
182                    msg: format!(
183                        "expect INDEX after INVERTED keyword for JSON2 type hint '{}'",
184                        path.join(".")
185                    )
186                }
187            );
188            ensure!(
189                !inverted_index,
190                InvalidSqlSnafu {
191                    msg: format!(
192                        "duplicated INVERTED INDEX option for JSON2 type hint '{}'",
193                        path.join(".")
194                    )
195                }
196            );
197            ensure!(
198                !path
199                    .iter()
200                    .any(|segment| segment == JSON2_REMAINDER_FIELD_NAME),
201                InvalidSqlSnafu {
202                    msg: format!(
203                        "JSON2 indexed type hint path cannot contain reserved field '{JSON2_REMAINDER_FIELD_NAME}'"
204                    )
205                }
206            );
207            inverted_index = true;
208        } else if let Token::Word(word) = parser.peek_token().token
209            && word.value.eq_ignore_ascii_case(SKIPPING)
210        {
211            return InvalidSqlSnafu {
212                msg: "JSON2 type hint SKIPPING INDEX is not supported yet".to_string(),
213            }
214            .fail();
215        } else if matches!(parser.peek_token().token, Token::Comma | Token::RParen) {
216            break;
217        } else {
218            return parser
219                .expected("JSON2 type hint option", parser.peek_token())
220                .context(SyntaxSnafu);
221        }
222    }
223
224    Ok(JsonTypeHint {
225        path,
226        data_type,
227        inverted_index,
228    })
229}
230
231fn parse_json2_path(parser: &mut Parser<'_>) -> Result<Vec<String>> {
232    let first = parser.parse_identifier().context(SyntaxSnafu)?;
233    let mut path = vec![first.value];
234
235    while parser.consume_token(&Token::Period) {
236        let segment = parser.parse_identifier().context(SyntaxSnafu)?;
237        path.push(segment.value);
238    }
239
240    ensure!(
241        !path.iter().any(|segment| segment.is_empty()),
242        InvalidSqlSnafu {
243            msg: "JSON2 type hint path segment cannot be empty".to_string(),
244        }
245    );
246
247    Ok(path)
248}
249
250fn validate_json2_type_hint_type(data_type: DataType) -> Result<DataType> {
251    match data_type {
252        DataType::Int64
253        | DataType::UInt64
254        | DataType::Float64
255        | DataType::String(None)
256        | DataType::BigInt(None)
257        | DataType::BigIntUnsigned(None)
258        | DataType::Double(ExactNumberInfo::None)
259        | DataType::Boolean => Ok(data_type),
260        _ => InvalidSqlSnafu {
261            msg: format!(
262                "unsupported JSON2 type hint data type: {data_type}; supported types: STRING, BIGINT, BIGINT UNSIGNED, DOUBLE, BOOLEAN; supported aliases: INT64, UINT64, FLOAT64"
263            ),
264        }
265        .fail(),
266    }
267}
268
269fn ensure_no_path_conflict(hints: &[JsonTypeHint], path: &[String]) -> Result<()> {
270    for hint in hints {
271        ensure!(
272            hint.path != path,
273            InvalidSqlSnafu {
274                msg: format!("duplicated JSON2 type hint path '{}'", path.join("."))
275            }
276        );
277        ensure!(
278            !hint.path.starts_with(path) && !path.starts_with(&hint.path),
279            InvalidSqlSnafu {
280                msg: format!(
281                    "JSON2 type hint path '{}' conflicts with '{}'",
282                    path.join("."),
283                    hint.path.join(".")
284                )
285            }
286        );
287    }
288    Ok(())
289}
290
291#[cfg(test)]
292mod tests {
293    use sqlparser::ast::{DataType, ExactNumberInfo};
294
295    use super::parse_json2_type_hint_path;
296    use crate::dialect::GreptimeDbDialect;
297    use crate::parser::{ParseOptions, ParserContext};
298    use crate::statements::create::Column;
299    use crate::statements::statement::Statement;
300
301    fn parse_json2_column(sql: &str) -> Column {
302        let Statement::CreateTable(mut create_table) =
303            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
304                .unwrap()
305                .remove(0)
306        else {
307            unreachable!()
308        };
309
310        create_table.columns.remove(0)
311    }
312
313    #[test]
314    fn test_parse_json2_type_hint_path() {
315        assert_eq!(
316            parse_json2_type_hint_path(r#"attrs."http.status_code""#).unwrap(),
317            vec!["attrs", "http.status_code"]
318        );
319        assert!(parse_json2_type_hint_path("user.id trailing").is_err());
320    }
321
322    #[test]
323    fn test_parse_json2_type_hints() {
324        let column = parse_json2_column(
325            r#"
326CREATE TABLE traces (
327    log_json_data JSON2 (
328        "service.name" STRING INVERTED INDEX,
329        http.method STRING,
330        status_code BIGINT,
331        comment STRING,
332    ),
333    ts TIMESTAMP TIME INDEX,
334)"#,
335        );
336
337        assert!(matches!(
338            column.column_def.data_type,
339            DataType::Custom(_, _)
340        ));
341        let hints = column.extensions.json2_options.unwrap().type_hints;
342        assert_eq!(hints.len(), 4);
343
344        assert_eq!(hints[0].path, vec!["service.name"]);
345        assert_eq!(hints[0].data_type, DataType::String(None));
346        assert!(hints[0].inverted_index);
347
348        assert_eq!(hints[1].path, vec!["http", "method"]);
349        assert_eq!(hints[1].data_type, DataType::String(None));
350        assert!(!hints[1].inverted_index);
351
352        assert_eq!(hints[2].path, vec!["status_code"]);
353        assert_eq!(hints[2].data_type, DataType::BigInt(None));
354
355        assert_eq!(hints[3].path, vec!["comment"]);
356        assert_eq!(hints[3].data_type, DataType::String(None));
357    }
358
359    #[test]
360    fn test_parse_json2_max_auto_expanded_paths() {
361        let column = parse_json2_column(
362            r#"
363CREATE TABLE traces (
364    log_json_data JSON2 (
365        http.method STRING,
366        max_auto_expanded_paths = 0
367    ),
368    ts TIMESTAMP TIME INDEX,
369)"#,
370        );
371
372        let options = column.extensions.json2_options.unwrap();
373        assert_eq!(options.max_auto_expanded_paths, Some(0));
374        assert_eq!(options.type_hints.len(), 1);
375
376        let empty = parse_json2_column(
377            r#"
378CREATE TABLE traces (
379    log_json_data JSON2 (),
380    ts TIMESTAMP TIME INDEX,
381)"#,
382        );
383        assert!(empty.extensions.json2_options.is_none());
384
385        let quoted = parse_json2_column(
386            r#"
387CREATE TABLE traces (
388    log_json_data JSON2 (
389        "max_auto_expanded_paths" STRING,
390        nested."!__remainder__!" STRING
391    ),
392    ts TIMESTAMP TIME INDEX,
393)"#,
394        );
395        let options = quoted.extensions.json2_options.unwrap();
396        assert_eq!(options.max_auto_expanded_paths, None);
397        assert_eq!(options.type_hints.len(), 2);
398    }
399
400    #[test]
401    fn test_parse_json2_max_auto_expanded_paths_rejects_invalid_options() {
402        for options in [
403            "max_auto_expanded_paths = 0, max_auto_expanded_paths = 1",
404            "max_auto_expanded_paths = -1",
405            "max_auto_expanded_paths = 1.5",
406            "max_auto_expanded_paths = 4294967296",
407            r#""!__remainder__!".value STRING"#,
408        ] {
409            let sql = format!(
410                "CREATE TABLE traces (log_json_data JSON2 ({options}), ts TIMESTAMP TIME INDEX)"
411            );
412            assert!(
413                ParserContext::create_with_dialect(
414                    &sql,
415                    &GreptimeDbDialect {},
416                    ParseOptions::default()
417                )
418                .is_err(),
419                "{options}"
420            );
421        }
422    }
423
424    #[test]
425    fn test_parse_json2_type_hint_defaults_to_nullable() {
426        let column = parse_json2_column(
427            r#"
428CREATE TABLE traces (
429    log_json_data JSON2 (http.method STRING),
430    ts TIMESTAMP TIME INDEX,
431)"#,
432        );
433
434        let hints = column.extensions.json2_options.unwrap().type_hints;
435        assert_eq!(hints.len(), 1);
436        assert_eq!(hints[0].data_type, DataType::String(None));
437    }
438
439    #[test]
440    fn test_parse_json2_type_hint_quoted_path_segments() {
441        let column = parse_json2_column(
442            r#"
443CREATE TABLE traces (
444    log_json_data JSON2 (
445        "a".b STRING,
446        "x"."y" STRING,
447        "a.b"."c" STRING,
448        a."b.c" STRING
449    ),
450    ts TIMESTAMP TIME INDEX,
451)"#,
452        );
453
454        let hints = column.extensions.json2_options.unwrap().type_hints;
455        assert_eq!(hints.len(), 4);
456        assert_eq!(hints[0].path, vec!["a", "b"]);
457        assert_eq!(hints[1].path, vec!["x", "y"]);
458        assert_eq!(hints[2].path, vec!["a.b", "c"]);
459        assert_eq!(hints[3].path, vec!["a", "b.c"]);
460    }
461
462    #[test]
463    fn test_parse_json2_type_hint_supported_types() {
464        for (sql_type, expected) in [
465            ("STRING", DataType::String(None)),
466            ("BIGINT", DataType::BigInt(None)),
467            ("BIGINT UNSIGNED", DataType::BigIntUnsigned(None)),
468            ("DOUBLE", DataType::Double(ExactNumberInfo::None)),
469            ("BOOLEAN", DataType::Boolean),
470            ("INT64", DataType::Int64),
471            ("UINT64", DataType::UInt64),
472            ("FLOAT64", DataType::Float64),
473        ] {
474            for sql_type in [sql_type.to_string(), sql_type.to_lowercase()] {
475                let column = parse_json2_column(&format!(
476                    "CREATE TABLE traces (j JSON2 (value {sql_type}), ts TIMESTAMP TIME INDEX)"
477                ));
478                let options = column.extensions.json2_options.unwrap();
479                let settings = options.build_json_settings().unwrap();
480                assert_eq!(settings.type_hints().len(), 1);
481                let hints = options.type_hints;
482                assert_eq!(hints[0].data_type, expected);
483            }
484        }
485    }
486
487    #[test]
488    fn test_parse_json2_type_hint_rejects_unsupported_types() {
489        for sql_type in [
490            "INT2",
491            "INT4",
492            "INT8",
493            "INT16",
494            "INT32",
495            "TINYINT",
496            "SMALLINT",
497            "INT",
498            "INTEGER",
499            "UINT8",
500            "UINT16",
501            "UINT32",
502            "TINYINT UNSIGNED",
503            "SMALLINT UNSIGNED",
504            "INT UNSIGNED",
505            "FLOAT",
506            "REAL",
507            "FLOAT4",
508            "FLOAT8",
509            "FLOAT32",
510            "BOOL",
511            "TEXT",
512            "VARCHAR(10)",
513            "CHAR(10)",
514            "TIMESTAMP",
515            "DECIMAL(10, 2)",
516            "STRING(10)",
517            "BIGINT(10)",
518        ] {
519            for sql in [
520                format!(
521                    "CREATE TABLE traces (j JSON2 (value {sql_type}), ts TIMESTAMP TIME INDEX)"
522                ),
523                format!("ALTER TABLE traces MODIFY COLUMN j JSON2 (value {sql_type})"),
524            ] {
525                let err = ParserContext::create_with_dialect(
526                    &sql,
527                    &GreptimeDbDialect {},
528                    ParseOptions::default(),
529                )
530                .unwrap_err();
531                assert!(
532                    err.to_string().contains(
533                        "supported types: STRING, BIGINT, BIGINT UNSIGNED, DOUBLE, BOOLEAN; supported aliases: INT64, UINT64, FLOAT64"
534                    ),
535                    "{sql}: {err}"
536                );
537            }
538        }
539    }
540
541    #[test]
542    fn test_parse_json2_type_hint_rejects_default() {
543        for default in ["-5", "abs(-1)"] {
544            let sql = format!(
545                "CREATE TABLE traces (log_json_data JSON2 (status_code BIGINT DEFAULT {default}), ts TIMESTAMP TIME INDEX)"
546            );
547            let err = ParserContext::create_with_dialect(
548                &sql,
549                &GreptimeDbDialect {},
550                ParseOptions::default(),
551            )
552            .unwrap_err();
553            assert!(err.to_string().contains("DEFAULT is not supported"));
554        }
555    }
556
557    #[test]
558    fn test_parse_json2_type_hint_rejects_duplicate_path() {
559        let result = ParserContext::create_with_dialect(
560            r#"
561CREATE TABLE traces (
562    log_json_data JSON2 (a.b STRING, a.b BIGINT),
563    ts TIMESTAMP TIME INDEX,
564)"#,
565            &GreptimeDbDialect {},
566            ParseOptions::default(),
567        );
568
569        assert!(result.is_err());
570        assert!(result.unwrap_err().to_string().contains("duplicated"));
571    }
572
573    #[test]
574    fn test_parse_json2_type_hint_rejects_parent_child_path() {
575        let result = ParserContext::create_with_dialect(
576            r#"
577CREATE TABLE traces (
578    log_json_data JSON2 (a STRING, a.b BIGINT),
579    ts TIMESTAMP TIME INDEX,
580)"#,
581            &GreptimeDbDialect {},
582            ParseOptions::default(),
583        );
584
585        assert!(result.is_err());
586        assert!(result.unwrap_err().to_string().contains("conflicts"));
587    }
588
589    #[test]
590    fn test_parse_json2_type_hint_rejects_nullability() {
591        for sql in [
592            r#"
593CREATE TABLE traces (
594    log_json_data JSON2 (a STRING NULL NULL),
595    ts TIMESTAMP TIME INDEX,
596)"#,
597            r#"
598CREATE TABLE traces (
599    log_json_data JSON2 (a STRING NOT NULL NOT NULL),
600    ts TIMESTAMP TIME INDEX,
601)"#,
602            r#"
603CREATE TABLE traces (
604    log_json_data JSON2 (a STRING NOT NULL NULL),
605    ts TIMESTAMP TIME INDEX,
606)"#,
607            r#"
608CREATE TABLE traces (
609    log_json_data JSON2 (a STRING NULL NOT NULL),
610    ts TIMESTAMP TIME INDEX,
611)"#,
612        ] {
613            let result = ParserContext::create_with_dialect(
614                sql,
615                &GreptimeDbDialect {},
616                ParseOptions::default(),
617            );
618
619            assert!(result.is_err());
620            assert!(
621                result
622                    .unwrap_err()
623                    .to_string()
624                    .contains("NULL/NOT NULL is not supported")
625            );
626        }
627    }
628}
sql/parsers/create_parser/json.rs

sql/parsers/create_parser/
json.rs
