1use std::collections::HashMap;
16use std::fmt::{Display, Formatter, Write};
17use std::ops::ControlFlow;
18
19use common_catalog::consts::FILE_ENGINE;
20use datatypes::json::{JSON2_DEFAULT_MAX_AUTO_EXPANDED_PATHS, JsonSettings};
21use datatypes::prelude::ConcreteDataType;
22use datatypes::schema::{CompressionOptions, FulltextOptions, SkippingIndexOptions};
23use itertools::Itertools;
24use serde::Serialize;
25use snafu::ResultExt;
26use sqlparser::ast::{ColumnOptionDef, DataType, Expr, visit_expressions_mut};
27use sqlparser_derive::{Visit, VisitMut};
28
29use crate::ast::{ColumnDef, Ident, ObjectName};
30use crate::error::{
31 InvalidFlowQuerySnafu, InvalidSqlSnafu, Result, SetCompressionOptionSnafu,
32 SetFulltextOptionSnafu, SetSkippingIndexOptionSnafu,
33};
34use crate::statements::query::Query as GtQuery;
35use crate::statements::statement::Statement;
36use crate::statements::tql::Tql;
37use crate::statements::{OptionMap, sql_data_type_to_concrete_data_type};
38
39const LINE_SEP: &str = ",\n";
40const COMMA_SEP: &str = ", ";
41const INDENT: usize = 2;
42pub const VECTOR_OPT_DIM: &str = "dim";
43
44macro_rules! format_indent {
45 ($fmt: expr, $arg: expr) => {
46 format!($fmt, format_args!("{: >1$}", "", INDENT), $arg)
47 };
48 ($arg: expr) => {
49 format_indent!("{}{}", $arg)
50 };
51}
52
53macro_rules! format_list_indent {
54 ($list: expr) => {
55 $list.iter().map(|e| format_indent!(e)).join(LINE_SEP)
56 };
57}
58
59macro_rules! format_list_comma {
60 ($list: expr) => {
61 $list.iter().map(|e| format!("{}", e)).join(COMMA_SEP)
62 };
63}
64
65#[cfg(feature = "enterprise")]
66pub mod trigger;
67
68fn format_table_constraint(constraints: &[TableConstraint]) -> String {
69 constraints.iter().map(|c| format_indent!(c)).join(LINE_SEP)
70}
71
72fn write_list<T: Display>(
73 f: &mut dyn Write,
74 values: &[T],
75 separator: &str,
76 prefix: &str,
77) -> std::fmt::Result {
78 for (index, value) in values.iter().enumerate() {
79 if index > 0 {
80 f.write_str(separator)?;
81 }
82 write!(f, "{prefix}{value}")?;
83 }
84 Ok(())
85}
86
87fn write_data_type(f: &mut dyn Write, data_type: &DataType) -> std::fmt::Result {
89 if let DataType::Custom(name, modifiers) = data_type {
90 write!(f, "{name}")?;
91 if !modifiers.is_empty() {
92 f.write_char('(')?;
93 write_list(f, modifiers, COMMA_SEP, "")?;
94 f.write_char(')')?;
95 }
96 Ok(())
97 } else {
98 write!(f, "{data_type}")
99 }
100}
101
102#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]
104pub enum TableConstraint {
105 PrimaryKey { columns: Vec<Ident> },
107 TimeIndex { column: Ident },
109}
110
111impl Display for TableConstraint {
112 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
113 match self {
114 TableConstraint::PrimaryKey { columns } => {
115 f.write_str("PRIMARY KEY (")?;
116 write_list(f, columns, COMMA_SEP, "")?;
117 f.write_str(")")
118 }
119 TableConstraint::TimeIndex { column } => {
120 write!(f, "TIME INDEX ({})", column)
121 }
122 }
123 }
124}
125
126#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]
127pub struct CreateTable {
128 pub if_not_exists: bool,
130 pub table_id: u32,
131 pub name: ObjectName,
133 pub columns: Vec<Column>,
134 pub engine: String,
135 pub constraints: Vec<TableConstraint>,
136 pub options: OptionMap,
138 pub partitions: Option<Partitions>,
139}
140
141#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]
143pub struct Column {
144 pub column_def: ColumnDef,
146 pub extensions: ColumnExtensions,
148}
149
150#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Default, Serialize)]
152pub struct ColumnExtensions {
153 pub vector_options: Option<OptionMap>,
155
156 pub fulltext_index_options: Option<OptionMap>,
158 pub skipping_index_options: Option<OptionMap>,
160 pub inverted_index_options: Option<OptionMap>,
164 pub compression_options: Option<OptionMap>,
166 pub json2_options: Option<Json2Options>,
168}
169
170#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Default, Serialize)]
172pub struct Json2Options {
173 pub(crate) max_auto_expanded_paths: Option<u32>,
175 pub(crate) type_hints: Vec<JsonTypeHint>,
177}
178
179impl Display for Json2Options {
180 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
181 f.write_str("(\n ")?;
182 if let Some(max) = self.max_auto_expanded_paths {
183 write!(f, "max_auto_expanded_paths = {max}")?;
184 }
185 for (index, hint) in self.type_hints.iter().enumerate() {
186 if index > 0 || self.max_auto_expanded_paths.is_some() {
187 f.write_str(",\n ")?;
188 }
189 for (index, segment) in hint.path.iter().enumerate() {
190 if index > 0 {
191 f.write_char('.')?;
192 }
193 f.write_char('"')?;
194 for part in segment.split_inclusive('"') {
195 f.write_str(part)?;
196 if part.ends_with('"') {
197 f.write_char('"')?;
198 }
199 }
200 f.write_char('"')?;
201 }
202 f.write_char(' ')?;
203 write_data_type(f, &hint.data_type)?;
204 if let Some(options) = &hint.skipping_index {
205 write!(
206 f,
207 " SKIPPING INDEX WITH (granularity = '{}', false_positive_rate = '{}', type = '{}')",
208 options.granularity,
209 options.false_positive_rate(),
210 options.index_type
211 )?;
212 }
213 if hint.inverted_index {
214 f.write_str(" INVERTED INDEX")?;
215 }
216 }
217 f.write_str("\n )")
218 }
219}
220
221impl Json2Options {
222 pub fn build_json_settings(&self) -> Result<JsonSettings> {
223 let type_hints = self
224 .type_hints
225 .iter()
226 .map(|hint| {
227 Ok(datatypes::json::JsonTypeHint {
228 path: hint.path.clone(),
229 data_type: sql_data_type_to_concrete_data_type(&hint.data_type)?,
230 inverted_index: hint.inverted_index,
231 skipping_index: hint.skipping_index.clone(),
232 })
233 })
234 .collect::<Result<Vec<_>>>()?;
235 let max_auto_expanded_paths = self
236 .max_auto_expanded_paths
237 .or(Some(JSON2_DEFAULT_MAX_AUTO_EXPANDED_PATHS));
238 JsonSettings::try_new(type_hints, max_auto_expanded_paths).map_err(Into::into)
239 }
240}
241
242#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]
243pub struct JsonTypeHint {
244 pub path: Vec<String>,
245 pub data_type: DataType,
246 pub inverted_index: bool,
247 pub skipping_index: Option<SkippingIndexOptions>,
249}
250
251impl Column {
252 pub fn name(&self) -> &Ident {
253 &self.column_def.name
254 }
255
256 pub fn data_type(&self) -> &DataType {
257 &self.column_def.data_type
258 }
259
260 pub fn mut_data_type(&mut self) -> &mut DataType {
261 &mut self.column_def.data_type
262 }
263
264 pub fn options(&self) -> &[ColumnOptionDef] {
265 &self.column_def.options
266 }
267
268 pub fn mut_options(&mut self) -> &mut Vec<ColumnOptionDef> {
269 &mut self.column_def.options
270 }
271}
272
273impl Display for Column {
274 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
275 self.write_sql(f, false)
276 }
277}
278
279impl Column {
280 fn write_sql(&self, f: &mut dyn Write, complete: bool) -> std::fmt::Result {
281 if !complete
282 && let Some(vector_options) = &self.extensions.vector_options
283 && let Some(dim) = vector_options.get(VECTOR_OPT_DIM)
284 {
285 write!(f, "{} VECTOR({})", self.column_def.name, dim)?;
286 } else {
287 write!(f, "{} ", self.column_def.name)?;
288 write_data_type(f, &self.column_def.data_type)?;
289 }
290 if let Some(options) = &self.extensions.json2_options {
291 write!(f, "{options}")?;
292 }
293 for option in &self.column_def.options {
294 write!(f, " {option}")?;
295 }
296
297 for (kind, options) in [
298 ("FULLTEXT", &self.extensions.fulltext_index_options),
299 ("SKIPPING", &self.extensions.skipping_index_options),
300 ("INVERTED", &self.extensions.inverted_index_options),
301 ] {
302 if let Some(options) = options {
303 write!(f, " {kind} INDEX")?;
304 if !options.is_empty() {
305 f.write_str(" WITH(")?;
306 options.write_sql(f, COMMA_SEP, "", complete)?;
307 f.write_str(")")?;
308 }
309 }
310 }
311
312 if let Some(compression_options) = &self.extensions.compression_options {
313 let options = compression_options.kv_pairs();
314 write!(f, " COMPRESSION WITH({})", format_list_comma!(options))?;
315 }
316
317 Ok(())
318 }
319}
320
321impl ColumnExtensions {
322 pub fn build_fulltext_options(&self) -> Result<Option<FulltextOptions>> {
323 let Some(options) = self.fulltext_index_options.as_ref() else {
324 return Ok(None);
325 };
326
327 let options: HashMap<String, String> = options.clone().into_map();
328 Ok(Some(options.try_into().context(SetFulltextOptionSnafu)?))
329 }
330
331 pub fn build_skipping_index_options(&self) -> Result<Option<SkippingIndexOptions>> {
332 let Some(options) = self.skipping_index_options.as_ref() else {
333 return Ok(None);
334 };
335
336 let options: HashMap<String, String> = options.clone().into_map();
337 Ok(Some(
338 options.try_into().context(SetSkippingIndexOptionSnafu)?,
339 ))
340 }
341
342 pub fn build_compression_options(&self) -> Result<Option<CompressionOptions>> {
343 let Some(options) = self.compression_options.as_ref() else {
344 return Ok(None);
345 };
346
347 let options: HashMap<String, String> = options.clone().into_map();
348 Ok(Some(options.try_into().context(SetCompressionOptionSnafu)?))
349 }
350
351 pub fn build_json_settings(&self) -> Result<Option<JsonSettings>> {
352 let Some(options) = &self.json2_options else {
353 return Ok(None);
354 };
355
356 options.build_json_settings().map(Some)
357 }
358
359 pub fn set_json_settings(&mut self, settings: JsonSettings) -> Result<()> {
360 let (type_hints, max_auto_expanded_paths) = settings.into_parts();
361 let type_hints = type_hints
362 .into_iter()
363 .map(|hint| {
364 let data_type = json_type_hint_sql_data_type(&hint.data_type)?;
365 Ok(JsonTypeHint {
366 path: hint.path,
367 data_type,
368 inverted_index: hint.inverted_index,
369 skipping_index: hint.skipping_index,
370 })
371 })
372 .collect::<Result<Vec<_>>>()?;
373 self.json2_options = (max_auto_expanded_paths.is_some() || !type_hints.is_empty())
374 .then_some(Json2Options {
375 max_auto_expanded_paths,
376 type_hints,
377 });
378 Ok(())
379 }
380}
381
382fn json_type_hint_sql_data_type(data_type: &ConcreteDataType) -> Result<DataType> {
383 let sql_type = match data_type {
384 ConcreteDataType::String(_) => DataType::String(None),
385 ConcreteDataType::Int64(_) => DataType::BigInt(None),
386 ConcreteDataType::UInt64(_) => DataType::BigIntUnsigned(None),
387 ConcreteDataType::Float64(_) => DataType::Double(sqlparser::ast::ExactNumberInfo::None),
388 ConcreteDataType::Boolean(_) => DataType::Boolean,
389 _ => {
390 return InvalidSqlSnafu {
391 msg: format!("unsupported JSON2 type hint data type: {data_type}"),
392 }
393 .fail();
394 }
395 };
396 Ok(sql_type)
397}
398
399#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]
406pub struct Partitions {
407 pub column_list: Vec<Ident>,
408 pub exprs: Vec<Expr>,
409}
410
411impl Partitions {
412 pub fn set_quote(&mut self, quote_style: char) {
414 self.column_list
415 .iter_mut()
416 .for_each(|c| c.quote_style = Some(quote_style));
417 let _ = visit_expressions_mut(&mut self.exprs, |expr| {
418 if let Expr::Identifier(ident) = expr {
419 ident.quote_style = Some(quote_style);
420 }
421 ControlFlow::<()>::Continue(())
422 });
423 }
424}
425
426impl Display for Partitions {
427 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
428 if !self.column_list.is_empty() {
429 f.write_str("PARTITION ON COLUMNS (")?;
430 write_list(f, &self.column_list, COMMA_SEP, "")?;
431 f.write_str(") (\n")?;
432 write_list(f, &self.exprs, LINE_SEP, " ")?;
433 f.write_str("\n)")?;
434 }
435 Ok(())
436 }
437}
438
439impl Display for CreateTable {
440 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
441 self.write_sql(f, false)
442 }
443}
444
445impl CreateTable {
446 pub fn consume_sql_size_budget(&self, remaining: &mut usize) -> bool {
451 struct Budget<'a>(&'a mut usize);
452 impl Write for Budget<'_> {
453 fn write_str(&mut self, value: &str) -> std::fmt::Result {
454 *self.0 = self.0.checked_sub(value.len()).ok_or(std::fmt::Error)?;
455 Ok(())
456 }
457 }
458 if self
459 .partitions
460 .as_ref()
461 .is_some_and(|p| p.column_list.is_empty() && !p.exprs.is_empty())
462 {
463 return false;
464 }
465 for column in &self.columns {
466 if let Some(options) = &column.extensions.vector_options {
467 let DataType::Custom(name, tokens) = &column.column_def.data_type else {
468 return false;
469 };
470 if options.len() != 1
471 || name.0.len() != 1
472 || tokens.len() != 1
473 || !name.0[0]
474 .as_ident()
475 .is_some_and(|name| name.value.eq_ignore_ascii_case("VECTOR"))
476 || !tokens[0].parse::<u32>().ok().is_some_and(|dim| {
477 options.get(VECTOR_OPT_DIM) == Some(dim.to_string().as_str())
478 })
479 {
480 return false;
481 }
482 }
483 }
484 self.write_sql(&mut Budget(remaining), true).is_ok()
485 }
486
487 fn write_sql(&self, f: &mut dyn Write, complete: bool) -> std::fmt::Result {
488 write!(f, "CREATE ")?;
489 if self.engine == FILE_ENGINE {
490 write!(f, "EXTERNAL ")?;
491 }
492 write!(f, "TABLE ")?;
493 if self.if_not_exists {
494 write!(f, "IF NOT EXISTS ")?;
495 }
496 writeln!(f, "{} (", self.name)?;
497 for (index, column) in self.columns.iter().enumerate() {
498 if index > 0 {
499 f.write_str(LINE_SEP)?;
500 }
501 f.write_str(" ")?;
502 column.write_sql(f, complete)?;
503 }
504 f.write_str(",\n")?;
505 write_list(f, &self.constraints, LINE_SEP, " ")?;
506 f.write_char('\n')?;
507 writeln!(f, ")")?;
508 if let Some(partitions) = &self.partitions {
509 writeln!(f, "{partitions}")?;
510 }
511 writeln!(f, "ENGINE={}", self.engine)?;
512 if !self.options.is_empty() {
513 f.write_str("WITH(\n")?;
514 self.options.write_sql(f, LINE_SEP, " ", complete)?;
515 f.write_str("\n)")?;
516 }
517 Ok(())
518 }
519}
520
521#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]
522pub struct CreateDatabase {
523 pub name: ObjectName,
524 pub if_not_exists: bool,
526 pub options: OptionMap,
527}
528
529impl CreateDatabase {
530 pub fn new(name: ObjectName, if_not_exists: bool, options: OptionMap) -> Self {
532 Self {
533 name,
534 if_not_exists,
535 options,
536 }
537 }
538}
539
540impl Display for CreateDatabase {
541 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
542 write!(f, "CREATE DATABASE ")?;
543 if self.if_not_exists {
544 write!(f, "IF NOT EXISTS ")?;
545 }
546 write!(f, "{}", self.name)?;
547 if !self.options.is_empty() {
548 let options = self.options.kv_pairs();
549 write!(f, "\nWITH(\n{}\n)", format_list_indent!(options))?;
550 }
551 Ok(())
552 }
553}
554
555#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]
556pub struct CreateExternalTable {
557 pub name: ObjectName,
559 pub columns: Vec<Column>,
560 pub constraints: Vec<TableConstraint>,
561 pub options: OptionMap,
563 pub if_not_exists: bool,
564 pub engine: String,
565}
566
567impl Display for CreateExternalTable {
568 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
569 write!(f, "CREATE EXTERNAL TABLE ")?;
570 if self.if_not_exists {
571 write!(f, "IF NOT EXISTS ")?;
572 }
573 writeln!(f, "{} (", self.name)?;
574 writeln!(f, "{},", format_list_indent!(self.columns))?;
575 writeln!(f, "{}", format_table_constraint(&self.constraints))?;
576 writeln!(f, ")")?;
577 writeln!(f, "ENGINE={}", self.engine)?;
578 if !self.options.is_empty() {
579 let options = self.options.kv_pairs();
580 write!(f, "WITH(\n{}\n)", format_list_indent!(options))?;
581 }
582 Ok(())
583 }
584}
585
586#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]
587pub struct CreateTableLike {
588 pub table_name: ObjectName,
590 pub source_name: ObjectName,
592}
593
594impl Display for CreateTableLike {
595 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
596 let table_name = &self.table_name;
597 let source_name = &self.source_name;
598 write!(f, r#"CREATE TABLE {table_name} LIKE {source_name}"#)
599 }
600}
601
602#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]
603pub struct CreateFlow {
604 pub flow_name: ObjectName,
606 pub sink_table_name: ObjectName,
608 pub or_replace: bool,
610 pub if_not_exists: bool,
612 pub expire_after: Option<i64>,
615 pub eval_interval: Option<i64>,
619 pub eval_offset: Option<i64>,
625 pub comment: Option<String>,
627 pub flow_options: OptionMap,
629 pub query: Box<SqlOrTql>,
631}
632
633#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]
635pub enum SqlOrTql {
636 Sql(GtQuery, String),
637 Tql(Tql, String),
638}
639
640impl std::fmt::Display for SqlOrTql {
641 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
642 match self {
643 Self::Sql(_, s) => write!(f, "{}", s),
644 Self::Tql(_, s) => write!(f, "{}", s),
645 }
646 }
647}
648
649impl SqlOrTql {
650 pub fn try_from_statement(
651 value: Statement,
652 original_query: &str,
653 ) -> std::result::Result<Self, crate::error::Error> {
654 match value {
655 Statement::Query(query) => Ok(Self::Sql(*query, original_query.to_string())),
656 Statement::Tql(tql) => Ok(Self::Tql(tql, original_query.to_string())),
657 _ => InvalidFlowQuerySnafu {
658 reason: format!("Expect either sql query or promql query, found {:?}", value),
659 }
660 .fail(),
661 }
662 }
663}
664
665impl Display for CreateFlow {
666 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
667 write!(f, "CREATE ")?;
668 if self.or_replace {
669 write!(f, "OR REPLACE ")?;
670 }
671 write!(f, "FLOW ")?;
672 if self.if_not_exists {
673 write!(f, "IF NOT EXISTS ")?;
674 }
675 writeln!(f, "{}", self.flow_name)?;
676 writeln!(f, "SINK TO {}", self.sink_table_name)?;
677 if let Some(expire_after) = &self.expire_after {
678 writeln!(f, "EXPIRE AFTER '{} s'", expire_after)?;
679 }
680 if let Some(eval_interval) = &self.eval_interval {
681 writeln!(f, "EVAL INTERVAL '{} s'", eval_interval)?;
682 }
683 if let Some(eval_offset) = &self.eval_offset
686 && *eval_offset != 0
687 {
688 writeln!(f, "EVAL OFFSET '{} s'", eval_offset)?;
689 }
690 if let Some(comment) = &self.comment {
691 writeln!(f, "COMMENT '{}'", comment)?;
692 }
693 if !self.flow_options.is_empty() {
694 let options = self.flow_options.kv_pairs();
695 writeln!(f, "WITH ({})", format_list_comma!(options))?;
696 }
697 write!(f, "AS {}", self.query)
698 }
699}
700
701#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]
703pub struct CreateView {
704 pub name: ObjectName,
706 pub columns: Vec<Ident>,
708 pub query: Box<Statement>,
711 pub or_replace: bool,
713 pub if_not_exists: bool,
715}
716
717impl Display for CreateView {
718 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
719 write!(f, "CREATE ")?;
720 if self.or_replace {
721 write!(f, "OR REPLACE ")?;
722 }
723 write!(f, "VIEW ")?;
724 if self.if_not_exists {
725 write!(f, "IF NOT EXISTS ")?;
726 }
727 write!(f, "{} ", self.name)?;
728 if !self.columns.is_empty() {
729 write!(f, "({}) ", format_list_comma!(self.columns))?;
730 }
731 write!(f, "AS {}", self.query)
732 }
733}
734
735#[cfg(test)]
736mod tests {
737 use std::assert_matches;
738
739 use datatypes::json::{JsonSettings, JsonTypeHint as DatatypeJsonTypeHint};
740 use datatypes::prelude::ConcreteDataType;
741
742 use super::*;
743 use crate::dialect::GreptimeDbDialect;
744 use crate::error::Error;
745 use crate::parser::{ParseOptions, ParserContext};
746 use crate::statements::statement::Statement;
747
748 #[test]
749 fn test_rewritten_create_sql_budget() {
750 let parse = |sql| {
751 let mut statements = ParserContext::create_with_dialect(
752 sql,
753 &GreptimeDbDialect {},
754 ParseOptions::default(),
755 )
756 .unwrap();
757 let Statement::CreateTable(create) = statements.remove(0) else {
758 panic!("CREATE expected")
759 };
760 create
761 };
762 let create = parse(
763 "CREATE TABLE demo (ts TIMESTAMP TIME INDEX, host STRING) ENGINE=metric WITH (on_physical_table='phy')",
764 );
765 let size = create.to_string().len();
766 let mut remaining = size;
767 assert!(create.consume_sql_size_budget(&mut remaining));
768 assert_eq!(remaining, 0);
769 assert!(!create.consume_sql_size_budget(&mut (size - 1)));
770
771 for location in ["table", "fulltext", "skipping", "inverted"] {
772 for value_type in ["secret", "array", "struct"] {
773 for length in [10, 8192] {
774 let mut create = create.clone();
775 let extensions = &mut create.columns[1].extensions;
776 let map = match location {
777 "table" => &mut create.options,
778 "fulltext" => extensions.fulltext_index_options.get_or_insert_default(),
779 "skipping" => extensions.skipping_index_options.get_or_insert_default(),
780 _ => extensions.inverted_index_options.get_or_insert_default(),
781 };
782 let value = "private-value".repeat(length);
783 match value_type {
784 "secret" => map.insert("secret_access_key".into(), value.clone()),
785 "array" => map.insert_options("payload", vec![value.as_str()].into()),
786 _ => map.insert_options(
787 "payload",
788 crate::util::OptionValue::try_new(Expr::Struct {
789 values: vec![Expr::Value(
790 sqlparser::ast::Value::SingleQuotedString(value.clone()).into(),
791 )],
792 fields: vec![],
793 })
794 .unwrap(),
795 ),
796 }
797 assert!(!create.to_string().contains(&value));
798 assert_eq!(
799 create.consume_sql_size_budget(&mut 4096),
800 length == 10,
801 "{location}/{value_type}"
802 );
803 }
804 }
805 }
806 let mut vector = parse("CREATE TABLE demo (ts TIMESTAMP TIME INDEX, v VECTOR(3))");
807 vector.columns[1].column_def.options.push(ColumnOptionDef {
808 name: None,
809 option: sqlparser::ast::ColumnOption::Comment("private-value".repeat(8192)),
810 });
811 assert!(vector.to_string().contains("private-value"));
812 assert!(!vector.consume_sql_size_budget(&mut 4096));
813 vector.columns[1].column_def.options.clear();
814 assert!(vector.consume_sql_size_budget(&mut 4096));
815 if let DataType::Custom(_, tokens) = &mut vector.columns[1].column_def.data_type {
816 tokens[0] = format!("{}3", "0".repeat(8192));
817 }
818 assert!(!vector.consume_sql_size_budget(&mut 4096));
819 vector.columns[1]
820 .extensions
821 .vector_options
822 .as_mut()
823 .unwrap()
824 .insert("extra".into(), "x".into());
825 assert!(!vector.consume_sql_size_budget(&mut 4096));
826
827 let mut create = create;
828 create.columns[1].extensions.json2_options = Some(Json2Options {
829 max_auto_expanded_paths: Some(1),
830 type_hints: vec![JsonTypeHint {
831 path: vec!["x".repeat(8192)],
832 data_type: DataType::Text,
833 inverted_index: true,
834 skipping_index: None,
835 }],
836 });
837 assert!(!create.consume_sql_size_budget(&mut 4096));
838 create.columns[1].extensions.json2_options = None;
839 create.partitions = Some(Partitions {
840 column_list: vec![],
841 exprs: vec![Expr::Identifier(Ident::new("hidden"))],
842 });
843 assert!(!create.consume_sql_size_budget(&mut 4096));
844 }
845
846 #[test]
847 fn test_json2_streamed_quote_escaping() {
848 let path = "a\\\"\"b";
849 let options = Json2Options {
850 max_auto_expanded_paths: None,
851 type_hints: vec![JsonTypeHint {
852 path: vec![path.into()],
853 data_type: DataType::Text,
854 inverted_index: false,
855 skipping_index: None,
856 }],
857 };
858 assert_eq!(
859 options.to_string(),
860 format!("(\n \"{}\" TEXT\n )", path.replace('"', "\"\""))
861 );
862 }
863
864 #[test]
865 fn test_display_create_table() {
866 let sql = r"create table if not exists demo(
867 host string,
868 ts timestamp,
869 cpu double default 0,
870 memory double,
871 TIME INDEX (ts),
872 PRIMARY KEY(host)
873 )
874 PARTITION ON COLUMNS (host) (
875 host = 'a',
876 host > 'a',
877 )
878 engine=mito
879 with(ttl='7d', storage='File');
880 ";
881 let result =
882 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
883 .unwrap();
884 assert_eq!(1, result.len());
885
886 match &result[0] {
887 Statement::CreateTable(c) => {
888 let new_sql = format!("\n{}", c);
889 assert_eq!(
890 r#"
891CREATE TABLE IF NOT EXISTS demo (
892 host STRING,
893 ts TIMESTAMP,
894 cpu DOUBLE DEFAULT 0,
895 memory DOUBLE,
896 TIME INDEX (ts),
897 PRIMARY KEY (host)
898)
899PARTITION ON COLUMNS (host) (
900 host = 'a',
901 host > 'a'
902)
903ENGINE=mito
904WITH(
905 storage = 'File',
906 ttl = '7d'
907)"#,
908 &new_sql
909 );
910
911 let new_result = ParserContext::create_with_dialect(
912 &new_sql,
913 &GreptimeDbDialect {},
914 ParseOptions::default(),
915 )
916 .unwrap();
917 assert_eq!(result, new_result);
918 }
919 _ => unreachable!(),
920 }
921 }
922
923 #[test]
924 fn test_display_empty_partition_column() {
925 let sql = r"create table if not exists demo(
926 host string,
927 ts timestamp,
928 cpu double default 0,
929 memory double,
930 TIME INDEX (ts),
931 PRIMARY KEY(ts, host)
932 );
933 ";
934 let result =
935 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
936 .unwrap();
937 assert_eq!(1, result.len());
938
939 match &result[0] {
940 Statement::CreateTable(c) => {
941 let new_sql = format!("\n{}", c);
942 assert_eq!(
943 r#"
944CREATE TABLE IF NOT EXISTS demo (
945 host STRING,
946 ts TIMESTAMP,
947 cpu DOUBLE DEFAULT 0,
948 memory DOUBLE,
949 TIME INDEX (ts),
950 PRIMARY KEY (ts, host)
951)
952ENGINE=mito
953"#,
954 &new_sql
955 );
956
957 let new_result = ParserContext::create_with_dialect(
958 &new_sql,
959 &GreptimeDbDialect {},
960 ParseOptions::default(),
961 )
962 .unwrap();
963 assert_eq!(result, new_result);
964 }
965 _ => unreachable!(),
966 }
967 }
968
969 #[test]
970 fn test_validate_table_options() {
971 let sql = r"create table if not exists demo(
972 host string,
973 ts timestamp,
974 cpu double default 0,
975 memory double,
976 TIME INDEX (ts),
977 PRIMARY KEY(host)
978 )
979 PARTITION ON COLUMNS (host) ()
980 engine=mito
981 with(ttl='7d', 'compaction.type'='world');
982";
983 let result =
984 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
985 .unwrap();
986 match &result[0] {
987 Statement::CreateTable(c) => {
988 assert_eq!(2, c.options.len());
989 }
990 _ => unreachable!(),
991 }
992
993 let sql = r"create table if not exists demo(
994 host string,
995 ts timestamp,
996 cpu double default 0,
997 memory double,
998 TIME INDEX (ts),
999 PRIMARY KEY(host)
1000 )
1001 PARTITION ON COLUMNS (host) ()
1002 engine=mito
1003 with(ttl='7d', hello='world');
1004";
1005 let result =
1006 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default());
1007 assert_matches!(result, Err(Error::InvalidTableOption { .. }));
1008
1009 let semantic = |with: &str| {
1011 let sql =
1012 format!("create table demo(host string, ts timestamp time index) with({with});");
1013 ParserContext::create_with_dialect(&sql, &GreptimeDbDialect {}, ParseOptions::default())
1014 };
1015 assert!(semantic("'greptime.semantic.signal_type'='metric'").is_ok());
1016 assert_matches!(
1018 semantic("'greptime.semantic.signal_type'='spans'"),
1019 Err(Error::InvalidTableOption { .. })
1020 );
1021 assert_matches!(
1023 semantic("'greptime.semantic.bogus'='x'"),
1024 Err(Error::InvalidTableOption { .. })
1025 );
1026 }
1027
1028 #[test]
1029 fn test_display_json2_type_hints_quotes_path_segments() {
1030 let sql = r#"CREATE TABLE traces (
1031 log_json_data JSON2 (
1032 "service.name" STRING,
1033 "a.b"."c" BIGINT,
1034 a."b.c" STRING
1035 ),
1036 ts TIMESTAMP TIME INDEX
1037 )"#;
1038 let result =
1039 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1040 .unwrap();
1041
1042 match &result[0] {
1043 Statement::CreateTable(c) => {
1044 let new_sql = format!("\n{}", c);
1045 assert_eq!(
1046 r#"
1047CREATE TABLE traces (
1048 log_json_data JSON2(
1049 "service.name" STRING,
1050 "a.b"."c" BIGINT,
1051 "a"."b.c" STRING
1052 ),
1053 ts TIMESTAMP NOT NULL,
1054 TIME INDEX (ts)
1055)
1056ENGINE=mito
1057"#,
1058 &new_sql
1059 );
1060
1061 let new_result = ParserContext::create_with_dialect(
1062 &new_sql,
1063 &GreptimeDbDialect {},
1064 ParseOptions::default(),
1065 )
1066 .unwrap();
1067 assert_eq!(result, new_result);
1068 }
1069 _ => unreachable!(),
1070 }
1071 }
1072
1073 #[test]
1074 fn test_parse_json2_max_auto_expanded_paths_option() -> Result<()> {
1075 let sql = r#"CREATE TABLE traces (
1076 log_json_data JSON2 (
1077 status_code BIGINT,
1078 max_auto_expanded_paths = 1
1079 ),
1080 ts TIMESTAMP TIME INDEX
1081 )"#;
1082 let result = ParserContext::create_with_dialect(
1083 sql,
1084 &GreptimeDbDialect {},
1085 ParseOptions::default(),
1086 )?;
1087 let Statement::CreateTable(create_table) = &result[0] else {
1088 unreachable!()
1089 };
1090 let settings = create_table.columns[0]
1091 .extensions
1092 .build_json_settings()?
1093 .unwrap();
1094 assert_eq!(settings.max_auto_expanded_paths(), Some(1));
1095 Ok(())
1096 }
1097
1098 #[test]
1099 fn test_display_json2_type_hints_quotes_numeric_segments() {
1100 let sql = r#"CREATE TABLE traces (
1101 log_json_data JSON2 (
1102 "1abc" STRING,
1103 a."2b" BIGINT
1104 ),
1105 ts TIMESTAMP TIME INDEX
1106 )"#;
1107 let result =
1108 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1109 .unwrap();
1110
1111 match &result[0] {
1112 Statement::CreateTable(c) => {
1113 let new_sql = format!("\n{}", c);
1114 assert_eq!(
1115 r#"
1116CREATE TABLE traces (
1117 log_json_data JSON2(
1118 "1abc" STRING,
1119 "a"."2b" BIGINT
1120 ),
1121 ts TIMESTAMP NOT NULL,
1122 TIME INDEX (ts)
1123)
1124ENGINE=mito
1125"#,
1126 &new_sql
1127 );
1128
1129 let new_result = ParserContext::create_with_dialect(
1130 &new_sql,
1131 &GreptimeDbDialect {},
1132 ParseOptions::default(),
1133 )
1134 .unwrap();
1135 assert_eq!(result, new_result);
1136 }
1137 _ => unreachable!(),
1138 }
1139 }
1140
1141 #[test]
1142 fn test_json2_type_hint_rejects_default() {
1143 let sql = r#"CREATE TABLE traces (
1144 log_json_data JSON2 (
1145 status_code BIGINT DEFAULT -5,
1146 duration DOUBLE DEFAULT +1.5,
1147 error BOOLEAN DEFAULT false,
1148 message STRING DEFAULT 'unknown'
1149 ),
1150 ts TIMESTAMP TIME INDEX
1151 )"#;
1152 let err =
1153 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1154 .unwrap_err();
1155 assert!(err.to_string().contains("DEFAULT is not supported"));
1156 }
1157
1158 #[test]
1159 fn test_json2_type_hint_rejects_not_null() {
1160 let sql = r#"CREATE TABLE traces (
1161 log_json_data JSON2 (
1162 status_code BIGINT NOT NULL DEFAULT NULL
1163 ),
1164 ts TIMESTAMP TIME INDEX
1165 )"#;
1166 let err =
1167 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1168 .unwrap_err();
1169 assert!(err.to_string().contains("NULL/NOT NULL is not supported"));
1170 }
1171
1172 #[test]
1173 fn test_set_json_settings_preserves_type_hint_sql_types() -> Result<()> {
1174 let mut extensions = super::ColumnExtensions::default();
1175 let settings = JsonSettings::try_new(
1176 vec![
1177 DatatypeJsonTypeHint {
1178 path: vec!["i".to_string()],
1179 data_type: ConcreteDataType::int64_datatype(),
1180 inverted_index: false,
1181 skipping_index: None,
1182 },
1183 DatatypeJsonTypeHint {
1184 path: vec!["f".to_string()],
1185 data_type: ConcreteDataType::float64_datatype(),
1186 inverted_index: false,
1187 skipping_index: None,
1188 },
1189 DatatypeJsonTypeHint {
1190 path: vec!["u".to_string()],
1191 data_type: ConcreteDataType::uint64_datatype(),
1192 inverted_index: false,
1193 skipping_index: None,
1194 },
1195 DatatypeJsonTypeHint {
1196 path: vec!["s".to_string()],
1197 data_type: ConcreteDataType::string_datatype(),
1198 inverted_index: false,
1199 skipping_index: None,
1200 },
1201 DatatypeJsonTypeHint {
1202 path: vec!["b".to_string()],
1203 data_type: ConcreteDataType::boolean_datatype(),
1204 inverted_index: false,
1205 skipping_index: None,
1206 },
1207 ],
1208 None,
1209 )?;
1210 extensions.set_json_settings(settings)?;
1211
1212 assert_eq!(
1213 extensions
1214 .json2_options
1215 .unwrap()
1216 .type_hints
1217 .iter()
1218 .map(|hint| hint.data_type.to_string())
1219 .collect::<Vec<_>>(),
1220 vec!["BIGINT", "DOUBLE", "BIGINT UNSIGNED", "STRING", "BOOLEAN"]
1221 );
1222 Ok(())
1223 }
1224
1225 #[test]
1226 fn test_set_json_settings_rejects_unsupported_type_hint_type() -> Result<()> {
1227 let err = JsonSettings::try_new(
1228 vec![DatatypeJsonTypeHint {
1229 path: vec!["u".to_string()],
1230 data_type: ConcreteDataType::date_datatype(),
1231 inverted_index: false,
1232 skipping_index: None,
1233 }],
1234 None,
1235 )
1236 .unwrap_err();
1237
1238 assert!(
1239 err.to_string()
1240 .contains("unsupported JSON2 type hint data type")
1241 );
1242 Ok(())
1243 }
1244
1245 #[test]
1246 fn test_set_empty_json_settings_omits_json2_options() -> Result<()> {
1247 let mut extensions = ColumnExtensions::default();
1248 extensions.set_json_settings(JsonSettings::default())?;
1249 assert!(extensions.json2_options.is_none());
1250 Ok(())
1251 }
1252
1253 #[test]
1254 fn test_display_create_database() {
1255 let sql = r"create database test;";
1256 let stmts =
1257 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1258 .unwrap();
1259 assert_eq!(1, stmts.len());
1260 assert_matches!(&stmts[0], Statement::CreateDatabase { .. });
1261
1262 match &stmts[0] {
1263 Statement::CreateDatabase(set) => {
1264 let new_sql = format!("\n{}", set);
1265 assert_eq!(
1266 r#"
1267CREATE DATABASE test"#,
1268 &new_sql
1269 );
1270 }
1271 _ => {
1272 unreachable!();
1273 }
1274 }
1275
1276 let sql = r"create database if not exists test;";
1277 let stmts =
1278 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1279 .unwrap();
1280 assert_eq!(1, stmts.len());
1281 assert_matches!(&stmts[0], Statement::CreateDatabase { .. });
1282
1283 match &stmts[0] {
1284 Statement::CreateDatabase(set) => {
1285 let new_sql = format!("\n{}", set);
1286 assert_eq!(
1287 r#"
1288CREATE DATABASE IF NOT EXISTS test"#,
1289 &new_sql
1290 );
1291 }
1292 _ => {
1293 unreachable!();
1294 }
1295 }
1296
1297 let sql = r#"CREATE DATABASE IF NOT EXISTS test WITH (ttl='1h');"#;
1298 let stmts =
1299 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1300 .unwrap();
1301 assert_eq!(1, stmts.len());
1302 assert_matches!(&stmts[0], Statement::CreateDatabase { .. });
1303
1304 match &stmts[0] {
1305 Statement::CreateDatabase(set) => {
1306 let new_sql = format!("\n{}", set);
1307 assert_eq!(
1308 r#"
1309CREATE DATABASE IF NOT EXISTS test
1310WITH(
1311 ttl = '1h'
1312)"#,
1313 &new_sql
1314 );
1315 }
1316 _ => {
1317 unreachable!();
1318 }
1319 }
1320 }
1321
1322 #[test]
1323 fn test_display_create_table_like() {
1324 let sql = r"create table t2 like t1;";
1325 let stmts =
1326 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1327 .unwrap();
1328 assert_eq!(1, stmts.len());
1329 assert_matches!(&stmts[0], Statement::CreateTableLike { .. });
1330
1331 match &stmts[0] {
1332 Statement::CreateTableLike(create) => {
1333 let new_sql = format!("\n{}", create);
1334 assert_eq!(
1335 r#"
1336CREATE TABLE t2 LIKE t1"#,
1337 &new_sql
1338 );
1339 }
1340 _ => {
1341 unreachable!();
1342 }
1343 }
1344 }
1345
1346 #[test]
1347 fn test_display_create_external_table() {
1348 let sql = r#"CREATE EXTERNAL TABLE city (
1349 host string,
1350 ts timestamp,
1351 cpu float64 default 0,
1352 memory float64,
1353 TIME INDEX (ts),
1354 PRIMARY KEY(host)
1355) WITH (location='/var/data/city.csv', format='csv');"#;
1356 let stmts =
1357 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1358 .unwrap();
1359 assert_eq!(1, stmts.len());
1360 assert_matches!(&stmts[0], Statement::CreateExternalTable { .. });
1361
1362 match &stmts[0] {
1363 Statement::CreateExternalTable(create) => {
1364 let new_sql = format!("\n{}", create);
1365 assert_eq!(
1366 r#"
1367CREATE EXTERNAL TABLE city (
1368 host STRING,
1369 ts TIMESTAMP,
1370 cpu DOUBLE DEFAULT 0,
1371 memory DOUBLE,
1372 TIME INDEX (ts),
1373 PRIMARY KEY (host)
1374)
1375ENGINE=file
1376WITH(
1377 format = 'csv',
1378 location = '/var/data/city.csv'
1379)"#,
1380 &new_sql
1381 );
1382 }
1383 _ => {
1384 unreachable!();
1385 }
1386 }
1387 }
1388
1389 #[test]
1390 fn test_display_create_flow() {
1391 let sql = r"CREATE FLOW filter_numbers
1392 SINK TO out_num_cnt
1393 AS SELECT number FROM numbers_input where number > 10;";
1394 let result =
1395 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1396 .unwrap();
1397 assert_eq!(1, result.len());
1398
1399 match &result[0] {
1400 Statement::CreateFlow(c) => {
1401 let new_sql = format!("\n{}", c);
1402 assert_eq!(
1403 r#"
1404CREATE FLOW filter_numbers
1405SINK TO out_num_cnt
1406AS SELECT number FROM numbers_input where number > 10"#,
1407 &new_sql
1408 );
1409
1410 let new_result = ParserContext::create_with_dialect(
1411 &new_sql,
1412 &GreptimeDbDialect {},
1413 ParseOptions::default(),
1414 )
1415 .unwrap();
1416 assert_eq!(result, new_result);
1417 }
1418 _ => unreachable!(),
1419 }
1420 }
1421}