1#[cfg(feature = "enterprise")]
16pub mod trigger;
17
18use std::collections::{HashMap, HashSet};
19
20use api::helper::ColumnDataTypeWrapper;
21use api::v1::alter_database_expr::Kind as AlterDatabaseKind;
22use api::v1::alter_table_expr::Kind as AlterTableKind;
23use api::v1::column_def::{options_from_column_schema, try_as_column_schema};
24use api::v1::{
25 AddColumn, AddColumns, AlterDatabaseExpr, AlterTableExpr, Analyzer, ColumnDataType,
26 ColumnDataTypeExtension, CreateFlowExpr, CreateTableExpr, CreateViewExpr, DropColumn,
27 DropColumns, DropDefaults, ExpireAfter, FulltextBackend as PbFulltextBackend, ModifyColumnType,
28 ModifyColumnTypes, RenameTable, SemanticType, SetDatabaseOptions, SetDefaults, SetFulltext,
29 SetIndex, SetIndexes, SetInverted, SetSkipping, SetTableOptions,
30 SkippingIndexType as PbSkippingIndexType, TableName, UnsetDatabaseOptions, UnsetFulltext,
31 UnsetIndex, UnsetIndexes, UnsetInverted, UnsetSkipping, UnsetTableOptions, set_index,
32 unset_index,
33};
34use common_datasource::object_store::LocalFileAccess;
35use common_error::ext::BoxedError;
36use common_grpc_expr::util::ColumnExpr;
37use common_time::Timezone;
38use datafusion::sql::planner::object_name_to_table_reference;
39use datatypes::schema::{
40 COLUMN_FULLTEXT_OPT_KEY_ANALYZER, COLUMN_FULLTEXT_OPT_KEY_BACKEND,
41 COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE, COLUMN_FULLTEXT_OPT_KEY_FALSE_POSITIVE_RATE,
42 COLUMN_FULLTEXT_OPT_KEY_GRANULARITY, COLUMN_SKIPPING_INDEX_OPT_KEY_FALSE_POSITIVE_RATE,
43 COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY, COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE, COMMENT_KEY,
44 ColumnDefaultConstraint, ColumnSchema, FulltextAnalyzer, FulltextBackend, Schema,
45 SkippingIndexType,
46};
47use file_engine::FileOptions;
48use query::sql::{
49 check_file_to_table_schema_compatibility, file_column_schemas_to_table,
50 infer_file_table_schema, prepare_file_table_files,
51};
52use session::context::QueryContextRef;
53use session::table_name::table_idents_to_full_name;
54use snafu::{OptionExt, ResultExt, ensure};
55use sql::ast::{
56 ColumnDef, ColumnOption, ColumnOptionDef, Expr, Ident, ObjectName, ObjectNamePartExt,
57};
58use sql::dialect::GreptimeDbDialect;
59use sql::parser::ParserContext;
60use sql::statements::alter::{
61 AlterDatabase, AlterDatabaseOperation, AlterTable, AlterTableOperation,
62};
63use sql::statements::create::{
64 Column as SqlColumn, ColumnExtensions, CreateExternalTable, CreateFlow, CreateTable,
65 CreateView, TableConstraint,
66};
67use sql::statements::{
68 OptionMap, column_to_schema, concrete_data_type_to_sql_data_type,
69 sql_column_def_to_grpc_column_def, sql_data_type_to_concrete_data_type, value_to_sql_value,
70};
71use sql::util::extract_tables_from_query;
72use store_api::mito_engine_options::{COMPACTION_OVERRIDE, COMPACTION_TYPE};
73use table::requests::{FILE_TABLE_META_KEY, TableOptions};
74use table::table_reference::TableReference;
75#[cfg(feature = "enterprise")]
76pub use trigger::to_create_trigger_task_expr;
77
78use crate::error::{
79 BuildCreateExprOnInsertionSnafu, ColumnDataTypeSnafu, ConvertColumnDefaultConstraintSnafu,
80 ConvertIdentifierSnafu, EncodeJsonSnafu, ExternalSnafu, FindNewColumnsOnInsertionSnafu,
81 IllegalPrimaryKeysDefSnafu, InferFileTableSchemaSnafu, InvalidColumnDefSnafu,
82 InvalidFlowNameSnafu, InvalidSqlSnafu, NotSupportedSnafu, ParseSqlSnafu, ParseSqlValueSnafu,
83 PrepareFileTableSnafu, Result, SchemaIncompatibleSnafu, UnrecognizedTableOptionSnafu,
84};
85
86pub fn create_table_expr_by_column_schemas(
87 table_name: &TableReference<'_>,
88 column_schemas: &[api::v1::ColumnSchema],
89 engine: &str,
90 desc: Option<&str>,
91) -> Result<CreateTableExpr> {
92 let column_exprs = ColumnExpr::from_column_schemas(column_schemas);
93 let expr = common_grpc_expr::util::build_create_table_expr(
94 None,
95 table_name,
96 column_exprs,
97 engine,
98 desc.unwrap_or("Created on insertion"),
99 )
100 .context(BuildCreateExprOnInsertionSnafu)?;
101
102 validate_create_expr(&expr)?;
103 Ok(expr)
104}
105
106pub fn extract_add_columns_expr(
107 schema: &Schema,
108 column_exprs: Vec<ColumnExpr>,
109) -> Result<Option<AddColumns>> {
110 let add_columns = common_grpc_expr::util::extract_new_columns(schema, column_exprs)
111 .context(FindNewColumnsOnInsertionSnafu)?;
112 if let Some(add_columns) = &add_columns {
113 validate_add_columns_expr(add_columns)?;
114 }
115 Ok(add_columns)
116}
117
118pub(crate) async fn create_external_expr(
142 create: CreateExternalTable,
143 query_ctx: &QueryContextRef,
144 local_file_access: &LocalFileAccess,
145) -> Result<CreateTableExpr> {
146 let (catalog_name, schema_name, table_name) =
147 table_idents_to_full_name(&create.name, query_ctx)
148 .map_err(BoxedError::new)
149 .context(ExternalSnafu)?;
150
151 let mut table_options = create.options.into_map();
152
153 let (object_store, files) = prepare_file_table_files(&table_options, local_file_access)
154 .await
155 .context(PrepareFileTableSnafu)?;
156
157 let file_column_schemas = infer_file_table_schema(&object_store, &files, &table_options)
158 .await
159 .context(InferFileTableSchemaSnafu)?
160 .column_schemas()
161 .to_vec();
162
163 let (time_index, primary_keys, table_column_schemas) = if !create.columns.is_empty() {
164 let time_index = find_time_index(&create.constraints)?;
166 let primary_keys = find_primary_keys(&create.columns, &create.constraints)?;
167 let column_schemas =
168 columns_to_column_schemas(&create.columns, &time_index, Some(&query_ctx.timezone()))?;
169 (time_index, primary_keys, column_schemas)
170 } else {
171 let (column_schemas, time_index) = file_column_schemas_to_table(&file_column_schemas);
173 let primary_keys = vec![];
174 (time_index, primary_keys, column_schemas)
175 };
176
177 check_file_to_table_schema_compatibility(&file_column_schemas, &table_column_schemas)
178 .context(SchemaIncompatibleSnafu)?;
179
180 let meta = FileOptions {
181 files,
182 file_column_schemas,
183 };
184 table_options.insert(
185 FILE_TABLE_META_KEY.to_string(),
186 serde_json::to_string(&meta).context(EncodeJsonSnafu)?,
187 );
188
189 let column_defs = column_schemas_to_defs(table_column_schemas, &primary_keys)?;
190 let expr = CreateTableExpr {
191 catalog_name,
192 schema_name,
193 table_name,
194 desc: String::default(),
195 column_defs,
196 time_index,
197 primary_keys,
198 create_if_not_exists: create.if_not_exists,
199 table_options,
200 table_id: None,
201 engine: create.engine.clone(),
202 };
203
204 Ok(expr)
205}
206
207pub fn create_to_expr(
209 create: &CreateTable,
210 query_ctx: &QueryContextRef,
211) -> Result<CreateTableExpr> {
212 let (catalog_name, schema_name, table_name) =
213 table_idents_to_full_name(&create.name, query_ctx)
214 .map_err(BoxedError::new)
215 .context(ExternalSnafu)?;
216
217 let time_index = find_time_index(&create.constraints)?;
218 let table_options = HashMap::from(
219 &TableOptions::try_from_iter(create.options.to_str_map())
220 .context(UnrecognizedTableOptionSnafu)?,
221 );
222
223 let mut table_options = table_options;
224 if table_options.contains_key(COMPACTION_TYPE) {
225 table_options.insert(COMPACTION_OVERRIDE.to_string(), "true".to_string());
226 }
227
228 let primary_keys = find_primary_keys(&create.columns, &create.constraints)?;
229
230 let expr = CreateTableExpr {
231 catalog_name,
232 schema_name,
233 table_name,
234 desc: String::default(),
235 column_defs: columns_to_expr(
236 &create.columns,
237 &time_index,
238 &primary_keys,
239 Some(&query_ctx.timezone()),
240 )?,
241 time_index,
242 primary_keys,
243 create_if_not_exists: create.if_not_exists,
244 table_options,
245 table_id: None,
246 engine: create.engine.clone(),
247 };
248
249 validate_create_expr(&expr)?;
250 Ok(expr)
251}
252
253pub fn expr_to_create(expr: &CreateTableExpr, quote_style: Option<char>) -> Result<CreateTable> {
261 let quote_style = quote_style.unwrap_or('`');
262
263 let table_name = ObjectName(vec![sql::ast::ObjectNamePart::Identifier(
265 sql::ast::Ident::with_quote(quote_style, &expr.table_name),
266 )]);
267
268 let mut columns = Vec::with_capacity(expr.column_defs.len());
270 for column_def in &expr.column_defs {
271 let column_schema = try_as_column_schema(column_def).context(InvalidColumnDefSnafu {
272 column: &column_def.name,
273 })?;
274
275 let mut options = Vec::new();
276
277 if column_def.is_nullable {
279 options.push(ColumnOptionDef {
280 name: None,
281 option: ColumnOption::Null,
282 });
283 } else {
284 options.push(ColumnOptionDef {
285 name: None,
286 option: ColumnOption::NotNull,
287 });
288 }
289
290 if let Some(default_constraint) = column_schema.default_constraint() {
292 let expr = match default_constraint {
293 ColumnDefaultConstraint::Value(v) => {
294 Expr::Value(value_to_sql_value(v).context(ParseSqlValueSnafu)?.into())
295 }
296 ColumnDefaultConstraint::Function(func_expr) => {
297 ParserContext::parse_function(func_expr, &GreptimeDbDialect {})
298 .context(ParseSqlSnafu)?
299 }
300 };
301 options.push(ColumnOptionDef {
302 name: None,
303 option: ColumnOption::Default(expr),
304 });
305 }
306
307 if !column_def.comment.is_empty() {
309 options.push(ColumnOptionDef {
310 name: None,
311 option: ColumnOption::Comment(column_def.comment.clone()),
312 });
313 }
314
315 let mut extensions = ColumnExtensions::default();
320
321 if let Ok(Some(opt)) = column_schema.fulltext_options()
323 && opt.enable
324 {
325 let mut map = HashMap::from([
326 (
327 COLUMN_FULLTEXT_OPT_KEY_ANALYZER.to_string(),
328 opt.analyzer.to_string(),
329 ),
330 (
331 COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE.to_string(),
332 opt.case_sensitive.to_string(),
333 ),
334 (
335 COLUMN_FULLTEXT_OPT_KEY_BACKEND.to_string(),
336 opt.backend.to_string(),
337 ),
338 ]);
339 if opt.backend == FulltextBackend::Bloom {
340 map.insert(
341 COLUMN_FULLTEXT_OPT_KEY_GRANULARITY.to_string(),
342 opt.granularity.to_string(),
343 );
344 map.insert(
345 COLUMN_FULLTEXT_OPT_KEY_FALSE_POSITIVE_RATE.to_string(),
346 opt.false_positive_rate().to_string(),
347 );
348 }
349 extensions.fulltext_index_options = Some(map.into());
350 }
351
352 if let Ok(Some(opt)) = column_schema.skipping_index_options() {
354 let map = HashMap::from([
355 (
356 COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY.to_string(),
357 opt.granularity.to_string(),
358 ),
359 (
360 COLUMN_SKIPPING_INDEX_OPT_KEY_FALSE_POSITIVE_RATE.to_string(),
361 opt.false_positive_rate().to_string(),
362 ),
363 (
364 COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE.to_string(),
365 opt.index_type.to_string(),
366 ),
367 ]);
368 extensions.skipping_index_options = Some(map.into());
369 }
370
371 if column_schema.is_inverted_indexed() {
373 extensions.inverted_index_options = Some(HashMap::new().into());
374 }
375
376 let sql_column = SqlColumn {
377 column_def: ColumnDef {
378 name: Ident::with_quote(quote_style, &column_def.name),
379 data_type: concrete_data_type_to_sql_data_type(&column_schema.data_type)
380 .context(ParseSqlSnafu)?,
381 options,
382 },
383 extensions,
384 };
385
386 columns.push(sql_column);
387 }
388
389 let mut constraints = Vec::new();
391
392 constraints.push(TableConstraint::TimeIndex {
394 column: Ident::with_quote(quote_style, &expr.time_index),
395 });
396
397 if !expr.primary_keys.is_empty() {
399 let primary_key_columns: Vec<Ident> = expr
400 .primary_keys
401 .iter()
402 .map(|pk| Ident::with_quote(quote_style, pk))
403 .collect();
404
405 constraints.push(TableConstraint::PrimaryKey {
406 columns: primary_key_columns,
407 });
408 }
409
410 let mut options = OptionMap::default();
412 for (key, value) in &expr.table_options {
413 options.insert(key.clone(), value.clone());
414 }
415
416 Ok(CreateTable {
417 if_not_exists: expr.create_if_not_exists,
418 table_id: expr.table_id.as_ref().map(|tid| tid.id).unwrap_or(0),
419 name: table_name,
420 columns,
421 engine: expr.engine.clone(),
422 constraints,
423 options,
424 partitions: None,
425 })
426}
427
428pub fn validate_create_expr(create: &CreateTableExpr) -> Result<()> {
430 let mut column_to_indices = HashMap::with_capacity(create.column_defs.len());
432 for (idx, column) in create.column_defs.iter().enumerate() {
433 if let Some(indices) = column_to_indices.get(&column.name) {
434 return InvalidSqlSnafu {
435 err_msg: format!(
436 "column name `{}` is duplicated at index {} and {}",
437 column.name, indices, idx
438 ),
439 }
440 .fail();
441 }
442 column_to_indices.insert(&column.name, idx);
443 }
444
445 let _ = column_to_indices
447 .get(&create.time_index)
448 .with_context(|| InvalidSqlSnafu {
449 err_msg: format!(
450 "column name `{}` is not found in column list",
451 create.time_index
452 ),
453 })?;
454
455 for pk in &create.primary_keys {
457 let _ = column_to_indices
458 .get(&pk)
459 .with_context(|| InvalidSqlSnafu {
460 err_msg: format!("column name `{}` is not found in column list", pk),
461 })?;
462 }
463
464 let mut pk_set = HashSet::new();
466 for pk in &create.primary_keys {
467 if !pk_set.insert(pk) {
468 return InvalidSqlSnafu {
469 err_msg: format!("column name `{}` is duplicated in primary keys", pk),
470 }
471 .fail();
472 }
473 }
474
475 if pk_set.contains(&create.time_index) {
477 return InvalidSqlSnafu {
478 err_msg: format!(
479 "column name `{}` is both primary key and time index",
480 create.time_index
481 ),
482 }
483 .fail();
484 }
485
486 for column in &create.column_defs {
487 if is_interval_type(&column.data_type()) {
489 return InvalidSqlSnafu {
490 err_msg: format!(
491 "column name `{}` is interval type, which is not supported",
492 column.name
493 ),
494 }
495 .fail();
496 }
497 if is_date_time_type(&column.data_type()) {
499 return InvalidSqlSnafu {
500 err_msg: format!(
501 "column name `{}` is datetime type, which is not supported, please use `timestamp` type instead",
502 column.name
503 ),
504 }
505 .fail();
506 }
507 }
508 Ok(())
509}
510
511fn validate_add_columns_expr(add_columns: &AddColumns) -> Result<()> {
512 for add_column in &add_columns.add_columns {
513 let Some(column_def) = &add_column.column_def else {
514 continue;
515 };
516 if is_date_time_type(&column_def.data_type()) {
517 return InvalidSqlSnafu {
518 err_msg: format!("column name `{}` is datetime type, which is not supported, please use `timestamp` type instead", column_def.name),
519 }
520 .fail();
521 }
522 if is_interval_type(&column_def.data_type()) {
523 return InvalidSqlSnafu {
524 err_msg: format!(
525 "column name `{}` is interval type, which is not supported",
526 column_def.name
527 ),
528 }
529 .fail();
530 }
531 }
532 Ok(())
533}
534
535fn is_date_time_type(data_type: &ColumnDataType) -> bool {
536 matches!(data_type, ColumnDataType::Datetime)
537}
538
539fn is_interval_type(data_type: &ColumnDataType) -> bool {
540 matches!(
541 data_type,
542 ColumnDataType::IntervalYearMonth
543 | ColumnDataType::IntervalDayTime
544 | ColumnDataType::IntervalMonthDayNano
545 )
546}
547
548fn find_primary_keys(
549 columns: &[SqlColumn],
550 constraints: &[TableConstraint],
551) -> Result<Vec<String>> {
552 let columns_pk = columns
553 .iter()
554 .filter_map(|x| {
555 if x.options()
556 .iter()
557 .any(|o| matches!(o.option, ColumnOption::PrimaryKey(_)))
558 {
559 Some(x.name().value.clone())
560 } else {
561 None
562 }
563 })
564 .collect::<Vec<String>>();
565
566 ensure!(
567 columns_pk.len() <= 1,
568 IllegalPrimaryKeysDefSnafu {
569 msg: "not allowed to inline multiple primary keys in columns options"
570 }
571 );
572
573 let constraints_pk = constraints
574 .iter()
575 .filter_map(|constraint| match constraint {
576 TableConstraint::PrimaryKey { columns, .. } => {
577 Some(columns.iter().map(|ident| ident.value.clone()))
578 }
579 _ => None,
580 })
581 .flatten()
582 .collect::<Vec<String>>();
583
584 ensure!(
585 columns_pk.is_empty() || constraints_pk.is_empty(),
586 IllegalPrimaryKeysDefSnafu {
587 msg: "found definitions of primary keys in multiple places"
588 }
589 );
590
591 let mut primary_keys = Vec::with_capacity(columns_pk.len() + constraints_pk.len());
592 primary_keys.extend(columns_pk);
593 primary_keys.extend(constraints_pk);
594 Ok(primary_keys)
595}
596
597pub fn find_time_index(constraints: &[TableConstraint]) -> Result<String> {
598 let time_index = constraints
599 .iter()
600 .filter_map(|constraint| match constraint {
601 TableConstraint::TimeIndex { column, .. } => Some(&column.value),
602 _ => None,
603 })
604 .collect::<Vec<&String>>();
605 ensure!(
606 time_index.len() == 1,
607 InvalidSqlSnafu {
608 err_msg: "must have one and only one TimeIndex columns",
609 }
610 );
611 Ok(time_index[0].clone())
612}
613
614fn columns_to_expr(
615 column_defs: &[SqlColumn],
616 time_index: &str,
617 primary_keys: &[String],
618 timezone: Option<&Timezone>,
619) -> Result<Vec<api::v1::ColumnDef>> {
620 let column_schemas = columns_to_column_schemas(column_defs, time_index, timezone)?;
621 column_schemas_to_defs(column_schemas, primary_keys)
622}
623
624fn columns_to_column_schemas(
625 columns: &[SqlColumn],
626 time_index: &str,
627 timezone: Option<&Timezone>,
628) -> Result<Vec<ColumnSchema>> {
629 columns
630 .iter()
631 .map(|c| column_to_schema(c, time_index, timezone).context(ParseSqlSnafu))
632 .collect::<Result<Vec<ColumnSchema>>>()
633}
634
635pub fn column_schemas_to_defs(
637 column_schemas: Vec<ColumnSchema>,
638 primary_keys: &[String],
639) -> Result<Vec<api::v1::ColumnDef>> {
640 let column_datatypes: Vec<(ColumnDataType, Option<ColumnDataTypeExtension>)> = column_schemas
641 .iter()
642 .map(|c| {
643 ColumnDataTypeWrapper::try_from(c.data_type.clone())
644 .map(|w| w.to_parts())
645 .context(ColumnDataTypeSnafu)
646 })
647 .collect::<Result<Vec<_>>>()?;
648
649 column_schemas
650 .iter()
651 .zip(column_datatypes)
652 .map(|(schema, datatype)| {
653 let semantic_type = if schema.is_time_index() {
654 SemanticType::Timestamp
655 } else if primary_keys.contains(&schema.name) {
656 SemanticType::Tag
657 } else {
658 SemanticType::Field
659 } as i32;
660 let comment = schema
661 .metadata()
662 .get(COMMENT_KEY)
663 .cloned()
664 .unwrap_or_default();
665
666 Ok(api::v1::ColumnDef {
667 name: schema.name.clone(),
668 data_type: datatype.0 as i32,
669 is_nullable: schema.is_nullable(),
670 default_constraint: match schema.default_constraint() {
671 None => vec![],
672 Some(v) => {
673 v.clone()
674 .try_into()
675 .context(ConvertColumnDefaultConstraintSnafu {
676 column_name: &schema.name,
677 })?
678 }
679 },
680 semantic_type,
681 comment,
682 datatype_extension: datatype.1,
683 options: options_from_column_schema(schema),
684 })
685 })
686 .collect()
687}
688
689#[derive(Debug, Clone, PartialEq, Eq)]
690pub struct RepartitionRequest {
691 pub catalog_name: String,
692 pub schema_name: String,
693 pub table_name: String,
694 pub source: RepartitionSource,
695 pub into_exprs: Vec<Expr>,
696 pub options: OptionMap,
697}
698
699#[derive(Debug, Clone, PartialEq, Eq)]
700pub enum RepartitionSource {
701 Partitions {
702 from_exprs: Vec<Expr>,
703 target_partition_columns: Option<Vec<String>>,
704 },
705 Unpartitioned {
706 partition_columns: Vec<String>,
707 },
708}
709
710pub(crate) fn to_repartition_request(
711 alter_table: AlterTable,
712 query_ctx: &QueryContextRef,
713) -> Result<RepartitionRequest> {
714 let AlterTable {
715 table_name,
716 alter_operation,
717 options,
718 } = alter_table;
719
720 let (catalog_name, schema_name, table_name) = table_idents_to_full_name(&table_name, query_ctx)
721 .map_err(BoxedError::new)
722 .context(ExternalSnafu)?;
723
724 let (source, into_exprs) = match alter_operation {
725 AlterTableOperation::Repartition { operation } => (
726 RepartitionSource::Partitions {
727 from_exprs: operation.from_exprs,
728 target_partition_columns: operation.partition_columns.map(|columns| {
729 columns
730 .into_iter()
731 .map(|ident| ident.value)
732 .collect::<Vec<_>>()
733 }),
734 },
735 operation.into_exprs,
736 ),
737 AlterTableOperation::Partition { partitions } => (
738 RepartitionSource::Unpartitioned {
739 partition_columns: partitions
740 .column_list
741 .into_iter()
742 .map(|ident| ident.value)
743 .collect(),
744 },
745 partitions.exprs,
746 ),
747 _ => {
748 return InvalidSqlSnafu {
749 err_msg: "expected REPARTITION or PARTITION operation",
750 }
751 .fail();
752 }
753 };
754
755 Ok(RepartitionRequest {
756 catalog_name,
757 schema_name,
758 table_name,
759 source,
760 into_exprs,
761 options,
762 })
763}
764
765pub(crate) fn to_alter_table_expr(
767 alter_table: AlterTable,
768 query_ctx: &QueryContextRef,
769) -> Result<AlterTableExpr> {
770 let (catalog_name, schema_name, table_name) =
771 table_idents_to_full_name(alter_table.table_name(), query_ctx)
772 .map_err(BoxedError::new)
773 .context(ExternalSnafu)?;
774
775 let kind = match alter_table.alter_operation {
776 AlterTableOperation::AddConstraint(_) => {
777 return NotSupportedSnafu {
778 feat: "ADD CONSTRAINT",
779 }
780 .fail();
781 }
782 AlterTableOperation::AddColumns { add_columns } => AlterTableKind::AddColumns(AddColumns {
783 add_columns: add_columns
784 .into_iter()
785 .map(|add_column| {
786 let column_def = sql_column_def_to_grpc_column_def(
787 &add_column.column_def,
788 Some(&query_ctx.timezone()),
789 )
790 .map_err(BoxedError::new)
791 .context(ExternalSnafu)?;
792 if is_interval_type(&column_def.data_type()) {
793 return NotSupportedSnafu {
794 feat: "Add column with interval type",
795 }
796 .fail();
797 }
798 Ok(AddColumn {
799 column_def: Some(column_def),
800 location: add_column.location.as_ref().map(From::from),
801 add_if_not_exists: add_column.add_if_not_exists,
802 })
803 })
804 .collect::<Result<Vec<AddColumn>>>()?,
805 }),
806 AlterTableOperation::ModifyColumnType {
807 column_name,
808 target_type,
809 } => {
810 let target_type =
811 sql_data_type_to_concrete_data_type(&target_type).context(ParseSqlSnafu)?;
812 let (target_type, target_type_extension) = ColumnDataTypeWrapper::try_from(target_type)
813 .map(|w| w.to_parts())
814 .context(ColumnDataTypeSnafu)?;
815 if is_interval_type(&target_type) {
816 return NotSupportedSnafu {
817 feat: "Modify column type to interval type",
818 }
819 .fail();
820 }
821 AlterTableKind::ModifyColumnTypes(ModifyColumnTypes {
822 modify_column_types: vec![ModifyColumnType {
823 column_name: column_name.value,
824 target_type: target_type as i32,
825 target_type_extension,
826 }],
827 })
828 }
829 AlterTableOperation::DropColumn { name } => AlterTableKind::DropColumns(DropColumns {
830 drop_columns: vec![DropColumn {
831 name: name.value.clone(),
832 }],
833 }),
834 AlterTableOperation::RenameTable { new_table_name } => {
835 AlterTableKind::RenameTable(RenameTable {
836 new_table_name: new_table_name.clone(),
837 })
838 }
839 AlterTableOperation::SetTableOptions { options } => {
840 AlterTableKind::SetTableOptions(SetTableOptions {
841 table_options: options.into_iter().map(Into::into).collect(),
842 })
843 }
844 AlterTableOperation::UnsetTableOptions { keys } => {
845 AlterTableKind::UnsetTableOptions(UnsetTableOptions { keys })
846 }
847 AlterTableOperation::Repartition { .. } => {
848 return NotSupportedSnafu {
849 feat: "ALTER TABLE ... REPARTITION",
850 }
851 .fail();
852 }
853 AlterTableOperation::Partition { .. } => {
854 return NotSupportedSnafu {
855 feat: "ALTER TABLE ... PARTITION ON COLUMNS",
856 }
857 .fail();
858 }
859 AlterTableOperation::SetIndex { options } => {
860 let option = match options {
861 sql::statements::alter::SetIndexOperation::Fulltext {
862 column_name,
863 options,
864 } => SetIndex {
865 options: Some(set_index::Options::Fulltext(SetFulltext {
866 column_name: column_name.value,
867 enable: options.enable,
868 analyzer: match options.analyzer {
869 FulltextAnalyzer::English => Analyzer::English.into(),
870 FulltextAnalyzer::Chinese => Analyzer::Chinese.into(),
871 },
872 case_sensitive: options.case_sensitive,
873 backend: match options.backend {
874 FulltextBackend::Bloom => PbFulltextBackend::Bloom.into(),
875 FulltextBackend::Tantivy => PbFulltextBackend::Tantivy.into(),
876 },
877 granularity: options.granularity as u64,
878 false_positive_rate: options.false_positive_rate(),
879 })),
880 },
881 sql::statements::alter::SetIndexOperation::Inverted { column_name } => SetIndex {
882 options: Some(set_index::Options::Inverted(SetInverted {
883 column_name: column_name.value,
884 })),
885 },
886 sql::statements::alter::SetIndexOperation::Skipping {
887 column_name,
888 options,
889 } => SetIndex {
890 options: Some(set_index::Options::Skipping(SetSkipping {
891 column_name: column_name.value,
892 enable: true,
893 granularity: options.granularity as u64,
894 false_positive_rate: options.false_positive_rate(),
895 skipping_index_type: match options.index_type {
896 SkippingIndexType::BloomFilter => {
897 PbSkippingIndexType::BloomFilter.into()
898 }
899 },
900 })),
901 },
902 };
903 AlterTableKind::SetIndexes(SetIndexes {
904 set_indexes: vec![option],
905 })
906 }
907 AlterTableOperation::UnsetIndex { options } => {
908 let option = match options {
909 sql::statements::alter::UnsetIndexOperation::Fulltext { column_name } => {
910 UnsetIndex {
911 options: Some(unset_index::Options::Fulltext(UnsetFulltext {
912 column_name: column_name.value,
913 })),
914 }
915 }
916 sql::statements::alter::UnsetIndexOperation::Inverted { column_name } => {
917 UnsetIndex {
918 options: Some(unset_index::Options::Inverted(UnsetInverted {
919 column_name: column_name.value,
920 })),
921 }
922 }
923 sql::statements::alter::UnsetIndexOperation::Skipping { column_name } => {
924 UnsetIndex {
925 options: Some(unset_index::Options::Skipping(UnsetSkipping {
926 column_name: column_name.value,
927 })),
928 }
929 }
930 };
931
932 AlterTableKind::UnsetIndexes(UnsetIndexes {
933 unset_indexes: vec![option],
934 })
935 }
936 AlterTableOperation::DropDefaults { columns } => {
937 AlterTableKind::DropDefaults(DropDefaults {
938 drop_defaults: columns
939 .into_iter()
940 .map(|col| {
941 let column_name = col.0.to_string();
942 Ok(api::v1::DropDefault { column_name })
943 })
944 .collect::<Result<Vec<_>>>()?,
945 })
946 }
947 AlterTableOperation::SetDefaults { defaults } => AlterTableKind::SetDefaults(SetDefaults {
948 set_defaults: defaults
949 .into_iter()
950 .map(|col| {
951 let column_name = col.column_name.to_string();
952 let default_constraint = serde_json::to_string(&col.default_constraint)
953 .context(EncodeJsonSnafu)?
954 .into_bytes();
955 Ok(api::v1::SetDefault {
956 column_name,
957 default_constraint,
958 })
959 })
960 .collect::<Result<Vec<_>>>()?,
961 }),
962 };
963
964 Ok(AlterTableExpr {
965 catalog_name,
966 schema_name,
967 table_name,
968 kind: Some(kind),
969 })
970}
971
972pub fn to_alter_database_expr(
974 alter_database: AlterDatabase,
975 query_ctx: &QueryContextRef,
976) -> Result<AlterDatabaseExpr> {
977 let catalog = query_ctx.current_catalog();
978 let schema = alter_database.database_name;
979
980 let kind = match alter_database.alter_operation {
981 AlterDatabaseOperation::SetDatabaseOption { options } => {
982 let options = options.into_iter().map(Into::into).collect();
983 AlterDatabaseKind::SetDatabaseOptions(SetDatabaseOptions {
984 set_database_options: options,
985 })
986 }
987 AlterDatabaseOperation::UnsetDatabaseOption { keys } => {
988 AlterDatabaseKind::UnsetDatabaseOptions(UnsetDatabaseOptions { keys })
989 }
990 };
991
992 Ok(AlterDatabaseExpr {
993 catalog_name: catalog.to_string(),
994 schema_name: schema.to_string(),
995 kind: Some(kind),
996 })
997}
998
999pub fn to_create_view_expr(
1001 stmt: CreateView,
1002 logical_plan: Vec<u8>,
1003 table_names: Vec<TableName>,
1004 columns: Vec<String>,
1005 plan_columns: Vec<String>,
1006 definition: String,
1007 query_ctx: QueryContextRef,
1008) -> Result<CreateViewExpr> {
1009 let (catalog_name, schema_name, view_name) = table_idents_to_full_name(&stmt.name, &query_ctx)
1010 .map_err(BoxedError::new)
1011 .context(ExternalSnafu)?;
1012
1013 let expr = CreateViewExpr {
1014 catalog_name,
1015 schema_name,
1016 view_name,
1017 logical_plan,
1018 create_if_not_exists: stmt.if_not_exists,
1019 or_replace: stmt.or_replace,
1020 table_names,
1021 columns,
1022 plan_columns,
1023 definition,
1024 };
1025
1026 Ok(expr)
1027}
1028
1029pub fn to_create_flow_task_expr(
1030 create_flow: CreateFlow,
1031 query_ctx: &QueryContextRef,
1032) -> Result<CreateFlowExpr> {
1033 let sink_table_ref = object_name_to_table_reference(create_flow.sink_table_name.clone(), true)
1035 .with_context(|_| ConvertIdentifierSnafu {
1036 ident: create_flow.sink_table_name.to_string(),
1037 })?;
1038 let catalog = sink_table_ref
1039 .catalog()
1040 .unwrap_or(query_ctx.current_catalog())
1041 .to_string();
1042 let schema = sink_table_ref
1043 .schema()
1044 .map(|s| s.to_owned())
1045 .unwrap_or(query_ctx.current_schema());
1046
1047 let sink_table_name = TableName {
1048 catalog_name: catalog,
1049 schema_name: schema,
1050 table_name: sink_table_ref.table().to_string(),
1051 };
1052
1053 let source_table_names = extract_tables_from_query(&create_flow.query)
1054 .map(|name| {
1055 let reference =
1056 object_name_to_table_reference(name.clone(), true).with_context(|_| {
1057 ConvertIdentifierSnafu {
1058 ident: name.to_string(),
1059 }
1060 })?;
1061 let catalog = reference
1062 .catalog()
1063 .unwrap_or(query_ctx.current_catalog())
1064 .to_string();
1065 let schema = reference
1066 .schema()
1067 .map(|s| s.to_string())
1068 .unwrap_or(query_ctx.current_schema());
1069
1070 let table_name = TableName {
1071 catalog_name: catalog,
1072 schema_name: schema,
1073 table_name: reference.table().to_string(),
1074 };
1075 Ok(table_name)
1076 })
1077 .collect::<Result<Vec<_>>>()?;
1078
1079 let eval_interval = create_flow.eval_interval;
1080
1081 Ok(CreateFlowExpr {
1082 catalog_name: query_ctx.current_catalog().to_string(),
1083 flow_name: sanitize_flow_name(create_flow.flow_name)?,
1084 source_table_names,
1085 sink_table_name: Some(sink_table_name),
1086 or_replace: create_flow.or_replace,
1087 create_if_not_exists: create_flow.if_not_exists,
1088 expire_after: create_flow.expire_after.map(|value| ExpireAfter { value }),
1089 eval_interval: eval_interval.map(|seconds| api::v1::EvalInterval { seconds }),
1090 comment: create_flow.comment.unwrap_or_default(),
1091 sql: create_flow.query.to_string(),
1092 flow_options: stringify_flow_options(create_flow.flow_options)?,
1093 })
1094}
1095
1096fn stringify_flow_options(flow_options: OptionMap) -> Result<HashMap<String, String>> {
1097 let options_len = flow_options.len();
1098 let flow_options = flow_options.into_map();
1099 ensure!(
1100 flow_options.len() == options_len,
1101 InvalidSqlSnafu {
1102 err_msg: "flow options only support scalar string-compatible values".to_string(),
1103 }
1104 );
1105 Ok(flow_options)
1106}
1107
1108fn sanitize_flow_name(mut flow_name: ObjectName) -> Result<String> {
1110 ensure!(
1111 flow_name.0.len() == 1,
1112 InvalidFlowNameSnafu {
1113 name: flow_name.to_string(),
1114 }
1115 );
1116 Ok(flow_name.0.swap_remove(0).to_string_unquoted())
1118}
1119
1120#[cfg(test)]
1121mod tests {
1122 use std::collections::HashMap;
1123
1124 use api::v1::{SetDatabaseOptions, UnsetDatabaseOptions};
1125 use datatypes::value::Value;
1126 use session::context::{QueryContext, QueryContextBuilder};
1127 use sql::dialect::GreptimeDbDialect;
1128 use sql::parser::{ParseOptions, ParserContext};
1129 use sql::statements::statement::Statement;
1130 use store_api::storage::ColumnDefaultConstraint;
1131
1132 use super::*;
1133
1134 #[test]
1135 fn test_create_flow_tql_expr() {
1136 let sql = r#"
1137CREATE FLOW calc_reqs SINK TO cnt_reqs AS
1138TQL EVAL (0, 15, '5s') count_values("status_code", http_requests);"#;
1139 let stmt =
1140 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default());
1141
1142 assert!(
1143 stmt.is_err(),
1144 "Expected error for invalid TQL EVAL parameters: {:#?}",
1145 stmt
1146 );
1147
1148 let sql = r#"
1149CREATE FLOW calc_reqs SINK TO cnt_reqs AS
1150TQL EVAL (now() - '15s'::interval, now(), '5s') count_values("status_code", http_requests);"#;
1151 let stmt =
1152 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1153 .unwrap()
1154 .pop()
1155 .unwrap();
1156
1157 let Statement::CreateFlow(create_flow) = stmt else {
1158 unreachable!()
1159 };
1160 let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1161
1162 let to_dot_sep =
1163 |c: TableName| format!("{}.{}.{}", c.catalog_name, c.schema_name, c.table_name);
1164 assert_eq!("calc_reqs", expr.flow_name);
1165 assert_eq!("greptime", expr.catalog_name);
1166 assert_eq!(
1167 "greptime.public.cnt_reqs",
1168 expr.sink_table_name.map(to_dot_sep).unwrap()
1169 );
1170 assert_eq!(1, expr.source_table_names.len());
1171 assert_eq!(
1172 "greptime.public.http_requests",
1173 to_dot_sep(expr.source_table_names[0].clone())
1174 );
1175 assert_eq!(
1176 r#"TQL EVAL (now() - '15s'::interval, now(), '5s') count_values("status_code", http_requests)"#,
1177 expr.sql
1178 );
1179
1180 let sql = r#"
1181CREATE FLOW calc_reqs SINK TO cnt_reqs AS
1182TQL EVAL (now() - '15s'::interval, now(), '5s') count_values("status_code", http_requests{__schema__="greptime_private"});"#;
1183 let stmt =
1184 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1185 .unwrap()
1186 .pop()
1187 .unwrap();
1188 let Statement::CreateFlow(create_flow) = stmt else {
1189 unreachable!()
1190 };
1191 let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1192 assert_eq!(1, expr.source_table_names.len());
1193 assert_eq!(
1194 "greptime.greptime_private.http_requests",
1195 to_dot_sep(expr.source_table_names[0].clone())
1196 );
1197
1198 let sql = r#"
1199CREATE FLOW calc_reqs SINK TO cnt_reqs AS
1200TQL EVAL (now() - '15s'::interval, now(), '5s') count_values("status_code", http_requests{__database__="greptime_private"});"#;
1201 let stmt =
1202 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1203 .unwrap()
1204 .pop()
1205 .unwrap();
1206 let Statement::CreateFlow(create_flow) = stmt else {
1207 unreachable!()
1208 };
1209 let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1210 assert_eq!(1, expr.source_table_names.len());
1211 assert_eq!(
1212 "greptime.greptime_private.http_requests",
1213 to_dot_sep(expr.source_table_names[0].clone())
1214 );
1215 }
1216
1217 #[test]
1218 fn test_create_flow_tql_cte_source_tables() {
1219 let sql = r#"
1220CREATE FLOW calc_cte
1221SINK TO metric_cte_sink
1222EVAL INTERVAL '1m'
1223AS
1224WITH tql(ts, the_value) AS (
1225 TQL EVAL (now() - '1m'::interval, now(), '5s') metric_cte
1226)
1227SELECT * FROM tql;
1228"#;
1229
1230 let stmt =
1231 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1232 .unwrap()
1233 .pop()
1234 .unwrap();
1235
1236 let Statement::CreateFlow(create_flow) = stmt else {
1237 unreachable!()
1238 };
1239 let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1240
1241 let to_dot_sep =
1242 |c: TableName| format!("{}.{}.{}", c.catalog_name, c.schema_name, c.table_name);
1243 assert_eq!(1, expr.source_table_names.len());
1244 assert_eq!(
1245 "greptime.public.metric_cte",
1246 to_dot_sep(expr.source_table_names[0].clone())
1247 );
1248 }
1249
1250 #[test]
1251 fn test_create_flow_tql_cte_source_tables_quoted_cte_name() {
1252 let sql = r#"
1253CREATE FLOW calc_cte
1254SINK TO metric_cte_sink
1255EVAL INTERVAL '1m'
1256AS
1257WITH "TQL"(ts, the_value) AS (
1258 TQL EVAL (now() - '1m'::interval, now(), '5s') metric_cte
1259)
1260SELECT * FROM "TQL";
1261"#;
1262
1263 let stmt =
1264 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1265 .unwrap()
1266 .pop()
1267 .unwrap();
1268
1269 let Statement::CreateFlow(create_flow) = stmt else {
1270 unreachable!()
1271 };
1272 let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1273
1274 let to_dot_sep =
1275 |c: TableName| format!("{}.{}.{}", c.catalog_name, c.schema_name, c.table_name);
1276 assert_eq!(1, expr.source_table_names.len());
1277 assert_eq!(
1278 "greptime.public.metric_cte",
1279 to_dot_sep(expr.source_table_names[0].clone())
1280 );
1281 }
1282
1283 #[test]
1284 fn test_create_flow_tql_cte_source_tables_same_name() {
1285 let sql = r#"
1286CREATE FLOW calc_cte
1287SINK TO metric_cte_sink
1288EVAL INTERVAL '1m'
1289AS
1290WITH tql(ts, the_value) AS (
1291 TQL EVAL (now() - '1m'::interval, now(), '5s') tql
1292)
1293SELECT * FROM tql;
1294"#;
1295
1296 let stmt =
1297 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1298 .unwrap()
1299 .pop()
1300 .unwrap();
1301
1302 let Statement::CreateFlow(create_flow) = stmt else {
1303 unreachable!()
1304 };
1305 let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1306
1307 let to_dot_sep =
1308 |c: TableName| format!("{}.{}.{}", c.catalog_name, c.schema_name, c.table_name);
1309 assert_eq!(1, expr.source_table_names.len());
1310 assert_eq!(
1311 "greptime.public.tql",
1312 to_dot_sep(expr.source_table_names[0].clone())
1313 );
1314 }
1315
1316 #[test]
1317 fn test_create_flow_expr() {
1318 let sql = r"
1319CREATE FLOW test_distinct_basic SINK TO out_distinct_basic AS
1320SELECT
1321 DISTINCT number as dis
1322FROM
1323 distinct_basic;";
1324 let stmt =
1325 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1326 .unwrap()
1327 .pop()
1328 .unwrap();
1329
1330 let Statement::CreateFlow(create_flow) = stmt else {
1331 unreachable!()
1332 };
1333 let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1334
1335 let to_dot_sep =
1336 |c: TableName| format!("{}.{}.{}", c.catalog_name, c.schema_name, c.table_name);
1337 assert_eq!("test_distinct_basic", expr.flow_name);
1338 assert_eq!("greptime", expr.catalog_name);
1339 assert_eq!(
1340 "greptime.public.out_distinct_basic",
1341 expr.sink_table_name.map(to_dot_sep).unwrap()
1342 );
1343 assert_eq!(1, expr.source_table_names.len());
1344 assert_eq!(
1345 "greptime.public.distinct_basic",
1346 to_dot_sep(expr.source_table_names[0].clone())
1347 );
1348 assert_eq!(
1349 r"SELECT
1350 DISTINCT number as dis
1351FROM
1352 distinct_basic",
1353 expr.sql
1354 );
1355
1356 let sql = r"
1357CREATE FLOW `task_2`
1358SINK TO schema_1.table_1
1359AS
1360SELECT max(c1), min(c2) FROM schema_2.table_2;";
1361 let stmt =
1362 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1363 .unwrap()
1364 .pop()
1365 .unwrap();
1366
1367 let Statement::CreateFlow(create_flow) = stmt else {
1368 unreachable!()
1369 };
1370 let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1371
1372 let to_dot_sep =
1373 |c: TableName| format!("{}.{}.{}", c.catalog_name, c.schema_name, c.table_name);
1374 assert_eq!("task_2", expr.flow_name);
1375 assert_eq!("greptime", expr.catalog_name);
1376 assert_eq!(
1377 "greptime.schema_1.table_1",
1378 expr.sink_table_name.map(to_dot_sep).unwrap()
1379 );
1380 assert_eq!(1, expr.source_table_names.len());
1381 assert_eq!(
1382 "greptime.schema_2.table_2",
1383 to_dot_sep(expr.source_table_names[0].clone())
1384 );
1385 assert_eq!("SELECT max(c1), min(c2) FROM schema_2.table_2", expr.sql);
1386 assert!(expr.flow_options.is_empty());
1387
1388 let sql = r"
1389CREATE FLOW task_3
1390SINK TO schema_1.table_1
1391WITH (defer_on_missing_source = 'true', foo = 'bar')
1392AS
1393SELECT max(c1), min(c2) FROM schema_2.table_2;";
1394 let stmt =
1395 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1396 .unwrap()
1397 .pop()
1398 .unwrap();
1399
1400 let Statement::CreateFlow(create_flow) = stmt else {
1401 unreachable!()
1402 };
1403 let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1404 assert_eq!(
1405 expr.flow_options,
1406 HashMap::from([
1407 ("defer_on_missing_source".to_string(), "true".to_string()),
1408 ("foo".to_string(), "bar".to_string()),
1409 ])
1410 );
1411
1412 let sql = r"
1413CREATE FLOW task_4
1414SINK TO schema_1.table_1
1415WITH (defer_on_missing_source = true)
1416AS
1417SELECT max(c1), min(c2) FROM schema_2.table_2;";
1418 let stmt =
1419 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1420 .unwrap()
1421 .pop()
1422 .unwrap();
1423
1424 let Statement::CreateFlow(create_flow) = stmt else {
1425 unreachable!()
1426 };
1427 let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1428 assert_eq!(
1429 expr.flow_options,
1430 HashMap::from([("defer_on_missing_source".to_string(), "true".to_string(),)])
1431 );
1432
1433 let sql = r"
1434CREATE FLOW task_5
1435SINK TO schema_1.table_1
1436WITH (defer_on_missing_source = [true])
1437AS
1438SELECT max(c1), min(c2) FROM schema_2.table_2;";
1439 let stmt =
1440 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1441 .unwrap()
1442 .pop()
1443 .unwrap();
1444
1445 let Statement::CreateFlow(create_flow) = stmt else {
1446 unreachable!()
1447 };
1448 let res = to_create_flow_task_expr(create_flow, &QueryContext::arc());
1449 assert!(res.is_err());
1450 assert!(
1451 res.unwrap_err()
1452 .to_string()
1453 .contains("flow options only support scalar string-compatible values")
1454 );
1455
1456 let sql = r"
1457CREATE FLOW abc.`task_2`
1458SINK TO schema_1.table_1
1459AS
1460SELECT max(c1), min(c2) FROM schema_2.table_2;";
1461 let stmt =
1462 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1463 .unwrap()
1464 .pop()
1465 .unwrap();
1466
1467 let Statement::CreateFlow(create_flow) = stmt else {
1468 unreachable!()
1469 };
1470 let res = to_create_flow_task_expr(create_flow, &QueryContext::arc());
1471
1472 assert!(res.is_err());
1473 assert!(
1474 res.unwrap_err()
1475 .to_string()
1476 .contains("Invalid flow name: abc.`task_2`")
1477 );
1478 }
1479
1480 #[test]
1481 fn test_create_to_expr() {
1482 let sql = "CREATE TABLE monitor (host STRING,ts TIMESTAMP,TIME INDEX (ts),PRIMARY KEY(host)) ENGINE=mito WITH(ttl='3days', write_buffer_size='1024KB');";
1483 let stmt =
1484 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1485 .unwrap()
1486 .pop()
1487 .unwrap();
1488
1489 let Statement::CreateTable(create_table) = stmt else {
1490 unreachable!()
1491 };
1492 let expr = create_to_expr(&create_table, &QueryContext::arc()).unwrap();
1493 assert_eq!("3days", expr.table_options.get("ttl").unwrap());
1494 assert_eq!(
1495 "1.0MiB",
1496 expr.table_options.get("write_buffer_size").unwrap()
1497 );
1498 }
1499
1500 #[test]
1501 fn test_invalid_create_to_expr() {
1502 let cases = [
1503 "CREATE TABLE monitor (host STRING primary key, ts TIMESTAMP TIME INDEX, some_column text, some_column string);",
1505 "CREATE TABLE monitor (host STRING, ts TIMESTAMP TIME INDEX, some_column STRING, PRIMARY KEY (some_column, host, some_column));",
1507 "CREATE TABLE monitor (host STRING, ts TIMESTAMP TIME INDEX, PRIMARY KEY (host, ts));",
1509 ];
1510
1511 for sql in cases {
1512 let stmt = ParserContext::create_with_dialect(
1513 sql,
1514 &GreptimeDbDialect {},
1515 ParseOptions::default(),
1516 )
1517 .unwrap()
1518 .pop()
1519 .unwrap();
1520 let Statement::CreateTable(create_table) = stmt else {
1521 unreachable!()
1522 };
1523 create_to_expr(&create_table, &QueryContext::arc()).unwrap_err();
1524 }
1525 }
1526
1527 #[test]
1528 fn test_create_to_expr_with_default_timestamp_value() {
1529 let sql = "CREATE TABLE monitor (v double,ts TIMESTAMP default '2024-01-30T00:01:01',TIME INDEX (ts)) engine=mito;";
1530 let stmt =
1531 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1532 .unwrap()
1533 .pop()
1534 .unwrap();
1535
1536 let Statement::CreateTable(create_table) = stmt else {
1537 unreachable!()
1538 };
1539
1540 let expr = create_to_expr(&create_table, &QueryContext::arc()).unwrap();
1542 let ts_column = &expr.column_defs[1];
1543 let constraint = assert_ts_column(ts_column);
1544 assert!(
1545 matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
1546 if ts.to_iso8601_string() == "2024-01-30 00:01:01+0000")
1547 );
1548
1549 let ctx = QueryContextBuilder::default()
1551 .timezone(Timezone::from_tz_string("+08:00").unwrap())
1552 .build()
1553 .into();
1554 let expr = create_to_expr(&create_table, &ctx).unwrap();
1555 let ts_column = &expr.column_defs[1];
1556 let constraint = assert_ts_column(ts_column);
1557 assert!(
1558 matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
1559 if ts.to_iso8601_string() == "2024-01-29 16:01:01+0000")
1560 );
1561 }
1562
1563 fn assert_ts_column(ts_column: &api::v1::ColumnDef) -> ColumnDefaultConstraint {
1564 assert_eq!("ts", ts_column.name);
1565 assert_eq!(
1566 ColumnDataType::TimestampMillisecond as i32,
1567 ts_column.data_type
1568 );
1569 assert!(!ts_column.default_constraint.is_empty());
1570
1571 ColumnDefaultConstraint::try_from(&ts_column.default_constraint[..]).unwrap()
1572 }
1573
1574 #[test]
1575 fn test_to_alter_expr() {
1576 let sql = "ALTER DATABASE greptime SET key1='value1', key2='value2';";
1577 let stmt =
1578 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1579 .unwrap()
1580 .pop()
1581 .unwrap();
1582
1583 let Statement::AlterDatabase(alter_database) = stmt else {
1584 unreachable!()
1585 };
1586
1587 let expr = to_alter_database_expr(alter_database, &QueryContext::arc()).unwrap();
1588 let kind = expr.kind.unwrap();
1589
1590 let AlterDatabaseKind::SetDatabaseOptions(SetDatabaseOptions {
1591 set_database_options,
1592 }) = kind
1593 else {
1594 unreachable!()
1595 };
1596
1597 assert_eq!(2, set_database_options.len());
1598 assert_eq!("key1", set_database_options[0].key);
1599 assert_eq!("value1", set_database_options[0].value);
1600 assert_eq!("key2", set_database_options[1].key);
1601 assert_eq!("value2", set_database_options[1].value);
1602
1603 let sql = "ALTER DATABASE greptime UNSET key1, key2;";
1604 let stmt =
1605 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1606 .unwrap()
1607 .pop()
1608 .unwrap();
1609
1610 let Statement::AlterDatabase(alter_database) = stmt else {
1611 unreachable!()
1612 };
1613
1614 let expr = to_alter_database_expr(alter_database, &QueryContext::arc()).unwrap();
1615 let kind = expr.kind.unwrap();
1616
1617 let AlterDatabaseKind::UnsetDatabaseOptions(UnsetDatabaseOptions { keys }) = kind else {
1618 unreachable!()
1619 };
1620
1621 assert_eq!(2, keys.len());
1622 assert!(keys.contains(&"key1".to_string()));
1623 assert!(keys.contains(&"key2".to_string()));
1624
1625 let sql = "ALTER TABLE monitor add column ts TIMESTAMP default '2024-01-30T00:01:01';";
1626 let stmt =
1627 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1628 .unwrap()
1629 .pop()
1630 .unwrap();
1631
1632 let Statement::AlterTable(alter_table) = stmt else {
1633 unreachable!()
1634 };
1635
1636 let expr = to_alter_table_expr(alter_table.clone(), &QueryContext::arc()).unwrap();
1638 let kind = expr.kind.unwrap();
1639
1640 let AlterTableKind::AddColumns(AddColumns { add_columns, .. }) = kind else {
1641 unreachable!()
1642 };
1643
1644 assert_eq!(1, add_columns.len());
1645 let ts_column = add_columns[0].column_def.clone().unwrap();
1646 let constraint = assert_ts_column(&ts_column);
1647 assert!(
1648 matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
1649 if ts.to_iso8601_string() == "2024-01-30 00:01:01+0000")
1650 );
1651
1652 let ctx = QueryContextBuilder::default()
1655 .timezone(Timezone::from_tz_string("+08:00").unwrap())
1656 .build()
1657 .into();
1658 let expr = to_alter_table_expr(alter_table, &ctx).unwrap();
1659 let kind = expr.kind.unwrap();
1660
1661 let AlterTableKind::AddColumns(AddColumns { add_columns, .. }) = kind else {
1662 unreachable!()
1663 };
1664
1665 assert_eq!(1, add_columns.len());
1666 let ts_column = add_columns[0].column_def.clone().unwrap();
1667 let constraint = assert_ts_column(&ts_column);
1668 assert!(
1669 matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
1670 if ts.to_iso8601_string() == "2024-01-29 16:01:01+0000")
1671 );
1672 }
1673
1674 #[test]
1675 fn test_to_alter_modify_column_type_expr() {
1676 let sql = "ALTER TABLE monitor MODIFY COLUMN mem_usage STRING;";
1677 let stmt =
1678 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1679 .unwrap()
1680 .pop()
1681 .unwrap();
1682
1683 let Statement::AlterTable(alter_table) = stmt else {
1684 unreachable!()
1685 };
1686
1687 let expr = to_alter_table_expr(alter_table.clone(), &QueryContext::arc()).unwrap();
1689 let kind = expr.kind.unwrap();
1690
1691 let AlterTableKind::ModifyColumnTypes(ModifyColumnTypes {
1692 modify_column_types,
1693 }) = kind
1694 else {
1695 unreachable!()
1696 };
1697
1698 assert_eq!(1, modify_column_types.len());
1699 let modify_column_type = &modify_column_types[0];
1700
1701 assert_eq!("mem_usage", modify_column_type.column_name);
1702 assert_eq!(
1703 ColumnDataType::String as i32,
1704 modify_column_type.target_type
1705 );
1706 assert!(modify_column_type.target_type_extension.is_none());
1707 }
1708
1709 #[test]
1710 fn test_to_repartition_request() {
1711 let sql = r#"
1712ALTER TABLE metrics REPARTITION (
1713 device_id < 100
1714) INTO (
1715 device_id < 100 AND area < 'South',
1716 device_id < 100 AND area >= 'South'
1717);"#;
1718 let stmt =
1719 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1720 .unwrap()
1721 .pop()
1722 .unwrap();
1723
1724 let Statement::AlterTable(alter_table) = stmt else {
1725 unreachable!()
1726 };
1727
1728 let request = to_repartition_request(alter_table, &QueryContext::arc()).unwrap();
1729 assert_eq!("greptime", request.catalog_name);
1730 assert_eq!("public", request.schema_name);
1731 assert_eq!("metrics", request.table_name);
1732 let RepartitionSource::Partitions {
1733 from_exprs,
1734 target_partition_columns,
1735 } = request.source
1736 else {
1737 unreachable!()
1738 };
1739 assert!(target_partition_columns.is_none());
1740 assert_eq!(
1741 from_exprs
1742 .into_iter()
1743 .map(|x| x.to_string())
1744 .collect::<Vec<_>>(),
1745 vec!["device_id < 100".to_string()]
1746 );
1747 assert_eq!(
1748 request
1749 .into_exprs
1750 .into_iter()
1751 .map(|x| x.to_string())
1752 .collect::<Vec<_>>(),
1753 vec![
1754 "device_id < 100 AND area < 'South'".to_string(),
1755 "device_id < 100 AND area >= 'South'".to_string()
1756 ]
1757 );
1758 }
1759
1760 #[test]
1761 fn test_to_repartition_request_with_target_partition_columns() {
1762 let sql = r#"
1763ALTER TABLE metrics REPARTITION (
1764 device_id < 100
1765) ON COLUMNS (device_id, area) INTO (
1766 device_id < 100 AND area < 'South',
1767 device_id < 100 AND area >= 'South'
1768);"#;
1769 let stmt =
1770 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1771 .unwrap()
1772 .pop()
1773 .unwrap();
1774
1775 let Statement::AlterTable(alter_table) = stmt else {
1776 unreachable!()
1777 };
1778
1779 let request = to_repartition_request(alter_table, &QueryContext::arc()).unwrap();
1780 let RepartitionSource::Partitions {
1781 target_partition_columns,
1782 ..
1783 } = request.source
1784 else {
1785 unreachable!()
1786 };
1787
1788 assert_eq!(
1789 target_partition_columns,
1790 Some(vec!["device_id".to_string(), "area".to_string()])
1791 );
1792 }
1793
1794 #[test]
1795 fn test_to_repartition_request_with_unpartitioned_source() {
1796 let sql = r#"
1797ALTER TABLE metrics PARTITION ON COLUMNS (device_id, area) (
1798 device_id < 100 AND area < 'South',
1799 device_id < 100 AND area >= 'South'
1800);"#;
1801 let stmt =
1802 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1803 .unwrap()
1804 .pop()
1805 .unwrap();
1806
1807 let Statement::AlterTable(alter_table) = stmt else {
1808 unreachable!()
1809 };
1810
1811 let request = to_repartition_request(alter_table, &QueryContext::arc()).unwrap();
1812 assert_eq!("greptime", request.catalog_name);
1813 assert_eq!("public", request.schema_name);
1814 assert_eq!("metrics", request.table_name);
1815 let RepartitionSource::Unpartitioned { partition_columns } = request.source else {
1816 unreachable!()
1817 };
1818 assert_eq!(partition_columns, vec!["device_id", "area"]);
1819 assert_eq!(
1820 request
1821 .into_exprs
1822 .into_iter()
1823 .map(|x| x.to_string())
1824 .collect::<Vec<_>>(),
1825 vec![
1826 "device_id < 100 AND area < 'South'".to_string(),
1827 "device_id < 100 AND area >= 'South'".to_string()
1828 ]
1829 );
1830 }
1831
1832 fn new_test_table_names() -> Vec<TableName> {
1833 vec![
1834 TableName {
1835 catalog_name: "greptime".to_string(),
1836 schema_name: "public".to_string(),
1837 table_name: "a_table".to_string(),
1838 },
1839 TableName {
1840 catalog_name: "greptime".to_string(),
1841 schema_name: "public".to_string(),
1842 table_name: "b_table".to_string(),
1843 },
1844 ]
1845 }
1846
1847 #[test]
1848 fn test_to_create_view_expr() {
1849 let sql = "CREATE VIEW test AS SELECT * FROM NUMBERS";
1850 let stmt =
1851 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1852 .unwrap()
1853 .pop()
1854 .unwrap();
1855
1856 let Statement::CreateView(stmt) = stmt else {
1857 unreachable!()
1858 };
1859
1860 let logical_plan = vec![1, 2, 3];
1861 let table_names = new_test_table_names();
1862 let columns = vec!["a".to_string()];
1863 let plan_columns = vec!["number".to_string()];
1864
1865 let expr = to_create_view_expr(
1866 stmt,
1867 logical_plan.clone(),
1868 table_names.clone(),
1869 columns.clone(),
1870 plan_columns.clone(),
1871 sql.to_string(),
1872 QueryContext::arc(),
1873 )
1874 .unwrap();
1875
1876 assert_eq!("greptime", expr.catalog_name);
1877 assert_eq!("public", expr.schema_name);
1878 assert_eq!("test", expr.view_name);
1879 assert!(!expr.create_if_not_exists);
1880 assert!(!expr.or_replace);
1881 assert_eq!(logical_plan, expr.logical_plan);
1882 assert_eq!(table_names, expr.table_names);
1883 assert_eq!(sql, expr.definition);
1884 assert_eq!(columns, expr.columns);
1885 assert_eq!(plan_columns, expr.plan_columns);
1886 }
1887
1888 #[test]
1889 fn test_to_create_view_expr_complex() {
1890 let sql = "CREATE OR REPLACE VIEW IF NOT EXISTS test.test_view AS SELECT * FROM NUMBERS";
1891 let stmt =
1892 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1893 .unwrap()
1894 .pop()
1895 .unwrap();
1896
1897 let Statement::CreateView(stmt) = stmt else {
1898 unreachable!()
1899 };
1900
1901 let logical_plan = vec![1, 2, 3];
1902 let table_names = new_test_table_names();
1903 let columns = vec!["a".to_string()];
1904 let plan_columns = vec!["number".to_string()];
1905
1906 let expr = to_create_view_expr(
1907 stmt,
1908 logical_plan.clone(),
1909 table_names.clone(),
1910 columns.clone(),
1911 plan_columns.clone(),
1912 sql.to_string(),
1913 QueryContext::arc(),
1914 )
1915 .unwrap();
1916
1917 assert_eq!("greptime", expr.catalog_name);
1918 assert_eq!("test", expr.schema_name);
1919 assert_eq!("test_view", expr.view_name);
1920 assert!(expr.create_if_not_exists);
1921 assert!(expr.or_replace);
1922 assert_eq!(logical_plan, expr.logical_plan);
1923 assert_eq!(table_names, expr.table_names);
1924 assert_eq!(sql, expr.definition);
1925 assert_eq!(columns, expr.columns);
1926 assert_eq!(plan_columns, expr.plan_columns);
1927 }
1928
1929 #[test]
1930 fn test_expr_to_create() {
1931 let sql = r#"CREATE TABLE IF NOT EXISTS `tt` (
1932 `timestamp` TIMESTAMP(9) NOT NULL,
1933 `ip_address` STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),
1934 `username` STRING NULL,
1935 `http_method` STRING NULL INVERTED INDEX,
1936 `request_line` STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false', false_positive_rate = '0.01', granularity = '10240'),
1937 `protocol` STRING NULL,
1938 `status_code` INT NULL INVERTED INDEX,
1939 `response_size` BIGINT NULL,
1940 `message` STRING NULL,
1941 TIME INDEX (`timestamp`),
1942 PRIMARY KEY (`username`, `status_code`)
1943)
1944ENGINE=mito
1945WITH(
1946 append_mode = 'true'
1947)"#;
1948 let stmt =
1949 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1950 .unwrap()
1951 .pop()
1952 .unwrap();
1953
1954 let Statement::CreateTable(original_create) = stmt else {
1955 unreachable!()
1956 };
1957
1958 let expr = create_to_expr(&original_create, &QueryContext::arc()).unwrap();
1960
1961 let create_table = expr_to_create(&expr, Some('`')).unwrap();
1962 let new_sql = format!("{:#}", create_table);
1963 assert_eq!(sql, new_sql);
1964 }
1965}