Skip to main content

operator/
expr_helper.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#[cfg(feature = "enterprise")]
16pub mod trigger;
17
18use std::collections::{HashMap, HashSet};
19
20use api::helper::ColumnDataTypeWrapper;
21use api::v1::alter_database_expr::Kind as AlterDatabaseKind;
22use api::v1::alter_table_expr::Kind as AlterTableKind;
23use api::v1::column_def::{options_from_column_schema, try_as_column_schema};
24use api::v1::{
25    AddColumn, AddColumns, AlterDatabaseExpr, AlterTableExpr, Analyzer, ColumnDataType,
26    ColumnDataTypeExtension, CreateFlowExpr, CreateTableExpr, CreateViewExpr, DropColumn,
27    DropColumns, DropDefaults, ExpireAfter, FulltextBackend as PbFulltextBackend, ModifyColumnType,
28    ModifyColumnTypes, RenameTable, SemanticType, SetDatabaseOptions, SetDefaults, SetFulltext,
29    SetIndex, SetIndexes, SetInverted, SetSkipping, SetTableOptions,
30    SkippingIndexType as PbSkippingIndexType, TableName, UnsetDatabaseOptions, UnsetFulltext,
31    UnsetIndex, UnsetIndexes, UnsetInverted, UnsetSkipping, UnsetTableOptions, set_index,
32    unset_index,
33};
34use common_datasource::object_store::LocalFileAccess;
35use common_error::ext::BoxedError;
36use common_grpc_expr::util::ColumnExpr;
37use common_time::Timezone;
38use datafusion::sql::planner::object_name_to_table_reference;
39use datatypes::schema::{
40    COLUMN_FULLTEXT_OPT_KEY_ANALYZER, COLUMN_FULLTEXT_OPT_KEY_BACKEND,
41    COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE, COLUMN_FULLTEXT_OPT_KEY_FALSE_POSITIVE_RATE,
42    COLUMN_FULLTEXT_OPT_KEY_GRANULARITY, COLUMN_SKIPPING_INDEX_OPT_KEY_FALSE_POSITIVE_RATE,
43    COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY, COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE, COMMENT_KEY,
44    ColumnDefaultConstraint, ColumnSchema, FulltextAnalyzer, FulltextBackend, Schema,
45    SkippingIndexType,
46};
47use file_engine::FileOptions;
48use query::sql::{
49    check_file_to_table_schema_compatibility, file_column_schemas_to_table,
50    infer_file_table_schema, prepare_file_table_files,
51};
52use session::context::QueryContextRef;
53use session::table_name::table_idents_to_full_name;
54use snafu::{OptionExt, ResultExt, ensure};
55use sql::ast::{
56    ColumnDef, ColumnOption, ColumnOptionDef, Expr, Ident, ObjectName, ObjectNamePartExt,
57};
58use sql::dialect::GreptimeDbDialect;
59use sql::parser::ParserContext;
60use sql::statements::alter::{
61    AlterDatabase, AlterDatabaseOperation, AlterTable, AlterTableOperation,
62};
63use sql::statements::create::{
64    Column as SqlColumn, ColumnExtensions, CreateExternalTable, CreateFlow, CreateTable,
65    CreateView, TableConstraint,
66};
67use sql::statements::{
68    OptionMap, column_to_schema, concrete_data_type_to_sql_data_type,
69    sql_column_def_to_grpc_column_def, sql_data_type_to_concrete_data_type, value_to_sql_value,
70};
71use sql::util::extract_tables_from_query;
72use store_api::mito_engine_options::{COMPACTION_OVERRIDE, COMPACTION_TYPE};
73use table::requests::{FILE_TABLE_META_KEY, TableOptions};
74use table::table_reference::TableReference;
75#[cfg(feature = "enterprise")]
76pub use trigger::to_create_trigger_task_expr;
77
78use crate::error::{
79    BuildCreateExprOnInsertionSnafu, ColumnDataTypeSnafu, ConvertColumnDefaultConstraintSnafu,
80    ConvertIdentifierSnafu, EncodeJsonSnafu, ExternalSnafu, FindNewColumnsOnInsertionSnafu,
81    IllegalPrimaryKeysDefSnafu, InferFileTableSchemaSnafu, InvalidColumnDefSnafu,
82    InvalidFlowNameSnafu, InvalidSqlSnafu, NotSupportedSnafu, ParseSqlSnafu, ParseSqlValueSnafu,
83    PrepareFileTableSnafu, Result, SchemaIncompatibleSnafu, UnrecognizedTableOptionSnafu,
84};
85
86pub fn create_table_expr_by_column_schemas(
87    table_name: &TableReference<'_>,
88    column_schemas: &[api::v1::ColumnSchema],
89    engine: &str,
90    desc: Option<&str>,
91) -> Result<CreateTableExpr> {
92    let column_exprs = ColumnExpr::from_column_schemas(column_schemas);
93    let expr = common_grpc_expr::util::build_create_table_expr(
94        None,
95        table_name,
96        column_exprs,
97        engine,
98        desc.unwrap_or("Created on insertion"),
99    )
100    .context(BuildCreateExprOnInsertionSnafu)?;
101
102    validate_create_expr(&expr)?;
103    Ok(expr)
104}
105
106pub fn extract_add_columns_expr(
107    schema: &Schema,
108    column_exprs: Vec<ColumnExpr>,
109) -> Result<Option<AddColumns>> {
110    let add_columns = common_grpc_expr::util::extract_new_columns(schema, column_exprs)
111        .context(FindNewColumnsOnInsertionSnafu)?;
112    if let Some(add_columns) = &add_columns {
113        validate_add_columns_expr(add_columns)?;
114    }
115    Ok(add_columns)
116}
117
118//   cpu float64,
119//   memory float64,
120//   TIME INDEX (ts),
121//   PRIMARY KEY(host)
122// ) WITH (location='/var/data/city.csv', format='csv');
123// ```
124// The user needs to specify the TIME INDEX column. If there is no suitable
125// column in the file to use as TIME INDEX, an additional placeholder column
126// needs to be created as the TIME INDEX, and a `DEFAULT <value>` constraint
127// should be added.
128//
129//
130// When the `CREATE EXTERNAL TABLE` statement is in inferred form, like
131// ```sql
132// CREATE EXTERNAL TABLE IF NOT EXISTS city WITH (location='/var/data/city.csv',format='csv');
133// ```
134// 1. If the TIME INDEX column can be inferred from metadata, use that column
135//    as the TIME INDEX. Otherwise,
136// 2. If a column named `greptime_timestamp` exists (with the requirement that
137//    the column is with type TIMESTAMP, otherwise an error is thrown), use
138//    that column as the TIME INDEX. Otherwise,
139// 3. Automatically create the `greptime_timestamp` column and add a `DEFAULT 0`
140//    constraint.
141pub(crate) async fn create_external_expr(
142    create: CreateExternalTable,
143    query_ctx: &QueryContextRef,
144    local_file_access: &LocalFileAccess,
145) -> Result<CreateTableExpr> {
146    let (catalog_name, schema_name, table_name) =
147        table_idents_to_full_name(&create.name, query_ctx)
148            .map_err(BoxedError::new)
149            .context(ExternalSnafu)?;
150
151    let mut table_options = create.options.into_map();
152
153    let (object_store, files) = prepare_file_table_files(&table_options, local_file_access)
154        .await
155        .context(PrepareFileTableSnafu)?;
156
157    let file_column_schemas = infer_file_table_schema(&object_store, &files, &table_options)
158        .await
159        .context(InferFileTableSchemaSnafu)?
160        .column_schemas()
161        .to_vec();
162
163    let (time_index, primary_keys, table_column_schemas) = if !create.columns.is_empty() {
164        // expanded form
165        let time_index = find_time_index(&create.constraints)?;
166        let primary_keys = find_primary_keys(&create.columns, &create.constraints)?;
167        let column_schemas =
168            columns_to_column_schemas(&create.columns, &time_index, Some(&query_ctx.timezone()))?;
169        (time_index, primary_keys, column_schemas)
170    } else {
171        // inferred form
172        let (column_schemas, time_index) = file_column_schemas_to_table(&file_column_schemas);
173        let primary_keys = vec![];
174        (time_index, primary_keys, column_schemas)
175    };
176
177    check_file_to_table_schema_compatibility(&file_column_schemas, &table_column_schemas)
178        .context(SchemaIncompatibleSnafu)?;
179
180    let meta = FileOptions {
181        files,
182        file_column_schemas,
183    };
184    table_options.insert(
185        FILE_TABLE_META_KEY.to_string(),
186        serde_json::to_string(&meta).context(EncodeJsonSnafu)?,
187    );
188
189    let column_defs = column_schemas_to_defs(table_column_schemas, &primary_keys)?;
190    let expr = CreateTableExpr {
191        catalog_name,
192        schema_name,
193        table_name,
194        desc: String::default(),
195        column_defs,
196        time_index,
197        primary_keys,
198        create_if_not_exists: create.if_not_exists,
199        table_options,
200        table_id: None,
201        engine: create.engine.clone(),
202    };
203
204    Ok(expr)
205}
206
207/// Convert `CreateTable` statement to [`CreateTableExpr`] gRPC request.
208pub fn create_to_expr(
209    create: &CreateTable,
210    query_ctx: &QueryContextRef,
211) -> Result<CreateTableExpr> {
212    let (catalog_name, schema_name, table_name) =
213        table_idents_to_full_name(&create.name, query_ctx)
214            .map_err(BoxedError::new)
215            .context(ExternalSnafu)?;
216
217    let time_index = find_time_index(&create.constraints)?;
218    let table_options = HashMap::from(
219        &TableOptions::try_from_iter(create.options.to_str_map())
220            .context(UnrecognizedTableOptionSnafu)?,
221    );
222
223    let mut table_options = table_options;
224    if table_options.contains_key(COMPACTION_TYPE) {
225        table_options.insert(COMPACTION_OVERRIDE.to_string(), "true".to_string());
226    }
227
228    let primary_keys = find_primary_keys(&create.columns, &create.constraints)?;
229
230    let expr = CreateTableExpr {
231        catalog_name,
232        schema_name,
233        table_name,
234        desc: String::default(),
235        column_defs: columns_to_expr(
236            &create.columns,
237            &time_index,
238            &primary_keys,
239            Some(&query_ctx.timezone()),
240        )?,
241        time_index,
242        primary_keys,
243        create_if_not_exists: create.if_not_exists,
244        table_options,
245        table_id: None,
246        engine: create.engine.clone(),
247    };
248
249    validate_create_expr(&expr)?;
250    Ok(expr)
251}
252
253/// Convert gRPC's [`CreateTableExpr`] back to `CreateTable` statement.
254/// You can use `create_table_expr_by_column_schemas` to create a `CreateTableExpr` from column schemas.
255///
256/// # Parameters
257///
258/// * `expr` - The `CreateTableExpr` to convert
259/// * `quote_style` - Optional quote style for identifiers (defaults to MySQL style ` backtick)
260pub fn expr_to_create(expr: &CreateTableExpr, quote_style: Option<char>) -> Result<CreateTable> {
261    let quote_style = quote_style.unwrap_or('`');
262
263    // Convert table name
264    let table_name = ObjectName(vec![sql::ast::ObjectNamePart::Identifier(
265        sql::ast::Ident::with_quote(quote_style, &expr.table_name),
266    )]);
267
268    // Convert columns
269    let mut columns = Vec::with_capacity(expr.column_defs.len());
270    for column_def in &expr.column_defs {
271        let column_schema = try_as_column_schema(column_def).context(InvalidColumnDefSnafu {
272            column: &column_def.name,
273        })?;
274
275        let mut options = Vec::new();
276
277        // Add NULL/NOT NULL constraint
278        if column_def.is_nullable {
279            options.push(ColumnOptionDef {
280                name: None,
281                option: ColumnOption::Null,
282            });
283        } else {
284            options.push(ColumnOptionDef {
285                name: None,
286                option: ColumnOption::NotNull,
287            });
288        }
289
290        // Add DEFAULT constraint if present
291        if let Some(default_constraint) = column_schema.default_constraint() {
292            let expr = match default_constraint {
293                ColumnDefaultConstraint::Value(v) => {
294                    Expr::Value(value_to_sql_value(v).context(ParseSqlValueSnafu)?.into())
295                }
296                ColumnDefaultConstraint::Function(func_expr) => {
297                    ParserContext::parse_function(func_expr, &GreptimeDbDialect {})
298                        .context(ParseSqlSnafu)?
299                }
300            };
301            options.push(ColumnOptionDef {
302                name: None,
303                option: ColumnOption::Default(expr),
304            });
305        }
306
307        // Add COMMENT if present
308        if !column_def.comment.is_empty() {
309            options.push(ColumnOptionDef {
310                name: None,
311                option: ColumnOption::Comment(column_def.comment.clone()),
312            });
313        }
314
315        // Note: We don't add inline PRIMARY KEY options here,
316        // we'll handle all primary keys as constraints instead for consistency
317
318        // Handle column extensions (fulltext, inverted index, skipping index)
319        let mut extensions = ColumnExtensions::default();
320
321        // Add fulltext index options if present
322        if let Ok(Some(opt)) = column_schema.fulltext_options()
323            && opt.enable
324        {
325            let mut map = HashMap::from([
326                (
327                    COLUMN_FULLTEXT_OPT_KEY_ANALYZER.to_string(),
328                    opt.analyzer.to_string(),
329                ),
330                (
331                    COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE.to_string(),
332                    opt.case_sensitive.to_string(),
333                ),
334                (
335                    COLUMN_FULLTEXT_OPT_KEY_BACKEND.to_string(),
336                    opt.backend.to_string(),
337                ),
338            ]);
339            if opt.backend == FulltextBackend::Bloom {
340                map.insert(
341                    COLUMN_FULLTEXT_OPT_KEY_GRANULARITY.to_string(),
342                    opt.granularity.to_string(),
343                );
344                map.insert(
345                    COLUMN_FULLTEXT_OPT_KEY_FALSE_POSITIVE_RATE.to_string(),
346                    opt.false_positive_rate().to_string(),
347                );
348            }
349            extensions.fulltext_index_options = Some(map.into());
350        }
351
352        // Add skipping index options if present
353        if let Ok(Some(opt)) = column_schema.skipping_index_options() {
354            let map = HashMap::from([
355                (
356                    COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY.to_string(),
357                    opt.granularity.to_string(),
358                ),
359                (
360                    COLUMN_SKIPPING_INDEX_OPT_KEY_FALSE_POSITIVE_RATE.to_string(),
361                    opt.false_positive_rate().to_string(),
362                ),
363                (
364                    COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE.to_string(),
365                    opt.index_type.to_string(),
366                ),
367            ]);
368            extensions.skipping_index_options = Some(map.into());
369        }
370
371        // Add inverted index options if present
372        if column_schema.is_inverted_indexed() {
373            extensions.inverted_index_options = Some(HashMap::new().into());
374        }
375
376        let sql_column = SqlColumn {
377            column_def: ColumnDef {
378                name: Ident::with_quote(quote_style, &column_def.name),
379                data_type: concrete_data_type_to_sql_data_type(&column_schema.data_type)
380                    .context(ParseSqlSnafu)?,
381                options,
382            },
383            extensions,
384        };
385
386        columns.push(sql_column);
387    }
388
389    // Convert constraints
390    let mut constraints = Vec::new();
391
392    // Add TIME INDEX constraint
393    constraints.push(TableConstraint::TimeIndex {
394        column: Ident::with_quote(quote_style, &expr.time_index),
395    });
396
397    // Add PRIMARY KEY constraint (always add as constraint for consistency)
398    if !expr.primary_keys.is_empty() {
399        let primary_key_columns: Vec<Ident> = expr
400            .primary_keys
401            .iter()
402            .map(|pk| Ident::with_quote(quote_style, pk))
403            .collect();
404
405        constraints.push(TableConstraint::PrimaryKey {
406            columns: primary_key_columns,
407        });
408    }
409
410    // Convert table options
411    let mut options = OptionMap::default();
412    for (key, value) in &expr.table_options {
413        options.insert(key.clone(), value.clone());
414    }
415
416    Ok(CreateTable {
417        if_not_exists: expr.create_if_not_exists,
418        table_id: expr.table_id.as_ref().map(|tid| tid.id).unwrap_or(0),
419        name: table_name,
420        columns,
421        engine: expr.engine.clone(),
422        constraints,
423        options,
424        partitions: None,
425    })
426}
427
428/// Validate the [`CreateTableExpr`] request.
429pub fn validate_create_expr(create: &CreateTableExpr) -> Result<()> {
430    // construct column list
431    let mut column_to_indices = HashMap::with_capacity(create.column_defs.len());
432    for (idx, column) in create.column_defs.iter().enumerate() {
433        if let Some(indices) = column_to_indices.get(&column.name) {
434            return InvalidSqlSnafu {
435                err_msg: format!(
436                    "column name `{}` is duplicated at index {} and {}",
437                    column.name, indices, idx
438                ),
439            }
440            .fail();
441        }
442        column_to_indices.insert(&column.name, idx);
443    }
444
445    // verify time_index exists
446    let _ = column_to_indices
447        .get(&create.time_index)
448        .with_context(|| InvalidSqlSnafu {
449            err_msg: format!(
450                "column name `{}` is not found in column list",
451                create.time_index
452            ),
453        })?;
454
455    // verify primary_key exists
456    for pk in &create.primary_keys {
457        let _ = column_to_indices
458            .get(&pk)
459            .with_context(|| InvalidSqlSnafu {
460                err_msg: format!("column name `{}` is not found in column list", pk),
461            })?;
462    }
463
464    // construct primary_key set
465    let mut pk_set = HashSet::new();
466    for pk in &create.primary_keys {
467        if !pk_set.insert(pk) {
468            return InvalidSqlSnafu {
469                err_msg: format!("column name `{}` is duplicated in primary keys", pk),
470            }
471            .fail();
472        }
473    }
474
475    // verify time index is not primary key
476    if pk_set.contains(&create.time_index) {
477        return InvalidSqlSnafu {
478            err_msg: format!(
479                "column name `{}` is both primary key and time index",
480                create.time_index
481            ),
482        }
483        .fail();
484    }
485
486    for column in &create.column_defs {
487        // verify do not contain interval type column issue #3235
488        if is_interval_type(&column.data_type()) {
489            return InvalidSqlSnafu {
490                err_msg: format!(
491                    "column name `{}` is interval type, which is not supported",
492                    column.name
493                ),
494            }
495            .fail();
496        }
497        // verify do not contain datetime type column issue #5489
498        if is_date_time_type(&column.data_type()) {
499            return InvalidSqlSnafu {
500                err_msg: format!(
501                    "column name `{}` is datetime type, which is not supported, please use `timestamp` type instead",
502                    column.name
503                ),
504            }
505            .fail();
506        }
507    }
508    Ok(())
509}
510
511fn validate_add_columns_expr(add_columns: &AddColumns) -> Result<()> {
512    for add_column in &add_columns.add_columns {
513        let Some(column_def) = &add_column.column_def else {
514            continue;
515        };
516        if is_date_time_type(&column_def.data_type()) {
517            return InvalidSqlSnafu {
518                    err_msg: format!("column name `{}` is datetime type, which is not supported, please use `timestamp` type instead", column_def.name),
519                }
520                .fail();
521        }
522        if is_interval_type(&column_def.data_type()) {
523            return InvalidSqlSnafu {
524                err_msg: format!(
525                    "column name `{}` is interval type, which is not supported",
526                    column_def.name
527                ),
528            }
529            .fail();
530        }
531    }
532    Ok(())
533}
534
535fn is_date_time_type(data_type: &ColumnDataType) -> bool {
536    matches!(data_type, ColumnDataType::Datetime)
537}
538
539fn is_interval_type(data_type: &ColumnDataType) -> bool {
540    matches!(
541        data_type,
542        ColumnDataType::IntervalYearMonth
543            | ColumnDataType::IntervalDayTime
544            | ColumnDataType::IntervalMonthDayNano
545    )
546}
547
548fn find_primary_keys(
549    columns: &[SqlColumn],
550    constraints: &[TableConstraint],
551) -> Result<Vec<String>> {
552    let columns_pk = columns
553        .iter()
554        .filter_map(|x| {
555            if x.options()
556                .iter()
557                .any(|o| matches!(o.option, ColumnOption::PrimaryKey(_)))
558            {
559                Some(x.name().value.clone())
560            } else {
561                None
562            }
563        })
564        .collect::<Vec<String>>();
565
566    ensure!(
567        columns_pk.len() <= 1,
568        IllegalPrimaryKeysDefSnafu {
569            msg: "not allowed to inline multiple primary keys in columns options"
570        }
571    );
572
573    let constraints_pk = constraints
574        .iter()
575        .filter_map(|constraint| match constraint {
576            TableConstraint::PrimaryKey { columns, .. } => {
577                Some(columns.iter().map(|ident| ident.value.clone()))
578            }
579            _ => None,
580        })
581        .flatten()
582        .collect::<Vec<String>>();
583
584    ensure!(
585        columns_pk.is_empty() || constraints_pk.is_empty(),
586        IllegalPrimaryKeysDefSnafu {
587            msg: "found definitions of primary keys in multiple places"
588        }
589    );
590
591    let mut primary_keys = Vec::with_capacity(columns_pk.len() + constraints_pk.len());
592    primary_keys.extend(columns_pk);
593    primary_keys.extend(constraints_pk);
594    Ok(primary_keys)
595}
596
597pub fn find_time_index(constraints: &[TableConstraint]) -> Result<String> {
598    let time_index = constraints
599        .iter()
600        .filter_map(|constraint| match constraint {
601            TableConstraint::TimeIndex { column, .. } => Some(&column.value),
602            _ => None,
603        })
604        .collect::<Vec<&String>>();
605    ensure!(
606        time_index.len() == 1,
607        InvalidSqlSnafu {
608            err_msg: "must have one and only one TimeIndex columns",
609        }
610    );
611    Ok(time_index[0].clone())
612}
613
614fn columns_to_expr(
615    column_defs: &[SqlColumn],
616    time_index: &str,
617    primary_keys: &[String],
618    timezone: Option<&Timezone>,
619) -> Result<Vec<api::v1::ColumnDef>> {
620    let column_schemas = columns_to_column_schemas(column_defs, time_index, timezone)?;
621    column_schemas_to_defs(column_schemas, primary_keys)
622}
623
624fn columns_to_column_schemas(
625    columns: &[SqlColumn],
626    time_index: &str,
627    timezone: Option<&Timezone>,
628) -> Result<Vec<ColumnSchema>> {
629    columns
630        .iter()
631        .map(|c| column_to_schema(c, time_index, timezone).context(ParseSqlSnafu))
632        .collect::<Result<Vec<ColumnSchema>>>()
633}
634
635// TODO(weny): refactor this function to use `try_as_column_def`
636pub fn column_schemas_to_defs(
637    column_schemas: Vec<ColumnSchema>,
638    primary_keys: &[String],
639) -> Result<Vec<api::v1::ColumnDef>> {
640    let column_datatypes: Vec<(ColumnDataType, Option<ColumnDataTypeExtension>)> = column_schemas
641        .iter()
642        .map(|c| {
643            ColumnDataTypeWrapper::try_from(c.data_type.clone())
644                .map(|w| w.to_parts())
645                .context(ColumnDataTypeSnafu)
646        })
647        .collect::<Result<Vec<_>>>()?;
648
649    column_schemas
650        .iter()
651        .zip(column_datatypes)
652        .map(|(schema, datatype)| {
653            let semantic_type = if schema.is_time_index() {
654                SemanticType::Timestamp
655            } else if primary_keys.contains(&schema.name) {
656                SemanticType::Tag
657            } else {
658                SemanticType::Field
659            } as i32;
660            let comment = schema
661                .metadata()
662                .get(COMMENT_KEY)
663                .cloned()
664                .unwrap_or_default();
665
666            Ok(api::v1::ColumnDef {
667                name: schema.name.clone(),
668                data_type: datatype.0 as i32,
669                is_nullable: schema.is_nullable(),
670                default_constraint: match schema.default_constraint() {
671                    None => vec![],
672                    Some(v) => {
673                        v.clone()
674                            .try_into()
675                            .context(ConvertColumnDefaultConstraintSnafu {
676                                column_name: &schema.name,
677                            })?
678                    }
679                },
680                semantic_type,
681                comment,
682                datatype_extension: datatype.1,
683                options: options_from_column_schema(schema),
684            })
685        })
686        .collect()
687}
688
689#[derive(Debug, Clone, PartialEq, Eq)]
690pub struct RepartitionRequest {
691    pub catalog_name: String,
692    pub schema_name: String,
693    pub table_name: String,
694    pub source: RepartitionSource,
695    pub into_exprs: Vec<Expr>,
696    pub options: OptionMap,
697}
698
699#[derive(Debug, Clone, PartialEq, Eq)]
700pub enum RepartitionSource {
701    Partitions {
702        from_exprs: Vec<Expr>,
703        target_partition_columns: Option<Vec<String>>,
704    },
705    Unpartitioned {
706        partition_columns: Vec<String>,
707    },
708}
709
710pub(crate) fn to_repartition_request(
711    alter_table: AlterTable,
712    query_ctx: &QueryContextRef,
713) -> Result<RepartitionRequest> {
714    let AlterTable {
715        table_name,
716        alter_operation,
717        options,
718    } = alter_table;
719
720    let (catalog_name, schema_name, table_name) = table_idents_to_full_name(&table_name, query_ctx)
721        .map_err(BoxedError::new)
722        .context(ExternalSnafu)?;
723
724    let (source, into_exprs) = match alter_operation {
725        AlterTableOperation::Repartition { operation } => (
726            RepartitionSource::Partitions {
727                from_exprs: operation.from_exprs,
728                target_partition_columns: operation.partition_columns.map(|columns| {
729                    columns
730                        .into_iter()
731                        .map(|ident| ident.value)
732                        .collect::<Vec<_>>()
733                }),
734            },
735            operation.into_exprs,
736        ),
737        AlterTableOperation::Partition { partitions } => (
738            RepartitionSource::Unpartitioned {
739                partition_columns: partitions
740                    .column_list
741                    .into_iter()
742                    .map(|ident| ident.value)
743                    .collect(),
744            },
745            partitions.exprs,
746        ),
747        _ => {
748            return InvalidSqlSnafu {
749                err_msg: "expected REPARTITION or PARTITION operation",
750            }
751            .fail();
752        }
753    };
754
755    Ok(RepartitionRequest {
756        catalog_name,
757        schema_name,
758        table_name,
759        source,
760        into_exprs,
761        options,
762    })
763}
764
765/// Converts a SQL alter table statement into a gRPC alter table expression.
766pub(crate) fn to_alter_table_expr(
767    alter_table: AlterTable,
768    query_ctx: &QueryContextRef,
769) -> Result<AlterTableExpr> {
770    let (catalog_name, schema_name, table_name) =
771        table_idents_to_full_name(alter_table.table_name(), query_ctx)
772            .map_err(BoxedError::new)
773            .context(ExternalSnafu)?;
774
775    let kind = match alter_table.alter_operation {
776        AlterTableOperation::AddConstraint(_) => {
777            return NotSupportedSnafu {
778                feat: "ADD CONSTRAINT",
779            }
780            .fail();
781        }
782        AlterTableOperation::AddColumns { add_columns } => AlterTableKind::AddColumns(AddColumns {
783            add_columns: add_columns
784                .into_iter()
785                .map(|add_column| {
786                    let column_def = sql_column_def_to_grpc_column_def(
787                        &add_column.column_def,
788                        Some(&query_ctx.timezone()),
789                    )
790                    .map_err(BoxedError::new)
791                    .context(ExternalSnafu)?;
792                    if is_interval_type(&column_def.data_type()) {
793                        return NotSupportedSnafu {
794                            feat: "Add column with interval type",
795                        }
796                        .fail();
797                    }
798                    Ok(AddColumn {
799                        column_def: Some(column_def),
800                        location: add_column.location.as_ref().map(From::from),
801                        add_if_not_exists: add_column.add_if_not_exists,
802                    })
803                })
804                .collect::<Result<Vec<AddColumn>>>()?,
805        }),
806        AlterTableOperation::ModifyColumnType {
807            column_name,
808            target_type,
809        } => {
810            let target_type =
811                sql_data_type_to_concrete_data_type(&target_type).context(ParseSqlSnafu)?;
812            let (target_type, target_type_extension) = ColumnDataTypeWrapper::try_from(target_type)
813                .map(|w| w.to_parts())
814                .context(ColumnDataTypeSnafu)?;
815            if is_interval_type(&target_type) {
816                return NotSupportedSnafu {
817                    feat: "Modify column type to interval type",
818                }
819                .fail();
820            }
821            AlterTableKind::ModifyColumnTypes(ModifyColumnTypes {
822                modify_column_types: vec![ModifyColumnType {
823                    column_name: column_name.value,
824                    target_type: target_type as i32,
825                    target_type_extension,
826                }],
827            })
828        }
829        AlterTableOperation::DropColumn { name } => AlterTableKind::DropColumns(DropColumns {
830            drop_columns: vec![DropColumn {
831                name: name.value.clone(),
832            }],
833        }),
834        AlterTableOperation::RenameTable { new_table_name } => {
835            AlterTableKind::RenameTable(RenameTable {
836                new_table_name: new_table_name.clone(),
837            })
838        }
839        AlterTableOperation::SetTableOptions { options } => {
840            AlterTableKind::SetTableOptions(SetTableOptions {
841                table_options: options.into_iter().map(Into::into).collect(),
842            })
843        }
844        AlterTableOperation::UnsetTableOptions { keys } => {
845            AlterTableKind::UnsetTableOptions(UnsetTableOptions { keys })
846        }
847        AlterTableOperation::Repartition { .. } => {
848            return NotSupportedSnafu {
849                feat: "ALTER TABLE ... REPARTITION",
850            }
851            .fail();
852        }
853        AlterTableOperation::Partition { .. } => {
854            return NotSupportedSnafu {
855                feat: "ALTER TABLE ... PARTITION ON COLUMNS",
856            }
857            .fail();
858        }
859        AlterTableOperation::SetIndex { options } => {
860            let option = match options {
861                sql::statements::alter::SetIndexOperation::Fulltext {
862                    column_name,
863                    options,
864                } => SetIndex {
865                    options: Some(set_index::Options::Fulltext(SetFulltext {
866                        column_name: column_name.value,
867                        enable: options.enable,
868                        analyzer: match options.analyzer {
869                            FulltextAnalyzer::English => Analyzer::English.into(),
870                            FulltextAnalyzer::Chinese => Analyzer::Chinese.into(),
871                        },
872                        case_sensitive: options.case_sensitive,
873                        backend: match options.backend {
874                            FulltextBackend::Bloom => PbFulltextBackend::Bloom.into(),
875                            FulltextBackend::Tantivy => PbFulltextBackend::Tantivy.into(),
876                        },
877                        granularity: options.granularity as u64,
878                        false_positive_rate: options.false_positive_rate(),
879                    })),
880                },
881                sql::statements::alter::SetIndexOperation::Inverted { column_name } => SetIndex {
882                    options: Some(set_index::Options::Inverted(SetInverted {
883                        column_name: column_name.value,
884                    })),
885                },
886                sql::statements::alter::SetIndexOperation::Skipping {
887                    column_name,
888                    options,
889                } => SetIndex {
890                    options: Some(set_index::Options::Skipping(SetSkipping {
891                        column_name: column_name.value,
892                        enable: true,
893                        granularity: options.granularity as u64,
894                        false_positive_rate: options.false_positive_rate(),
895                        skipping_index_type: match options.index_type {
896                            SkippingIndexType::BloomFilter => {
897                                PbSkippingIndexType::BloomFilter.into()
898                            }
899                        },
900                    })),
901                },
902            };
903            AlterTableKind::SetIndexes(SetIndexes {
904                set_indexes: vec![option],
905            })
906        }
907        AlterTableOperation::UnsetIndex { options } => {
908            let option = match options {
909                sql::statements::alter::UnsetIndexOperation::Fulltext { column_name } => {
910                    UnsetIndex {
911                        options: Some(unset_index::Options::Fulltext(UnsetFulltext {
912                            column_name: column_name.value,
913                        })),
914                    }
915                }
916                sql::statements::alter::UnsetIndexOperation::Inverted { column_name } => {
917                    UnsetIndex {
918                        options: Some(unset_index::Options::Inverted(UnsetInverted {
919                            column_name: column_name.value,
920                        })),
921                    }
922                }
923                sql::statements::alter::UnsetIndexOperation::Skipping { column_name } => {
924                    UnsetIndex {
925                        options: Some(unset_index::Options::Skipping(UnsetSkipping {
926                            column_name: column_name.value,
927                        })),
928                    }
929                }
930            };
931
932            AlterTableKind::UnsetIndexes(UnsetIndexes {
933                unset_indexes: vec![option],
934            })
935        }
936        AlterTableOperation::DropDefaults { columns } => {
937            AlterTableKind::DropDefaults(DropDefaults {
938                drop_defaults: columns
939                    .into_iter()
940                    .map(|col| {
941                        let column_name = col.0.to_string();
942                        Ok(api::v1::DropDefault { column_name })
943                    })
944                    .collect::<Result<Vec<_>>>()?,
945            })
946        }
947        AlterTableOperation::SetDefaults { defaults } => AlterTableKind::SetDefaults(SetDefaults {
948            set_defaults: defaults
949                .into_iter()
950                .map(|col| {
951                    let column_name = col.column_name.to_string();
952                    let default_constraint = serde_json::to_string(&col.default_constraint)
953                        .context(EncodeJsonSnafu)?
954                        .into_bytes();
955                    Ok(api::v1::SetDefault {
956                        column_name,
957                        default_constraint,
958                    })
959                })
960                .collect::<Result<Vec<_>>>()?,
961        }),
962    };
963
964    Ok(AlterTableExpr {
965        catalog_name,
966        schema_name,
967        table_name,
968        kind: Some(kind),
969    })
970}
971
972/// Try to cast the `[AlterDatabase]` statement into gRPC `[AlterDatabaseExpr]`.
973pub fn to_alter_database_expr(
974    alter_database: AlterDatabase,
975    query_ctx: &QueryContextRef,
976) -> Result<AlterDatabaseExpr> {
977    let catalog = query_ctx.current_catalog();
978    let schema = alter_database.database_name;
979
980    let kind = match alter_database.alter_operation {
981        AlterDatabaseOperation::SetDatabaseOption { options } => {
982            let options = options.into_iter().map(Into::into).collect();
983            AlterDatabaseKind::SetDatabaseOptions(SetDatabaseOptions {
984                set_database_options: options,
985            })
986        }
987        AlterDatabaseOperation::UnsetDatabaseOption { keys } => {
988            AlterDatabaseKind::UnsetDatabaseOptions(UnsetDatabaseOptions { keys })
989        }
990    };
991
992    Ok(AlterDatabaseExpr {
993        catalog_name: catalog.to_string(),
994        schema_name: schema.to_string(),
995        kind: Some(kind),
996    })
997}
998
999/// Try to cast the `[CreateViewExpr]` statement into gRPC `[CreateViewExpr]`.
1000pub fn to_create_view_expr(
1001    stmt: CreateView,
1002    logical_plan: Vec<u8>,
1003    table_names: Vec<TableName>,
1004    columns: Vec<String>,
1005    plan_columns: Vec<String>,
1006    definition: String,
1007    query_ctx: QueryContextRef,
1008) -> Result<CreateViewExpr> {
1009    let (catalog_name, schema_name, view_name) = table_idents_to_full_name(&stmt.name, &query_ctx)
1010        .map_err(BoxedError::new)
1011        .context(ExternalSnafu)?;
1012
1013    let expr = CreateViewExpr {
1014        catalog_name,
1015        schema_name,
1016        view_name,
1017        logical_plan,
1018        create_if_not_exists: stmt.if_not_exists,
1019        or_replace: stmt.or_replace,
1020        table_names,
1021        columns,
1022        plan_columns,
1023        definition,
1024    };
1025
1026    Ok(expr)
1027}
1028
1029pub fn to_create_flow_task_expr(
1030    create_flow: CreateFlow,
1031    query_ctx: &QueryContextRef,
1032) -> Result<CreateFlowExpr> {
1033    // retrieve sink table name
1034    let sink_table_ref = object_name_to_table_reference(create_flow.sink_table_name.clone(), true)
1035        .with_context(|_| ConvertIdentifierSnafu {
1036            ident: create_flow.sink_table_name.to_string(),
1037        })?;
1038    let catalog = sink_table_ref
1039        .catalog()
1040        .unwrap_or(query_ctx.current_catalog())
1041        .to_string();
1042    let schema = sink_table_ref
1043        .schema()
1044        .map(|s| s.to_owned())
1045        .unwrap_or(query_ctx.current_schema());
1046
1047    let sink_table_name = TableName {
1048        catalog_name: catalog,
1049        schema_name: schema,
1050        table_name: sink_table_ref.table().to_string(),
1051    };
1052
1053    let source_table_names = extract_tables_from_query(&create_flow.query)
1054        .map(|name| {
1055            let reference =
1056                object_name_to_table_reference(name.clone(), true).with_context(|_| {
1057                    ConvertIdentifierSnafu {
1058                        ident: name.to_string(),
1059                    }
1060                })?;
1061            let catalog = reference
1062                .catalog()
1063                .unwrap_or(query_ctx.current_catalog())
1064                .to_string();
1065            let schema = reference
1066                .schema()
1067                .map(|s| s.to_string())
1068                .unwrap_or(query_ctx.current_schema());
1069
1070            let table_name = TableName {
1071                catalog_name: catalog,
1072                schema_name: schema,
1073                table_name: reference.table().to_string(),
1074            };
1075            Ok(table_name)
1076        })
1077        .collect::<Result<Vec<_>>>()?;
1078
1079    let eval_interval = create_flow.eval_interval;
1080
1081    Ok(CreateFlowExpr {
1082        catalog_name: query_ctx.current_catalog().to_string(),
1083        flow_name: sanitize_flow_name(create_flow.flow_name)?,
1084        source_table_names,
1085        sink_table_name: Some(sink_table_name),
1086        or_replace: create_flow.or_replace,
1087        create_if_not_exists: create_flow.if_not_exists,
1088        expire_after: create_flow.expire_after.map(|value| ExpireAfter { value }),
1089        eval_interval: eval_interval.map(|seconds| api::v1::EvalInterval { seconds }),
1090        comment: create_flow.comment.unwrap_or_default(),
1091        sql: create_flow.query.to_string(),
1092        flow_options: stringify_flow_options(create_flow.flow_options)?,
1093    })
1094}
1095
1096fn stringify_flow_options(flow_options: OptionMap) -> Result<HashMap<String, String>> {
1097    let options_len = flow_options.len();
1098    let flow_options = flow_options.into_map();
1099    ensure!(
1100        flow_options.len() == options_len,
1101        InvalidSqlSnafu {
1102            err_msg: "flow options only support scalar string-compatible values".to_string(),
1103        }
1104    );
1105    Ok(flow_options)
1106}
1107
1108/// sanitize the flow name, remove possible quotes
1109fn sanitize_flow_name(mut flow_name: ObjectName) -> Result<String> {
1110    ensure!(
1111        flow_name.0.len() == 1,
1112        InvalidFlowNameSnafu {
1113            name: flow_name.to_string(),
1114        }
1115    );
1116    // safety: we've checked flow_name.0 has exactly one element.
1117    Ok(flow_name.0.swap_remove(0).to_string_unquoted())
1118}
1119
1120#[cfg(test)]
1121mod tests {
1122    use std::collections::HashMap;
1123
1124    use api::v1::{SetDatabaseOptions, UnsetDatabaseOptions};
1125    use datatypes::value::Value;
1126    use session::context::{QueryContext, QueryContextBuilder};
1127    use sql::dialect::GreptimeDbDialect;
1128    use sql::parser::{ParseOptions, ParserContext};
1129    use sql::statements::statement::Statement;
1130    use store_api::storage::ColumnDefaultConstraint;
1131
1132    use super::*;
1133
1134    #[test]
1135    fn test_create_flow_tql_expr() {
1136        let sql = r#"
1137CREATE FLOW calc_reqs SINK TO cnt_reqs AS
1138TQL EVAL (0, 15, '5s') count_values("status_code", http_requests);"#;
1139        let stmt =
1140            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default());
1141
1142        assert!(
1143            stmt.is_err(),
1144            "Expected error for invalid TQL EVAL parameters: {:#?}",
1145            stmt
1146        );
1147
1148        let sql = r#"
1149CREATE FLOW calc_reqs SINK TO cnt_reqs AS
1150TQL EVAL (now() - '15s'::interval, now(), '5s') count_values("status_code", http_requests);"#;
1151        let stmt =
1152            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1153                .unwrap()
1154                .pop()
1155                .unwrap();
1156
1157        let Statement::CreateFlow(create_flow) = stmt else {
1158            unreachable!()
1159        };
1160        let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1161
1162        let to_dot_sep =
1163            |c: TableName| format!("{}.{}.{}", c.catalog_name, c.schema_name, c.table_name);
1164        assert_eq!("calc_reqs", expr.flow_name);
1165        assert_eq!("greptime", expr.catalog_name);
1166        assert_eq!(
1167            "greptime.public.cnt_reqs",
1168            expr.sink_table_name.map(to_dot_sep).unwrap()
1169        );
1170        assert_eq!(1, expr.source_table_names.len());
1171        assert_eq!(
1172            "greptime.public.http_requests",
1173            to_dot_sep(expr.source_table_names[0].clone())
1174        );
1175        assert_eq!(
1176            r#"TQL EVAL (now() - '15s'::interval, now(), '5s') count_values("status_code", http_requests)"#,
1177            expr.sql
1178        );
1179
1180        let sql = r#"
1181CREATE FLOW calc_reqs SINK TO cnt_reqs AS
1182TQL EVAL (now() - '15s'::interval, now(), '5s') count_values("status_code", http_requests{__schema__="greptime_private"});"#;
1183        let stmt =
1184            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1185                .unwrap()
1186                .pop()
1187                .unwrap();
1188        let Statement::CreateFlow(create_flow) = stmt else {
1189            unreachable!()
1190        };
1191        let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1192        assert_eq!(1, expr.source_table_names.len());
1193        assert_eq!(
1194            "greptime.greptime_private.http_requests",
1195            to_dot_sep(expr.source_table_names[0].clone())
1196        );
1197
1198        let sql = r#"
1199CREATE FLOW calc_reqs SINK TO cnt_reqs AS
1200TQL EVAL (now() - '15s'::interval, now(), '5s') count_values("status_code", http_requests{__database__="greptime_private"});"#;
1201        let stmt =
1202            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1203                .unwrap()
1204                .pop()
1205                .unwrap();
1206        let Statement::CreateFlow(create_flow) = stmt else {
1207            unreachable!()
1208        };
1209        let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1210        assert_eq!(1, expr.source_table_names.len());
1211        assert_eq!(
1212            "greptime.greptime_private.http_requests",
1213            to_dot_sep(expr.source_table_names[0].clone())
1214        );
1215    }
1216
1217    #[test]
1218    fn test_create_flow_tql_cte_source_tables() {
1219        let sql = r#"
1220CREATE FLOW calc_cte
1221SINK TO metric_cte_sink
1222EVAL INTERVAL '1m'
1223AS
1224WITH tql(ts, the_value) AS (
1225  TQL EVAL (now() - '1m'::interval, now(), '5s') metric_cte
1226)
1227SELECT * FROM tql;
1228"#;
1229
1230        let stmt =
1231            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1232                .unwrap()
1233                .pop()
1234                .unwrap();
1235
1236        let Statement::CreateFlow(create_flow) = stmt else {
1237            unreachable!()
1238        };
1239        let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1240
1241        let to_dot_sep =
1242            |c: TableName| format!("{}.{}.{}", c.catalog_name, c.schema_name, c.table_name);
1243        assert_eq!(1, expr.source_table_names.len());
1244        assert_eq!(
1245            "greptime.public.metric_cte",
1246            to_dot_sep(expr.source_table_names[0].clone())
1247        );
1248    }
1249
1250    #[test]
1251    fn test_create_flow_tql_cte_source_tables_quoted_cte_name() {
1252        let sql = r#"
1253CREATE FLOW calc_cte
1254SINK TO metric_cte_sink
1255EVAL INTERVAL '1m'
1256AS
1257WITH "TQL"(ts, the_value) AS (
1258  TQL EVAL (now() - '1m'::interval, now(), '5s') metric_cte
1259)
1260SELECT * FROM "TQL";
1261"#;
1262
1263        let stmt =
1264            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1265                .unwrap()
1266                .pop()
1267                .unwrap();
1268
1269        let Statement::CreateFlow(create_flow) = stmt else {
1270            unreachable!()
1271        };
1272        let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1273
1274        let to_dot_sep =
1275            |c: TableName| format!("{}.{}.{}", c.catalog_name, c.schema_name, c.table_name);
1276        assert_eq!(1, expr.source_table_names.len());
1277        assert_eq!(
1278            "greptime.public.metric_cte",
1279            to_dot_sep(expr.source_table_names[0].clone())
1280        );
1281    }
1282
1283    #[test]
1284    fn test_create_flow_tql_cte_source_tables_same_name() {
1285        let sql = r#"
1286CREATE FLOW calc_cte
1287SINK TO metric_cte_sink
1288EVAL INTERVAL '1m'
1289AS
1290WITH tql(ts, the_value) AS (
1291  TQL EVAL (now() - '1m'::interval, now(), '5s') tql
1292)
1293SELECT * FROM tql;
1294"#;
1295
1296        let stmt =
1297            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1298                .unwrap()
1299                .pop()
1300                .unwrap();
1301
1302        let Statement::CreateFlow(create_flow) = stmt else {
1303            unreachable!()
1304        };
1305        let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1306
1307        let to_dot_sep =
1308            |c: TableName| format!("{}.{}.{}", c.catalog_name, c.schema_name, c.table_name);
1309        assert_eq!(1, expr.source_table_names.len());
1310        assert_eq!(
1311            "greptime.public.tql",
1312            to_dot_sep(expr.source_table_names[0].clone())
1313        );
1314    }
1315
1316    #[test]
1317    fn test_create_flow_expr() {
1318        let sql = r"
1319CREATE FLOW test_distinct_basic SINK TO out_distinct_basic AS
1320SELECT
1321    DISTINCT number as dis
1322FROM
1323    distinct_basic;";
1324        let stmt =
1325            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1326                .unwrap()
1327                .pop()
1328                .unwrap();
1329
1330        let Statement::CreateFlow(create_flow) = stmt else {
1331            unreachable!()
1332        };
1333        let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1334
1335        let to_dot_sep =
1336            |c: TableName| format!("{}.{}.{}", c.catalog_name, c.schema_name, c.table_name);
1337        assert_eq!("test_distinct_basic", expr.flow_name);
1338        assert_eq!("greptime", expr.catalog_name);
1339        assert_eq!(
1340            "greptime.public.out_distinct_basic",
1341            expr.sink_table_name.map(to_dot_sep).unwrap()
1342        );
1343        assert_eq!(1, expr.source_table_names.len());
1344        assert_eq!(
1345            "greptime.public.distinct_basic",
1346            to_dot_sep(expr.source_table_names[0].clone())
1347        );
1348        assert_eq!(
1349            r"SELECT
1350    DISTINCT number as dis
1351FROM
1352    distinct_basic",
1353            expr.sql
1354        );
1355
1356        let sql = r"
1357CREATE FLOW `task_2`
1358SINK TO schema_1.table_1
1359AS
1360SELECT max(c1), min(c2) FROM schema_2.table_2;";
1361        let stmt =
1362            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1363                .unwrap()
1364                .pop()
1365                .unwrap();
1366
1367        let Statement::CreateFlow(create_flow) = stmt else {
1368            unreachable!()
1369        };
1370        let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1371
1372        let to_dot_sep =
1373            |c: TableName| format!("{}.{}.{}", c.catalog_name, c.schema_name, c.table_name);
1374        assert_eq!("task_2", expr.flow_name);
1375        assert_eq!("greptime", expr.catalog_name);
1376        assert_eq!(
1377            "greptime.schema_1.table_1",
1378            expr.sink_table_name.map(to_dot_sep).unwrap()
1379        );
1380        assert_eq!(1, expr.source_table_names.len());
1381        assert_eq!(
1382            "greptime.schema_2.table_2",
1383            to_dot_sep(expr.source_table_names[0].clone())
1384        );
1385        assert_eq!("SELECT max(c1), min(c2) FROM schema_2.table_2", expr.sql);
1386        assert!(expr.flow_options.is_empty());
1387
1388        let sql = r"
1389CREATE FLOW task_3
1390SINK TO schema_1.table_1
1391WITH (defer_on_missing_source = 'true', foo = 'bar')
1392AS
1393SELECT max(c1), min(c2) FROM schema_2.table_2;";
1394        let stmt =
1395            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1396                .unwrap()
1397                .pop()
1398                .unwrap();
1399
1400        let Statement::CreateFlow(create_flow) = stmt else {
1401            unreachable!()
1402        };
1403        let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1404        assert_eq!(
1405            expr.flow_options,
1406            HashMap::from([
1407                ("defer_on_missing_source".to_string(), "true".to_string()),
1408                ("foo".to_string(), "bar".to_string()),
1409            ])
1410        );
1411
1412        let sql = r"
1413CREATE FLOW task_4
1414SINK TO schema_1.table_1
1415WITH (defer_on_missing_source = true)
1416AS
1417SELECT max(c1), min(c2) FROM schema_2.table_2;";
1418        let stmt =
1419            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1420                .unwrap()
1421                .pop()
1422                .unwrap();
1423
1424        let Statement::CreateFlow(create_flow) = stmt else {
1425            unreachable!()
1426        };
1427        let expr = to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
1428        assert_eq!(
1429            expr.flow_options,
1430            HashMap::from([("defer_on_missing_source".to_string(), "true".to_string(),)])
1431        );
1432
1433        let sql = r"
1434CREATE FLOW task_5
1435SINK TO schema_1.table_1
1436WITH (defer_on_missing_source = [true])
1437AS
1438SELECT max(c1), min(c2) FROM schema_2.table_2;";
1439        let stmt =
1440            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1441                .unwrap()
1442                .pop()
1443                .unwrap();
1444
1445        let Statement::CreateFlow(create_flow) = stmt else {
1446            unreachable!()
1447        };
1448        let res = to_create_flow_task_expr(create_flow, &QueryContext::arc());
1449        assert!(res.is_err());
1450        assert!(
1451            res.unwrap_err()
1452                .to_string()
1453                .contains("flow options only support scalar string-compatible values")
1454        );
1455
1456        let sql = r"
1457CREATE FLOW abc.`task_2`
1458SINK TO schema_1.table_1
1459AS
1460SELECT max(c1), min(c2) FROM schema_2.table_2;";
1461        let stmt =
1462            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1463                .unwrap()
1464                .pop()
1465                .unwrap();
1466
1467        let Statement::CreateFlow(create_flow) = stmt else {
1468            unreachable!()
1469        };
1470        let res = to_create_flow_task_expr(create_flow, &QueryContext::arc());
1471
1472        assert!(res.is_err());
1473        assert!(
1474            res.unwrap_err()
1475                .to_string()
1476                .contains("Invalid flow name: abc.`task_2`")
1477        );
1478    }
1479
1480    #[test]
1481    fn test_create_to_expr() {
1482        let sql = "CREATE TABLE monitor (host STRING,ts TIMESTAMP,TIME INDEX (ts),PRIMARY KEY(host)) ENGINE=mito WITH(ttl='3days', write_buffer_size='1024KB');";
1483        let stmt =
1484            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1485                .unwrap()
1486                .pop()
1487                .unwrap();
1488
1489        let Statement::CreateTable(create_table) = stmt else {
1490            unreachable!()
1491        };
1492        let expr = create_to_expr(&create_table, &QueryContext::arc()).unwrap();
1493        assert_eq!("3days", expr.table_options.get("ttl").unwrap());
1494        assert_eq!(
1495            "1.0MiB",
1496            expr.table_options.get("write_buffer_size").unwrap()
1497        );
1498    }
1499
1500    #[test]
1501    fn test_invalid_create_to_expr() {
1502        let cases = [
1503            // duplicate column declaration
1504            "CREATE TABLE monitor (host STRING primary key, ts TIMESTAMP TIME INDEX, some_column text, some_column string);",
1505            // duplicate primary key
1506            "CREATE TABLE monitor (host STRING, ts TIMESTAMP TIME INDEX, some_column STRING, PRIMARY KEY (some_column, host, some_column));",
1507            // time index is primary key
1508            "CREATE TABLE monitor (host STRING, ts TIMESTAMP TIME INDEX, PRIMARY KEY (host, ts));",
1509        ];
1510
1511        for sql in cases {
1512            let stmt = ParserContext::create_with_dialect(
1513                sql,
1514                &GreptimeDbDialect {},
1515                ParseOptions::default(),
1516            )
1517            .unwrap()
1518            .pop()
1519            .unwrap();
1520            let Statement::CreateTable(create_table) = stmt else {
1521                unreachable!()
1522            };
1523            create_to_expr(&create_table, &QueryContext::arc()).unwrap_err();
1524        }
1525    }
1526
1527    #[test]
1528    fn test_create_to_expr_with_default_timestamp_value() {
1529        let sql = "CREATE TABLE monitor (v double,ts TIMESTAMP default '2024-01-30T00:01:01',TIME INDEX (ts)) engine=mito;";
1530        let stmt =
1531            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1532                .unwrap()
1533                .pop()
1534                .unwrap();
1535
1536        let Statement::CreateTable(create_table) = stmt else {
1537            unreachable!()
1538        };
1539
1540        // query context with system timezone UTC.
1541        let expr = create_to_expr(&create_table, &QueryContext::arc()).unwrap();
1542        let ts_column = &expr.column_defs[1];
1543        let constraint = assert_ts_column(ts_column);
1544        assert!(
1545            matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
1546                         if ts.to_iso8601_string() == "2024-01-30 00:01:01+0000")
1547        );
1548
1549        // query context with timezone `+08:00`
1550        let ctx = QueryContextBuilder::default()
1551            .timezone(Timezone::from_tz_string("+08:00").unwrap())
1552            .build()
1553            .into();
1554        let expr = create_to_expr(&create_table, &ctx).unwrap();
1555        let ts_column = &expr.column_defs[1];
1556        let constraint = assert_ts_column(ts_column);
1557        assert!(
1558            matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
1559                         if ts.to_iso8601_string() == "2024-01-29 16:01:01+0000")
1560        );
1561    }
1562
1563    fn assert_ts_column(ts_column: &api::v1::ColumnDef) -> ColumnDefaultConstraint {
1564        assert_eq!("ts", ts_column.name);
1565        assert_eq!(
1566            ColumnDataType::TimestampMillisecond as i32,
1567            ts_column.data_type
1568        );
1569        assert!(!ts_column.default_constraint.is_empty());
1570
1571        ColumnDefaultConstraint::try_from(&ts_column.default_constraint[..]).unwrap()
1572    }
1573
1574    #[test]
1575    fn test_to_alter_expr() {
1576        let sql = "ALTER DATABASE greptime SET key1='value1', key2='value2';";
1577        let stmt =
1578            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1579                .unwrap()
1580                .pop()
1581                .unwrap();
1582
1583        let Statement::AlterDatabase(alter_database) = stmt else {
1584            unreachable!()
1585        };
1586
1587        let expr = to_alter_database_expr(alter_database, &QueryContext::arc()).unwrap();
1588        let kind = expr.kind.unwrap();
1589
1590        let AlterDatabaseKind::SetDatabaseOptions(SetDatabaseOptions {
1591            set_database_options,
1592        }) = kind
1593        else {
1594            unreachable!()
1595        };
1596
1597        assert_eq!(2, set_database_options.len());
1598        assert_eq!("key1", set_database_options[0].key);
1599        assert_eq!("value1", set_database_options[0].value);
1600        assert_eq!("key2", set_database_options[1].key);
1601        assert_eq!("value2", set_database_options[1].value);
1602
1603        let sql = "ALTER DATABASE greptime UNSET key1, key2;";
1604        let stmt =
1605            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1606                .unwrap()
1607                .pop()
1608                .unwrap();
1609
1610        let Statement::AlterDatabase(alter_database) = stmt else {
1611            unreachable!()
1612        };
1613
1614        let expr = to_alter_database_expr(alter_database, &QueryContext::arc()).unwrap();
1615        let kind = expr.kind.unwrap();
1616
1617        let AlterDatabaseKind::UnsetDatabaseOptions(UnsetDatabaseOptions { keys }) = kind else {
1618            unreachable!()
1619        };
1620
1621        assert_eq!(2, keys.len());
1622        assert!(keys.contains(&"key1".to_string()));
1623        assert!(keys.contains(&"key2".to_string()));
1624
1625        let sql = "ALTER TABLE monitor add column ts TIMESTAMP default '2024-01-30T00:01:01';";
1626        let stmt =
1627            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1628                .unwrap()
1629                .pop()
1630                .unwrap();
1631
1632        let Statement::AlterTable(alter_table) = stmt else {
1633            unreachable!()
1634        };
1635
1636        // query context with system timezone UTC.
1637        let expr = to_alter_table_expr(alter_table.clone(), &QueryContext::arc()).unwrap();
1638        let kind = expr.kind.unwrap();
1639
1640        let AlterTableKind::AddColumns(AddColumns { add_columns, .. }) = kind else {
1641            unreachable!()
1642        };
1643
1644        assert_eq!(1, add_columns.len());
1645        let ts_column = add_columns[0].column_def.clone().unwrap();
1646        let constraint = assert_ts_column(&ts_column);
1647        assert!(
1648            matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
1649                         if ts.to_iso8601_string() == "2024-01-30 00:01:01+0000")
1650        );
1651
1652        //
1653        // query context with timezone `+08:00`
1654        let ctx = QueryContextBuilder::default()
1655            .timezone(Timezone::from_tz_string("+08:00").unwrap())
1656            .build()
1657            .into();
1658        let expr = to_alter_table_expr(alter_table, &ctx).unwrap();
1659        let kind = expr.kind.unwrap();
1660
1661        let AlterTableKind::AddColumns(AddColumns { add_columns, .. }) = kind else {
1662            unreachable!()
1663        };
1664
1665        assert_eq!(1, add_columns.len());
1666        let ts_column = add_columns[0].column_def.clone().unwrap();
1667        let constraint = assert_ts_column(&ts_column);
1668        assert!(
1669            matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
1670                         if ts.to_iso8601_string() == "2024-01-29 16:01:01+0000")
1671        );
1672    }
1673
1674    #[test]
1675    fn test_to_alter_modify_column_type_expr() {
1676        let sql = "ALTER TABLE monitor MODIFY COLUMN mem_usage STRING;";
1677        let stmt =
1678            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1679                .unwrap()
1680                .pop()
1681                .unwrap();
1682
1683        let Statement::AlterTable(alter_table) = stmt else {
1684            unreachable!()
1685        };
1686
1687        // query context with system timezone UTC.
1688        let expr = to_alter_table_expr(alter_table.clone(), &QueryContext::arc()).unwrap();
1689        let kind = expr.kind.unwrap();
1690
1691        let AlterTableKind::ModifyColumnTypes(ModifyColumnTypes {
1692            modify_column_types,
1693        }) = kind
1694        else {
1695            unreachable!()
1696        };
1697
1698        assert_eq!(1, modify_column_types.len());
1699        let modify_column_type = &modify_column_types[0];
1700
1701        assert_eq!("mem_usage", modify_column_type.column_name);
1702        assert_eq!(
1703            ColumnDataType::String as i32,
1704            modify_column_type.target_type
1705        );
1706        assert!(modify_column_type.target_type_extension.is_none());
1707    }
1708
1709    #[test]
1710    fn test_to_repartition_request() {
1711        let sql = r#"
1712ALTER TABLE metrics REPARTITION (
1713  device_id < 100
1714) INTO (
1715  device_id < 100 AND area < 'South',
1716  device_id < 100 AND area >= 'South'
1717);"#;
1718        let stmt =
1719            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1720                .unwrap()
1721                .pop()
1722                .unwrap();
1723
1724        let Statement::AlterTable(alter_table) = stmt else {
1725            unreachable!()
1726        };
1727
1728        let request = to_repartition_request(alter_table, &QueryContext::arc()).unwrap();
1729        assert_eq!("greptime", request.catalog_name);
1730        assert_eq!("public", request.schema_name);
1731        assert_eq!("metrics", request.table_name);
1732        let RepartitionSource::Partitions {
1733            from_exprs,
1734            target_partition_columns,
1735        } = request.source
1736        else {
1737            unreachable!()
1738        };
1739        assert!(target_partition_columns.is_none());
1740        assert_eq!(
1741            from_exprs
1742                .into_iter()
1743                .map(|x| x.to_string())
1744                .collect::<Vec<_>>(),
1745            vec!["device_id < 100".to_string()]
1746        );
1747        assert_eq!(
1748            request
1749                .into_exprs
1750                .into_iter()
1751                .map(|x| x.to_string())
1752                .collect::<Vec<_>>(),
1753            vec![
1754                "device_id < 100 AND area < 'South'".to_string(),
1755                "device_id < 100 AND area >= 'South'".to_string()
1756            ]
1757        );
1758    }
1759
1760    #[test]
1761    fn test_to_repartition_request_with_target_partition_columns() {
1762        let sql = r#"
1763ALTER TABLE metrics REPARTITION (
1764  device_id < 100
1765) ON COLUMNS (device_id, area) INTO (
1766  device_id < 100 AND area < 'South',
1767  device_id < 100 AND area >= 'South'
1768);"#;
1769        let stmt =
1770            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1771                .unwrap()
1772                .pop()
1773                .unwrap();
1774
1775        let Statement::AlterTable(alter_table) = stmt else {
1776            unreachable!()
1777        };
1778
1779        let request = to_repartition_request(alter_table, &QueryContext::arc()).unwrap();
1780        let RepartitionSource::Partitions {
1781            target_partition_columns,
1782            ..
1783        } = request.source
1784        else {
1785            unreachable!()
1786        };
1787
1788        assert_eq!(
1789            target_partition_columns,
1790            Some(vec!["device_id".to_string(), "area".to_string()])
1791        );
1792    }
1793
1794    #[test]
1795    fn test_to_repartition_request_with_unpartitioned_source() {
1796        let sql = r#"
1797ALTER TABLE metrics PARTITION ON COLUMNS (device_id, area) (
1798  device_id < 100 AND area < 'South',
1799  device_id < 100 AND area >= 'South'
1800);"#;
1801        let stmt =
1802            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1803                .unwrap()
1804                .pop()
1805                .unwrap();
1806
1807        let Statement::AlterTable(alter_table) = stmt else {
1808            unreachable!()
1809        };
1810
1811        let request = to_repartition_request(alter_table, &QueryContext::arc()).unwrap();
1812        assert_eq!("greptime", request.catalog_name);
1813        assert_eq!("public", request.schema_name);
1814        assert_eq!("metrics", request.table_name);
1815        let RepartitionSource::Unpartitioned { partition_columns } = request.source else {
1816            unreachable!()
1817        };
1818        assert_eq!(partition_columns, vec!["device_id", "area"]);
1819        assert_eq!(
1820            request
1821                .into_exprs
1822                .into_iter()
1823                .map(|x| x.to_string())
1824                .collect::<Vec<_>>(),
1825            vec![
1826                "device_id < 100 AND area < 'South'".to_string(),
1827                "device_id < 100 AND area >= 'South'".to_string()
1828            ]
1829        );
1830    }
1831
1832    fn new_test_table_names() -> Vec<TableName> {
1833        vec![
1834            TableName {
1835                catalog_name: "greptime".to_string(),
1836                schema_name: "public".to_string(),
1837                table_name: "a_table".to_string(),
1838            },
1839            TableName {
1840                catalog_name: "greptime".to_string(),
1841                schema_name: "public".to_string(),
1842                table_name: "b_table".to_string(),
1843            },
1844        ]
1845    }
1846
1847    #[test]
1848    fn test_to_create_view_expr() {
1849        let sql = "CREATE VIEW test AS SELECT * FROM NUMBERS";
1850        let stmt =
1851            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1852                .unwrap()
1853                .pop()
1854                .unwrap();
1855
1856        let Statement::CreateView(stmt) = stmt else {
1857            unreachable!()
1858        };
1859
1860        let logical_plan = vec![1, 2, 3];
1861        let table_names = new_test_table_names();
1862        let columns = vec!["a".to_string()];
1863        let plan_columns = vec!["number".to_string()];
1864
1865        let expr = to_create_view_expr(
1866            stmt,
1867            logical_plan.clone(),
1868            table_names.clone(),
1869            columns.clone(),
1870            plan_columns.clone(),
1871            sql.to_string(),
1872            QueryContext::arc(),
1873        )
1874        .unwrap();
1875
1876        assert_eq!("greptime", expr.catalog_name);
1877        assert_eq!("public", expr.schema_name);
1878        assert_eq!("test", expr.view_name);
1879        assert!(!expr.create_if_not_exists);
1880        assert!(!expr.or_replace);
1881        assert_eq!(logical_plan, expr.logical_plan);
1882        assert_eq!(table_names, expr.table_names);
1883        assert_eq!(sql, expr.definition);
1884        assert_eq!(columns, expr.columns);
1885        assert_eq!(plan_columns, expr.plan_columns);
1886    }
1887
1888    #[test]
1889    fn test_to_create_view_expr_complex() {
1890        let sql = "CREATE OR REPLACE VIEW IF NOT EXISTS test.test_view AS SELECT * FROM NUMBERS";
1891        let stmt =
1892            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1893                .unwrap()
1894                .pop()
1895                .unwrap();
1896
1897        let Statement::CreateView(stmt) = stmt else {
1898            unreachable!()
1899        };
1900
1901        let logical_plan = vec![1, 2, 3];
1902        let table_names = new_test_table_names();
1903        let columns = vec!["a".to_string()];
1904        let plan_columns = vec!["number".to_string()];
1905
1906        let expr = to_create_view_expr(
1907            stmt,
1908            logical_plan.clone(),
1909            table_names.clone(),
1910            columns.clone(),
1911            plan_columns.clone(),
1912            sql.to_string(),
1913            QueryContext::arc(),
1914        )
1915        .unwrap();
1916
1917        assert_eq!("greptime", expr.catalog_name);
1918        assert_eq!("test", expr.schema_name);
1919        assert_eq!("test_view", expr.view_name);
1920        assert!(expr.create_if_not_exists);
1921        assert!(expr.or_replace);
1922        assert_eq!(logical_plan, expr.logical_plan);
1923        assert_eq!(table_names, expr.table_names);
1924        assert_eq!(sql, expr.definition);
1925        assert_eq!(columns, expr.columns);
1926        assert_eq!(plan_columns, expr.plan_columns);
1927    }
1928
1929    #[test]
1930    fn test_expr_to_create() {
1931        let sql = r#"CREATE TABLE IF NOT EXISTS `tt` (
1932  `timestamp` TIMESTAMP(9) NOT NULL,
1933  `ip_address` STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),
1934  `username` STRING NULL,
1935  `http_method` STRING NULL INVERTED INDEX,
1936  `request_line` STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false', false_positive_rate = '0.01', granularity = '10240'),
1937  `protocol` STRING NULL,
1938  `status_code` INT NULL INVERTED INDEX,
1939  `response_size` BIGINT NULL,
1940  `message` STRING NULL,
1941  TIME INDEX (`timestamp`),
1942  PRIMARY KEY (`username`, `status_code`)
1943)
1944ENGINE=mito
1945WITH(
1946  append_mode = 'true'
1947)"#;
1948        let stmt =
1949            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
1950                .unwrap()
1951                .pop()
1952                .unwrap();
1953
1954        let Statement::CreateTable(original_create) = stmt else {
1955            unreachable!()
1956        };
1957
1958        // Convert CreateTable -> CreateTableExpr -> CreateTable
1959        let expr = create_to_expr(&original_create, &QueryContext::arc()).unwrap();
1960
1961        let create_table = expr_to_create(&expr, Some('`')).unwrap();
1962        let new_sql = format!("{:#}", create_table);
1963        assert_eq!(sql, new_sql);
1964    }
1965}