Skip to main content

operator/statement/
ddl.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{HashMap, HashSet};
16use std::sync::Arc;
17use std::time::Duration;
18
19use api::helper::ColumnDataTypeWrapper;
20use api::v1::alter_table_expr::Kind;
21use api::v1::meta::CreateFlowTask as PbCreateFlowTask;
22use api::v1::repartition::Source;
23use api::v1::{
24    AlterDatabaseExpr, AlterTableExpr, CreateFlowExpr, CreateTableExpr, CreateViewExpr,
25    PartitionedSource, Repartition, TargetPartitionColumns, UnpartitionedSource, column_def,
26};
27#[cfg(feature = "enterprise")]
28use api::v1::{
29    CreateTriggerExpr as PbCreateTriggerExpr, meta::CreateTriggerTask as PbCreateTriggerTask,
30};
31use catalog::CatalogManagerRef;
32use chrono::Utc;
33use common_base::regex_pattern::NAME_PATTERN_REG;
34use common_catalog::consts::{
35    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, is_ddl_reserved_table, is_readonly_schema,
36    is_readonly_table,
37};
38use common_catalog::{format_full_flow_name, format_full_table_name};
39use common_error::ext::BoxedError;
40#[cfg(feature = "enterprise")]
41use common_meta::cache_invalidator::CacheInvalidatorRef;
42use common_meta::cache_invalidator::Context;
43use common_meta::ddl::create_flow::{
44    DEFER_ON_MISSING_SOURCE_KEY, FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType,
45    INTERNAL_EVAL_OFFSET_KEY, INTERNAL_EVAL_SCHEDULE_KEY,
46};
47use common_meta::instruction::CacheIdent;
48#[cfg(feature = "enterprise")]
49use common_meta::key::TableMetadataManagerRef;
50use common_meta::key::schema_name::{SchemaName, SchemaNameKey};
51#[cfg(feature = "enterprise")]
52use common_meta::procedure_executor::ProcedureExecutorRef;
53#[cfg(feature = "enterprise")]
54use common_meta::rpc::ddl::trigger::CreateTriggerTask;
55#[cfg(feature = "enterprise")]
56use common_meta::rpc::ddl::trigger::DropTriggerTask;
57use common_meta::rpc::ddl::{
58    CreateFlowTask, CreatorGrantIntent, DdlTask, DropFlowTask, DropViewTask, SubmitDdlTaskRequest,
59    SubmitDdlTaskResponse, TriggerReason,
60};
61use common_query::Output;
62use common_recordbatch::{RecordBatch, RecordBatches};
63use common_sql::convert::sql_value_to_value;
64use common_telemetry::{debug, info, tracing, warn};
65use common_time::{Timestamp, Timezone};
66use datafusion_common::tree_node::TreeNodeVisitor;
67use datafusion_expr::LogicalPlan;
68use datatypes::prelude::ConcreteDataType;
69use datatypes::schema::{ColumnSchema, Schema};
70use datatypes::value::Value;
71use datatypes::vectors::{StringVector, VectorRef};
72use humantime::parse_duration;
73use partition::expr::{Operand, PartitionExpr, RestrictedOp};
74use partition::multi_dim::MultiDimPartitionRule;
75use query::parser::QueryStatement;
76use query::plan::extract_and_rewrite_full_table_names;
77use query::query_engine::DefaultSerializer;
78use query::sql::create_table_stmt;
79use session::context::QueryContextRef;
80use session::table_name::table_idents_to_full_name;
81use snafu::{OptionExt, ResultExt, ensure};
82use sql::parser::{ParseOptions, ParserContext};
83use sql::parsers::utils::is_tql;
84use sql::statements::OptionMap;
85#[cfg(feature = "enterprise")]
86use sql::statements::alter::trigger::AlterTrigger;
87use sql::statements::alter::{AlterDatabase, AlterTable, AlterTableOperation};
88#[cfg(feature = "enterprise")]
89use sql::statements::create::trigger::CreateTrigger;
90use sql::statements::create::{
91    CreateExternalTable, CreateFlow, CreateTable, CreateTableLike, CreateView, Partitions,
92};
93use sql::statements::statement::Statement;
94use sqlparser::ast::{Expr, Ident, UnaryOperator, Value as ParserValue};
95use store_api::metric_engine_consts::{LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME};
96use store_api::mito_engine_options::APPEND_MODE_KEY;
97use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
98use table::TableRef;
99use table::dist_table::DistTable;
100use table::metadata::{self, TableId, TableInfo, TableMeta, TableType};
101use table::requests::{
102    AlterKind, AlterTableRequest, AnnotationContext, COMMENT_KEY, DDL_TIMEOUT, DDL_WAIT,
103    TableOptions, validate_and_normalize_annotation_options,
104};
105use table::table_name::TableName;
106use table::table_reference::TableReference;
107
108use crate::error::{
109    self, AlterExprToRequestSnafu, BuildDfLogicalPlanSnafu, CatalogSnafu, ColumnDataTypeSnafu,
110    ColumnNotFoundSnafu, ConvertSchemaSnafu, CreateLogicalTablesSnafu,
111    DeserializePartitionExprSnafu, EmptyDdlExprSnafu, ExternalSnafu, ExtractTableNamesSnafu,
112    FlowNotFoundSnafu, InvalidPartitionRuleSnafu, InvalidPartitionSnafu, InvalidSqlSnafu,
113    InvalidTableNameSnafu, InvalidViewNameSnafu, InvalidViewStmtSnafu, NotSupportedSnafu,
114    PartitionExprToPbSnafu, Result, SchemaInUseSnafu, SchemaNotFoundSnafu, SchemaReadOnlySnafu,
115    SerializePartitionExprSnafu, SubstraitCodecSnafu, TableAlreadyExistsSnafu,
116    TableDdlReservedSnafu, TableMetadataManagerSnafu, TableNotFoundSnafu, TableReadOnlySnafu,
117    UnrecognizedTableOptionSnafu, ViewAlreadyExistsSnafu,
118};
119use crate::expr_helper::{self, RepartitionRequest, RepartitionSource};
120use crate::statement::StatementExecutor;
121use crate::statement::show::create_partitions_stmt;
122use crate::utils::{to_executor_context, to_executor_context_with_origin_frontend};
123
124#[derive(Debug, Clone, Copy)]
125struct DdlSubmitOptions {
126    wait: bool,
127    timeout: Duration,
128}
129
130const ALLOWED_FLOW_OPTIONS: [&str; 2] = [
131    DEFER_ON_MISSING_SOURCE_KEY,
132    FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY,
133];
134
135fn build_procedure_id_output(procedure_id: Vec<u8>) -> Result<Output> {
136    let procedure_id = String::from_utf8_lossy(&procedure_id).to_string();
137    let vector: VectorRef = Arc::new(StringVector::from(vec![procedure_id]));
138    let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
139        "Procedure ID",
140        vector.data_type(),
141        false,
142    )]));
143    let batch =
144        RecordBatch::new(schema.clone(), vec![vector]).context(error::BuildRecordBatchSnafu)?;
145    let batches =
146        RecordBatches::try_new(schema, vec![batch]).context(error::BuildRecordBatchSnafu)?;
147    Ok(Output::new_with_record_batches(batches))
148}
149
150fn parse_ddl_options(options: &OptionMap) -> Result<DdlSubmitOptions> {
151    let wait = match options.get(DDL_WAIT) {
152        Some(value) => value.parse::<bool>().map_err(|_| {
153            InvalidSqlSnafu {
154                err_msg: format!("invalid DDL option '{DDL_WAIT}': '{value}'"),
155            }
156            .build()
157        })?,
158        None => SubmitDdlTaskRequest::default_wait(),
159    };
160
161    let timeout = match options.get(DDL_TIMEOUT) {
162        Some(value) => parse_duration(value).map_err(|err| {
163            InvalidSqlSnafu {
164                err_msg: format!("invalid DDL option '{DDL_TIMEOUT}': '{value}': {err}"),
165            }
166            .build()
167        })?,
168        None => SubmitDdlTaskRequest::default_timeout(),
169    };
170
171    Ok(DdlSubmitOptions { wait, timeout })
172}
173
174fn supported_flow_options() -> String {
175    ALLOWED_FLOW_OPTIONS.join(", ")
176}
177
178fn normalize_flow_bool_option(key: &str, value: &str) -> Result<String> {
179    value
180        .trim()
181        .to_ascii_lowercase()
182        .parse::<bool>()
183        .map(|value| value.to_string())
184        .map_err(|_| {
185            InvalidSqlSnafu {
186                err_msg: format!("invalid flow option '{key}': '{value}'"),
187            }
188            .build()
189        })
190}
191
192fn validate_and_normalize_flow_options(
193    options: HashMap<String, String>,
194    eval_interval: Option<i64>,
195) -> Result<HashMap<String, String>> {
196    // Reject non-positive eval_interval (zero or negative).
197    if let Some(secs) = eval_interval
198        && secs <= 0
199    {
200        return InvalidSqlSnafu {
201            err_msg: format!("EVAL INTERVAL must be positive, got {secs} seconds"),
202        }
203        .fail();
204    }
205
206    options
207        .into_iter()
208        .map(|(key, value)| {
209            if matches!(
210                key.as_str(),
211                FlowType::FLOW_TYPE_KEY | INTERNAL_EVAL_OFFSET_KEY | INTERNAL_EVAL_SCHEDULE_KEY
212            ) {
213                return InvalidSqlSnafu {
214                    err_msg: format!("flow option '{key}' is reserved for internal use"),
215                }
216                .fail();
217            }
218
219            let normalized_value = match key.as_str() {
220                DEFER_ON_MISSING_SOURCE_KEY | FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY => {
221                    normalize_flow_bool_option(&key, &value)?
222                }
223                _ => {
224                    return InvalidSqlSnafu {
225                        err_msg: format!(
226                            "unknown flow option '{key}', supported options: {}",
227                            supported_flow_options()
228                        ),
229                    }
230                    .fail();
231                }
232            };
233
234            Ok((key, normalized_value))
235        })
236        .collect()
237}
238
239/// Validates `EVAL OFFSET` semantics at the operator boundary: an offset is
240/// only legal together with `EVAL INTERVAL` and must lie in
241/// `[0, eval_interval)`. Never modulo-normalized.
242fn validate_eval_offset(
243    eval_offset_secs: Option<i64>,
244    eval_interval_secs: Option<i64>,
245) -> Result<()> {
246    if let Some(offset_secs) = eval_offset_secs {
247        let Some(eval_interval_secs) = eval_interval_secs else {
248            return InvalidSqlSnafu {
249                err_msg: "EVAL OFFSET requires EVAL INTERVAL to be specified".to_string(),
250            }
251            .fail();
252        };
253        if !(0..eval_interval_secs).contains(&offset_secs) {
254            return InvalidSqlSnafu {
255                err_msg: format!(
256                    "EVAL OFFSET must be in range [0, EVAL INTERVAL), got {offset_secs} seconds with EVAL INTERVAL {eval_interval_secs} seconds"
257                ),
258            }
259            .fail();
260        }
261    }
262    Ok(())
263}
264
265fn determine_flow_type_for_source_state(
266    flow_name: &str,
267    flow_options: &HashMap<String, String>,
268    has_missing_source_table: bool,
269    has_instant_ttl_source_table: bool,
270    force_batching: bool,
271) -> Result<Option<FlowType>> {
272    if has_instant_ttl_source_table && force_batching {
273        // The batching scheduler cannot read instant-TTL source tables, so
274        // reject this combination even when another source table is missing.
275        return InvalidSqlSnafu {
276            err_msg: format!(
277                "flow '{}' with EVAL INTERVAL requires the batching scheduler, but source tables with ttl=instant are not supported under batching mode; use a TTL longer than the flush interval",
278                flow_name
279            ),
280        }
281        .fail();
282    }
283
284    if has_missing_source_table {
285        let defer_on_missing_source = flow_options
286            .get(DEFER_ON_MISSING_SOURCE_KEY)
287            .is_some_and(|value| value == "true");
288        ensure!(
289            defer_on_missing_source,
290            InvalidSqlSnafu {
291                err_msg: format!(
292                    "missing source tables for flow '{}'; use WITH ({DEFER_ON_MISSING_SOURCE_KEY} = true) to create a pending flow",
293                    flow_name
294                )
295            }
296        );
297        info!(
298            "Flow `{}` is created as a pending batching flow because source tables are missing and defer_on_missing_source=true",
299            flow_name
300        );
301        return Ok(Some(FlowType::Batching));
302    }
303
304    if has_instant_ttl_source_table {
305        return Ok(Some(FlowType::Streaming));
306    }
307
308    Ok(None)
309}
310
311impl StatementExecutor {
312    pub fn catalog_manager(&self) -> CatalogManagerRef {
313        self.catalog_manager.clone()
314    }
315
316    #[tracing::instrument(skip_all)]
317    pub async fn create_table(&self, stmt: CreateTable, ctx: QueryContextRef) -> Result<TableRef> {
318        let (catalog, schema, _table) = table_idents_to_full_name(&stmt.name, &ctx)
319            .map_err(BoxedError::new)
320            .context(error::ExternalSnafu)?;
321
322        let schema_options = self
323            .table_metadata_manager
324            .schema_manager()
325            .get(SchemaNameKey {
326                catalog: &catalog,
327                schema: &schema,
328            })
329            .await
330            .context(TableMetadataManagerSnafu)?
331            .map(|v| v.into_inner());
332
333        let create_expr = &mut expr_helper::create_to_expr(&stmt, &ctx)?;
334        // Don't inherit schema-level TTL/compaction options into table options:
335        // TTL is applied during compaction, and `compaction.*` is handled separately.
336        if let Some(schema_options) = schema_options {
337            for (key, value) in schema_options.extra_options.iter() {
338                if key.starts_with("compaction.") {
339                    continue;
340                }
341                create_expr
342                    .table_options
343                    .entry(key.clone())
344                    .or_insert(value.clone());
345            }
346        }
347
348        self.create_table_inner(create_expr, stmt.partitions, ctx, TriggerReason::Manual)
349            .await
350    }
351
352    #[tracing::instrument(skip_all)]
353    pub async fn create_table_like(
354        &self,
355        stmt: CreateTableLike,
356        ctx: QueryContextRef,
357    ) -> Result<TableRef> {
358        let (catalog, schema, table) = table_idents_to_full_name(&stmt.source_name, &ctx)
359            .map_err(BoxedError::new)
360            .context(error::ExternalSnafu)?;
361        let table_ref = self
362            .catalog_manager
363            .table(&catalog, &schema, &table, Some(&ctx))
364            .await
365            .context(CatalogSnafu)?
366            .context(TableNotFoundSnafu { table_name: &table })?;
367        let partition_info = self
368            .partition_manager
369            .find_physical_partition_info(table_ref.table_info().table_id())
370            .await
371            .context(error::FindTablePartitionRuleSnafu { table_name: table })?;
372
373        // CREATE TABLE LIKE also inherits database level options.
374        let schema_options = self
375            .table_metadata_manager
376            .schema_manager()
377            .get(SchemaNameKey {
378                catalog: &catalog,
379                schema: &schema,
380            })
381            .await
382            .context(TableMetadataManagerSnafu)?
383            .map(|v| v.into_inner());
384
385        let quote_style = ctx.quote_style();
386        let mut create_stmt =
387            create_table_stmt(&table_ref.table_info(), schema_options, quote_style)
388                .context(error::ParseQuerySnafu)?;
389        create_stmt.name = stmt.table_name;
390        create_stmt.if_not_exists = false;
391
392        let table_info = table_ref.table_info();
393        let partitions = create_partitions_stmt(&table_info, &partition_info.partitions)?.and_then(
394            |mut partitions| {
395                if !partitions.column_list.is_empty() {
396                    partitions.set_quote(quote_style);
397                    Some(partitions)
398                } else {
399                    None
400                }
401            },
402        );
403
404        let create_expr = &mut expr_helper::create_to_expr(&create_stmt, &ctx)?;
405        self.create_table_inner(create_expr, partitions, ctx, TriggerReason::Manual)
406            .await
407    }
408
409    #[tracing::instrument(skip_all)]
410    pub async fn create_external_table(
411        &self,
412        create_expr: CreateExternalTable,
413        ctx: QueryContextRef,
414    ) -> Result<TableRef> {
415        let create_expr =
416            &mut expr_helper::create_external_expr(create_expr, &ctx, &self.local_file_access)
417                .await?;
418        self.create_table_inner(create_expr, None, ctx, TriggerReason::Manual)
419            .await
420    }
421
422    /// Creates the declared-edge table with its canonical schema, entering
423    /// below the user-DDL guard ([`ensure_table_definition_writable`]).
424    /// `create_if_not_exists` makes concurrent first inserts race safely.
425    pub async fn create_declared_relationships_table(
426        &self,
427        catalog: &str,
428        query_ctx: QueryContextRef,
429    ) -> Result<TableRef> {
430        let mut expr = super::semantic_graph::build_declared_relationships_expr(catalog);
431        self.create_non_logic_table(&mut expr, None, query_ctx, TriggerReason::AutoCreate)
432            .await
433    }
434
435    #[tracing::instrument(skip_all)]
436    pub async fn create_table_inner(
437        &self,
438        create_table: &mut CreateTableExpr,
439        partitions: Option<Partitions>,
440        query_ctx: QueryContextRef,
441        trigger_reason: TriggerReason,
442    ) -> Result<TableRef> {
443        ensure_table_definition_writable(&create_table.schema_name, &create_table.table_name)?;
444
445        if create_table.engine == METRIC_ENGINE_NAME
446            && create_table
447                .table_options
448                .contains_key(LOGICAL_TABLE_METADATA_KEY)
449        {
450            if let Some(partitions) = partitions.as_ref()
451                && !partitions.exprs.is_empty()
452            {
453                self.validate_logical_table_partition_rule(create_table, partitions, &query_ctx)
454                    .await?;
455            }
456            // Create logical tables
457            self.create_logical_tables(
458                std::slice::from_ref(create_table),
459                query_ctx,
460                trigger_reason,
461            )
462            .await?
463            .into_iter()
464            .next()
465            .context(error::UnexpectedSnafu {
466                violated: "expected to create logical tables",
467            })
468        } else {
469            // Create other normal table
470            self.create_non_logic_table(create_table, partitions, query_ctx, trigger_reason)
471                .await
472        }
473    }
474
475    #[tracing::instrument(skip_all)]
476    pub async fn create_non_logic_table(
477        &self,
478        create_table: &mut CreateTableExpr,
479        partitions: Option<Partitions>,
480        query_ctx: QueryContextRef,
481        trigger_reason: TriggerReason,
482    ) -> Result<TableRef> {
483        let _timer = crate::metrics::DIST_CREATE_TABLE.start_timer();
484
485        // Check if schema exists
486        let schema = self
487            .table_metadata_manager
488            .schema_manager()
489            .get(SchemaNameKey::new(
490                &create_table.catalog_name,
491                &create_table.schema_name,
492            ))
493            .await
494            .context(TableMetadataManagerSnafu)?;
495        ensure!(
496            schema.is_some(),
497            SchemaNotFoundSnafu {
498                schema_info: &create_table.schema_name,
499            }
500        );
501
502        // if table exists.
503        if let Some(table) = self
504            .catalog_manager
505            .table(
506                &create_table.catalog_name,
507                &create_table.schema_name,
508                &create_table.table_name,
509                Some(&query_ctx),
510            )
511            .await
512            .context(CatalogSnafu)?
513        {
514            return if create_table.create_if_not_exists {
515                Ok(table)
516            } else {
517                TableAlreadyExistsSnafu {
518                    table: format_full_table_name(
519                        &create_table.catalog_name,
520                        &create_table.schema_name,
521                        &create_table.table_name,
522                    ),
523                }
524                .fail()
525            };
526        }
527
528        ensure!(
529            NAME_PATTERN_REG.is_match(&create_table.table_name),
530            InvalidTableNameSnafu {
531                table_name: &create_table.table_name,
532            }
533        );
534
535        let table_name = TableName::new(
536            &create_table.catalog_name,
537            &create_table.schema_name,
538            &create_table.table_name,
539        );
540
541        let (partitions, partition_cols) = parse_partitions(create_table, partitions, &query_ctx)?;
542        let mut table_info = create_table_info(create_table, partition_cols)?;
543
544        let resp = self
545            .create_table_procedure(
546                create_table.clone(),
547                partitions,
548                table_info.clone(),
549                query_ctx,
550                trigger_reason,
551            )
552            .await?;
553
554        let table_id = resp
555            .table_ids
556            .into_iter()
557            .next()
558            .context(error::UnexpectedSnafu {
559                violated: "expected table_id",
560            })?;
561        info!("Successfully created table '{table_name}' with table id {table_id}");
562
563        table_info.ident.table_id = table_id;
564
565        let table_info = Arc::new(table_info);
566        create_table.table_id = Some(api::v1::TableId { id: table_id });
567
568        let table = DistTable::table(table_info);
569
570        Ok(table)
571    }
572
573    #[tracing::instrument(skip_all)]
574    pub async fn create_logical_tables(
575        &self,
576        create_table_exprs: &[CreateTableExpr],
577        query_context: QueryContextRef,
578        trigger_reason: TriggerReason,
579    ) -> Result<Vec<TableRef>> {
580        let _timer = crate::metrics::DIST_CREATE_TABLES.start_timer();
581        ensure!(
582            !create_table_exprs.is_empty(),
583            EmptyDdlExprSnafu {
584                name: "create logic tables"
585            }
586        );
587
588        // Check table names
589        for create_table in create_table_exprs {
590            ensure!(
591                NAME_PATTERN_REG.is_match(&create_table.table_name),
592                InvalidTableNameSnafu {
593                    table_name: &create_table.table_name,
594                }
595            );
596        }
597
598        let raw_tables_info = create_table_exprs
599            .iter()
600            .map(|create| create_table_info(create, vec![]))
601            .collect::<Result<Vec<_>>>()?;
602        let tables_data = create_table_exprs
603            .iter()
604            .cloned()
605            .zip(raw_tables_info.iter().cloned())
606            .collect::<Vec<_>>();
607
608        let resp = self
609            .create_logical_tables_procedure(tables_data, query_context.clone(), trigger_reason)
610            .await?;
611
612        let table_ids = resp.table_ids;
613        ensure!(
614            table_ids.len() == raw_tables_info.len(),
615            CreateLogicalTablesSnafu {
616                reason: format!(
617                    "The number of tables is inconsistent with the expected number to be created, expected: {}, actual: {}",
618                    raw_tables_info.len(),
619                    table_ids.len()
620                )
621            }
622        );
623        info!("Successfully created logical tables: {:?}", table_ids);
624
625        // Reacquire table infos from catalog so logical tables inherit the latest partition
626        // metadata (e.g. partition_key_indices) from their physical tables.
627        // And the returned table info also included extra partition columns that are in physical table but not in logical table's create table expr
628        let mut tables_info = Vec::with_capacity(table_ids.len());
629        for (table_id, create_table) in table_ids.iter().zip(create_table_exprs.iter()) {
630            let table = self
631                .catalog_manager
632                .table(
633                    &create_table.catalog_name,
634                    &create_table.schema_name,
635                    &create_table.table_name,
636                    Some(&query_context),
637                )
638                .await
639                .context(CatalogSnafu)?
640                .with_context(|| TableNotFoundSnafu {
641                    table_name: format_full_table_name(
642                        &create_table.catalog_name,
643                        &create_table.schema_name,
644                        &create_table.table_name,
645                    ),
646                })?;
647
648            let table_info = table.table_info();
649            // Safety check: ensure we are returning the table info that matches the newly created table id.
650            ensure!(
651                table_info.table_id() == *table_id,
652                CreateLogicalTablesSnafu {
653                    reason: format!(
654                        "Table id mismatch after creation, expected {}, got {} for table {}",
655                        table_id,
656                        table_info.table_id(),
657                        format_full_table_name(
658                            &create_table.catalog_name,
659                            &create_table.schema_name,
660                            &create_table.table_name
661                        )
662                    )
663                }
664            );
665
666            tables_info.push(table_info);
667        }
668
669        Ok(tables_info.into_iter().map(DistTable::table).collect())
670    }
671
672    async fn validate_logical_table_partition_rule(
673        &self,
674        create_table: &CreateTableExpr,
675        partitions: &Partitions,
676        query_ctx: &QueryContextRef,
677    ) -> Result<()> {
678        let (_, mut logical_partition_exprs) =
679            parse_partitions_for_logical_validation(create_table, partitions, query_ctx)?;
680
681        let physical_table_name = create_table
682            .table_options
683            .get(LOGICAL_TABLE_METADATA_KEY)
684            .with_context(|| CreateLogicalTablesSnafu {
685                reason: format!(
686                    "expect `{LOGICAL_TABLE_METADATA_KEY}` option on creating logical table"
687                ),
688            })?;
689
690        let physical_table = self
691            .catalog_manager
692            .table(
693                &create_table.catalog_name,
694                &create_table.schema_name,
695                physical_table_name,
696                Some(query_ctx),
697            )
698            .await
699            .context(CatalogSnafu)?
700            .context(TableNotFoundSnafu {
701                table_name: physical_table_name.clone(),
702            })?;
703
704        let physical_table_info = physical_table.table_info();
705        let (partition_rule, _) = self
706            .partition_manager
707            .find_table_partition_rule(&physical_table_info)
708            .await
709            .context(error::FindTablePartitionRuleSnafu {
710                table_name: physical_table_name.clone(),
711            })?;
712
713        let multi_dim_rule = partition_rule
714            .as_ref()
715            .as_any()
716            .downcast_ref::<MultiDimPartitionRule>()
717            .context(InvalidPartitionRuleSnafu {
718                reason: "physical table partition rule is not range-based",
719            })?;
720
721        // TODO(ruihang): project physical partition exprs to logical partition column
722        let mut physical_partition_exprs = multi_dim_rule.exprs().to_vec();
723        logical_partition_exprs.sort_unstable();
724        physical_partition_exprs.sort_unstable();
725
726        ensure!(
727            physical_partition_exprs == logical_partition_exprs,
728            InvalidPartitionRuleSnafu {
729                reason: format!(
730                    "logical table partition rule must match the corresponding physical table's\n logical table partition exprs:\t\t {:?}\n physical table partition exprs:\t {:?}",
731                    logical_partition_exprs, physical_partition_exprs
732                ),
733            }
734        );
735
736        Ok(())
737    }
738
739    #[cfg(feature = "enterprise")]
740    #[tracing::instrument(skip_all)]
741    pub async fn create_trigger(
742        &self,
743        stmt: CreateTrigger,
744        query_context: QueryContextRef,
745    ) -> Result<Output> {
746        let expr = expr_helper::to_create_trigger_task_expr(stmt, &query_context)?;
747        self.create_trigger_inner(expr, query_context).await
748    }
749
750    #[cfg(feature = "enterprise")]
751    pub async fn create_trigger_inner(
752        &self,
753        expr: PbCreateTriggerExpr,
754        query_context: QueryContextRef,
755    ) -> Result<Output> {
756        self.create_trigger_procedure(expr, query_context).await?;
757        Ok(Output::new_with_affected_rows(0))
758    }
759
760    #[cfg(feature = "enterprise")]
761    async fn create_trigger_procedure(
762        &self,
763        expr: PbCreateTriggerExpr,
764        query_context: QueryContextRef,
765    ) -> Result<SubmitDdlTaskResponse> {
766        let task = CreateTriggerTask::try_from(PbCreateTriggerTask {
767            create_trigger: Some(expr),
768        })
769        .context(error::InvalidExprSnafu)?;
770
771        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
772        let request = SubmitDdlTaskRequest::new(DdlTask::new_create_trigger(task));
773
774        self.procedure_executor
775            .submit_ddl_task(executor_context, request)
776            .await
777            .context(error::ExecuteDdlSnafu)
778    }
779
780    #[tracing::instrument(skip_all)]
781    pub async fn create_flow(
782        &self,
783        stmt: CreateFlow,
784        query_context: QueryContextRef,
785    ) -> Result<Output> {
786        // TODO(ruihang): do some verification
787        let eval_offset_secs = stmt.eval_offset;
788        let expr = expr_helper::to_create_flow_task_expr(stmt, &query_context)?;
789
790        // The typed `EVAL OFFSET` comes straight from the parsed SQL AST; it is
791        // the only trusted source of the offset at this boundary.
792        self.create_flow_procedure(expr, eval_offset_secs, query_context)
793            .await?;
794        Ok(Output::new_with_affected_rows(0))
795    }
796
797    /// Direct gRPC entry point for `CREATE FLOW`.
798    pub async fn create_flow_inner(
799        &self,
800        expr: CreateFlowExpr,
801        query_context: QueryContextRef,
802    ) -> Result<Output> {
803        self.create_flow_procedure(expr, None, query_context)
804            .await?;
805        Ok(Output::new_with_affected_rows(0))
806    }
807
808    async fn create_flow_procedure(
809        &self,
810        mut expr: CreateFlowExpr,
811        eval_offset_secs: Option<i64>,
812        query_context: QueryContextRef,
813    ) -> Result<SubmitDdlTaskResponse> {
814        let eval_interval_secs = expr.eval_interval.as_ref().map(|e| e.seconds);
815
816        // Reject non-positive eval_interval (zero or negative).
817        if let Some(secs) = eval_interval_secs
818            && secs <= 0
819        {
820            return InvalidSqlSnafu {
821                err_msg: format!("EVAL INTERVAL must be positive, got {secs} seconds"),
822            }
823            .fail();
824        }
825
826        // Validate EVAL OFFSET semantics at the operator boundary (defense in
827        // depth; the parser already enforces them deterministically). The
828        // offset arrives as a typed parameter, never from `flow_options`.
829        validate_eval_offset(eval_offset_secs, eval_interval_secs)?;
830
831        // Validate user options. `validate_and_normalize_flow_options` also
832        // rejects the internal transport keys, so a spoofed key smuggled into
833        // the expr can never be honored here.
834        expr.flow_options =
835            validate_and_normalize_flow_options(expr.flow_options, eval_interval_secs)?;
836
837        let flow_type = self
838            .determine_flow_type(&expr, query_context.clone())
839            .await?;
840        info!("determined flow={} type: {:#?}", expr.flow_name, flow_type);
841
842        expr.flow_options
843            .insert(FlowType::FLOW_TYPE_KEY.to_string(), flow_type.to_string());
844
845        let mut task = CreateFlowTask::try_from(PbCreateFlowTask {
846            create_flow: Some(expr),
847        })
848        .context(error::InvalidExprSnafu)?;
849        task.eval_offset_secs = eval_offset_secs;
850        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
851        let request = SubmitDdlTaskRequest::new(DdlTask::new_create_flow(task));
852
853        self.procedure_executor
854            .submit_ddl_task(executor_context, request)
855            .await
856            .context(error::ExecuteDdlSnafu)
857    }
858
859    /// Determines the flow type from source-table state, schedule requirements,
860    /// and SQL shape.
861    async fn determine_flow_type(
862        &self,
863        expr: &CreateFlowExpr,
864        query_ctx: QueryContextRef,
865    ) -> Result<FlowType> {
866        let has_eval_interval = expr.eval_interval.is_some();
867
868        let mut has_missing_source_table = false;
869        let mut has_instant_ttl_source_table = false;
870
871        for src_table_name in &expr.source_table_names {
872            let table = self
873                .catalog_manager()
874                .table(
875                    &src_table_name.catalog_name,
876                    &src_table_name.schema_name,
877                    &src_table_name.table_name,
878                    Some(&query_ctx),
879                )
880                .await
881                .map_err(BoxedError::new)
882                .context(ExternalSnafu)?;
883
884            let Some(table) = table else {
885                has_missing_source_table = true;
886                continue;
887            };
888
889            if table.table_info().meta.options.ttl == Some(common_time::TimeToLive::Instant) {
890                warn!(
891                    "Source table `{}` for flow `{}`'s ttl=instant, {}",
892                    format_full_table_name(
893                        &src_table_name.catalog_name,
894                        &src_table_name.schema_name,
895                        &src_table_name.table_name
896                    ),
897                    expr.flow_name,
898                    if has_eval_interval {
899                        "rejecting flow because EVAL INTERVAL requires the batching scheduler which does not support instant TTL"
900                    } else {
901                        "fallback to streaming mode"
902                    }
903                );
904                has_instant_ttl_source_table = true;
905            }
906        }
907
908        if let Some(flow_type) = determine_flow_type_for_source_state(
909            &expr.flow_name,
910            &expr.flow_options,
911            has_missing_source_table,
912            has_instant_ttl_source_table,
913            has_eval_interval,
914        )? {
915            return Ok(flow_type);
916        }
917
918        // A flow with `EVAL INTERVAL` (and therefore possibly `EVAL OFFSET`)
919        // always uses the batching scheduler: the fixed epoch-phase schedule is
920        // only honored by the batching scheduler, regardless of the SQL shape.
921        // Non-aggregate SQL is supported as an explicit full-query flow (the
922        // batching engine requires a time-window expression or EVAL INTERVAL).
923        if has_eval_interval {
924            return Ok(FlowType::Batching);
925        }
926
927        let engine = &self.query_engine;
928        let stmts = ParserContext::create_with_dialect(
929            &expr.sql,
930            query_ctx.sql_dialect(),
931            ParseOptions::default(),
932        )
933        .map_err(BoxedError::new)
934        .context(ExternalSnafu)?;
935
936        ensure!(
937            stmts.len() == 1,
938            InvalidSqlSnafu {
939                err_msg: format!("Expect only one statement, found {}", stmts.len())
940            }
941        );
942        let stmt = &stmts[0];
943
944        if is_tql(query_ctx.sql_dialect(), &expr.sql)
945            .map_err(BoxedError::new)
946            .context(ExternalSnafu)?
947        {
948            return Ok(FlowType::Batching);
949        }
950
951        // support tql parse too
952        let plan = match stmt {
953            // prom ql is only supported in batching mode
954            Statement::Tql(_) => return Ok(FlowType::Batching),
955            _ => engine
956                .planner()
957                .plan(&QueryStatement::Sql(stmt.clone()), query_ctx)
958                .await
959                .map_err(BoxedError::new)
960                .context(ExternalSnafu)?,
961        };
962
963        /// Visitor to find aggregation or distinct
964        struct FindAggr {
965            is_aggr: bool,
966        }
967
968        impl TreeNodeVisitor<'_> for FindAggr {
969            type Node = LogicalPlan;
970            fn f_down(
971                &mut self,
972                node: &Self::Node,
973            ) -> datafusion_common::Result<datafusion_common::tree_node::TreeNodeRecursion>
974            {
975                match node {
976                    LogicalPlan::Aggregate(_) | LogicalPlan::Distinct(_) => {
977                        self.is_aggr = true;
978                        return Ok(datafusion_common::tree_node::TreeNodeRecursion::Stop);
979                    }
980                    _ => (),
981                }
982                Ok(datafusion_common::tree_node::TreeNodeRecursion::Continue)
983            }
984        }
985
986        let mut find_aggr = FindAggr { is_aggr: false };
987
988        plan.visit_with_subqueries(&mut find_aggr)
989            .context(BuildDfLogicalPlanSnafu)?;
990        if find_aggr.is_aggr {
991            Ok(FlowType::Batching)
992        } else {
993            Ok(FlowType::Streaming)
994        }
995    }
996
997    #[tracing::instrument(skip_all)]
998    pub async fn create_view(
999        &self,
1000        create_view: CreateView,
1001        ctx: QueryContextRef,
1002    ) -> Result<TableRef> {
1003        // convert input into logical plan
1004        let logical_plan = match &*create_view.query {
1005            Statement::Query(query) => {
1006                self.plan(
1007                    &QueryStatement::Sql(Statement::Query(query.clone())),
1008                    ctx.clone(),
1009                )
1010                .await?
1011            }
1012            Statement::Tql(query) => self.plan_tql(query.clone(), &ctx).await?,
1013            _ => {
1014                return InvalidViewStmtSnafu {}.fail();
1015            }
1016        };
1017        // Save the definition for `show create view`.
1018        let definition = create_view.to_string();
1019
1020        // Save the columns in plan, it may changed when the schemas of tables in plan
1021        // are altered.
1022        let schema: Schema = logical_plan
1023            .schema()
1024            .clone()
1025            .try_into()
1026            .context(ConvertSchemaSnafu)?;
1027        let plan_columns: Vec<_> = schema
1028            .column_schemas()
1029            .iter()
1030            .map(|c| c.name.clone())
1031            .collect();
1032
1033        let columns: Vec<_> = create_view
1034            .columns
1035            .iter()
1036            .map(|ident| ident.to_string())
1037            .collect();
1038
1039        // Validate columns
1040        if !columns.is_empty() {
1041            ensure!(
1042                columns.len() == plan_columns.len(),
1043                error::ViewColumnsMismatchSnafu {
1044                    view_name: create_view.name.to_string(),
1045                    expected: plan_columns.len(),
1046                    actual: columns.len(),
1047                }
1048            );
1049        }
1050
1051        // Extract the table names from the original plan
1052        // and rewrite them as fully qualified names.
1053        let (table_names, plan) = extract_and_rewrite_full_table_names(logical_plan, ctx.clone())
1054            .context(ExtractTableNamesSnafu)?;
1055
1056        let table_names = table_names.into_iter().map(|t| t.into()).collect();
1057
1058        // TODO(dennis): we don't save the optimized plan yet,
1059        // because there are some serialization issue with our own defined plan node (such as `MergeScanLogicalPlan`).
1060        // When the issues are fixed, we can use the `optimized_plan` instead.
1061        // let optimized_plan = self.optimize_logical_plan(logical_plan)?.unwrap_df_plan();
1062
1063        // encode logical plan
1064        let encoded_plan = DFLogicalSubstraitConvertor
1065            .encode(&plan, DefaultSerializer)
1066            .context(SubstraitCodecSnafu)?;
1067
1068        let expr = expr_helper::to_create_view_expr(
1069            create_view,
1070            encoded_plan.to_vec(),
1071            table_names,
1072            columns,
1073            plan_columns,
1074            definition,
1075            ctx.clone(),
1076        )?;
1077
1078        // TODO(dennis): validate the logical plan
1079        self.create_view_by_expr(expr, ctx).await
1080    }
1081
1082    pub async fn create_view_by_expr(
1083        &self,
1084        expr: CreateViewExpr,
1085        ctx: QueryContextRef,
1086    ) -> Result<TableRef> {
1087        // A view could otherwise squat a reserved name and block the canonical
1088        // table's first-write creation.
1089        ensure_table_definition_writable(&expr.schema_name, &expr.view_name)?;
1090        ensure! {
1091            !(expr.create_if_not_exists & expr.or_replace),
1092            InvalidSqlSnafu {
1093                err_msg: "syntax error Create Or Replace and If Not Exist cannot be used together",
1094            }
1095        };
1096        let _timer = crate::metrics::DIST_CREATE_VIEW.start_timer();
1097
1098        let schema_exists = self
1099            .table_metadata_manager
1100            .schema_manager()
1101            .exists(SchemaNameKey::new(&expr.catalog_name, &expr.schema_name))
1102            .await
1103            .context(TableMetadataManagerSnafu)?;
1104
1105        ensure!(
1106            schema_exists,
1107            SchemaNotFoundSnafu {
1108                schema_info: &expr.schema_name,
1109            }
1110        );
1111
1112        // if view or table exists.
1113        if let Some(table) = self
1114            .catalog_manager
1115            .table(
1116                &expr.catalog_name,
1117                &expr.schema_name,
1118                &expr.view_name,
1119                Some(&ctx),
1120            )
1121            .await
1122            .context(CatalogSnafu)?
1123        {
1124            let table_type = table.table_info().table_type;
1125
1126            match (table_type, expr.create_if_not_exists, expr.or_replace) {
1127                (TableType::View, true, false) => {
1128                    return Ok(table);
1129                }
1130                (TableType::View, false, false) => {
1131                    return ViewAlreadyExistsSnafu {
1132                        name: format_full_table_name(
1133                            &expr.catalog_name,
1134                            &expr.schema_name,
1135                            &expr.view_name,
1136                        ),
1137                    }
1138                    .fail();
1139                }
1140                (TableType::View, _, true) => {
1141                    // Try to replace an exists view
1142                }
1143                _ => {
1144                    return TableAlreadyExistsSnafu {
1145                        table: format_full_table_name(
1146                            &expr.catalog_name,
1147                            &expr.schema_name,
1148                            &expr.view_name,
1149                        ),
1150                    }
1151                    .fail();
1152                }
1153            }
1154        }
1155
1156        ensure!(
1157            NAME_PATTERN_REG.is_match(&expr.view_name),
1158            InvalidViewNameSnafu {
1159                name: expr.view_name.clone(),
1160            }
1161        );
1162
1163        let view_name = TableName::new(&expr.catalog_name, &expr.schema_name, &expr.view_name);
1164
1165        let mut view_info = TableInfo {
1166            ident: metadata::TableIdent {
1167                // The view id of distributed table is assigned by Meta, set "0" here as a placeholder.
1168                table_id: 0,
1169                version: 0,
1170            },
1171            name: expr.view_name.clone(),
1172            desc: None,
1173            catalog_name: expr.catalog_name.clone(),
1174            schema_name: expr.schema_name.clone(),
1175            // The meta doesn't make sense for views, so using a default one.
1176            meta: TableMeta::empty(),
1177            table_type: TableType::View,
1178        };
1179
1180        let executor_context = to_executor_context(ctx, TriggerReason::Manual);
1181        let request = SubmitDdlTaskRequest::new(DdlTask::new_create_view(expr, view_info.clone()));
1182
1183        let resp = self
1184            .procedure_executor
1185            .submit_ddl_task(executor_context, request)
1186            .await
1187            .context(error::ExecuteDdlSnafu)?;
1188
1189        debug!(
1190            "Submit creating view '{view_name}' task response: {:?}",
1191            resp
1192        );
1193
1194        let view_id = resp
1195            .table_ids
1196            .into_iter()
1197            .next()
1198            .context(error::UnexpectedSnafu {
1199                violated: "expected table_id",
1200            })?;
1201        info!("Successfully created view '{view_name}' with view id {view_id}");
1202
1203        view_info.ident.table_id = view_id;
1204
1205        let view_info = Arc::new(view_info);
1206
1207        let table = DistTable::table(view_info);
1208
1209        // Invalidates local cache ASAP.
1210        self.cache_invalidator
1211            .invalidate(
1212                &Context::default(),
1213                &[
1214                    CacheIdent::TableId(view_id),
1215                    CacheIdent::TableName(view_name.clone()),
1216                ],
1217            )
1218            .await
1219            .context(error::InvalidateTableCacheSnafu)?;
1220
1221        Ok(table)
1222    }
1223
1224    #[tracing::instrument(skip_all)]
1225    pub async fn drop_flow(
1226        &self,
1227        catalog_name: String,
1228        flow_name: String,
1229        drop_if_exists: bool,
1230        query_context: QueryContextRef,
1231    ) -> Result<Output> {
1232        if let Some(flow) = self
1233            .flow_metadata_manager
1234            .flow_name_manager()
1235            .get(&catalog_name, &flow_name)
1236            .await
1237            .context(error::TableMetadataManagerSnafu)?
1238        {
1239            let flow_id = flow.flow_id();
1240            let task = DropFlowTask {
1241                catalog_name,
1242                flow_name,
1243                flow_id,
1244                drop_if_exists,
1245            };
1246            self.drop_flow_procedure(task, query_context).await?;
1247
1248            Ok(Output::new_with_affected_rows(0))
1249        } else if drop_if_exists {
1250            Ok(Output::new_with_affected_rows(0))
1251        } else {
1252            FlowNotFoundSnafu {
1253                flow_name: format_full_flow_name(&catalog_name, &flow_name),
1254            }
1255            .fail()
1256        }
1257    }
1258
1259    async fn drop_flow_procedure(
1260        &self,
1261        expr: DropFlowTask,
1262        query_context: QueryContextRef,
1263    ) -> Result<SubmitDdlTaskResponse> {
1264        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
1265        let request = SubmitDdlTaskRequest::new(DdlTask::new_drop_flow(expr));
1266
1267        self.procedure_executor
1268            .submit_ddl_task(executor_context, request)
1269            .await
1270            .context(error::ExecuteDdlSnafu)
1271    }
1272
1273    #[cfg(feature = "enterprise")]
1274    #[tracing::instrument(skip_all)]
1275    pub(super) async fn drop_trigger(
1276        &self,
1277        catalog_name: String,
1278        trigger_name: String,
1279        drop_if_exists: bool,
1280        query_context: QueryContextRef,
1281    ) -> Result<Output> {
1282        let task = DropTriggerTask {
1283            catalog_name,
1284            trigger_name,
1285            drop_if_exists,
1286        };
1287        self.drop_trigger_procedure(task, query_context).await?;
1288        Ok(Output::new_with_affected_rows(0))
1289    }
1290
1291    #[cfg(feature = "enterprise")]
1292    async fn drop_trigger_procedure(
1293        &self,
1294        expr: DropTriggerTask,
1295        query_context: QueryContextRef,
1296    ) -> Result<SubmitDdlTaskResponse> {
1297        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
1298        let request = SubmitDdlTaskRequest::new(DdlTask::new_drop_trigger(expr));
1299
1300        self.procedure_executor
1301            .submit_ddl_task(executor_context, request)
1302            .await
1303            .context(error::ExecuteDdlSnafu)
1304    }
1305
1306    /// Drop a view
1307    #[tracing::instrument(skip_all)]
1308    pub async fn drop_view(
1309        &self,
1310        catalog: String,
1311        schema: String,
1312        view: String,
1313        drop_if_exists: bool,
1314        query_context: QueryContextRef,
1315    ) -> Result<Output> {
1316        let view_info = if let Some(view) = self
1317            .catalog_manager
1318            .table(&catalog, &schema, &view, None)
1319            .await
1320            .context(CatalogSnafu)?
1321        {
1322            view.table_info()
1323        } else if drop_if_exists {
1324            // DROP VIEW IF EXISTS meets view not found - ignored
1325            return Ok(Output::new_with_affected_rows(0));
1326        } else {
1327            return TableNotFoundSnafu {
1328                table_name: format_full_table_name(&catalog, &schema, &view),
1329            }
1330            .fail();
1331        };
1332
1333        // Ensure the exists one is view, we can't drop other table types
1334        ensure!(
1335            view_info.table_type == TableType::View,
1336            error::InvalidViewSnafu {
1337                msg: "not a view",
1338                view_name: format_full_table_name(&catalog, &schema, &view),
1339            }
1340        );
1341
1342        let view_id = view_info.table_id();
1343        let view_name = TableName::new(&catalog, &schema, &view);
1344
1345        let task = DropViewTask {
1346            catalog,
1347            schema,
1348            view,
1349            view_id,
1350            drop_if_exists,
1351        };
1352
1353        self.drop_view_procedure(task, query_context).await?;
1354
1355        // Invalidates local cache ASAP.
1356        self.cache_invalidator
1357            .invalidate(
1358                &Context::default(),
1359                &[
1360                    CacheIdent::TableId(view_id),
1361                    CacheIdent::TableName(view_name),
1362                ],
1363            )
1364            .await
1365            .context(error::InvalidateTableCacheSnafu)?;
1366
1367        Ok(Output::new_with_affected_rows(0))
1368    }
1369
1370    /// Submit [DropViewTask] to procedure executor.
1371    async fn drop_view_procedure(
1372        &self,
1373        expr: DropViewTask,
1374        query_context: QueryContextRef,
1375    ) -> Result<SubmitDdlTaskResponse> {
1376        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
1377        let request = SubmitDdlTaskRequest::new(DdlTask::new_drop_view(expr));
1378
1379        self.procedure_executor
1380            .submit_ddl_task(executor_context, request)
1381            .await
1382            .context(error::ExecuteDdlSnafu)
1383    }
1384
1385    #[tracing::instrument(skip_all)]
1386    pub async fn alter_logical_tables(
1387        &self,
1388        alter_table_exprs: Vec<AlterTableExpr>,
1389        query_context: QueryContextRef,
1390        trigger_reason: TriggerReason,
1391    ) -> Result<Output> {
1392        let _timer = crate::metrics::DIST_ALTER_TABLES.start_timer();
1393        ensure!(
1394            !alter_table_exprs.is_empty(),
1395            EmptyDdlExprSnafu {
1396                name: "alter logical tables"
1397            }
1398        );
1399
1400        // group by physical table id
1401        let mut groups: HashMap<TableId, Vec<AlterTableExpr>> = HashMap::new();
1402        for expr in alter_table_exprs {
1403            // Get table_id from catalog_manager
1404            let catalog = if expr.catalog_name.is_empty() {
1405                query_context.current_catalog()
1406            } else {
1407                &expr.catalog_name
1408            };
1409            let schema = if expr.schema_name.is_empty() {
1410                query_context.current_schema()
1411            } else {
1412                expr.schema_name.clone()
1413            };
1414            let table_name = &expr.table_name;
1415            let table = self
1416                .catalog_manager
1417                .table(catalog, &schema, table_name, Some(&query_context))
1418                .await
1419                .context(CatalogSnafu)?
1420                .with_context(|| TableNotFoundSnafu {
1421                    table_name: format_full_table_name(catalog, &schema, table_name),
1422                })?;
1423            let table_id = table.table_info().ident.table_id;
1424            let physical_table_id = self
1425                .table_metadata_manager
1426                .table_route_manager()
1427                .get_physical_table_id(table_id)
1428                .await
1429                .context(TableMetadataManagerSnafu)?;
1430            groups.entry(physical_table_id).or_default().push(expr);
1431        }
1432
1433        // Submit procedure for each physical table
1434        let mut handles = Vec::with_capacity(groups.len());
1435        for (_physical_table_id, exprs) in groups {
1436            let fut =
1437                self.alter_logical_tables_procedure(exprs, query_context.clone(), trigger_reason);
1438            handles.push(fut);
1439        }
1440        let _results = futures::future::try_join_all(handles).await?;
1441
1442        Ok(Output::new_with_affected_rows(0))
1443    }
1444
1445    #[tracing::instrument(skip_all)]
1446    pub async fn drop_table(
1447        &self,
1448        table_name: TableName,
1449        drop_if_exists: bool,
1450        query_context: QueryContextRef,
1451    ) -> Result<Output> {
1452        // Reserved for grpc call
1453        self.drop_tables(&[table_name], drop_if_exists, query_context)
1454            .await
1455    }
1456
1457    #[tracing::instrument(skip_all)]
1458    pub async fn drop_tables(
1459        &self,
1460        table_names: &[TableName],
1461        drop_if_exists: bool,
1462        query_context: QueryContextRef,
1463    ) -> Result<Output> {
1464        let mut tables = Vec::with_capacity(table_names.len());
1465        for table_name in table_names {
1466            ensure_table_writable(&table_name.schema_name, &table_name.table_name)?;
1467
1468            if let Some(table) = self
1469                .catalog_manager
1470                .table(
1471                    &table_name.catalog_name,
1472                    &table_name.schema_name,
1473                    &table_name.table_name,
1474                    Some(&query_context),
1475                )
1476                .await
1477                .context(CatalogSnafu)?
1478            {
1479                tables.push(table.table_info().table_id());
1480            } else if drop_if_exists {
1481                // DROP TABLE IF EXISTS meets table not found - ignored
1482                continue;
1483            } else {
1484                return TableNotFoundSnafu {
1485                    table_name: table_name.to_string(),
1486                }
1487                .fail();
1488            }
1489        }
1490
1491        for (table_name, table_id) in table_names.iter().zip(tables.into_iter()) {
1492            self.drop_table_procedure(table_name, table_id, drop_if_exists, query_context.clone())
1493                .await?;
1494
1495            // Invalidates local cache ASAP.
1496            self.cache_invalidator
1497                .invalidate(
1498                    &Context::default(),
1499                    &[
1500                        CacheIdent::TableId(table_id),
1501                        CacheIdent::TableName(table_name.clone()),
1502                    ],
1503                )
1504                .await
1505                .context(error::InvalidateTableCacheSnafu)?;
1506        }
1507        Ok(Output::new_with_affected_rows(0))
1508    }
1509
1510    #[cfg(feature = "enterprise")]
1511    #[tracing::instrument(skip_all)]
1512    pub async fn undrop_table(
1513        &self,
1514        table_name: TableName,
1515        query_context: QueryContextRef,
1516    ) -> Result<Output> {
1517        execute_undrop_table(
1518            &self.table_metadata_manager,
1519            &self.procedure_executor,
1520            &self.cache_invalidator,
1521            table_name,
1522            query_context,
1523        )
1524        .await
1525    }
1526
1527    #[tracing::instrument(skip_all)]
1528    pub async fn drop_database(
1529        &self,
1530        catalog: String,
1531        schema: String,
1532        drop_if_exists: bool,
1533        query_context: QueryContextRef,
1534    ) -> Result<Output> {
1535        ensure!(
1536            !is_readonly_schema(&schema),
1537            SchemaReadOnlySnafu { name: schema }
1538        );
1539
1540        if self
1541            .catalog_manager
1542            .schema_exists(&catalog, &schema, None)
1543            .await
1544            .context(CatalogSnafu)?
1545        {
1546            if schema == query_context.current_schema() {
1547                SchemaInUseSnafu { name: schema }.fail()
1548            } else {
1549                self.drop_database_procedure(catalog, schema, drop_if_exists, query_context)
1550                    .await?;
1551
1552                Ok(Output::new_with_affected_rows(0))
1553            }
1554        } else if drop_if_exists {
1555            // DROP TABLE IF EXISTS meets table not found - ignored
1556            Ok(Output::new_with_affected_rows(0))
1557        } else {
1558            SchemaNotFoundSnafu {
1559                schema_info: schema,
1560            }
1561            .fail()
1562        }
1563    }
1564
1565    #[tracing::instrument(skip_all)]
1566    pub async fn truncate_table(
1567        &self,
1568        table_name: TableName,
1569        time_ranges: Vec<(Timestamp, Timestamp)>,
1570        query_context: QueryContextRef,
1571    ) -> Result<Output> {
1572        ensure_table_writable(&table_name.schema_name, &table_name.table_name)?;
1573
1574        let table = self
1575            .catalog_manager
1576            .table(
1577                &table_name.catalog_name,
1578                &table_name.schema_name,
1579                &table_name.table_name,
1580                Some(&query_context),
1581            )
1582            .await
1583            .context(CatalogSnafu)?
1584            .with_context(|| TableNotFoundSnafu {
1585                table_name: table_name.to_string(),
1586            })?;
1587        let table_id = table.table_info().table_id();
1588        self.truncate_table_procedure(&table_name, table_id, time_ranges, query_context)
1589            .await?;
1590
1591        Ok(Output::new_with_affected_rows(0))
1592    }
1593
1594    #[tracing::instrument(skip_all)]
1595    pub async fn alter_table(
1596        &self,
1597        alter_table: AlterTable,
1598        query_context: QueryContextRef,
1599    ) -> Result<Output> {
1600        if matches!(
1601            alter_table.alter_operation(),
1602            AlterTableOperation::Repartition { .. } | AlterTableOperation::Partition { .. }
1603        ) {
1604            let request = expr_helper::to_repartition_request(alter_table, &query_context)?;
1605            return self.repartition_table(request, &query_context).await;
1606        }
1607
1608        let expr = expr_helper::to_alter_table_expr(alter_table, &query_context)?;
1609        self.alter_table_inner(expr, query_context, TriggerReason::Manual)
1610            .await
1611    }
1612
1613    #[tracing::instrument(skip_all)]
1614    pub async fn repartition_table(
1615        &self,
1616        request: RepartitionRequest,
1617        query_context: &QueryContextRef,
1618    ) -> Result<Output> {
1619        // Check if the schema is read-only.
1620        ensure_table_definition_writable(&request.schema_name, &request.table_name)?;
1621
1622        let table_ref = TableReference::full(
1623            &request.catalog_name,
1624            &request.schema_name,
1625            &request.table_name,
1626        );
1627        // Get the table from the catalog.
1628        let table = self
1629            .catalog_manager
1630            .table(
1631                &request.catalog_name,
1632                &request.schema_name,
1633                &request.table_name,
1634                Some(query_context),
1635            )
1636            .await
1637            .context(CatalogSnafu)?
1638            .with_context(|| TableNotFoundSnafu {
1639                table_name: table_ref.to_string(),
1640            })?;
1641        let table_id = table.table_info().ident.table_id;
1642        // Get existing partition expressions from the table route.
1643        let (physical_table_id, physical_table_route) = self
1644            .table_metadata_manager
1645            .table_route_manager()
1646            .get_physical_table_route(table_id)
1647            .await
1648            .context(TableMetadataManagerSnafu)?;
1649
1650        ensure!(
1651            physical_table_id == table_id,
1652            NotSupportedSnafu {
1653                feat: "REPARTITION on logical tables"
1654            }
1655        );
1656
1657        let table_info = table.table_info();
1658        let existing_partition_columns = table_info.meta.partition_columns().collect::<Vec<_>>();
1659        let column_schemas = table_info.meta.schema.column_schemas();
1660        // `REPARTITION ... ON COLUMNS` uses overwrite semantics: the provided
1661        // columns are the full target partition columns, not an extension of the
1662        // current ones. Therefore source expressions are converted with the
1663        // existing partition columns, while target expressions and the final
1664        // partition rule are validated against this effective target column set.
1665        let target_partition_columns = match &request.source {
1666            RepartitionSource::Partitions {
1667                target_partition_columns,
1668                ..
1669            } => {
1670                ensure!(
1671                    !existing_partition_columns.is_empty(),
1672                    InvalidPartitionRuleSnafu {
1673                        reason: format!(
1674                            "table {} does not have partition columns, cannot repartition",
1675                            table_ref
1676                        )
1677                    }
1678                );
1679
1680                if let Some(target_partition_columns) = target_partition_columns {
1681                    ensure!(
1682                        !target_partition_columns.is_empty(),
1683                        InvalidPartitionRuleSnafu {
1684                            reason: "ON COLUMNS requires at least one partition column"
1685                        }
1686                    );
1687                    validate_and_collect_partition_columns(
1688                        target_partition_columns,
1689                        column_schemas,
1690                    )?
1691                } else {
1692                    existing_partition_columns.clone()
1693                }
1694            }
1695            RepartitionSource::Unpartitioned { partition_columns } => {
1696                ensure!(
1697                    !partition_columns.is_empty(),
1698                    InvalidPartitionRuleSnafu {
1699                        reason: "PARTITION ON COLUMNS requires at least one partition column"
1700                    }
1701                );
1702                ensure!(
1703                    existing_partition_columns.is_empty(),
1704                    InvalidPartitionRuleSnafu {
1705                        reason: format!("table {} already has partition columns", table_ref)
1706                    }
1707                );
1708                partition_columns
1709                    .iter()
1710                    .map(|column_name| {
1711                        column_schemas
1712                            .iter()
1713                            .find(|column| &column.name == column_name)
1714                            .with_context(|| ColumnNotFoundSnafu { msg: column_name })
1715                    })
1716                    .collect::<Result<Vec<_>>>()?
1717            }
1718        };
1719
1720        let from_column_name_and_type = column_name_and_type(&existing_partition_columns);
1721        let target_column_name_and_type = column_name_and_type(&target_partition_columns);
1722        let target_partition_column_names = target_partition_columns
1723            .iter()
1724            .map(|column| column.name.clone())
1725            .collect::<Vec<_>>();
1726        let timezone = query_context.timezone();
1727        // Convert SQL Exprs to PartitionExprs.
1728        let from_partition_exprs = match &request.source {
1729            RepartitionSource::Partitions { from_exprs, .. } => from_exprs
1730                .iter()
1731                .map(|expr| convert_one_expr(expr, &from_column_name_and_type, &timezone))
1732                .collect::<Result<Vec<_>>>()?,
1733            RepartitionSource::Unpartitioned { .. } => vec![],
1734        };
1735
1736        let mut into_partition_exprs = request
1737            .into_exprs
1738            .iter()
1739            .map(|expr| convert_one_expr(expr, &target_column_name_and_type, &timezone))
1740            .collect::<Result<Vec<_>>>()?;
1741
1742        // `MERGE PARTITION` (and some `REPARTITION`) generates a single `OR` expression from
1743        // multiple source partitions; try to simplify it for better readability and stability.
1744        if matches!(&request.source, RepartitionSource::Partitions { .. })
1745            && from_partition_exprs.len() > 1
1746            && into_partition_exprs.len() == 1
1747            && let Some(expr) = into_partition_exprs.pop()
1748        {
1749            into_partition_exprs.push(partition::simplify::simplify_merged_partition_expr(expr));
1750        }
1751
1752        // Parse existing partition expressions from region routes.
1753        let mut existing_partition_exprs =
1754            Vec::with_capacity(physical_table_route.region_routes.len());
1755        for route in &physical_table_route.region_routes {
1756            let expr_json = route.region.partition_expr();
1757            if !expr_json.is_empty() {
1758                match PartitionExpr::from_json_str(&expr_json) {
1759                    Ok(Some(expr)) => existing_partition_exprs.push(expr),
1760                    Ok(None) => {
1761                        // Empty
1762                    }
1763                    Err(e) => {
1764                        return Err(e).context(DeserializePartitionExprSnafu);
1765                    }
1766                }
1767            }
1768        }
1769
1770        // Validate that from_partition_exprs are a subset of existing partition exprs.
1771        // We compare PartitionExpr directly since it implements Eq.
1772        if matches!(&request.source, RepartitionSource::Partitions { .. }) {
1773            for from_expr in &from_partition_exprs {
1774                ensure!(
1775                    existing_partition_exprs.contains(from_expr),
1776                    InvalidPartitionRuleSnafu {
1777                        reason: format!(
1778                            "partition expression '{}' does not exist in table {}",
1779                            from_expr, table_ref
1780                        )
1781                    }
1782                );
1783            }
1784        }
1785
1786        // Build the new partition expressions:
1787        // new_exprs = existing_exprs - from_exprs + into_exprs
1788        let new_partition_exprs: Vec<PartitionExpr> = match &request.source {
1789            RepartitionSource::Partitions { .. } => existing_partition_exprs
1790                .into_iter()
1791                .filter(|expr| !from_partition_exprs.contains(expr))
1792                .chain(into_partition_exprs.clone().into_iter())
1793                .collect(),
1794            RepartitionSource::Unpartitioned { .. } => into_partition_exprs.clone(),
1795        };
1796        ensure_partition_expr_columns_in_target(
1797            &new_partition_exprs,
1798            &target_partition_column_names.iter().collect(),
1799        )?;
1800        let new_partition_exprs_len = new_partition_exprs.len();
1801        let from_partition_exprs_len = from_partition_exprs.len();
1802
1803        // Validate the new partition expressions using MultiDimPartitionRule and PartitionChecker.
1804        let _ = MultiDimPartitionRule::try_new(
1805            target_partition_column_names,
1806            vec![],
1807            new_partition_exprs,
1808            true,
1809        )
1810        .context(InvalidPartitionSnafu)?;
1811
1812        let ddl_options = parse_ddl_options(&request.options)?;
1813        let serialize_exprs = |exprs: Vec<PartitionExpr>| -> Result<Vec<String>> {
1814            let mut json_exprs = Vec::with_capacity(exprs.len());
1815            for expr in exprs {
1816                json_exprs.push(expr.as_json_str().context(SerializePartitionExprSnafu)?);
1817            }
1818            Ok(json_exprs)
1819        };
1820        let from_partition_exprs_json = serialize_exprs(from_partition_exprs)?;
1821        let into_partition_exprs_json = serialize_exprs(into_partition_exprs)?;
1822        let source = match &request.source {
1823            RepartitionSource::Partitions {
1824                target_partition_columns,
1825                ..
1826            } => Source::PartitionExprs(PartitionedSource {
1827                exprs: from_partition_exprs_json,
1828                target_partition_columns: target_partition_columns
1829                    .clone()
1830                    .map(|columns| TargetPartitionColumns { columns }),
1831            }),
1832            RepartitionSource::Unpartitioned { partition_columns } => {
1833                Source::Unpartitioned(UnpartitionedSource {
1834                    partition_columns: partition_columns.clone(),
1835                })
1836            }
1837        };
1838        let repartition = Repartition {
1839            into_partition_exprs: into_partition_exprs_json,
1840            source: Some(source),
1841            ..Default::default()
1842        };
1843        let executor_context = to_executor_context(query_context.clone(), TriggerReason::Manual);
1844        let mut req = SubmitDdlTaskRequest::new(DdlTask::new_alter_table(AlterTableExpr {
1845            catalog_name: request.catalog_name.clone(),
1846            schema_name: request.schema_name.clone(),
1847            table_name: request.table_name.clone(),
1848            kind: Some(Kind::Repartition(repartition)),
1849        }));
1850        req.wait = ddl_options.wait;
1851        req.timeout = ddl_options.timeout;
1852
1853        info!(
1854            "Submitting repartition task for table {} (table_id={}), from {} to {} partitions, timeout: {:?}, wait: {}",
1855            table_ref,
1856            table_id,
1857            from_partition_exprs_len,
1858            new_partition_exprs_len,
1859            ddl_options.timeout,
1860            ddl_options.wait
1861        );
1862
1863        let response = self
1864            .procedure_executor
1865            .submit_ddl_task(executor_context, req)
1866            .await
1867            .context(error::ExecuteDdlSnafu)?;
1868
1869        if !ddl_options.wait {
1870            return build_procedure_id_output(response.key);
1871        }
1872
1873        // Only invalidate cache if wait is true.
1874        let invalidate_keys = vec![
1875            CacheIdent::TableId(table_id),
1876            CacheIdent::TableName(TableName::new(
1877                request.catalog_name,
1878                request.schema_name,
1879                request.table_name,
1880            )),
1881        ];
1882
1883        // Invalidates local cache ASAP.
1884        self.cache_invalidator
1885            .invalidate(&Context::default(), &invalidate_keys)
1886            .await
1887            .context(error::InvalidateTableCacheSnafu)?;
1888
1889        Ok(Output::new_with_affected_rows(0))
1890    }
1891
1892    #[tracing::instrument(skip_all)]
1893    pub async fn alter_table_inner(
1894        &self,
1895        expr: AlterTableExpr,
1896        query_context: QueryContextRef,
1897        trigger_reason: TriggerReason,
1898    ) -> Result<Output> {
1899        ensure_table_definition_writable(&expr.schema_name, &expr.table_name)?;
1900
1901        let catalog_name = if expr.catalog_name.is_empty() {
1902            DEFAULT_CATALOG_NAME.to_string()
1903        } else {
1904            expr.catalog_name.clone()
1905        };
1906
1907        let schema_name = if expr.schema_name.is_empty() {
1908            DEFAULT_SCHEMA_NAME.to_string()
1909        } else {
1910            expr.schema_name.clone()
1911        };
1912
1913        let table_name = expr.table_name.clone();
1914
1915        let table = self
1916            .catalog_manager
1917            .table(
1918                &catalog_name,
1919                &schema_name,
1920                &table_name,
1921                Some(&query_context),
1922            )
1923            .await
1924            .context(CatalogSnafu)?
1925            .with_context(|| TableNotFoundSnafu {
1926                table_name: format_full_table_name(&catalog_name, &schema_name, &table_name),
1927            })?;
1928
1929        let table_id = table.table_info().ident.table_id;
1930        let need_alter = verify_alter(table_id, table.table_info(), expr.clone())?;
1931        if !need_alter {
1932            return Ok(Output::new_with_affected_rows(0));
1933        }
1934        info!(
1935            "Table info before alter is {:?}, expr: {:?}",
1936            table.table_info(),
1937            expr
1938        );
1939
1940        let physical_table_id = self
1941            .table_metadata_manager
1942            .table_route_manager()
1943            .get_physical_table_id(table_id)
1944            .await
1945            .context(TableMetadataManagerSnafu)?;
1946
1947        let executor_context = to_executor_context(query_context, trigger_reason);
1948
1949        let (req, invalidate_keys) = if physical_table_id == table_id {
1950            // This is physical table
1951            let req = SubmitDdlTaskRequest::new(DdlTask::new_alter_table(expr));
1952
1953            let invalidate_keys = vec![
1954                CacheIdent::TableId(table_id),
1955                CacheIdent::TableName(TableName::new(catalog_name, schema_name, table_name)),
1956            ];
1957
1958            (req, invalidate_keys)
1959        } else {
1960            // This is logical table. Annotation alters only rewrite its own
1961            // metadata; `AlterLogicalTablesProcedure` only handles column adds.
1962            let annotation_alter = match expr.kind.as_ref() {
1963                Some(kind) => common_grpc_expr::annotation_alter_family(kind)
1964                    .context(AlterExprToRequestSnafu)?
1965                    .is_some_and(|family| family.allows_logical_tables()),
1966                None => false,
1967            };
1968            let task = if annotation_alter {
1969                DdlTask::new_alter_table(expr)
1970            } else {
1971                DdlTask::new_alter_logical_tables(vec![expr])
1972            };
1973            let req = SubmitDdlTaskRequest::new(task);
1974
1975            let mut invalidate_keys = vec![
1976                CacheIdent::TableId(physical_table_id),
1977                CacheIdent::TableId(table_id),
1978                CacheIdent::TableName(TableName::new(catalog_name, schema_name, table_name)),
1979            ];
1980
1981            let physical_table = self
1982                .table_metadata_manager
1983                .table_info_manager()
1984                .get(physical_table_id)
1985                .await
1986                .context(TableMetadataManagerSnafu)?
1987                .map(|x| x.into_inner());
1988            if let Some(physical_table) = physical_table {
1989                let physical_table_name = TableName::new(
1990                    physical_table.table_info.catalog_name,
1991                    physical_table.table_info.schema_name,
1992                    physical_table.table_info.name,
1993                );
1994                invalidate_keys.push(CacheIdent::TableName(physical_table_name));
1995            }
1996
1997            (req, invalidate_keys)
1998        };
1999
2000        self.procedure_executor
2001            .submit_ddl_task(executor_context, req)
2002            .await
2003            .context(error::ExecuteDdlSnafu)?;
2004
2005        // Invalidates local cache ASAP.
2006        self.cache_invalidator
2007            .invalidate(&Context::default(), &invalidate_keys)
2008            .await
2009            .context(error::InvalidateTableCacheSnafu)?;
2010
2011        Ok(Output::new_with_affected_rows(0))
2012    }
2013
2014    #[cfg(feature = "enterprise")]
2015    #[tracing::instrument(skip_all)]
2016    pub async fn alter_trigger(
2017        &self,
2018        _alter_expr: AlterTrigger,
2019        _query_context: QueryContextRef,
2020    ) -> Result<Output> {
2021        crate::error::NotSupportedSnafu {
2022            feat: "alter trigger",
2023        }
2024        .fail()
2025    }
2026
2027    #[tracing::instrument(skip_all)]
2028    pub async fn alter_database(
2029        &self,
2030        alter_expr: AlterDatabase,
2031        query_context: QueryContextRef,
2032    ) -> Result<Output> {
2033        let alter_expr = expr_helper::to_alter_database_expr(alter_expr, &query_context)?;
2034        self.alter_database_inner(alter_expr, query_context).await
2035    }
2036
2037    #[tracing::instrument(skip_all)]
2038    pub async fn alter_database_inner(
2039        &self,
2040        alter_expr: AlterDatabaseExpr,
2041        query_context: QueryContextRef,
2042    ) -> Result<Output> {
2043        ensure!(
2044            !is_readonly_schema(&alter_expr.schema_name),
2045            SchemaReadOnlySnafu {
2046                name: query_context.current_schema().clone()
2047            }
2048        );
2049
2050        let exists = self
2051            .catalog_manager
2052            .schema_exists(&alter_expr.catalog_name, &alter_expr.schema_name, None)
2053            .await
2054            .context(CatalogSnafu)?;
2055        ensure!(
2056            exists,
2057            SchemaNotFoundSnafu {
2058                schema_info: alter_expr.schema_name,
2059            }
2060        );
2061
2062        let cache_ident = [CacheIdent::SchemaName(SchemaName {
2063            catalog_name: alter_expr.catalog_name.clone(),
2064            schema_name: alter_expr.schema_name.clone(),
2065        })];
2066
2067        self.alter_database_procedure(alter_expr, query_context)
2068            .await?;
2069
2070        // Invalidates local cache ASAP.
2071        self.cache_invalidator
2072            .invalidate(&Context::default(), &cache_ident)
2073            .await
2074            .context(error::InvalidateTableCacheSnafu)?;
2075
2076        Ok(Output::new_with_affected_rows(0))
2077    }
2078
2079    async fn create_table_procedure(
2080        &self,
2081        create_table: CreateTableExpr,
2082        partitions: Vec<PartitionExpr>,
2083        table_info: TableInfo,
2084        query_context: QueryContextRef,
2085        trigger_reason: TriggerReason,
2086    ) -> Result<SubmitDdlTaskResponse> {
2087        let partitions = partitions
2088            .into_iter()
2089            .map(|expr| expr.as_pb_partition().context(PartitionExprToPbSnafu))
2090            .collect::<Result<Vec<_>>>()?;
2091
2092        let executor_context = to_executor_context_with_origin_frontend(
2093            query_context,
2094            &self.origin_frontend_addr,
2095            trigger_reason,
2096        );
2097        let request = SubmitDdlTaskRequest::new(DdlTask::new_create_table(
2098            create_table,
2099            partitions,
2100            table_info,
2101        ));
2102
2103        self.procedure_executor
2104            .submit_ddl_task(executor_context, request)
2105            .await
2106            .context(error::ExecuteDdlSnafu)
2107    }
2108
2109    async fn create_logical_tables_procedure(
2110        &self,
2111        tables_data: Vec<(CreateTableExpr, TableInfo)>,
2112        query_context: QueryContextRef,
2113        trigger_reason: TriggerReason,
2114    ) -> Result<SubmitDdlTaskResponse> {
2115        let executor_context = to_executor_context_with_origin_frontend(
2116            query_context,
2117            &self.origin_frontend_addr,
2118            trigger_reason,
2119        );
2120        let request = SubmitDdlTaskRequest::new(DdlTask::new_create_logical_tables(tables_data));
2121
2122        self.procedure_executor
2123            .submit_ddl_task(executor_context, request)
2124            .await
2125            .context(error::ExecuteDdlSnafu)
2126    }
2127
2128    async fn alter_logical_tables_procedure(
2129        &self,
2130        tables_data: Vec<AlterTableExpr>,
2131        query_context: QueryContextRef,
2132        trigger_reason: TriggerReason,
2133    ) -> Result<SubmitDdlTaskResponse> {
2134        let executor_context = to_executor_context(query_context, trigger_reason);
2135        let request = SubmitDdlTaskRequest::new(DdlTask::new_alter_logical_tables(tables_data));
2136
2137        self.procedure_executor
2138            .submit_ddl_task(executor_context, request)
2139            .await
2140            .context(error::ExecuteDdlSnafu)
2141    }
2142
2143    async fn drop_table_procedure(
2144        &self,
2145        table_name: &TableName,
2146        table_id: TableId,
2147        drop_if_exists: bool,
2148        query_context: QueryContextRef,
2149    ) -> Result<SubmitDdlTaskResponse> {
2150        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2151        let request = SubmitDdlTaskRequest::new(DdlTask::new_drop_table(
2152            table_name.catalog_name.clone(),
2153            table_name.schema_name.clone(),
2154            table_name.table_name.clone(),
2155            table_id,
2156            drop_if_exists,
2157        ));
2158
2159        self.procedure_executor
2160            .submit_ddl_task(executor_context, request)
2161            .await
2162            .context(error::ExecuteDdlSnafu)
2163    }
2164
2165    async fn drop_database_procedure(
2166        &self,
2167        catalog: String,
2168        schema: String,
2169        drop_if_exists: bool,
2170        query_context: QueryContextRef,
2171    ) -> Result<SubmitDdlTaskResponse> {
2172        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2173        let request =
2174            SubmitDdlTaskRequest::new(DdlTask::new_drop_database(catalog, schema, drop_if_exists));
2175
2176        self.procedure_executor
2177            .submit_ddl_task(executor_context, request)
2178            .await
2179            .context(error::ExecuteDdlSnafu)
2180    }
2181
2182    async fn alter_database_procedure(
2183        &self,
2184        alter_expr: AlterDatabaseExpr,
2185        query_context: QueryContextRef,
2186    ) -> Result<SubmitDdlTaskResponse> {
2187        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2188        let request = SubmitDdlTaskRequest::new(DdlTask::new_alter_database(alter_expr));
2189
2190        self.procedure_executor
2191            .submit_ddl_task(executor_context, request)
2192            .await
2193            .context(error::ExecuteDdlSnafu)
2194    }
2195
2196    async fn truncate_table_procedure(
2197        &self,
2198        table_name: &TableName,
2199        table_id: TableId,
2200        time_ranges: Vec<(Timestamp, Timestamp)>,
2201        query_context: QueryContextRef,
2202    ) -> Result<SubmitDdlTaskResponse> {
2203        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2204        let request = SubmitDdlTaskRequest::new(DdlTask::new_truncate_table(
2205            table_name.catalog_name.clone(),
2206            table_name.schema_name.clone(),
2207            table_name.table_name.clone(),
2208            table_id,
2209            time_ranges,
2210        ));
2211
2212        self.procedure_executor
2213            .submit_ddl_task(executor_context, request)
2214            .await
2215            .context(error::ExecuteDdlSnafu)
2216    }
2217
2218    #[tracing::instrument(skip_all)]
2219    pub async fn create_database(
2220        &self,
2221        database: &str,
2222        create_if_not_exists: bool,
2223        options: HashMap<String, String>,
2224        query_context: QueryContextRef,
2225    ) -> Result<Output> {
2226        let catalog = query_context.current_catalog();
2227        ensure!(
2228            NAME_PATTERN_REG.is_match(catalog),
2229            error::UnexpectedSnafu {
2230                violated: format!("Invalid catalog name: {}", catalog)
2231            }
2232        );
2233
2234        ensure!(
2235            NAME_PATTERN_REG.is_match(database),
2236            error::UnexpectedSnafu {
2237                violated: format!("Invalid database name: {}", database)
2238            }
2239        );
2240
2241        #[cfg(feature = "enterprise")]
2242        let creator = match &self.create_database_handler {
2243            Some(handler) => handler.creator(&query_context).context(ExternalSnafu)?,
2244            None => None,
2245        };
2246        #[cfg(not(feature = "enterprise"))]
2247        let creator = None;
2248
2249        let output = if !self
2250            .catalog_manager
2251            .schema_exists(catalog, database, None)
2252            .await
2253            .context(CatalogSnafu)?
2254            && !self.catalog_manager.is_reserved_schema_name(database)
2255        {
2256            self.create_database_procedure(
2257                catalog.to_string(),
2258                database.to_string(),
2259                create_if_not_exists,
2260                options,
2261                query_context.clone(),
2262                creator,
2263            )
2264            .await?;
2265
2266            Output::new_with_affected_rows(1)
2267        } else if create_if_not_exists {
2268            Output::new_with_affected_rows(1)
2269        } else {
2270            return error::SchemaExistsSnafu { name: database }.fail();
2271        };
2272
2273        #[cfg(feature = "enterprise")]
2274        if let Some(handler) = &self.create_database_handler {
2275            handler
2276                .refresh_current_user(&query_context)
2277                .await
2278                .context(ExternalSnafu)?;
2279        }
2280
2281        Ok(output)
2282    }
2283
2284    async fn create_database_procedure(
2285        &self,
2286        catalog: String,
2287        database: String,
2288        create_if_not_exists: bool,
2289        options: HashMap<String, String>,
2290        query_context: QueryContextRef,
2291        creator: Option<CreatorGrantIntent>,
2292    ) -> Result<SubmitDdlTaskResponse> {
2293        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2294        let request = SubmitDdlTaskRequest::new(DdlTask::new_create_database(
2295            catalog,
2296            database,
2297            create_if_not_exists,
2298            options,
2299            creator,
2300        ));
2301
2302        self.procedure_executor
2303            .submit_ddl_task(executor_context, request)
2304            .await
2305            .context(error::ExecuteDdlSnafu)
2306    }
2307}
2308
2309/// Parse partition statement [Partitions] into [PartitionExpr] and partition columns.
2310pub fn parse_partitions(
2311    create_table: &CreateTableExpr,
2312    partitions: Option<Partitions>,
2313    query_ctx: &QueryContextRef,
2314) -> Result<(Vec<PartitionExpr>, Vec<String>)> {
2315    // If partitions are not defined by user, use the timestamp column (which has to be existed) as
2316    // the partition column, and create only one partition.
2317    let partition_columns = find_partition_columns(&partitions)?;
2318    let partition_exprs =
2319        find_partition_entries(create_table, &partitions, &partition_columns, query_ctx)?;
2320
2321    // Validates partition
2322    let exprs = partition_exprs.clone();
2323    MultiDimPartitionRule::try_new(partition_columns.clone(), vec![], exprs, true)
2324        .context(InvalidPartitionSnafu)?;
2325
2326    Ok((partition_exprs, partition_columns))
2327}
2328
2329fn parse_partitions_for_logical_validation(
2330    create_table: &CreateTableExpr,
2331    partitions: &Partitions,
2332    query_ctx: &QueryContextRef,
2333) -> Result<(Vec<String>, Vec<PartitionExpr>)> {
2334    let partition_columns = partitions
2335        .column_list
2336        .iter()
2337        .map(|ident| ident.value.clone())
2338        .collect::<Vec<_>>();
2339
2340    let column_name_and_type = partition_columns
2341        .iter()
2342        .map(|pc| {
2343            let column = create_table
2344                .column_defs
2345                .iter()
2346                .find(|c| &c.name == pc)
2347                .context(ColumnNotFoundSnafu { msg: pc.clone() })?;
2348            let column_name = &column.name;
2349            let data_type = ConcreteDataType::from(
2350                ColumnDataTypeWrapper::try_new(column.data_type, column.datatype_extension.clone())
2351                    .context(ColumnDataTypeSnafu)?,
2352            );
2353            Ok((column_name, data_type))
2354        })
2355        .collect::<Result<HashMap<_, _>>>()?;
2356
2357    let mut partition_exprs = Vec::with_capacity(partitions.exprs.len());
2358    for expr in &partitions.exprs {
2359        let partition_expr = convert_one_expr(expr, &column_name_and_type, &query_ctx.timezone())?;
2360        partition_exprs.push(partition_expr);
2361    }
2362
2363    MultiDimPartitionRule::try_new(
2364        partition_columns.clone(),
2365        vec![],
2366        partition_exprs.clone(),
2367        true,
2368    )
2369    .context(InvalidPartitionSnafu)?;
2370
2371    Ok((partition_columns, partition_exprs))
2372}
2373
2374/// Verifies an alter and returns whether it is necessary to perform the alter.
2375///
2376/// # Returns
2377///
2378/// Returns true if the alter need to be porformed; otherwise, it returns false.
2379pub fn verify_alter(
2380    table_id: TableId,
2381    table_info: Arc<TableInfo>,
2382    expr: AlterTableExpr,
2383) -> Result<bool> {
2384    let request: AlterTableRequest =
2385        common_grpc_expr::alter_expr_to_request(table_id, expr, Some(&table_info.meta))
2386            .context(AlterExprToRequestSnafu)?;
2387
2388    let AlterTableRequest {
2389        table_name,
2390        alter_kind,
2391        ..
2392    } = &request;
2393
2394    if let AlterKind::RenameTable { new_table_name } = alter_kind {
2395        ensure!(
2396            NAME_PATTERN_REG.is_match(new_table_name),
2397            error::UnexpectedSnafu {
2398                violated: format!("Invalid table name: {}", new_table_name)
2399            }
2400        );
2401        // Renaming INTO a computed graph table's name would let the overlay
2402        // shadow the renamed physical table, orphaning its data.
2403        ensure_table_definition_writable(&table_info.schema_name, new_table_name)?;
2404    } else if let AlterKind::AddColumns { columns } = alter_kind {
2405        // If all the columns are marked as add_if_not_exists and they already exist in the table,
2406        // there is no need to perform the alter.
2407        let column_names: HashSet<_> = table_info
2408            .meta
2409            .schema
2410            .column_schemas()
2411            .iter()
2412            .map(|schema| &schema.name)
2413            .collect();
2414        if columns.iter().all(|column| {
2415            column_names.contains(&column.column_schema.name) && column.add_if_not_exists
2416        }) {
2417            return Ok(false);
2418        }
2419    }
2420
2421    let new_meta = table_info
2422        .meta
2423        .builder_with_alter_kind(table_name, &request.alter_kind)
2424        .context(error::TableSnafu)?
2425        .build()
2426        .context(error::BuildTableMetaSnafu { table_name })?;
2427
2428    validate_json2_columns_append_mode(&new_meta.schema, &new_meta.options)?;
2429
2430    Ok(true)
2431}
2432
2433pub fn create_table_info(
2434    create_table: &CreateTableExpr,
2435    partition_columns: Vec<String>,
2436) -> Result<TableInfo> {
2437    let mut column_schemas = Vec::with_capacity(create_table.column_defs.len());
2438    let mut column_name_to_index_map = HashMap::new();
2439
2440    for (idx, column) in create_table.column_defs.iter().enumerate() {
2441        let schema =
2442            column_def::try_as_column_schema(column).context(error::InvalidColumnDefSnafu {
2443                column: &column.name,
2444            })?;
2445        let schema = schema.with_time_index(column.name == create_table.time_index);
2446
2447        column_schemas.push(schema);
2448        let _ = column_name_to_index_map.insert(column.name.clone(), idx);
2449    }
2450
2451    let next_column_id = column_schemas.len() as u32;
2452    let schema = Arc::new(Schema::try_new(column_schemas).context(ConvertSchemaSnafu)?);
2453
2454    let primary_key_indices = create_table
2455        .primary_keys
2456        .iter()
2457        .map(|name| {
2458            column_name_to_index_map
2459                .get(name)
2460                .cloned()
2461                .context(ColumnNotFoundSnafu { msg: name })
2462        })
2463        .collect::<Result<Vec<_>>>()?;
2464
2465    let partition_key_indices = partition_columns
2466        .into_iter()
2467        .map(|col_name| {
2468            column_name_to_index_map
2469                .get(&col_name)
2470                .cloned()
2471                .context(ColumnNotFoundSnafu { msg: col_name })
2472        })
2473        .collect::<Result<Vec<_>>>()?;
2474
2475    let mut table_options = TableOptions::try_from_iter(&create_table.table_options)
2476        .context(UnrecognizedTableOptionSnafu)?;
2477
2478    validate_json2_columns_append_mode(&schema, &table_options)?;
2479
2480    validate_and_normalize_annotations(&mut table_options, &schema, &partition_key_indices)?;
2481
2482    let meta = TableMeta {
2483        schema,
2484        primary_key_indices,
2485        value_indices: vec![],
2486        engine: create_table.engine.clone(),
2487        next_column_id,
2488        options: table_options,
2489        created_on: Utc::now(),
2490        updated_on: Utc::now(),
2491        partition_key_indices,
2492        column_ids: vec![],
2493    };
2494
2495    let desc = if create_table.desc.is_empty() {
2496        create_table.table_options.get(COMMENT_KEY).cloned()
2497    } else {
2498        Some(create_table.desc.clone())
2499    };
2500
2501    let table_info = TableInfo {
2502        ident: metadata::TableIdent {
2503            // The table id of distributed table is assigned by Meta, set "0" here as a placeholder.
2504            table_id: 0,
2505            version: 0,
2506        },
2507        name: create_table.table_name.clone(),
2508        desc,
2509        catalog_name: create_table.catalog_name.clone(),
2510        schema_name: create_table.schema_name.clone(),
2511        meta,
2512        table_type: TableType::Base,
2513    };
2514    Ok(table_info)
2515}
2516
2517fn validate_json2_columns_append_mode(schema: &Schema, table_options: &TableOptions) -> Result<()> {
2518    let append_mode = table_options
2519        .extra_options
2520        .get(APPEND_MODE_KEY)
2521        .is_some_and(|value| value == "true");
2522
2523    for column in schema.column_schemas() {
2524        if column.data_type.is_json2() {
2525            ensure!(
2526                append_mode,
2527                InvalidSqlSnafu {
2528                    err_msg: format!(
2529                        "JSON2 column `{}` requires {}='true'",
2530                        column.name, APPEND_MODE_KEY
2531                    ),
2532                }
2533            );
2534        }
2535    }
2536
2537    Ok(())
2538}
2539
2540/// Rejects DDL against read-only schemas and computed entity-graph tables.
2541fn ensure_table_writable(schema: &str, table: &str) -> Result<()> {
2542    ensure!(
2543        !is_readonly_schema(schema),
2544        SchemaReadOnlySnafu {
2545            name: schema.to_string()
2546        }
2547    );
2548    ensure!(
2549        !is_readonly_table(schema, table),
2550        TableReadOnlySnafu {
2551            name: table.to_string()
2552        }
2553    );
2554    Ok(())
2555}
2556
2557/// [`ensure_table_writable`] plus the definition guard for system-defined
2558/// tables: user CREATE, ALTER and RENAME-into are rejected so the canonical
2559/// schema cannot be squatted or mutated. DROP and TRUNCATE stay allowed — the
2560/// next INSERT recreates the table canonically, which is also the recovery
2561/// path if the canonical definition changes across an upgrade.
2562fn ensure_table_definition_writable(schema: &str, table: &str) -> Result<()> {
2563    ensure_table_writable(schema, table)?;
2564    ensure!(
2565        !is_ddl_reserved_table(schema, table),
2566        TableDdlReservedSnafu {
2567            name: table.to_string()
2568        }
2569    );
2570    Ok(())
2571}
2572
2573/// CREATE-side annotation validation: one rule source in the table crate,
2574/// mapped onto this crate's existing error variants so client-visible codes
2575/// and messages stay put.
2576fn validate_and_normalize_annotations(
2577    options: &mut TableOptions,
2578    schema: &Schema,
2579    partition_key_indices: &[usize],
2580) -> Result<()> {
2581    use table::requests::AnnotationValidationError as CheckError;
2582    let cx = AnnotationContext {
2583        schema,
2584        partition_key_indices,
2585    };
2586    validate_and_normalize_annotation_options(options, &cx).map_err(|e| match e {
2587        CheckError::ColumnNotFound { column } => ColumnNotFoundSnafu { msg: column }.build(),
2588        e @ (CheckError::UnknownKey { .. }
2589        | CheckError::InvalidValue { .. }
2590        | CheckError::InvalidPartitionNumHint { .. }
2591        | CheckError::ColumnNotStringForm { .. }) => InvalidSqlSnafu {
2592            err_msg: e.to_string(),
2593        }
2594        .build(),
2595        e @ (CheckError::NotSingleColumn
2596        | CheckError::PartitionMetadataConflict
2597        | CheckError::TimeIndexConflict) => InvalidPartitionRuleSnafu {
2598            reason: e.to_string(),
2599        }
2600        .build(),
2601    })
2602}
2603
2604fn find_partition_columns(partitions: &Option<Partitions>) -> Result<Vec<String>> {
2605    let columns = if let Some(partitions) = partitions {
2606        partitions
2607            .column_list
2608            .iter()
2609            .map(|x| x.value.clone())
2610            .collect::<Vec<_>>()
2611    } else {
2612        vec![]
2613    };
2614    Ok(columns)
2615}
2616
2617/// Parse [Partitions] into a group of partition entries.
2618///
2619/// Returns a list of [PartitionExpr], each of which defines a partition.
2620fn find_partition_entries(
2621    create_table: &CreateTableExpr,
2622    partitions: &Option<Partitions>,
2623    partition_columns: &[String],
2624    query_ctx: &QueryContextRef,
2625) -> Result<Vec<PartitionExpr>> {
2626    let Some(partitions) = partitions else {
2627        return Ok(vec![]);
2628    };
2629
2630    // extract concrete data type of partition columns
2631    let column_name_and_type = partition_columns
2632        .iter()
2633        .map(|pc| {
2634            let column = create_table
2635                .column_defs
2636                .iter()
2637                .find(|c| &c.name == pc)
2638                // unwrap is safe here because we have checked that partition columns are defined
2639                .unwrap();
2640            let column_name = &column.name;
2641            let data_type = ConcreteDataType::from(
2642                ColumnDataTypeWrapper::try_new(column.data_type, column.datatype_extension.clone())
2643                    .context(ColumnDataTypeSnafu)?,
2644            );
2645            Ok((column_name, data_type))
2646        })
2647        .collect::<Result<HashMap<_, _>>>()?;
2648
2649    // Transform parser expr to partition expr
2650    let mut partition_exprs = Vec::with_capacity(partitions.exprs.len());
2651    for partition in &partitions.exprs {
2652        let partition_expr =
2653            convert_one_expr(partition, &column_name_and_type, &query_ctx.timezone())?;
2654        partition_exprs.push(partition_expr);
2655    }
2656
2657    Ok(partition_exprs)
2658}
2659
2660fn column_name_and_type<'a>(
2661    partition_columns: &'a [&'a ColumnSchema],
2662) -> HashMap<&'a String, ConcreteDataType> {
2663    partition_columns
2664        .iter()
2665        .map(|column| (&column.name, column.data_type.clone()))
2666        .collect()
2667}
2668
2669fn validate_and_collect_partition_columns<'a>(
2670    column_names: &[String],
2671    column_schemas: &'a [ColumnSchema],
2672) -> Result<Vec<&'a ColumnSchema>> {
2673    let mut seen = HashSet::with_capacity(column_names.len());
2674    column_names
2675        .iter()
2676        .map(|column_name| {
2677            ensure!(
2678                seen.insert(column_name),
2679                InvalidPartitionRuleSnafu {
2680                    reason: format!("duplicate partition column '{}'", column_name)
2681                }
2682            );
2683            column_schemas
2684                .iter()
2685                .find(|column| &column.name == column_name)
2686                .with_context(|| ColumnNotFoundSnafu { msg: column_name })
2687        })
2688        .collect()
2689}
2690
2691fn ensure_partition_expr_columns_in_target(
2692    partition_exprs: &[PartitionExpr],
2693    target_partition_columns: &HashSet<&String>,
2694) -> Result<()> {
2695    for expr in partition_exprs {
2696        ensure_partition_operand_columns_in_target(&expr.lhs, target_partition_columns)?;
2697        ensure_partition_operand_columns_in_target(&expr.rhs, target_partition_columns)?;
2698    }
2699
2700    Ok(())
2701}
2702
2703fn ensure_partition_operand_columns_in_target(
2704    operand: &Operand,
2705    target_partition_columns: &HashSet<&String>,
2706) -> Result<()> {
2707    match operand {
2708        Operand::Column(column) => ensure!(
2709            target_partition_columns.contains(column),
2710            InvalidPartitionRuleSnafu {
2711                reason: format!(
2712                    "partition expression references column '{}' that is not in target partition columns",
2713                    column
2714                )
2715            }
2716        ),
2717        Operand::Expr(expr) => {
2718            ensure_partition_operand_columns_in_target(&expr.lhs, target_partition_columns)?;
2719            ensure_partition_operand_columns_in_target(&expr.rhs, target_partition_columns)?;
2720        }
2721        Operand::Value(_) => {}
2722    }
2723
2724    Ok(())
2725}
2726
2727fn convert_one_expr(
2728    expr: &Expr,
2729    column_name_and_type: &HashMap<&String, ConcreteDataType>,
2730    timezone: &Timezone,
2731) -> Result<PartitionExpr> {
2732    let Expr::BinaryOp { left, op, right } = expr else {
2733        return InvalidPartitionRuleSnafu {
2734            reason: "partition rule must be a binary expression",
2735        }
2736        .fail();
2737    };
2738
2739    let op =
2740        RestrictedOp::try_from_parser(&op.clone()).with_context(|| InvalidPartitionRuleSnafu {
2741            reason: format!("unsupported operator in partition expr {op}"),
2742        })?;
2743
2744    // convert leaf node.
2745    let (lhs, op, rhs) = match (left.as_ref(), right.as_ref()) {
2746        // col, val
2747        (Expr::Identifier(ident), Expr::Value(value)) => {
2748            let (column_name, data_type) = convert_identifier(ident, column_name_and_type)?;
2749            let value = convert_value(&value.value, data_type, timezone, None)?;
2750            (Operand::Column(column_name), op, Operand::Value(value))
2751        }
2752        (Expr::Identifier(ident), Expr::UnaryOp { op: unary_op, expr })
2753            if let Expr::Value(v) = &**expr =>
2754        {
2755            let (column_name, data_type) = convert_identifier(ident, column_name_and_type)?;
2756            let value = convert_value(&v.value, data_type, timezone, Some(*unary_op))?;
2757            (Operand::Column(column_name), op, Operand::Value(value))
2758        }
2759        // val, col
2760        (Expr::Value(value), Expr::Identifier(ident)) => {
2761            let (column_name, data_type) = convert_identifier(ident, column_name_and_type)?;
2762            let value = convert_value(&value.value, data_type, timezone, None)?;
2763            (Operand::Value(value), op, Operand::Column(column_name))
2764        }
2765        (Expr::UnaryOp { op: unary_op, expr }, Expr::Identifier(ident))
2766            if let Expr::Value(v) = &**expr =>
2767        {
2768            let (column_name, data_type) = convert_identifier(ident, column_name_and_type)?;
2769            let value = convert_value(&v.value, data_type, timezone, Some(*unary_op))?;
2770            (Operand::Value(value), op, Operand::Column(column_name))
2771        }
2772        (Expr::BinaryOp { .. }, Expr::BinaryOp { .. }) => {
2773            // sub-expr must against another sub-expr
2774            let lhs = convert_one_expr(left, column_name_and_type, timezone)?;
2775            let rhs = convert_one_expr(right, column_name_and_type, timezone)?;
2776            (Operand::Expr(lhs), op, Operand::Expr(rhs))
2777        }
2778        _ => {
2779            return InvalidPartitionRuleSnafu {
2780                reason: format!("invalid partition expr {expr}"),
2781            }
2782            .fail();
2783        }
2784    };
2785
2786    Ok(PartitionExpr::new(lhs, op, rhs))
2787}
2788
2789fn convert_identifier(
2790    ident: &Ident,
2791    column_name_and_type: &HashMap<&String, ConcreteDataType>,
2792) -> Result<(String, ConcreteDataType)> {
2793    let column_name = ident.value.clone();
2794    let data_type = column_name_and_type
2795        .get(&column_name)
2796        .cloned()
2797        .with_context(|| ColumnNotFoundSnafu { msg: &column_name })?;
2798    Ok((column_name, data_type))
2799}
2800
2801fn convert_value(
2802    value: &ParserValue,
2803    data_type: ConcreteDataType,
2804    timezone: &Timezone,
2805    unary_op: Option<UnaryOperator>,
2806) -> Result<Value> {
2807    sql_value_to_value(
2808        &ColumnSchema::new("<NONAME>", data_type, true),
2809        value,
2810        Some(timezone),
2811        unary_op,
2812        false,
2813    )
2814    .context(error::SqlCommonSnafu)
2815}
2816
2817#[cfg(feature = "enterprise")]
2818async fn execute_undrop_table(
2819    table_metadata_manager: &TableMetadataManagerRef,
2820    procedure_executor: &ProcedureExecutorRef,
2821    cache_invalidator: &CacheInvalidatorRef,
2822    table_name: TableName,
2823    query_context: QueryContextRef,
2824) -> Result<Output> {
2825    // Undropping restores a table definition and could resurrect a
2826    // pre-canonical shape of a DDL-reserved table; rejected like CREATE.
2827    ensure_table_definition_writable(&table_name.schema_name, &table_name.table_name)?;
2828
2829    let dropped = table_metadata_manager
2830        .get_dropped_table(&table_name)
2831        .await
2832        .context(TableMetadataManagerSnafu)?
2833        .with_context(|| TableNotFoundSnafu {
2834            table_name: table_name.to_string(),
2835        })?;
2836
2837    let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2838    let request = SubmitDdlTaskRequest::new(DdlTask::new_undrop_table(dropped.table_id));
2839    procedure_executor
2840        .submit_ddl_task(executor_context, request)
2841        .await
2842        .context(error::ExecuteDdlSnafu)?;
2843
2844    if let Err(err) = cache_invalidator
2845        .invalidate(
2846            &Context::default(),
2847            &[
2848                CacheIdent::TableId(dropped.table_id),
2849                CacheIdent::TableName(table_name),
2850            ],
2851        )
2852        .await
2853    {
2854        warn!(
2855            "Failed to invalidate cache after restoring table '{}' (id={}): {}",
2856            dropped.table_name, dropped.table_id, err
2857        );
2858    }
2859
2860    Ok(Output::new_with_affected_rows(0))
2861}
2862
2863#[cfg(test)]
2864mod test {
2865    #[cfg(feature = "enterprise")]
2866    use std::sync::{Arc, Mutex};
2867    use std::time::Duration;
2868
2869    #[cfg(feature = "enterprise")]
2870    use api::v1::meta::{ProcedureDetailResponse, ReconcileRequest, ReconcileResponse};
2871    #[cfg(feature = "enterprise")]
2872    use common_meta::cache_invalidator::{CacheInvalidator, CacheInvalidatorRef};
2873    #[cfg(feature = "enterprise")]
2874    use common_meta::instruction::CacheIdent;
2875    #[cfg(feature = "enterprise")]
2876    use common_meta::key::table_route::TableRouteValue;
2877    #[cfg(feature = "enterprise")]
2878    use common_meta::key::test_utils::new_test_table_info_with_name;
2879    #[cfg(feature = "enterprise")]
2880    use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
2881    #[cfg(feature = "enterprise")]
2882    use common_meta::kv_backend::memory::MemoryKvBackend;
2883    #[cfg(feature = "enterprise")]
2884    use common_meta::procedure_executor::{
2885        ExecutorContext, ProcedureExecutor, ProcedureExecutorRef,
2886    };
2887    #[cfg(feature = "enterprise")]
2888    use common_meta::rpc::ddl::{DdlTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse};
2889    #[cfg(feature = "enterprise")]
2890    use common_meta::rpc::procedure::{
2891        MigrateRegionRequest, MigrateRegionResponse, ProcedureStateResponse,
2892    };
2893    use session::context::{QueryContext, QueryContextBuilder};
2894    use sql::dialect::GreptimeDbDialect;
2895    use sql::parser::{ParseOptions, ParserContext};
2896    use sql::statements::statement::Statement;
2897    use sqlparser::parser::Parser;
2898    use table::requests::REPARTITION_COLUMN_HINT_KEY;
2899
2900    use super::*;
2901    use crate::expr_helper;
2902
2903    #[cfg(feature = "enterprise")]
2904    #[derive(Default)]
2905    struct RecordingProcedureExecutor {
2906        requests: Mutex<Vec<SubmitDdlTaskRequest>>,
2907        contexts: Mutex<Vec<ExecutorContext>>,
2908        fail: bool,
2909    }
2910
2911    #[cfg(feature = "enterprise")]
2912    #[async_trait::async_trait]
2913    impl ProcedureExecutor for RecordingProcedureExecutor {
2914        async fn submit_ddl_task(
2915            &self,
2916            ctx: ExecutorContext,
2917            request: SubmitDdlTaskRequest,
2918        ) -> common_meta::error::Result<SubmitDdlTaskResponse> {
2919            self.contexts.lock().unwrap().push(ctx);
2920            self.requests.lock().unwrap().push(request);
2921            if self.fail {
2922                return common_meta::error::UnsupportedSnafu {
2923                    operation: "test submit failure",
2924                }
2925                .fail();
2926            }
2927            Ok(SubmitDdlTaskResponse::default())
2928        }
2929
2930        async fn migrate_region(
2931            &self,
2932            _: &ExecutorContext,
2933            _: MigrateRegionRequest,
2934        ) -> common_meta::error::Result<MigrateRegionResponse> {
2935            unimplemented!()
2936        }
2937        async fn reconcile(
2938            &self,
2939            _: &ExecutorContext,
2940            _: ReconcileRequest,
2941        ) -> common_meta::error::Result<ReconcileResponse> {
2942            unimplemented!()
2943        }
2944        async fn query_procedure_state(
2945            &self,
2946            _: &ExecutorContext,
2947            _: &str,
2948        ) -> common_meta::error::Result<ProcedureStateResponse> {
2949            unimplemented!()
2950        }
2951        async fn list_procedures(
2952            &self,
2953            _: &ExecutorContext,
2954        ) -> common_meta::error::Result<ProcedureDetailResponse> {
2955            unimplemented!()
2956        }
2957    }
2958
2959    #[cfg(feature = "enterprise")]
2960    #[derive(Default)]
2961    struct RecordingCacheInvalidator {
2962        invalidations: Mutex<Vec<Vec<CacheIdent>>>,
2963        fail: bool,
2964    }
2965
2966    #[cfg(feature = "enterprise")]
2967    #[async_trait::async_trait]
2968    impl CacheInvalidator for RecordingCacheInvalidator {
2969        async fn invalidate(
2970            &self,
2971            _: &Context,
2972            caches: &[CacheIdent],
2973        ) -> common_meta::error::Result<()> {
2974            self.invalidations.lock().unwrap().push(caches.to_vec());
2975            if self.fail {
2976                return common_meta::error::UnsupportedSnafu {
2977                    operation: "test cache failure",
2978                }
2979                .fail();
2980            }
2981            Ok(())
2982        }
2983
2984        fn invalidate_all(&self) -> common_meta::error::Result<()> {
2985            Ok(())
2986        }
2987    }
2988
2989    #[cfg(feature = "enterprise")]
2990    async fn dropped_table_manager(table_id: TableId, name: &TableName) -> TableMetadataManagerRef {
2991        let backend = Arc::new(MemoryKvBackend::default());
2992        let manager = Arc::new(TableMetadataManager::new(backend));
2993        let mut info = new_test_table_info_with_name(table_id, &name.table_name);
2994        info.catalog_name = name.catalog_name.clone();
2995        info.schema_name = name.schema_name.clone();
2996        let route = TableRouteValue::physical(vec![]);
2997        manager
2998            .create_table_metadata(info, route.clone(), HashMap::new())
2999            .await
3000            .unwrap();
3001        manager
3002            .delete_table_metadata(table_id, name, &route, &HashMap::new(), None)
3003            .await
3004            .unwrap();
3005        manager
3006    }
3007
3008    #[cfg(feature = "enterprise")]
3009    #[tokio::test]
3010    async fn test_undrop_table_execution_path() {
3011        let name = TableName::new("greptime", "public", "metrics");
3012        let manager = dropped_table_manager(42, &name).await;
3013        // A same-name live table must not redirect restoration to its ID.
3014        let mut live = new_test_table_info_with_name(99, "metrics");
3015        live.catalog_name = "greptime".to_string();
3016        live.schema_name = "public".to_string();
3017        manager
3018            .create_table_metadata(live, TableRouteValue::physical(vec![]), HashMap::new())
3019            .await
3020            .unwrap();
3021        let procedure = Arc::new(RecordingProcedureExecutor::default());
3022        let cache = Arc::new(RecordingCacheInvalidator::default());
3023
3024        execute_undrop_table(
3025            &manager,
3026            &(procedure.clone() as ProcedureExecutorRef),
3027            &(cache.clone() as CacheInvalidatorRef),
3028            name.clone(),
3029            QueryContext::arc(),
3030        )
3031        .await
3032        .unwrap();
3033
3034        let requests = procedure.requests.lock().unwrap();
3035        assert!(matches!(&requests[0].task, DdlTask::UndropTable(task) if task.table_id == 42));
3036        let contexts = procedure.contexts.lock().unwrap();
3037        assert_eq!(
3038            contexts[0].event_input.as_ref().map(|input| input.reason),
3039            Some(TriggerReason::Manual)
3040        );
3041        assert_eq!(
3042            cache.invalidations.lock().unwrap()[0],
3043            vec![CacheIdent::TableId(42), CacheIdent::TableName(name)]
3044        );
3045    }
3046
3047    #[cfg(feature = "enterprise")]
3048    #[tokio::test]
3049    async fn test_undrop_table_missing_tombstone_submits_nothing() {
3050        let manager = Arc::new(TableMetadataManager::new(Arc::new(
3051            MemoryKvBackend::default(),
3052        )));
3053        let procedure = Arc::new(RecordingProcedureExecutor::default());
3054        let cache = Arc::new(RecordingCacheInvalidator::default());
3055        let err = execute_undrop_table(
3056            &manager,
3057            &(procedure.clone() as ProcedureExecutorRef),
3058            &(cache as CacheInvalidatorRef),
3059            TableName::new("greptime", "public", "missing"),
3060            QueryContext::arc(),
3061        )
3062        .await
3063        .unwrap_err();
3064        assert!(matches!(err, crate::error::Error::TableNotFound { .. }));
3065        assert!(procedure.requests.lock().unwrap().is_empty());
3066    }
3067
3068    #[cfg(feature = "enterprise")]
3069    #[tokio::test]
3070    async fn test_undrop_table_submit_failure_does_not_invalidate() {
3071        let name = TableName::new("greptime", "public", "metrics");
3072        let manager = dropped_table_manager(42, &name).await;
3073        let procedure = Arc::new(RecordingProcedureExecutor {
3074            fail: true,
3075            ..Default::default()
3076        });
3077        let cache = Arc::new(RecordingCacheInvalidator::default());
3078        execute_undrop_table(
3079            &manager,
3080            &(procedure as ProcedureExecutorRef),
3081            &(cache.clone() as CacheInvalidatorRef),
3082            name,
3083            QueryContext::arc(),
3084        )
3085        .await
3086        .unwrap_err();
3087        assert!(cache.invalidations.lock().unwrap().is_empty());
3088    }
3089
3090    #[cfg(feature = "enterprise")]
3091    #[tokio::test]
3092    async fn test_undrop_table_cache_failure_returns_success_after_submit() {
3093        let name = TableName::new("greptime", "public", "metrics");
3094        let manager = dropped_table_manager(42, &name).await;
3095        let procedure = Arc::new(RecordingProcedureExecutor::default());
3096        let cache = Arc::new(RecordingCacheInvalidator {
3097            fail: true,
3098            ..Default::default()
3099        });
3100        execute_undrop_table(
3101            &manager,
3102            &(procedure.clone() as ProcedureExecutorRef),
3103            &(cache.clone() as CacheInvalidatorRef),
3104            name.clone(),
3105            QueryContext::arc(),
3106        )
3107        .await
3108        .unwrap();
3109        assert_eq!(procedure.requests.lock().unwrap().len(), 1);
3110        assert_eq!(
3111            cache.invalidations.lock().unwrap()[0],
3112            vec![CacheIdent::TableId(42), CacheIdent::TableName(name)]
3113        );
3114    }
3115
3116    #[test]
3117    fn test_parse_ddl_options() {
3118        let options = OptionMap::from([
3119            ("timeout".to_string(), "5m".to_string()),
3120            ("wait".to_string(), "false".to_string()),
3121        ]);
3122        let ddl_options = parse_ddl_options(&options).unwrap();
3123        assert!(!ddl_options.wait);
3124        assert_eq!(Duration::from_secs(300), ddl_options.timeout);
3125    }
3126
3127    #[test]
3128    fn test_validate_and_normalize_annotations() {
3129        let schema = Schema::new(vec![
3130            ColumnSchema::new("service_name", ConcreteDataType::string_datatype(), true),
3131            ColumnSchema::new("host_id", ConcreteDataType::string_datatype(), true),
3132            ColumnSchema::new("value", ConcreteDataType::float64_datatype(), true),
3133            ColumnSchema::new("payload", ConcreteDataType::binary_datatype(), true),
3134        ]);
3135        let opts = |pairs: &[(&str, &str)]| TableOptions {
3136            extra_options: pairs
3137                .iter()
3138                .map(|(k, v)| (k.to_string(), v.to_string()))
3139                .collect(),
3140            ..Default::default()
3141        };
3142        let check = |pairs: &[(&str, &str)]| {
3143            let mut options = opts(pairs);
3144            validate_and_normalize_annotations(&mut options, &schema, &[]).map(|()| options)
3145        };
3146
3147        // Any existing column with a string form may be an id, tag or field.
3148        for pairs in [
3149            &[(
3150                "greptime.semantic.entity.process.id",
3151                "service_name,host_id",
3152            )][..],
3153            &[("greptime.semantic.entity.service.id", "value")][..],
3154        ] {
3155            assert!(check(pairs).is_ok());
3156        }
3157
3158        // Missing columns keep this crate's error variant and status code.
3159        let missing = check(&[("greptime.semantic.entity.service.id", "nope")]).unwrap_err();
3160        assert!(matches!(missing, error::Error::ColumnNotFound { .. }));
3161        assert_eq!(
3162            common_error::status_code::StatusCode::InvalidArguments,
3163            common_error::ext::ErrorExt::status_code(&missing)
3164        );
3165
3166        let binary_id = check(&[("greptime.semantic.entity.service.id", "payload")]).unwrap_err();
3167        assert!(matches!(binary_id, error::Error::InvalidSql { .. }));
3168
3169        // The SQL parser checks keys and value domains, but gRPC expressions
3170        // bypass it.
3171        let bad_value = check(&[("greptime.semantic.signal_type", "garbage")]).unwrap_err();
3172        assert!(matches!(bad_value, error::Error::InvalidSql { .. }));
3173
3174        let unknown_key = check(&[("greptime.semantic.nonsense", "x")]).unwrap_err();
3175        assert!(matches!(unknown_key, error::Error::InvalidSql { .. }));
3176    }
3177
3178    #[test]
3179    fn test_validate_and_normalize_flow_options_empty() {
3180        assert!(
3181            validate_and_normalize_flow_options(HashMap::new(), None)
3182                .unwrap()
3183                .is_empty()
3184        );
3185    }
3186
3187    #[test]
3188    fn test_validate_and_normalize_flow_options_valid() {
3189        let options = HashMap::from([
3190            (DEFER_ON_MISSING_SOURCE_KEY.to_string(), "TRUE".to_string()),
3191            (
3192                FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(),
3193                "FALSE".to_string(),
3194            ),
3195        ]);
3196
3197        assert_eq!(
3198            validate_and_normalize_flow_options(options, None).unwrap(),
3199            HashMap::from([
3200                (DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string(),),
3201                (
3202                    FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(),
3203                    "false".to_string(),
3204                )
3205            ])
3206        );
3207    }
3208
3209    #[test]
3210    fn test_validate_and_normalize_flow_options_unknown_option() {
3211        let err = validate_and_normalize_flow_options(
3212            HashMap::from([("foo".to_string(), "bar".to_string())]),
3213            None,
3214        )
3215        .unwrap_err();
3216
3217        assert!(
3218            err.to_string()
3219                .contains("unknown flow option 'foo', supported options:")
3220        );
3221    }
3222
3223    #[test]
3224    fn test_validate_and_normalize_flow_options_reserved_option() {
3225        let err = validate_and_normalize_flow_options(
3226            HashMap::from([(
3227                FlowType::FLOW_TYPE_KEY.to_string(),
3228                FlowType::BATCHING.to_string(),
3229            )]),
3230            None,
3231        )
3232        .unwrap_err();
3233
3234        assert!(
3235            err.to_string()
3236                .contains("flow option 'flow_type' is reserved for internal use")
3237        );
3238    }
3239
3240    #[test]
3241    fn test_validate_and_normalize_flow_options_invalid_bool() {
3242        let err = validate_and_normalize_flow_options(
3243            HashMap::from([(
3244                DEFER_ON_MISSING_SOURCE_KEY.to_string(),
3245                "not-a-bool".to_string(),
3246            )]),
3247            None,
3248        )
3249        .unwrap_err();
3250
3251        assert!(
3252            err.to_string()
3253                .contains("invalid flow option 'defer_on_missing_source': 'not-a-bool'")
3254        );
3255    }
3256
3257    #[test]
3258    fn test_validate_and_normalize_flow_options_rejects_redacted_invalid_input() {
3259        let sql = r"
3260CREATE FLOW task_6
3261SINK TO schema_1.table_1
3262WITH (access_key_id = [true])
3263AS
3264SELECT max(c1), min(c2) FROM schema_2.table_2;";
3265        let stmt =
3266            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
3267                .unwrap()
3268                .pop()
3269                .unwrap();
3270
3271        let Statement::CreateFlow(create_flow) = stmt else {
3272            unreachable!()
3273        };
3274        let expr =
3275            expr_helper::to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
3276        let err = validate_and_normalize_flow_options(expr.flow_options, None).unwrap_err();
3277
3278        assert!(
3279            err.to_string()
3280                .contains("unknown flow option 'access_key_id'")
3281        );
3282    }
3283
3284    // --- Schedule option tests ---
3285
3286    #[test]
3287    fn test_eval_interval_rejected_non_positive() {
3288        // Zero eval_interval should be rejected.
3289        let err = validate_and_normalize_flow_options(HashMap::new(), Some(0)).unwrap_err();
3290        assert!(err.to_string().contains("EVAL INTERVAL must be positive"));
3291
3292        // Negative eval_interval should be rejected.
3293        let err = validate_and_normalize_flow_options(HashMap::new(), Some(-5)).unwrap_err();
3294        assert!(err.to_string().contains("EVAL INTERVAL must be positive"));
3295
3296        // Positive eval_interval should be accepted.
3297        let result = validate_and_normalize_flow_options(HashMap::new(), Some(300));
3298        assert!(result.is_ok());
3299    }
3300
3301    #[test]
3302    fn test_schedule_and_internal_keys_rejected_as_unknown_options() {
3303        for key in [
3304            "eval_interval_anchor",
3305            "eval_interval_start",
3306            "eval_interval_missed_tick_policy",
3307            "eval_interval_catchup_max_runs",
3308            "eval_interval_catchup_max_lag",
3309        ] {
3310            let err = validate_and_normalize_flow_options(
3311                HashMap::from([(key.to_string(), "value".to_string())]),
3312                Some(300),
3313            )
3314            .unwrap_err();
3315
3316            assert!(
3317                err.to_string()
3318                    .contains(&format!("unknown flow option '{key}'")),
3319                "unexpected error for {key}: {err}"
3320            );
3321        }
3322    }
3323
3324    #[test]
3325    fn test_internal_transport_keys_rejected_as_reserved() {
3326        for key in [
3327            "__greptime_internal_eval_schedule",
3328            "__greptime_internal_eval_offset_secs",
3329        ] {
3330            let err = validate_and_normalize_flow_options(
3331                HashMap::from([(key.to_string(), "120".to_string())]),
3332                Some(300),
3333            )
3334            .unwrap_err();
3335
3336            assert!(
3337                err.to_string()
3338                    .contains(&format!("flow option '{key}' is reserved for internal use")),
3339                "unexpected error for {key}: {err}"
3340            );
3341        }
3342    }
3343
3344    #[test]
3345    fn test_determine_flow_type_for_source_state_missing_sources_require_opt_in() {
3346        let err =
3347            determine_flow_type_for_source_state("my_flow", &HashMap::new(), true, false, false)
3348                .unwrap_err();
3349
3350        assert!(err.to_string().contains(
3351            "missing source tables for flow 'my_flow'; use WITH (defer_on_missing_source = true) to create a pending flow"
3352        ));
3353    }
3354
3355    #[test]
3356    fn test_determine_flow_type_for_source_state_missing_sources_prefer_batching() {
3357        let flow_options =
3358            HashMap::from([(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string())]);
3359
3360        assert_eq!(
3361            determine_flow_type_for_source_state("my_flow", &flow_options, true, true, false)
3362                .unwrap(),
3363            Some(FlowType::Batching)
3364        );
3365        let err = determine_flow_type_for_source_state("my_flow", &flow_options, true, true, true)
3366            .unwrap_err();
3367        assert!(err.to_string().contains(
3368            "flow 'my_flow' with EVAL INTERVAL requires the batching scheduler, but source tables with ttl=instant are not supported under batching mode"
3369        ));
3370    }
3371
3372    #[test]
3373    fn test_determine_flow_type_for_source_state_instant_ttl_without_missing_sources() {
3374        assert_eq!(
3375            determine_flow_type_for_source_state("my_flow", &HashMap::new(), false, true, false)
3376                .unwrap(),
3377            Some(FlowType::Streaming)
3378        );
3379    }
3380
3381    #[test]
3382    fn test_determine_flow_type_for_source_state_instant_ttl_with_eval_interval_is_rejected() {
3383        // A flow with `EVAL INTERVAL` is forced onto the batching scheduler,
3384        // which cannot read instant-TTL source tables: reject instead of
3385        // silently falling back to streaming (where the schedule/offset would
3386        // be ignored).
3387        let err =
3388            determine_flow_type_for_source_state("my_flow", &HashMap::new(), false, true, true)
3389                .unwrap_err();
3390        assert!(
3391            err.to_string().contains(
3392                "flow 'my_flow' with EVAL INTERVAL requires the batching scheduler, but source tables with ttl=instant are not supported under batching mode"
3393            ),
3394            "unexpected error: {err}"
3395        );
3396    }
3397
3398    #[test]
3399    fn test_name_is_match() {
3400        assert!(!NAME_PATTERN_REG.is_match("/adaf"));
3401        assert!(!NAME_PATTERN_REG.is_match("🈲"));
3402        assert!(NAME_PATTERN_REG.is_match("hello"));
3403        assert!(NAME_PATTERN_REG.is_match("test@"));
3404        assert!(!NAME_PATTERN_REG.is_match("@test"));
3405        assert!(NAME_PATTERN_REG.is_match("test#"));
3406        assert!(!NAME_PATTERN_REG.is_match("#test"));
3407        assert!(!NAME_PATTERN_REG.is_match("@"));
3408        assert!(!NAME_PATTERN_REG.is_match("#"));
3409    }
3410
3411    #[test]
3412    fn test_partition_expr_equivalence_with_swapped_operands() {
3413        let column_name = "device_id".to_string();
3414        let column_name_and_type =
3415            HashMap::from([(&column_name, ConcreteDataType::int32_datatype())]);
3416        let timezone = Timezone::from_tz_string("UTC").unwrap();
3417        let dialect = GreptimeDbDialect {};
3418
3419        let mut parser = Parser::new(&dialect)
3420            .try_with_sql("device_id < 100")
3421            .unwrap();
3422        let expr_left = parser.parse_expr().unwrap();
3423
3424        let mut parser = Parser::new(&dialect)
3425            .try_with_sql("100 > device_id")
3426            .unwrap();
3427        let expr_right = parser.parse_expr().unwrap();
3428
3429        let partition_left =
3430            convert_one_expr(&expr_left, &column_name_and_type, &timezone).unwrap();
3431        let partition_right =
3432            convert_one_expr(&expr_right, &column_name_and_type, &timezone).unwrap();
3433
3434        assert_eq!(partition_left, partition_right);
3435        assert!([partition_left.clone()].contains(&partition_right));
3436
3437        let mut physical_partition_exprs = vec![partition_left];
3438        let mut logical_partition_exprs = vec![partition_right];
3439        physical_partition_exprs.sort_unstable();
3440        logical_partition_exprs.sort_unstable();
3441        assert_eq!(physical_partition_exprs, logical_partition_exprs);
3442    }
3443
3444    #[test]
3445    fn test_repartition_target_partition_columns_are_overwrite_context() {
3446        let device_id = ColumnSchema::new("device_id", ConcreteDataType::int32_datatype(), true);
3447        let area = ColumnSchema::new("area", ConcreteDataType::string_datatype(), true);
3448        let existing_partition_columns = vec![&device_id];
3449        let target_partition_columns = vec![&device_id, &area];
3450        let existing_column_name_and_type = column_name_and_type(&existing_partition_columns);
3451        let target_column_name_and_type = column_name_and_type(&target_partition_columns);
3452        let timezone = Timezone::from_tz_string("UTC").unwrap();
3453        let dialect = GreptimeDbDialect {};
3454
3455        let mut parser = Parser::new(&dialect)
3456            .try_with_sql("device_id < 100 AND area < 'South'")
3457            .unwrap();
3458        let expr = parser.parse_expr().unwrap();
3459
3460        let err = convert_one_expr(&expr, &existing_column_name_and_type, &timezone).unwrap_err();
3461        assert!(err.to_string().contains("area"));
3462
3463        let partition_expr = convert_one_expr(&expr, &target_column_name_and_type, &timezone)
3464            .expect("target columns should overwrite the conversion context");
3465        let partition_expr = partition_expr.to_string();
3466        assert!(partition_expr.contains("device_id"));
3467        assert!(partition_expr.contains("area"));
3468        assert!(partition_expr.contains("South"));
3469    }
3470
3471    #[test]
3472    fn test_repartition_rejects_remaining_expr_outside_target_columns() {
3473        let device_id = "device_id".to_string();
3474        let area = "area".to_string();
3475        let timezone = Timezone::from_tz_string("UTC").unwrap();
3476        let column_name_and_type = HashMap::from([
3477            (&device_id, ConcreteDataType::int32_datatype()),
3478            (&area, ConcreteDataType::string_datatype()),
3479        ]);
3480        let dialect = GreptimeDbDialect {};
3481        let mut parser = Parser::new(&dialect)
3482            .try_with_sql("device_id >= 100")
3483            .unwrap();
3484        let remaining_old_expr = convert_one_expr(
3485            &parser.parse_expr().unwrap(),
3486            &column_name_and_type,
3487            &timezone,
3488        )
3489        .unwrap();
3490        let target_partition_columns = HashSet::from([&area]);
3491
3492        let err = ensure_partition_expr_columns_in_target(
3493            &[remaining_old_expr],
3494            &target_partition_columns,
3495        )
3496        .unwrap_err();
3497
3498        assert!(err.to_string().contains("device_id"));
3499        assert!(err.to_string().contains("target partition columns"));
3500    }
3501
3502    #[test]
3503    fn test_repartition_rejects_duplicate_target_partition_columns() {
3504        let device_id = ColumnSchema::new("device_id", ConcreteDataType::int32_datatype(), true);
3505        let column_schemas = vec![device_id];
3506        let target_partition_columns = vec!["device_id".to_string(), "device_id".to_string()];
3507
3508        let err =
3509            validate_and_collect_partition_columns(&target_partition_columns, &column_schemas)
3510                .unwrap_err();
3511
3512        assert!(err.to_string().contains("duplicate partition column"));
3513        assert!(err.to_string().contains("device_id"));
3514    }
3515
3516    fn create_expr_from_sql(sql: &str) -> CreateTableExpr {
3517        let result =
3518            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
3519                .unwrap();
3520
3521        match &result[0] {
3522            Statement::CreateTable(create) => {
3523                expr_helper::create_to_expr(create, &QueryContext::arc()).unwrap()
3524            }
3525            _ => unreachable!(),
3526        }
3527    }
3528
3529    #[test]
3530    fn test_create_table_with_repartition_partition_num_hint() {
3531        let key = table::requests::REPARTITION_PARTITION_NUM_HINT_KEY;
3532        for (value, expected) in [
3533            (" 8 ", Some("8")),
3534            ("0", None),
3535            ("-1", None),
3536            ("4294967296", None),
3537        ] {
3538            let expr = create_expr_from_sql(&format!(
3539                "CREATE TABLE metrics (host STRING, ts TIMESTAMP TIME INDEX, PRIMARY KEY(host)) WITH ('{key}' = '{value}')"
3540            ));
3541            let result = create_table_info(&expr, vec![]);
3542            if let Some(expected) = expected {
3543                let info = result.unwrap();
3544                assert_eq!(
3545                    info.meta.options.extra_options.get(key).map(String::as_str),
3546                    Some(expected)
3547                );
3548            } else {
3549                assert!(
3550                    result
3551                        .unwrap_err()
3552                        .to_string()
3553                        .contains("expects a positive integer")
3554                );
3555            }
3556        }
3557    }
3558
3559    #[test]
3560    fn test_create_table_with_repartition_column_hint() {
3561        let expr = create_expr_from_sql(
3562            r"
3563CREATE TABLE metrics (
3564  host STRING,
3565  ts TIMESTAMP TIME INDEX,
3566  cpu DOUBLE,
3567  PRIMARY KEY(host)
3568)
3569WITH ('repartition.column.hint' = ' host ')",
3570        );
3571
3572        let table_info = create_table_info(&expr, vec![]).unwrap();
3573        assert_eq!(
3574            table_info
3575                .meta
3576                .options
3577                .extra_options
3578                .get(REPARTITION_COLUMN_HINT_KEY),
3579            Some(&"host".to_string())
3580        );
3581    }
3582
3583    #[test]
3584    fn test_create_table_info_rejects_non_timestamp_time_index() {
3585        let expr = CreateTableExpr {
3586            catalog_name: "greptime".to_string(),
3587            schema_name: "public".to_string(),
3588            table_name: "demo".to_string(),
3589            desc: String::new(),
3590            column_defs: vec![api::v1::ColumnDef {
3591                name: "host".to_string(),
3592                data_type: api::v1::ColumnDataType::String as i32,
3593                is_nullable: true,
3594                default_constraint: vec![],
3595                semantic_type: 0,
3596                comment: String::new(),
3597                datatype_extension: None,
3598                options: None,
3599            }],
3600            time_index: "host".to_string(),
3601            primary_keys: vec![],
3602            create_if_not_exists: false,
3603            table_options: HashMap::new(),
3604            table_id: None,
3605            engine: "mito".to_string(),
3606        };
3607
3608        let err = create_table_info(&expr, vec![]).unwrap_err();
3609        assert_eq!(
3610            common_error::ext::ErrorExt::status_code(&err),
3611            common_error::status_code::StatusCode::InvalidArguments
3612        );
3613    }
3614
3615    #[test]
3616    fn test_verify_alter_guards_entity_column_types() {
3617        let expr = create_expr_from_sql(
3618            "CREATE TABLE t (svc STRING, ts TIMESTAMP TIME INDEX) \
3619             WITH ('greptime.semantic.entity.service.id'='svc');",
3620        );
3621        let table_info = Arc::new(create_table_info(&expr, vec![]).unwrap());
3622        let alter = |kind| AlterTableExpr {
3623            catalog_name: "greptime".to_string(),
3624            schema_name: "public".to_string(),
3625            table_name: "t".to_string(),
3626            kind: Some(kind),
3627        };
3628
3629        let modify = alter(Kind::ModifyColumnTypes(api::v1::ModifyColumnTypes {
3630            modify_column_types: vec![api::v1::ModifyColumnType {
3631                column_name: "svc".to_string(),
3632                target_type: api::v1::ColumnDataType::Binary as i32,
3633                target_type_extension: None,
3634            }],
3635        }));
3636        let err = verify_alter(1, table_info.clone(), modify).unwrap_err();
3637        let msg = common_error::ext::ErrorExt::output_msg(&err);
3638        assert!(
3639            msg.contains("must keep a type that renders as a string"),
3640            "{msg}"
3641        );
3642
3643        // Dropping a declared column stays allowed: the derivation skips the
3644        // stale declaration.
3645        let drop = alter(Kind::DropColumns(api::v1::DropColumns {
3646            drop_columns: vec![api::v1::DropColumn {
3647                name: "svc".to_string(),
3648            }],
3649        }));
3650        assert!(verify_alter(1, table_info, drop).unwrap());
3651    }
3652
3653    #[test]
3654    fn test_json2_requires_append_mode() {
3655        let cases = [
3656            "CREATE TABLE monitor (payload JSON2, ts TIMESTAMP TIME INDEX);",
3657            "CREATE TABLE monitor (payload JSON2, ts TIMESTAMP TIME INDEX) WITH (append_mode='false');",
3658        ];
3659
3660        for sql in cases {
3661            let expr = create_expr_from_sql(sql);
3662            let err = create_table_info(&expr, vec![]).unwrap_err();
3663            assert!(
3664                err.to_string()
3665                    .contains("JSON2 column `payload` requires append_mode='true'"),
3666                "{err}"
3667            );
3668        }
3669
3670        let expr = create_expr_from_sql(
3671            "CREATE TABLE monitor (payload JSON2, ts TIMESTAMP TIME INDEX) WITH (append_mode='true');",
3672        );
3673        create_table_info(&expr, vec![]).unwrap();
3674    }
3675
3676    #[test]
3677    fn test_create_table_with_empty_repartition_column_hint() {
3678        let expr = create_expr_from_sql(
3679            r"
3680CREATE TABLE metrics (
3681  host STRING,
3682  ts TIMESTAMP TIME INDEX,
3683  cpu DOUBLE,
3684  PRIMARY KEY(host)
3685)
3686WITH ('repartition.column.hint' = '')",
3687        );
3688
3689        let err = create_table_info(&expr, vec![]).unwrap_err();
3690        assert!(
3691            err.to_string()
3692                .contains("repartition.column.hint expects exactly one column name")
3693        );
3694    }
3695
3696    #[test]
3697    fn test_create_table_with_multiple_repartition_column_hints() {
3698        let expr = create_expr_from_sql(
3699            r"
3700CREATE TABLE metrics (
3701  host STRING,
3702  region_id STRING,
3703  ts TIMESTAMP TIME INDEX,
3704  cpu DOUBLE,
3705  PRIMARY KEY(host)
3706)
3707WITH ('repartition.column.hint' = 'host,region_id')",
3708        );
3709
3710        let err = create_table_info(&expr, vec![]).unwrap_err();
3711        assert!(
3712            err.to_string()
3713                .contains("repartition.column.hint expects exactly one column name")
3714        );
3715    }
3716
3717    #[test]
3718    fn test_create_table_with_missing_repartition_column_hint() {
3719        let expr = create_expr_from_sql(
3720            r"
3721CREATE TABLE metrics (
3722  host STRING,
3723  ts TIMESTAMP TIME INDEX,
3724  cpu DOUBLE,
3725  PRIMARY KEY(host)
3726)
3727WITH ('repartition.column.hint' = 'region_id')",
3728        );
3729
3730        let err = create_table_info(&expr, vec![]).unwrap_err();
3731        assert!(
3732            err.to_string()
3733                .contains("Cannot find column by name: region")
3734        );
3735    }
3736
3737    #[test]
3738    fn test_create_table_with_time_index_repartition_column_hint() {
3739        let expr = create_expr_from_sql(
3740            r"
3741CREATE TABLE metrics (
3742  host STRING,
3743  ts TIMESTAMP TIME INDEX,
3744  cpu DOUBLE,
3745  PRIMARY KEY(host)
3746)
3747WITH ('repartition.column.hint' = 'ts')",
3748        );
3749
3750        let err = create_table_info(&expr, vec![]).unwrap_err();
3751        assert!(
3752            err.to_string()
3753                .contains("cannot set repartition.column.hint to the time index column")
3754        );
3755    }
3756
3757    #[test]
3758    fn test_create_partitioned_table_with_repartition_column_hint() {
3759        let expr = create_expr_from_sql(
3760            r"
3761CREATE TABLE metrics (
3762  host STRING,
3763  ts TIMESTAMP TIME INDEX,
3764  cpu DOUBLE,
3765  PRIMARY KEY(host)
3766)
3767WITH ('repartition.column.hint' = 'host')",
3768        );
3769
3770        let err = create_table_info(&expr, vec!["host".to_string()]).unwrap_err();
3771        assert!(
3772            err.to_string()
3773                .contains("cannot set repartition.column.hint on a table with partition metadata")
3774        );
3775    }
3776
3777    #[tokio::test]
3778    #[ignore = "TODO(ruihang): WIP new partition rule"]
3779    async fn test_parse_partitions() {
3780        common_telemetry::init_default_ut_logging();
3781        let cases = [
3782            (
3783                r"
3784CREATE TABLE rcx ( a INT, b STRING, c TIMESTAMP, TIME INDEX (c) )
3785PARTITION ON COLUMNS (b) (
3786  b < 'hz',
3787  b >= 'hz' AND b < 'sh',
3788  b >= 'sh'
3789)
3790ENGINE=mito",
3791                r#"[{"column_list":["b"],"value_list":["{\"Value\":{\"String\":\"hz\"}}"]},{"column_list":["b"],"value_list":["{\"Value\":{\"String\":\"sh\"}}"]},{"column_list":["b"],"value_list":["\"MaxValue\""]}]"#,
3792            ),
3793            (
3794                r"
3795CREATE TABLE rcx ( a INT, b STRING, c TIMESTAMP, TIME INDEX (c) )
3796PARTITION BY RANGE COLUMNS (b, a) (
3797  PARTITION r0 VALUES LESS THAN ('hz', 10),
3798  b < 'hz' AND a < 10,
3799  b >= 'hz' AND b < 'sh' AND a >= 10 AND a < 20,
3800  b >= 'sh' AND a >= 20
3801)
3802ENGINE=mito",
3803                r#"[{"column_list":["b","a"],"value_list":["{\"Value\":{\"String\":\"hz\"}}","{\"Value\":{\"Int32\":10}}"]},{"column_list":["b","a"],"value_list":["{\"Value\":{\"String\":\"sh\"}}","{\"Value\":{\"Int32\":20}}"]},{"column_list":["b","a"],"value_list":["\"MaxValue\"","\"MaxValue\""]}]"#,
3804            ),
3805        ];
3806        let ctx = QueryContextBuilder::default().build().into();
3807        for (sql, expected) in cases {
3808            let result = ParserContext::create_with_dialect(
3809                sql,
3810                &GreptimeDbDialect {},
3811                ParseOptions::default(),
3812            )
3813            .unwrap();
3814            match &result[0] {
3815                Statement::CreateTable(c) => {
3816                    let expr = expr_helper::create_to_expr(c, &QueryContext::arc()).unwrap();
3817                    let (partitions, _) =
3818                        parse_partitions(&expr, c.partitions.clone(), &ctx).unwrap();
3819                    let json = serde_json::to_string(&partitions).unwrap();
3820                    assert_eq!(json, expected);
3821                }
3822                _ => unreachable!(),
3823            }
3824        }
3825    }
3826}