Skip to main content

operator/
statement.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15mod admin;
16mod comment;
17mod copy_database;
18mod copy_query_to;
19mod copy_table_from;
20mod copy_table_to;
21mod cursor;
22mod database_copy;
23pub mod ddl;
24mod describe;
25mod dml;
26pub mod export_database;
27pub mod export_logical_tables;
28mod kill;
29pub mod semantic_graph;
30mod set;
31mod show;
32mod tql;
33
34use std::collections::HashMap;
35use std::sync::Arc;
36
37use api::v1::RowInsertRequests;
38use catalog::CatalogManagerRef;
39use catalog::kvbackend::KvBackendCatalogManager;
40use catalog::process_manager::ProcessManagerRef;
41use client::RecordBatches;
42use client::error::{ExternalSnafu as ClientExternalSnafu, Result as ClientResult};
43use client::inserter::{InsertOptions, Inserter};
44use common_datasource::object_store::LocalFileAccess;
45use common_error::ext::BoxedError;
46use common_meta::cache_invalidator::CacheInvalidatorRef;
47use common_meta::key::flow::{FlowMetadataManager, FlowMetadataManagerRef};
48use common_meta::key::schema_name::SchemaNameKey;
49use common_meta::key::view_info::{ViewInfoManager, ViewInfoManagerRef};
50use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
51use common_meta::kv_backend::KvBackendRef;
52use common_meta::procedure_executor::ProcedureExecutorRef;
53#[cfg(feature = "enterprise")]
54use common_meta::rpc::ddl::CreatorGrantIntent;
55use common_query::Output;
56use common_telemetry::{debug, tracing, warn};
57use common_time::Timestamp;
58use common_time::range::TimestampRange;
59use datafusion_expr::LogicalPlan;
60use datatypes::prelude::ConcreteDataType;
61use datatypes::schema::ColumnSchema;
62use humantime::format_duration;
63use itertools::Itertools;
64use partition::manager::PartitionRuleManagerRef;
65use query::QueryEngineRef;
66use query::parser::QueryStatement;
67use session::context::{Channel, QueryContextBuilder, QueryContextRef};
68use session::table_name::table_idents_to_full_name;
69use set::{set_query_timeout, set_read_preference, set_skip_wal};
70use snafu::{OptionExt, ResultExt, ensure};
71use sql::ast::ObjectNamePartExt;
72use sql::statements::OptionMap;
73use sql::statements::copy::{
74    CopyDatabase, CopyDatabaseArgument, CopyQueryToArgument, CopyTable, CopyTableArgument,
75};
76use sql::statements::set_variables::SetVariables;
77use sql::statements::show::ShowCreateTableVariant;
78use sql::statements::statement::Statement;
79use sql::util::format_raw_object_name;
80use sqlparser::ast::ObjectName;
81use store_api::mito_engine_options::{APPEND_MODE_KEY, TTL_KEY};
82use table::TableRef;
83use table::requests::{CopyDatabaseRequest, CopyDirection, CopyQueryToRequest, CopyTableRequest};
84use table::table_name::TableName;
85use table::table_reference::TableReference;
86
87pub use self::admin::{
88    AdminEventRecorderHandle, AdminFunctionLayer, AdminFunctionLayerRef,
89    AdminFunctionRecordingLayer, AdminFunctionRequest, AdminFunctionResponse, AdminFunctionService,
90    AdminFunctionServiceRef, admin_output_schema,
91};
92use self::set::{
93    set_bytea_output, set_datestyle, set_intervalstyle, set_timezone, validate_client_encoding,
94};
95use crate::error::{
96    self, CatalogSnafu, ExecLogicalPlanSnafu, ExternalSnafu, InvalidSqlSnafu, NotSupportedSnafu,
97    PlanStatementSnafu, Result, SchemaNotFoundSnafu, SqlCommonSnafu, TableMetadataManagerSnafu,
98    TableNotFoundSnafu, UnexpectedSnafu, UpgradeCatalogManagerRefSnafu,
99};
100use crate::insert::InserterRef;
101use crate::statement::copy_database::{COPY_DATABASE_TIME_END_KEY, COPY_DATABASE_TIME_START_KEY};
102use crate::statement::set::set_allow_query_fallback;
103
104/// A configurator that customizes or enhances a [`StatementExecutor`].
105#[async_trait::async_trait]
106pub trait StatementExecutorConfigurator: Send + Sync {
107    async fn configure(
108        &self,
109        executor: StatementExecutor,
110        ctx: ExecutorConfigureContext,
111    ) -> std::result::Result<StatementExecutor, BoxedError>;
112}
113
114pub type StatementExecutorConfiguratorRef = Arc<dyn StatementExecutorConfigurator>;
115
116#[cfg(feature = "enterprise")]
117#[async_trait::async_trait]
118pub trait CreateDatabaseHandler: Send + Sync {
119    fn creator(
120        &self,
121        query_ctx: &QueryContextRef,
122    ) -> std::result::Result<Option<CreatorGrantIntent>, BoxedError>;
123
124    async fn refresh_current_user(
125        &self,
126        query_ctx: &QueryContextRef,
127    ) -> std::result::Result<(), BoxedError>;
128}
129
130#[cfg(feature = "enterprise")]
131pub type CreateDatabaseHandlerRef = Arc<dyn CreateDatabaseHandler>;
132
133pub struct ExecutorConfigureContext {
134    pub kv_backend: KvBackendRef,
135}
136
137#[derive(Clone)]
138pub struct StatementExecutor {
139    catalog_manager: CatalogManagerRef,
140    query_engine: QueryEngineRef,
141    procedure_executor: ProcedureExecutorRef,
142    table_metadata_manager: TableMetadataManagerRef,
143    flow_metadata_manager: FlowMetadataManagerRef,
144    view_info_manager: ViewInfoManagerRef,
145    partition_manager: PartitionRuleManagerRef,
146    cache_invalidator: CacheInvalidatorRef,
147    inserter: InserterRef,
148    process_manager: Option<ProcessManagerRef>,
149    origin_frontend_addr: String,
150    admin_function_service: AdminFunctionServiceRef,
151    pub(crate) local_file_access: LocalFileAccess,
152    #[cfg(feature = "enterprise")]
153    create_database_handler: Option<CreateDatabaseHandlerRef>,
154    #[cfg(feature = "enterprise")]
155    trigger_querier: Option<TriggerQuerierRef>,
156}
157
158pub type StatementExecutorRef = Arc<StatementExecutor>;
159
160/// Trait for querying trigger info, such as `SHOW CREATE TRIGGER` etc.
161#[cfg(feature = "enterprise")]
162#[async_trait::async_trait]
163pub trait TriggerQuerier: Send + Sync {
164    // Query the `SHOW CREATE TRIGGER` statement for the given trigger.
165    async fn show_create_trigger(
166        &self,
167        catalog: &str,
168        trigger: &str,
169        query_ctx: &QueryContextRef,
170    ) -> std::result::Result<Output, BoxedError>;
171
172    fn as_any(&self) -> &dyn std::any::Any;
173}
174
175#[cfg(feature = "enterprise")]
176pub type TriggerQuerierRef = Arc<dyn TriggerQuerier>;
177
178impl StatementExecutor {
179    #[allow(clippy::too_many_arguments)]
180    pub fn new(
181        catalog_manager: CatalogManagerRef,
182        query_engine: QueryEngineRef,
183        procedure_executor: ProcedureExecutorRef,
184        kv_backend: KvBackendRef,
185        cache_invalidator: CacheInvalidatorRef,
186        inserter: InserterRef,
187        partition_manager: PartitionRuleManagerRef,
188        process_manager: Option<ProcessManagerRef>,
189        origin_frontend_addr: String,
190        local_file_access: LocalFileAccess,
191    ) -> Self {
192        let admin_function_service = admin::new_admin_function_service(query_engine.clone());
193        Self {
194            catalog_manager,
195            query_engine,
196            procedure_executor,
197            table_metadata_manager: Arc::new(TableMetadataManager::new(kv_backend.clone())),
198            flow_metadata_manager: Arc::new(FlowMetadataManager::new(kv_backend.clone())),
199            view_info_manager: Arc::new(ViewInfoManager::new(kv_backend.clone())),
200            partition_manager,
201            cache_invalidator,
202            inserter,
203            process_manager,
204            origin_frontend_addr,
205            admin_function_service,
206            local_file_access,
207            #[cfg(feature = "enterprise")]
208            create_database_handler: None,
209            #[cfg(feature = "enterprise")]
210            trigger_querier: None,
211        }
212    }
213
214    /// Adds a layer around the ADMIN function execution service.
215    ///
216    /// The last added layer is the outermost layer.
217    pub fn with_admin_function_layer(mut self, layer: AdminFunctionLayerRef) -> Self {
218        self.admin_function_service = layer.layer(self.admin_function_service);
219        self
220    }
221
222    #[cfg(feature = "enterprise")]
223    pub fn with_trigger_querier(mut self, querier: TriggerQuerierRef) -> Self {
224        self.trigger_querier = Some(querier);
225        self
226    }
227
228    #[cfg(feature = "enterprise")]
229    pub fn with_create_database_handler(mut self, handler: CreateDatabaseHandlerRef) -> Self {
230        self.create_database_handler = Some(handler);
231        self
232    }
233
234    #[cfg(feature = "testing")]
235    pub async fn execute_stmt(
236        &self,
237        stmt: QueryStatement,
238        query_ctx: QueryContextRef,
239    ) -> Result<Output> {
240        match stmt {
241            QueryStatement::Sql(stmt) => self.execute_sql(stmt, query_ctx).await,
242            QueryStatement::Promql(_, _) => self.plan_exec(stmt, query_ctx).await,
243        }
244    }
245
246    #[tracing::instrument(skip_all)]
247    pub async fn execute_sql(&self, stmt: Statement, query_ctx: QueryContextRef) -> Result<Output> {
248        match stmt {
249            Statement::Query(_) | Statement::Explain(_) | Statement::Delete(_) => {
250                self.plan_exec(QueryStatement::Sql(stmt), query_ctx).await
251            }
252
253            Statement::DeclareCursor(declare_cursor) => {
254                self.declare_cursor(declare_cursor, query_ctx).await
255            }
256            Statement::FetchCursor(fetch_cursor) => {
257                self.fetch_cursor(fetch_cursor, query_ctx).await
258            }
259            Statement::CloseCursor(close_cursor) => {
260                self.close_cursor(close_cursor, query_ctx).await
261            }
262
263            Statement::Insert(insert) => self.insert(insert, query_ctx).await,
264
265            Statement::Tql(tql) => self.execute_tql(tql, query_ctx).await,
266
267            Statement::DescribeTable(stmt) => self.describe_table(stmt, query_ctx).await,
268
269            Statement::ShowDatabases(stmt) => self.show_databases(stmt, query_ctx).await,
270
271            Statement::ShowTables(stmt) => self.show_tables(stmt, query_ctx).await,
272
273            Statement::ShowTableStatus(stmt) => self.show_table_status(stmt, query_ctx).await,
274
275            Statement::ShowCollation(kind) => self.show_collation(kind, query_ctx).await,
276
277            Statement::ShowCharset(kind) => self.show_charset(kind, query_ctx).await,
278
279            Statement::ShowViews(stmt) => self.show_views(stmt, query_ctx).await,
280
281            Statement::ShowFlows(stmt) => self.show_flows(stmt, query_ctx).await,
282            Statement::ShowFlowStatus(stmt) => self.show_flow_status(stmt, query_ctx).await,
283
284            #[cfg(feature = "enterprise")]
285            Statement::ShowTriggers(stmt) => self.show_triggers(stmt, query_ctx).await,
286
287            Statement::Copy(sql::statements::copy::Copy::CopyQueryTo(stmt)) => {
288                let query_output = self
289                    .plan_exec(QueryStatement::Sql(*stmt.query), query_ctx)
290                    .await?;
291                let req = to_copy_query_request(stmt.arg)?;
292
293                self.copy_query_to(req, query_output)
294                    .await
295                    .map(Output::new_with_affected_rows)
296            }
297
298            Statement::Copy(sql::statements::copy::Copy::CopyTable(stmt)) => {
299                let req = to_copy_table_request(stmt, query_ctx.clone())?;
300                match req.direction {
301                    CopyDirection::Export => self
302                        .copy_table_to(req, query_ctx)
303                        .await
304                        .map(Output::new_with_affected_rows),
305                    CopyDirection::Import => self.copy_table_from(req, query_ctx).await,
306                }
307            }
308
309            Statement::Copy(sql::statements::copy::Copy::CopyDatabase(copy_database)) => {
310                match copy_database {
311                    CopyDatabase::To(arg) => {
312                        self.copy_database_to(
313                            to_copy_database_request(arg, &query_ctx)?,
314                            query_ctx.clone(),
315                        )
316                        .await
317                    }
318                    CopyDatabase::From(arg) => {
319                        self.copy_database_from(
320                            to_copy_database_request(arg, &query_ctx)?,
321                            query_ctx,
322                        )
323                        .await
324                    }
325                }
326            }
327
328            Statement::CreateTable(stmt) => {
329                let _ = self.create_table(stmt, query_ctx).await?;
330                Ok(Output::new_with_affected_rows(0))
331            }
332            Statement::CreateTableLike(stmt) => {
333                let _ = self.create_table_like(stmt, query_ctx).await?;
334                Ok(Output::new_with_affected_rows(0))
335            }
336            Statement::CreateExternalTable(stmt) => {
337                let _ = self.create_external_table(stmt, query_ctx).await?;
338                Ok(Output::new_with_affected_rows(0))
339            }
340            Statement::CreateFlow(stmt) => self.create_flow(stmt, query_ctx).await,
341            #[cfg(feature = "enterprise")]
342            Statement::CreateTrigger(stmt) => self.create_trigger(stmt, query_ctx).await,
343            Statement::DropFlow(stmt) => {
344                self.drop_flow(
345                    query_ctx.current_catalog().to_string(),
346                    format_raw_object_name(stmt.flow_name()),
347                    stmt.drop_if_exists(),
348                    query_ctx,
349                )
350                .await
351            }
352            #[cfg(feature = "enterprise")]
353            Statement::DropTrigger(stmt) => {
354                self.drop_trigger(
355                    query_ctx.current_catalog().to_string(),
356                    format_raw_object_name(stmt.trigger_name()),
357                    stmt.drop_if_exists(),
358                    query_ctx,
359                )
360                .await
361            }
362            Statement::CreateView(stmt) => {
363                let _ = self.create_view(stmt, query_ctx).await?;
364                Ok(Output::new_with_affected_rows(0))
365            }
366            Statement::DropView(stmt) => {
367                let (catalog_name, schema_name, view_name) =
368                    table_idents_to_full_name(&stmt.view_name, &query_ctx)
369                        .map_err(BoxedError::new)
370                        .context(ExternalSnafu)?;
371
372                self.drop_view(
373                    catalog_name,
374                    schema_name,
375                    view_name,
376                    stmt.drop_if_exists,
377                    query_ctx,
378                )
379                .await
380            }
381            Statement::AlterTable(alter_table) => self.alter_table(alter_table, query_ctx).await,
382
383            Statement::AlterDatabase(alter_database) => {
384                self.alter_database(alter_database, query_ctx).await
385            }
386
387            #[cfg(feature = "enterprise")]
388            Statement::AlterTrigger(alter_trigger) => {
389                self.alter_trigger(alter_trigger, query_ctx).await
390            }
391
392            Statement::DropTable(stmt) => {
393                let mut table_names = Vec::with_capacity(stmt.table_names().len());
394                for table_name_stmt in stmt.table_names() {
395                    let (catalog, schema, table) =
396                        table_idents_to_full_name(table_name_stmt, &query_ctx)
397                            .map_err(BoxedError::new)
398                            .context(ExternalSnafu)?;
399                    table_names.push(TableName::new(catalog, schema, table));
400                }
401                self.drop_tables(&table_names[..], stmt.drop_if_exists(), query_ctx.clone())
402                    .await
403            }
404            #[cfg(feature = "enterprise")]
405            Statement::UndropTable(stmt) => {
406                let (catalog, schema, table) =
407                    table_idents_to_full_name(stmt.table_name(), &query_ctx)
408                        .map_err(BoxedError::new)
409                        .context(ExternalSnafu)?;
410                self.undrop_table(TableName::new(catalog, schema, table), query_ctx)
411                    .await
412            }
413            Statement::DropDatabase(stmt) => {
414                self.drop_database(
415                    query_ctx.current_catalog().to_string(),
416                    format_raw_object_name(stmt.name()),
417                    stmt.drop_if_exists(),
418                    query_ctx,
419                )
420                .await
421            }
422            Statement::TruncateTable(stmt) => {
423                let (catalog, schema, table) =
424                    table_idents_to_full_name(stmt.table_name(), &query_ctx)
425                        .map_err(BoxedError::new)
426                        .context(ExternalSnafu)?;
427                let table_name = TableName::new(catalog, schema, table);
428                let time_ranges = self
429                    .convert_truncate_time_ranges(&table_name, stmt.time_ranges(), &query_ctx)
430                    .await?;
431                self.truncate_table(table_name, time_ranges, query_ctx)
432                    .await
433            }
434            Statement::CreateDatabase(stmt) => {
435                self.create_database(
436                    &format_raw_object_name(&stmt.name),
437                    stmt.if_not_exists,
438                    stmt.options.into_map(),
439                    query_ctx,
440                )
441                .await
442            }
443            Statement::ShowCreateDatabase(show) => {
444                let (catalog, database) =
445                    idents_to_full_database_name(&show.database_name, &query_ctx)
446                        .map_err(BoxedError::new)
447                        .context(ExternalSnafu)?;
448                let table_metadata_manager = self
449                    .catalog_manager
450                    .as_any()
451                    .downcast_ref::<KvBackendCatalogManager>()
452                    .map(|manager| manager.table_metadata_manager_ref().clone())
453                    .context(UpgradeCatalogManagerRefSnafu)?;
454                let opts: HashMap<String, String> = table_metadata_manager
455                    .schema_manager()
456                    .get(SchemaNameKey::new(&catalog, &database))
457                    .await
458                    .context(TableMetadataManagerSnafu)?
459                    .context(SchemaNotFoundSnafu {
460                        schema_info: &database,
461                    })?
462                    .into_inner()
463                    .into();
464
465                self.show_create_database(&database, opts.into()).await
466            }
467            Statement::ShowCreateTable(show) => {
468                let (catalog, schema, table) =
469                    table_idents_to_full_name(&show.table_name, &query_ctx)
470                        .map_err(BoxedError::new)
471                        .context(ExternalSnafu)?;
472
473                let table_ref = self
474                    .catalog_manager
475                    .table(&catalog, &schema, &table, Some(&query_ctx))
476                    .await
477                    .context(CatalogSnafu)?
478                    .context(TableNotFoundSnafu { table_name: &table })?;
479                let table_name = TableName::new(catalog, schema, table);
480
481                match show.variant {
482                    ShowCreateTableVariant::Original => {
483                        self.show_create_table(table_name, table_ref, query_ctx)
484                            .await
485                    }
486                    ShowCreateTableVariant::PostgresForeignTable => {
487                        self.show_create_table_for_pg(table_name, table_ref, query_ctx)
488                            .await
489                    }
490                }
491            }
492            Statement::ShowCreateFlow(show) => self.show_create_flow(show, query_ctx).await,
493            Statement::ShowCreateView(show) => self.show_create_view(show, query_ctx).await,
494            #[cfg(feature = "enterprise")]
495            Statement::ShowCreateTrigger(show) => self.show_create_trigger(show, query_ctx).await,
496            Statement::SetVariables(set_var) => self.set_variables(set_var, query_ctx),
497            Statement::ShowVariables(show_variable) => self.show_variable(show_variable, query_ctx),
498            Statement::Comment(stmt) => self.comment(stmt, query_ctx).await,
499            Statement::ShowColumns(show_columns) => {
500                self.show_columns(show_columns, query_ctx).await
501            }
502            Statement::ShowIndex(show_index) => self.show_index(show_index, query_ctx).await,
503            Statement::ShowRegion(show_region) => self.show_region(show_region, query_ctx).await,
504            Statement::ShowStatus(_) => self.show_status(query_ctx).await,
505            Statement::ShowSearchPath(_) => self.show_search_path(query_ctx).await,
506            Statement::Use(db) => self.use_database(db, query_ctx).await,
507            Statement::Admin(admin) => self.execute_admin_command(admin, query_ctx).await,
508            Statement::Kill(kill) => self.execute_kill(query_ctx, kill).await,
509            Statement::ShowProcesslist(show) => self.show_processlist(show, query_ctx).await,
510        }
511    }
512
513    pub async fn use_database(&self, db: String, query_ctx: QueryContextRef) -> Result<Output> {
514        let catalog = query_ctx.current_catalog();
515        ensure!(
516            self.catalog_manager
517                .schema_exists(catalog, db.as_ref(), Some(&query_ctx))
518                .await
519                .context(CatalogSnafu)?,
520            SchemaNotFoundSnafu { schema_info: &db }
521        );
522
523        query_ctx.set_current_schema(&db);
524
525        Ok(Output::new_with_record_batches(RecordBatches::empty()))
526    }
527
528    fn set_variables(&self, set_var: SetVariables, query_ctx: QueryContextRef) -> Result<Output> {
529        let var_name = set_var.variable.to_string().to_uppercase();
530
531        debug!(
532            "Trying to set {}={} for session: {} ",
533            var_name,
534            set_var.value.iter().map(|e| e.to_string()).join(", "),
535            query_ctx.conn_info()
536        );
537
538        match var_name.as_str() {
539            "READ_PREFERENCE" => set_read_preference(set_var.value, query_ctx)?,
540            "SKIP_WAL" => set_skip_wal(set_var.value, query_ctx)?,
541
542            "@@TIME_ZONE" | "@@SESSION.TIME_ZONE" | "TIMEZONE" | "TIME_ZONE" => {
543                set_timezone(set_var.value, query_ctx)?
544            }
545
546            "BYTEA_OUTPUT" => set_bytea_output(set_var.value, query_ctx)?,
547
548            // Same as "bytea_output", we just ignore it here.
549            // Not harmful since it only relates to how date is viewed in client app's output.
550            // The tracked issue is https://github.com/GreptimeTeam/greptimedb/issues/3442.
551            "DATESTYLE" => set_datestyle(set_var.value, query_ctx)?,
552            "INTERVALSTYLE" => set_intervalstyle(set_var.value, query_ctx)?,
553
554            // Allow query to fallback when failed to push down.
555            "ALLOW_QUERY_FALLBACK" => set_allow_query_fallback(set_var.value, query_ctx)?,
556
557            "CLIENT_ENCODING" => validate_client_encoding(set_var)?,
558            "@@SESSION.MAX_EXECUTION_TIME" | "MAX_EXECUTION_TIME" => match query_ctx.channel() {
559                Channel::Mysql => set_query_timeout(set_var.value, query_ctx)?,
560                Channel::Postgres => {
561                    warn!(
562                        "Unsupported set variable {} for channel {:?}",
563                        var_name,
564                        query_ctx.channel()
565                    );
566                    query_ctx.set_warning(format!("Unsupported set variable {}", var_name))
567                }
568                _ => {
569                    return NotSupportedSnafu {
570                        feat: format!("Unsupported set variable {}", var_name),
571                    }
572                    .fail();
573                }
574            },
575            "STATEMENT_TIMEOUT" => match query_ctx.channel() {
576                Channel::Postgres => set_query_timeout(set_var.value, query_ctx)?,
577                Channel::Mysql => {
578                    warn!(
579                        "Unsupported set variable {} for channel {:?}",
580                        var_name,
581                        query_ctx.channel()
582                    );
583                    query_ctx.set_warning(format!("Unsupported set variable {}", var_name));
584                }
585                _ => {
586                    return NotSupportedSnafu {
587                        feat: format!("Unsupported set variable {}", var_name),
588                    }
589                    .fail();
590                }
591            },
592            "SEARCH_PATH" => {
593                if query_ctx.channel() == Channel::Postgres {
594                    let search_path = set_var.search_path().context(NotSupportedSnafu {
595                        feat: "Unsupported search path in set variable statement",
596                    })?;
597                    query_ctx.set_current_schema(search_path);
598                } else {
599                    return NotSupportedSnafu {
600                        feat: format!("Unsupported set variable {}", var_name),
601                    }
602                    .fail();
603                }
604            }
605            _ => {
606                if query_ctx.channel() == Channel::Postgres || query_ctx.channel() == Channel::Mysql
607                {
608                    // For unknown SET statements, we give a warning with success.
609                    // This prevents the SET call from becoming a blocker of MySQL/Postgres clients'
610                    // connection establishment.
611                    warn!(
612                        "Unsupported set variable {} for channel {:?}",
613                        var_name,
614                        query_ctx.channel()
615                    );
616                    query_ctx.set_warning(format!("Unsupported set variable {}", var_name));
617                } else {
618                    return NotSupportedSnafu {
619                        feat: format!("Unsupported set variable {}", var_name),
620                    }
621                    .fail();
622                }
623            }
624        }
625        Ok(Output::new_with_affected_rows(0))
626    }
627
628    #[tracing::instrument(skip_all)]
629    pub async fn plan(
630        &self,
631        stmt: &QueryStatement,
632        query_ctx: QueryContextRef,
633    ) -> Result<LogicalPlan> {
634        self.query_engine
635            .planner()
636            .plan(stmt, query_ctx)
637            .await
638            .context(PlanStatementSnafu)
639    }
640
641    /// Execute [`LogicalPlan`] directly.
642    #[tracing::instrument(skip_all)]
643    pub async fn exec_plan(&self, plan: LogicalPlan, query_ctx: QueryContextRef) -> Result<Output> {
644        self.query_engine
645            .execute(plan, query_ctx)
646            .await
647            .context(ExecLogicalPlanSnafu)
648    }
649
650    pub fn optimize_logical_plan(&self, plan: LogicalPlan) -> Result<LogicalPlan> {
651        self.query_engine
652            .planner()
653            .optimize(plan)
654            .context(PlanStatementSnafu)
655    }
656
657    #[tracing::instrument(skip_all)]
658    async fn plan_exec(&self, stmt: QueryStatement, query_ctx: QueryContextRef) -> Result<Output> {
659        let plan = self.plan(&stmt, query_ctx.clone()).await?;
660        self.exec_plan(plan, query_ctx).await
661    }
662
663    async fn get_table(&self, table_ref: &TableReference<'_>) -> Result<TableRef> {
664        let TableReference {
665            catalog,
666            schema,
667            table,
668        } = table_ref;
669        self.catalog_manager
670            .table(catalog, schema, table, None)
671            .await
672            .context(CatalogSnafu)?
673            .with_context(|| TableNotFoundSnafu {
674                table_name: table_ref.to_string(),
675            })
676    }
677
678    pub fn procedure_executor(&self) -> &ProcedureExecutorRef {
679        &self.procedure_executor
680    }
681
682    pub fn cache_invalidator(&self) -> &CacheInvalidatorRef {
683        &self.cache_invalidator
684    }
685
686    /// Convert truncate time ranges for the given table from sql values to timestamps
687    ///
688    pub async fn convert_truncate_time_ranges(
689        &self,
690        table_name: &TableName,
691        sql_values_time_range: &[(sqlparser::ast::Value, sqlparser::ast::Value)],
692        query_ctx: &QueryContextRef,
693    ) -> Result<Vec<(Timestamp, Timestamp)>> {
694        if sql_values_time_range.is_empty() {
695            return Ok(vec![]);
696        }
697        let table = self.get_table(&table_name.table_ref()).await?;
698        let info = table.table_info();
699        let time_index_dt = info
700            .meta
701            .schema
702            .timestamp_column()
703            .context(UnexpectedSnafu {
704                violated: "Table must have a timestamp column",
705            })?;
706
707        let time_unit = time_index_dt
708            .data_type
709            .as_timestamp()
710            .with_context(|| UnexpectedSnafu {
711                violated: format!(
712                    "Table {}'s time index column must be a timestamp type, found: {:?}",
713                    table_name, time_index_dt
714                ),
715            })?
716            .unit();
717
718        let start_column = ColumnSchema::new(
719            "range_start",
720            ConcreteDataType::timestamp_datatype(time_unit),
721            false,
722        );
723        let end_column = ColumnSchema::new(
724            "range_end",
725            ConcreteDataType::timestamp_datatype(time_unit),
726            false,
727        );
728        let mut time_ranges = Vec::with_capacity(sql_values_time_range.len());
729        for (start, end) in sql_values_time_range {
730            let start = common_sql::convert::sql_value_to_value(
731                &start_column,
732                start,
733                Some(&query_ctx.timezone()),
734                None,
735                false,
736            )
737            .context(SqlCommonSnafu)
738            .and_then(|v| {
739                if let datatypes::value::Value::Timestamp(t) = v {
740                    Ok(t)
741                } else {
742                    error::InvalidSqlSnafu {
743                        err_msg: format!("Expected a timestamp value, found {v:?}"),
744                    }
745                    .fail()
746                }
747            })?;
748
749            let end = common_sql::convert::sql_value_to_value(
750                &end_column,
751                end,
752                Some(&query_ctx.timezone()),
753                None,
754                false,
755            )
756            .context(SqlCommonSnafu)
757            .and_then(|v| {
758                if let datatypes::value::Value::Timestamp(t) = v {
759                    Ok(t)
760                } else {
761                    error::InvalidSqlSnafu {
762                        err_msg: format!("Expected a timestamp value, found {v:?}"),
763                    }
764                    .fail()
765                }
766            })?;
767            time_ranges.push((start, end));
768        }
769        Ok(time_ranges)
770    }
771
772    /// Returns the inserter for the statement executor.
773    pub(crate) fn inserter(&self) -> &InserterRef {
774        &self.inserter
775    }
776}
777
778fn to_copy_query_request(stmt: CopyQueryToArgument) -> Result<CopyQueryToRequest> {
779    let CopyQueryToArgument {
780        with,
781        connection,
782        location,
783    } = stmt;
784
785    Ok(CopyQueryToRequest {
786        location,
787        with: with.into_map(),
788        connection: connection.into_map(),
789    })
790}
791
792// Verifies time related format is valid
793fn verify_time_related_format(with: &OptionMap) -> Result<()> {
794    let time_format = with.get(common_datasource::file_format::TIME_FORMAT);
795    let date_format = with.get(common_datasource::file_format::DATE_FORMAT);
796    let timestamp_format = with.get(common_datasource::file_format::TIMESTAMP_FORMAT);
797    let file_format = with.get(common_datasource::file_format::FORMAT_TYPE);
798
799    if !matches!(file_format, Some(f) if f.eq_ignore_ascii_case("csv")) {
800        ensure!(
801            time_format.is_none() && date_format.is_none() && timestamp_format.is_none(),
802            error::TimestampFormatNotSupportedSnafu {
803                format: "<unknown>".to_string(),
804                file_format: file_format.unwrap_or_default(),
805            }
806        );
807    }
808
809    for (key, format_opt) in [
810        (common_datasource::file_format::TIME_FORMAT, time_format),
811        (common_datasource::file_format::DATE_FORMAT, date_format),
812        (
813            common_datasource::file_format::TIMESTAMP_FORMAT,
814            timestamp_format,
815        ),
816    ] {
817        if let Some(format) = format_opt {
818            chrono::format::strftime::StrftimeItems::new(format)
819                .parse()
820                .map_err(|_| error::InvalidCopyParameterSnafu { key, value: format }.build())?;
821        }
822    }
823
824    Ok(())
825}
826
827fn to_copy_table_request(stmt: CopyTable, query_ctx: QueryContextRef) -> Result<CopyTableRequest> {
828    let direction = match stmt {
829        CopyTable::To(_) => CopyDirection::Export,
830        CopyTable::From(_) => CopyDirection::Import,
831    };
832
833    let CopyTableArgument {
834        location,
835        connection,
836        with,
837        table_name,
838        limit,
839        ..
840    } = match stmt {
841        CopyTable::To(arg) => arg,
842        CopyTable::From(arg) => arg,
843    };
844    let (catalog_name, schema_name, table_name) =
845        table_idents_to_full_name(&table_name, &query_ctx)
846            .map_err(BoxedError::new)
847            .context(ExternalSnafu)?;
848
849    let timestamp_range = timestamp_range_from_option_map(&with, &query_ctx)?;
850
851    verify_time_related_format(&with)?;
852
853    let pattern = with
854        .get(common_datasource::file_format::FILE_PATTERN)
855        .map(|x| x.to_string());
856
857    Ok(CopyTableRequest {
858        catalog_name,
859        schema_name,
860        table_name,
861        location,
862        with: with.into_map(),
863        connection: connection.into_map(),
864        pattern,
865        direction,
866        timestamp_range,
867        limit,
868    })
869}
870
871/// Converts [CopyDatabaseArgument] to [CopyDatabaseRequest].
872/// This function extracts the necessary info including catalog/database name, time range, etc.
873fn to_copy_database_request(
874    arg: CopyDatabaseArgument,
875    query_ctx: &QueryContextRef,
876) -> Result<CopyDatabaseRequest> {
877    let (catalog_name, database_name) = idents_to_full_database_name(&arg.database_name, query_ctx)
878        .map_err(BoxedError::new)
879        .context(ExternalSnafu)?;
880    let time_range = timestamp_range_from_option_map(&arg.with, query_ctx)?;
881
882    Ok(CopyDatabaseRequest {
883        catalog_name,
884        schema_name: database_name,
885        location: arg.location,
886        with: arg.with.into_map(),
887        connection: arg.connection.into_map(),
888        time_range,
889    })
890}
891
892/// Extracts timestamp range from OptionMap with keys `start_time` and `end_time`.
893/// The timestamp ranges should be a valid timestamp string as defined in [Timestamp::from_str].
894/// The timezone used for conversion will respect that inside `query_ctx`.
895fn timestamp_range_from_option_map(
896    options: &OptionMap,
897    query_ctx: &QueryContextRef,
898) -> Result<Option<TimestampRange>> {
899    let start_timestamp = extract_timestamp(options, COPY_DATABASE_TIME_START_KEY, query_ctx)?;
900    let end_timestamp = extract_timestamp(options, COPY_DATABASE_TIME_END_KEY, query_ctx)?;
901    let time_range = match (start_timestamp, end_timestamp) {
902        (Some(start), Some(end)) => Some(TimestampRange::new(start, end).with_context(|| {
903            error::InvalidTimestampRangeSnafu {
904                start: start.to_iso8601_string(),
905                end: end.to_iso8601_string(),
906            }
907        })?),
908        (Some(start), None) => Some(TimestampRange::from_start(start)),
909        (None, Some(end)) => Some(TimestampRange::until_end(end, false)), // exclusive end
910        (None, None) => None,
911    };
912    Ok(time_range)
913}
914
915/// Extracts timestamp from a [HashMap<String, String>] with given key.
916fn extract_timestamp(
917    map: &OptionMap,
918    key: &str,
919    query_ctx: &QueryContextRef,
920) -> Result<Option<Timestamp>> {
921    map.get(key)
922        .map(|v| {
923            Timestamp::from_str(v, Some(&query_ctx.timezone()))
924                .map_err(|_| error::InvalidCopyParameterSnafu { key, value: v }.build())
925        })
926        .transpose()
927}
928
929fn idents_to_full_database_name(
930    obj_name: &ObjectName,
931    query_ctx: &QueryContextRef,
932) -> Result<(String, String)> {
933    match &obj_name.0[..] {
934        [database] => Ok((
935            query_ctx.current_catalog().to_owned(),
936            database.to_string_unquoted(),
937        )),
938        [catalog, database] => Ok((catalog.to_string_unquoted(), database.to_string_unquoted())),
939        _ => InvalidSqlSnafu {
940            err_msg: format!(
941                "expect database name to be <catalog>.<database>, <database>, found: {obj_name}",
942            ),
943        }
944        .fail(),
945    }
946}
947
948/// The [`Inserter`] implementation for the statement executor.
949pub struct InserterImpl {
950    statement_executor: StatementExecutorRef,
951    options: Option<InsertOptions>,
952}
953
954impl InserterImpl {
955    pub fn new(statement_executor: StatementExecutorRef, options: Option<InsertOptions>) -> Self {
956        Self {
957            statement_executor,
958            options,
959        }
960    }
961}
962
963#[async_trait::async_trait]
964impl Inserter for InserterImpl {
965    async fn insert_rows(
966        &self,
967        context: &client::inserter::Context<'_>,
968        requests: RowInsertRequests,
969    ) -> ClientResult<()> {
970        let mut ctx_builder = QueryContextBuilder::default()
971            .current_catalog(context.catalog.to_string())
972            .current_schema(context.schema.to_string());
973        if let Some(options) = self.options.as_ref() {
974            ctx_builder = ctx_builder
975                .set_extension(
976                    TTL_KEY.to_string(),
977                    format_duration(options.ttl).to_string(),
978                )
979                .set_extension(APPEND_MODE_KEY.to_string(), options.append_mode.to_string());
980        }
981        let query_ctx = ctx_builder.build().into();
982
983        self.statement_executor
984            .inserter()
985            .handle_row_inserts(
986                requests,
987                query_ctx,
988                self.statement_executor.as_ref(),
989                false,
990                false,
991            )
992            .await
993            .map_err(BoxedError::new)
994            .context(ClientExternalSnafu)
995            .map(|_| ())
996    }
997
998    fn set_options(&mut self, options: &InsertOptions) {
999        self.options = Some(*options);
1000    }
1001}
1002
1003#[cfg(test)]
1004mod tests {
1005    use std::assert_matches;
1006    use std::collections::HashMap;
1007
1008    use common_time::range::TimestampRange;
1009    use common_time::{Timestamp, Timezone};
1010    use session::context::QueryContextBuilder;
1011    use sql::statements::OptionMap;
1012
1013    use crate::error;
1014    use crate::statement::copy_database::{
1015        COPY_DATABASE_TIME_END_KEY, COPY_DATABASE_TIME_START_KEY,
1016    };
1017    use crate::statement::{timestamp_range_from_option_map, verify_time_related_format};
1018
1019    fn check_timestamp_range((start, end): (&str, &str)) -> error::Result<Option<TimestampRange>> {
1020        let query_ctx = QueryContextBuilder::default()
1021            .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
1022            .build()
1023            .into();
1024        let map = OptionMap::from(
1025            [
1026                (COPY_DATABASE_TIME_START_KEY.to_string(), start.to_string()),
1027                (COPY_DATABASE_TIME_END_KEY.to_string(), end.to_string()),
1028            ]
1029            .into_iter()
1030            .collect::<HashMap<_, _>>(),
1031        );
1032        timestamp_range_from_option_map(&map, &query_ctx)
1033    }
1034
1035    #[test]
1036    fn test_timestamp_range_from_option_map() {
1037        assert_eq!(
1038            Some(
1039                TimestampRange::new(
1040                    Timestamp::new_second(1649635200),
1041                    Timestamp::new_second(1649664000),
1042                )
1043                .unwrap(),
1044            ),
1045            check_timestamp_range(("2022-04-11 08:00:00", "2022-04-11 16:00:00"),).unwrap()
1046        );
1047
1048        assert_matches!(
1049            check_timestamp_range(("2022-04-11 08:00:00", "2022-04-11 07:00:00")).unwrap_err(),
1050            error::Error::InvalidTimestampRange { .. }
1051        );
1052    }
1053
1054    #[test]
1055    fn test_verify_timestamp_format() {
1056        let map = OptionMap::from(
1057            [
1058                (
1059                    common_datasource::file_format::TIMESTAMP_FORMAT.to_string(),
1060                    "%Y-%m-%d %H:%M:%S".to_string(),
1061                ),
1062                (
1063                    common_datasource::file_format::FORMAT_TYPE.to_string(),
1064                    "csv".to_string(),
1065                ),
1066            ]
1067            .into_iter()
1068            .collect::<HashMap<_, _>>(),
1069        );
1070        assert!(verify_time_related_format(&map).is_ok());
1071
1072        let map = OptionMap::from(
1073            [
1074                (
1075                    common_datasource::file_format::TIMESTAMP_FORMAT.to_string(),
1076                    "%Y-%m-%d %H:%M:%S".to_string(),
1077                ),
1078                (
1079                    common_datasource::file_format::FORMAT_TYPE.to_string(),
1080                    "json".to_string(),
1081                ),
1082            ]
1083            .into_iter()
1084            .collect::<HashMap<_, _>>(),
1085        );
1086
1087        assert_matches!(
1088            verify_time_related_format(&map).unwrap_err(),
1089            error::Error::TimestampFormatNotSupported { .. }
1090        );
1091        let map = OptionMap::from(
1092            [
1093                (
1094                    common_datasource::file_format::TIMESTAMP_FORMAT.to_string(),
1095                    "%111112".to_string(),
1096                ),
1097                (
1098                    common_datasource::file_format::FORMAT_TYPE.to_string(),
1099                    "csv".to_string(),
1100                ),
1101            ]
1102            .into_iter()
1103            .collect::<HashMap<_, _>>(),
1104        );
1105
1106        assert_matches!(
1107            verify_time_related_format(&map).unwrap_err(),
1108            error::Error::InvalidCopyParameter { .. }
1109        );
1110    }
1111}