1mod admin;
16mod comment;
17mod copy_database;
18mod copy_query_to;
19mod copy_table_from;
20mod copy_table_to;
21mod cursor;
22mod database_copy;
23pub mod ddl;
24mod describe;
25mod dml;
26pub mod export_database;
27pub mod export_logical_tables;
28mod kill;
29pub mod semantic_graph;
30mod set;
31mod show;
32mod tql;
33
34use std::collections::HashMap;
35use std::sync::Arc;
36
37use api::v1::RowInsertRequests;
38use catalog::CatalogManagerRef;
39use catalog::kvbackend::KvBackendCatalogManager;
40use catalog::process_manager::ProcessManagerRef;
41use client::RecordBatches;
42use client::error::{ExternalSnafu as ClientExternalSnafu, Result as ClientResult};
43use client::inserter::{InsertOptions, Inserter};
44use common_datasource::object_store::LocalFileAccess;
45use common_error::ext::BoxedError;
46use common_meta::cache_invalidator::CacheInvalidatorRef;
47use common_meta::key::flow::{FlowMetadataManager, FlowMetadataManagerRef};
48use common_meta::key::schema_name::SchemaNameKey;
49use common_meta::key::view_info::{ViewInfoManager, ViewInfoManagerRef};
50use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
51use common_meta::kv_backend::KvBackendRef;
52use common_meta::procedure_executor::ProcedureExecutorRef;
53#[cfg(feature = "enterprise")]
54use common_meta::rpc::ddl::CreatorGrantIntent;
55use common_query::Output;
56use common_telemetry::{debug, tracing, warn};
57use common_time::Timestamp;
58use common_time::range::TimestampRange;
59use datafusion_expr::LogicalPlan;
60use datatypes::prelude::ConcreteDataType;
61use datatypes::schema::ColumnSchema;
62use humantime::format_duration;
63use itertools::Itertools;
64use partition::manager::PartitionRuleManagerRef;
65use query::QueryEngineRef;
66use query::parser::QueryStatement;
67use session::context::{Channel, QueryContextBuilder, QueryContextRef};
68use session::table_name::table_idents_to_full_name;
69use set::{set_query_timeout, set_read_preference, set_skip_wal};
70use snafu::{OptionExt, ResultExt, ensure};
71use sql::ast::ObjectNamePartExt;
72use sql::statements::OptionMap;
73use sql::statements::copy::{
74 CopyDatabase, CopyDatabaseArgument, CopyQueryToArgument, CopyTable, CopyTableArgument,
75};
76use sql::statements::set_variables::SetVariables;
77use sql::statements::show::ShowCreateTableVariant;
78use sql::statements::statement::Statement;
79use sql::util::format_raw_object_name;
80use sqlparser::ast::ObjectName;
81use store_api::mito_engine_options::{APPEND_MODE_KEY, TTL_KEY};
82use table::TableRef;
83use table::requests::{CopyDatabaseRequest, CopyDirection, CopyQueryToRequest, CopyTableRequest};
84use table::table_name::TableName;
85use table::table_reference::TableReference;
86
87pub use self::admin::{
88 AdminEventRecorderHandle, AdminFunctionLayer, AdminFunctionLayerRef,
89 AdminFunctionRecordingLayer, AdminFunctionRequest, AdminFunctionResponse, AdminFunctionService,
90 AdminFunctionServiceRef, admin_output_schema,
91};
92use self::set::{
93 set_bytea_output, set_datestyle, set_intervalstyle, set_timezone, validate_client_encoding,
94};
95use crate::error::{
96 self, CatalogSnafu, ExecLogicalPlanSnafu, ExternalSnafu, InvalidSqlSnafu, NotSupportedSnafu,
97 PlanStatementSnafu, Result, SchemaNotFoundSnafu, SqlCommonSnafu, TableMetadataManagerSnafu,
98 TableNotFoundSnafu, UnexpectedSnafu, UpgradeCatalogManagerRefSnafu,
99};
100use crate::insert::InserterRef;
101use crate::statement::copy_database::{COPY_DATABASE_TIME_END_KEY, COPY_DATABASE_TIME_START_KEY};
102use crate::statement::set::set_allow_query_fallback;
103
104#[async_trait::async_trait]
106pub trait StatementExecutorConfigurator: Send + Sync {
107 async fn configure(
108 &self,
109 executor: StatementExecutor,
110 ctx: ExecutorConfigureContext,
111 ) -> std::result::Result<StatementExecutor, BoxedError>;
112}
113
114pub type StatementExecutorConfiguratorRef = Arc<dyn StatementExecutorConfigurator>;
115
116#[cfg(feature = "enterprise")]
117#[async_trait::async_trait]
118pub trait CreateDatabaseHandler: Send + Sync {
119 fn creator(
120 &self,
121 query_ctx: &QueryContextRef,
122 ) -> std::result::Result<Option<CreatorGrantIntent>, BoxedError>;
123
124 async fn refresh_current_user(
125 &self,
126 query_ctx: &QueryContextRef,
127 ) -> std::result::Result<(), BoxedError>;
128}
129
130#[cfg(feature = "enterprise")]
131pub type CreateDatabaseHandlerRef = Arc<dyn CreateDatabaseHandler>;
132
133pub struct ExecutorConfigureContext {
134 pub kv_backend: KvBackendRef,
135}
136
137#[derive(Clone)]
138pub struct StatementExecutor {
139 catalog_manager: CatalogManagerRef,
140 query_engine: QueryEngineRef,
141 procedure_executor: ProcedureExecutorRef,
142 table_metadata_manager: TableMetadataManagerRef,
143 flow_metadata_manager: FlowMetadataManagerRef,
144 view_info_manager: ViewInfoManagerRef,
145 partition_manager: PartitionRuleManagerRef,
146 cache_invalidator: CacheInvalidatorRef,
147 inserter: InserterRef,
148 process_manager: Option<ProcessManagerRef>,
149 origin_frontend_addr: String,
150 admin_function_service: AdminFunctionServiceRef,
151 pub(crate) local_file_access: LocalFileAccess,
152 #[cfg(feature = "enterprise")]
153 create_database_handler: Option<CreateDatabaseHandlerRef>,
154 #[cfg(feature = "enterprise")]
155 trigger_querier: Option<TriggerQuerierRef>,
156}
157
158pub type StatementExecutorRef = Arc<StatementExecutor>;
159
160#[cfg(feature = "enterprise")]
162#[async_trait::async_trait]
163pub trait TriggerQuerier: Send + Sync {
164 async fn show_create_trigger(
166 &self,
167 catalog: &str,
168 trigger: &str,
169 query_ctx: &QueryContextRef,
170 ) -> std::result::Result<Output, BoxedError>;
171
172 fn as_any(&self) -> &dyn std::any::Any;
173}
174
175#[cfg(feature = "enterprise")]
176pub type TriggerQuerierRef = Arc<dyn TriggerQuerier>;
177
178impl StatementExecutor {
179 #[allow(clippy::too_many_arguments)]
180 pub fn new(
181 catalog_manager: CatalogManagerRef,
182 query_engine: QueryEngineRef,
183 procedure_executor: ProcedureExecutorRef,
184 kv_backend: KvBackendRef,
185 cache_invalidator: CacheInvalidatorRef,
186 inserter: InserterRef,
187 partition_manager: PartitionRuleManagerRef,
188 process_manager: Option<ProcessManagerRef>,
189 origin_frontend_addr: String,
190 local_file_access: LocalFileAccess,
191 ) -> Self {
192 let admin_function_service = admin::new_admin_function_service(query_engine.clone());
193 Self {
194 catalog_manager,
195 query_engine,
196 procedure_executor,
197 table_metadata_manager: Arc::new(TableMetadataManager::new(kv_backend.clone())),
198 flow_metadata_manager: Arc::new(FlowMetadataManager::new(kv_backend.clone())),
199 view_info_manager: Arc::new(ViewInfoManager::new(kv_backend.clone())),
200 partition_manager,
201 cache_invalidator,
202 inserter,
203 process_manager,
204 origin_frontend_addr,
205 admin_function_service,
206 local_file_access,
207 #[cfg(feature = "enterprise")]
208 create_database_handler: None,
209 #[cfg(feature = "enterprise")]
210 trigger_querier: None,
211 }
212 }
213
214 pub fn with_admin_function_layer(mut self, layer: AdminFunctionLayerRef) -> Self {
218 self.admin_function_service = layer.layer(self.admin_function_service);
219 self
220 }
221
222 #[cfg(feature = "enterprise")]
223 pub fn with_trigger_querier(mut self, querier: TriggerQuerierRef) -> Self {
224 self.trigger_querier = Some(querier);
225 self
226 }
227
228 #[cfg(feature = "enterprise")]
229 pub fn with_create_database_handler(mut self, handler: CreateDatabaseHandlerRef) -> Self {
230 self.create_database_handler = Some(handler);
231 self
232 }
233
234 #[cfg(feature = "testing")]
235 pub async fn execute_stmt(
236 &self,
237 stmt: QueryStatement,
238 query_ctx: QueryContextRef,
239 ) -> Result<Output> {
240 match stmt {
241 QueryStatement::Sql(stmt) => self.execute_sql(stmt, query_ctx).await,
242 QueryStatement::Promql(_, _) => self.plan_exec(stmt, query_ctx).await,
243 }
244 }
245
246 #[tracing::instrument(skip_all)]
247 pub async fn execute_sql(&self, stmt: Statement, query_ctx: QueryContextRef) -> Result<Output> {
248 match stmt {
249 Statement::Query(_) | Statement::Explain(_) | Statement::Delete(_) => {
250 self.plan_exec(QueryStatement::Sql(stmt), query_ctx).await
251 }
252
253 Statement::DeclareCursor(declare_cursor) => {
254 self.declare_cursor(declare_cursor, query_ctx).await
255 }
256 Statement::FetchCursor(fetch_cursor) => {
257 self.fetch_cursor(fetch_cursor, query_ctx).await
258 }
259 Statement::CloseCursor(close_cursor) => {
260 self.close_cursor(close_cursor, query_ctx).await
261 }
262
263 Statement::Insert(insert) => self.insert(insert, query_ctx).await,
264
265 Statement::Tql(tql) => self.execute_tql(tql, query_ctx).await,
266
267 Statement::DescribeTable(stmt) => self.describe_table(stmt, query_ctx).await,
268
269 Statement::ShowDatabases(stmt) => self.show_databases(stmt, query_ctx).await,
270
271 Statement::ShowTables(stmt) => self.show_tables(stmt, query_ctx).await,
272
273 Statement::ShowTableStatus(stmt) => self.show_table_status(stmt, query_ctx).await,
274
275 Statement::ShowCollation(kind) => self.show_collation(kind, query_ctx).await,
276
277 Statement::ShowCharset(kind) => self.show_charset(kind, query_ctx).await,
278
279 Statement::ShowViews(stmt) => self.show_views(stmt, query_ctx).await,
280
281 Statement::ShowFlows(stmt) => self.show_flows(stmt, query_ctx).await,
282 Statement::ShowFlowStatus(stmt) => self.show_flow_status(stmt, query_ctx).await,
283
284 #[cfg(feature = "enterprise")]
285 Statement::ShowTriggers(stmt) => self.show_triggers(stmt, query_ctx).await,
286
287 Statement::Copy(sql::statements::copy::Copy::CopyQueryTo(stmt)) => {
288 let query_output = self
289 .plan_exec(QueryStatement::Sql(*stmt.query), query_ctx)
290 .await?;
291 let req = to_copy_query_request(stmt.arg)?;
292
293 self.copy_query_to(req, query_output)
294 .await
295 .map(Output::new_with_affected_rows)
296 }
297
298 Statement::Copy(sql::statements::copy::Copy::CopyTable(stmt)) => {
299 let req = to_copy_table_request(stmt, query_ctx.clone())?;
300 match req.direction {
301 CopyDirection::Export => self
302 .copy_table_to(req, query_ctx)
303 .await
304 .map(Output::new_with_affected_rows),
305 CopyDirection::Import => self.copy_table_from(req, query_ctx).await,
306 }
307 }
308
309 Statement::Copy(sql::statements::copy::Copy::CopyDatabase(copy_database)) => {
310 match copy_database {
311 CopyDatabase::To(arg) => {
312 self.copy_database_to(
313 to_copy_database_request(arg, &query_ctx)?,
314 query_ctx.clone(),
315 )
316 .await
317 }
318 CopyDatabase::From(arg) => {
319 self.copy_database_from(
320 to_copy_database_request(arg, &query_ctx)?,
321 query_ctx,
322 )
323 .await
324 }
325 }
326 }
327
328 Statement::CreateTable(stmt) => {
329 let _ = self.create_table(stmt, query_ctx).await?;
330 Ok(Output::new_with_affected_rows(0))
331 }
332 Statement::CreateTableLike(stmt) => {
333 let _ = self.create_table_like(stmt, query_ctx).await?;
334 Ok(Output::new_with_affected_rows(0))
335 }
336 Statement::CreateExternalTable(stmt) => {
337 let _ = self.create_external_table(stmt, query_ctx).await?;
338 Ok(Output::new_with_affected_rows(0))
339 }
340 Statement::CreateFlow(stmt) => self.create_flow(stmt, query_ctx).await,
341 #[cfg(feature = "enterprise")]
342 Statement::CreateTrigger(stmt) => self.create_trigger(stmt, query_ctx).await,
343 Statement::DropFlow(stmt) => {
344 self.drop_flow(
345 query_ctx.current_catalog().to_string(),
346 format_raw_object_name(stmt.flow_name()),
347 stmt.drop_if_exists(),
348 query_ctx,
349 )
350 .await
351 }
352 #[cfg(feature = "enterprise")]
353 Statement::DropTrigger(stmt) => {
354 self.drop_trigger(
355 query_ctx.current_catalog().to_string(),
356 format_raw_object_name(stmt.trigger_name()),
357 stmt.drop_if_exists(),
358 query_ctx,
359 )
360 .await
361 }
362 Statement::CreateView(stmt) => {
363 let _ = self.create_view(stmt, query_ctx).await?;
364 Ok(Output::new_with_affected_rows(0))
365 }
366 Statement::DropView(stmt) => {
367 let (catalog_name, schema_name, view_name) =
368 table_idents_to_full_name(&stmt.view_name, &query_ctx)
369 .map_err(BoxedError::new)
370 .context(ExternalSnafu)?;
371
372 self.drop_view(
373 catalog_name,
374 schema_name,
375 view_name,
376 stmt.drop_if_exists,
377 query_ctx,
378 )
379 .await
380 }
381 Statement::AlterTable(alter_table) => self.alter_table(alter_table, query_ctx).await,
382
383 Statement::AlterDatabase(alter_database) => {
384 self.alter_database(alter_database, query_ctx).await
385 }
386
387 #[cfg(feature = "enterprise")]
388 Statement::AlterTrigger(alter_trigger) => {
389 self.alter_trigger(alter_trigger, query_ctx).await
390 }
391
392 Statement::DropTable(stmt) => {
393 let mut table_names = Vec::with_capacity(stmt.table_names().len());
394 for table_name_stmt in stmt.table_names() {
395 let (catalog, schema, table) =
396 table_idents_to_full_name(table_name_stmt, &query_ctx)
397 .map_err(BoxedError::new)
398 .context(ExternalSnafu)?;
399 table_names.push(TableName::new(catalog, schema, table));
400 }
401 self.drop_tables(&table_names[..], stmt.drop_if_exists(), query_ctx.clone())
402 .await
403 }
404 #[cfg(feature = "enterprise")]
405 Statement::UndropTable(stmt) => {
406 let (catalog, schema, table) =
407 table_idents_to_full_name(stmt.table_name(), &query_ctx)
408 .map_err(BoxedError::new)
409 .context(ExternalSnafu)?;
410 self.undrop_table(TableName::new(catalog, schema, table), query_ctx)
411 .await
412 }
413 Statement::DropDatabase(stmt) => {
414 self.drop_database(
415 query_ctx.current_catalog().to_string(),
416 format_raw_object_name(stmt.name()),
417 stmt.drop_if_exists(),
418 query_ctx,
419 )
420 .await
421 }
422 Statement::TruncateTable(stmt) => {
423 let (catalog, schema, table) =
424 table_idents_to_full_name(stmt.table_name(), &query_ctx)
425 .map_err(BoxedError::new)
426 .context(ExternalSnafu)?;
427 let table_name = TableName::new(catalog, schema, table);
428 let time_ranges = self
429 .convert_truncate_time_ranges(&table_name, stmt.time_ranges(), &query_ctx)
430 .await?;
431 self.truncate_table(table_name, time_ranges, query_ctx)
432 .await
433 }
434 Statement::CreateDatabase(stmt) => {
435 self.create_database(
436 &format_raw_object_name(&stmt.name),
437 stmt.if_not_exists,
438 stmt.options.into_map(),
439 query_ctx,
440 )
441 .await
442 }
443 Statement::ShowCreateDatabase(show) => {
444 let (catalog, database) =
445 idents_to_full_database_name(&show.database_name, &query_ctx)
446 .map_err(BoxedError::new)
447 .context(ExternalSnafu)?;
448 let table_metadata_manager = self
449 .catalog_manager
450 .as_any()
451 .downcast_ref::<KvBackendCatalogManager>()
452 .map(|manager| manager.table_metadata_manager_ref().clone())
453 .context(UpgradeCatalogManagerRefSnafu)?;
454 let opts: HashMap<String, String> = table_metadata_manager
455 .schema_manager()
456 .get(SchemaNameKey::new(&catalog, &database))
457 .await
458 .context(TableMetadataManagerSnafu)?
459 .context(SchemaNotFoundSnafu {
460 schema_info: &database,
461 })?
462 .into_inner()
463 .into();
464
465 self.show_create_database(&database, opts.into()).await
466 }
467 Statement::ShowCreateTable(show) => {
468 let (catalog, schema, table) =
469 table_idents_to_full_name(&show.table_name, &query_ctx)
470 .map_err(BoxedError::new)
471 .context(ExternalSnafu)?;
472
473 let table_ref = self
474 .catalog_manager
475 .table(&catalog, &schema, &table, Some(&query_ctx))
476 .await
477 .context(CatalogSnafu)?
478 .context(TableNotFoundSnafu { table_name: &table })?;
479 let table_name = TableName::new(catalog, schema, table);
480
481 match show.variant {
482 ShowCreateTableVariant::Original => {
483 self.show_create_table(table_name, table_ref, query_ctx)
484 .await
485 }
486 ShowCreateTableVariant::PostgresForeignTable => {
487 self.show_create_table_for_pg(table_name, table_ref, query_ctx)
488 .await
489 }
490 }
491 }
492 Statement::ShowCreateFlow(show) => self.show_create_flow(show, query_ctx).await,
493 Statement::ShowCreateView(show) => self.show_create_view(show, query_ctx).await,
494 #[cfg(feature = "enterprise")]
495 Statement::ShowCreateTrigger(show) => self.show_create_trigger(show, query_ctx).await,
496 Statement::SetVariables(set_var) => self.set_variables(set_var, query_ctx),
497 Statement::ShowVariables(show_variable) => self.show_variable(show_variable, query_ctx),
498 Statement::Comment(stmt) => self.comment(stmt, query_ctx).await,
499 Statement::ShowColumns(show_columns) => {
500 self.show_columns(show_columns, query_ctx).await
501 }
502 Statement::ShowIndex(show_index) => self.show_index(show_index, query_ctx).await,
503 Statement::ShowRegion(show_region) => self.show_region(show_region, query_ctx).await,
504 Statement::ShowStatus(_) => self.show_status(query_ctx).await,
505 Statement::ShowSearchPath(_) => self.show_search_path(query_ctx).await,
506 Statement::Use(db) => self.use_database(db, query_ctx).await,
507 Statement::Admin(admin) => self.execute_admin_command(admin, query_ctx).await,
508 Statement::Kill(kill) => self.execute_kill(query_ctx, kill).await,
509 Statement::ShowProcesslist(show) => self.show_processlist(show, query_ctx).await,
510 }
511 }
512
513 pub async fn use_database(&self, db: String, query_ctx: QueryContextRef) -> Result<Output> {
514 let catalog = query_ctx.current_catalog();
515 ensure!(
516 self.catalog_manager
517 .schema_exists(catalog, db.as_ref(), Some(&query_ctx))
518 .await
519 .context(CatalogSnafu)?,
520 SchemaNotFoundSnafu { schema_info: &db }
521 );
522
523 query_ctx.set_current_schema(&db);
524
525 Ok(Output::new_with_record_batches(RecordBatches::empty()))
526 }
527
528 fn set_variables(&self, set_var: SetVariables, query_ctx: QueryContextRef) -> Result<Output> {
529 let var_name = set_var.variable.to_string().to_uppercase();
530
531 debug!(
532 "Trying to set {}={} for session: {} ",
533 var_name,
534 set_var.value.iter().map(|e| e.to_string()).join(", "),
535 query_ctx.conn_info()
536 );
537
538 match var_name.as_str() {
539 "READ_PREFERENCE" => set_read_preference(set_var.value, query_ctx)?,
540 "SKIP_WAL" => set_skip_wal(set_var.value, query_ctx)?,
541
542 "@@TIME_ZONE" | "@@SESSION.TIME_ZONE" | "TIMEZONE" | "TIME_ZONE" => {
543 set_timezone(set_var.value, query_ctx)?
544 }
545
546 "BYTEA_OUTPUT" => set_bytea_output(set_var.value, query_ctx)?,
547
548 "DATESTYLE" => set_datestyle(set_var.value, query_ctx)?,
552 "INTERVALSTYLE" => set_intervalstyle(set_var.value, query_ctx)?,
553
554 "ALLOW_QUERY_FALLBACK" => set_allow_query_fallback(set_var.value, query_ctx)?,
556
557 "CLIENT_ENCODING" => validate_client_encoding(set_var)?,
558 "@@SESSION.MAX_EXECUTION_TIME" | "MAX_EXECUTION_TIME" => match query_ctx.channel() {
559 Channel::Mysql => set_query_timeout(set_var.value, query_ctx)?,
560 Channel::Postgres => {
561 warn!(
562 "Unsupported set variable {} for channel {:?}",
563 var_name,
564 query_ctx.channel()
565 );
566 query_ctx.set_warning(format!("Unsupported set variable {}", var_name))
567 }
568 _ => {
569 return NotSupportedSnafu {
570 feat: format!("Unsupported set variable {}", var_name),
571 }
572 .fail();
573 }
574 },
575 "STATEMENT_TIMEOUT" => match query_ctx.channel() {
576 Channel::Postgres => set_query_timeout(set_var.value, query_ctx)?,
577 Channel::Mysql => {
578 warn!(
579 "Unsupported set variable {} for channel {:?}",
580 var_name,
581 query_ctx.channel()
582 );
583 query_ctx.set_warning(format!("Unsupported set variable {}", var_name));
584 }
585 _ => {
586 return NotSupportedSnafu {
587 feat: format!("Unsupported set variable {}", var_name),
588 }
589 .fail();
590 }
591 },
592 "SEARCH_PATH" => {
593 if query_ctx.channel() == Channel::Postgres {
594 let search_path = set_var.search_path().context(NotSupportedSnafu {
595 feat: "Unsupported search path in set variable statement",
596 })?;
597 query_ctx.set_current_schema(search_path);
598 } else {
599 return NotSupportedSnafu {
600 feat: format!("Unsupported set variable {}", var_name),
601 }
602 .fail();
603 }
604 }
605 _ => {
606 if query_ctx.channel() == Channel::Postgres || query_ctx.channel() == Channel::Mysql
607 {
608 warn!(
612 "Unsupported set variable {} for channel {:?}",
613 var_name,
614 query_ctx.channel()
615 );
616 query_ctx.set_warning(format!("Unsupported set variable {}", var_name));
617 } else {
618 return NotSupportedSnafu {
619 feat: format!("Unsupported set variable {}", var_name),
620 }
621 .fail();
622 }
623 }
624 }
625 Ok(Output::new_with_affected_rows(0))
626 }
627
628 #[tracing::instrument(skip_all)]
629 pub async fn plan(
630 &self,
631 stmt: &QueryStatement,
632 query_ctx: QueryContextRef,
633 ) -> Result<LogicalPlan> {
634 self.query_engine
635 .planner()
636 .plan(stmt, query_ctx)
637 .await
638 .context(PlanStatementSnafu)
639 }
640
641 #[tracing::instrument(skip_all)]
643 pub async fn exec_plan(&self, plan: LogicalPlan, query_ctx: QueryContextRef) -> Result<Output> {
644 self.query_engine
645 .execute(plan, query_ctx)
646 .await
647 .context(ExecLogicalPlanSnafu)
648 }
649
650 pub fn optimize_logical_plan(&self, plan: LogicalPlan) -> Result<LogicalPlan> {
651 self.query_engine
652 .planner()
653 .optimize(plan)
654 .context(PlanStatementSnafu)
655 }
656
657 #[tracing::instrument(skip_all)]
658 async fn plan_exec(&self, stmt: QueryStatement, query_ctx: QueryContextRef) -> Result<Output> {
659 let plan = self.plan(&stmt, query_ctx.clone()).await?;
660 self.exec_plan(plan, query_ctx).await
661 }
662
663 async fn get_table(&self, table_ref: &TableReference<'_>) -> Result<TableRef> {
664 let TableReference {
665 catalog,
666 schema,
667 table,
668 } = table_ref;
669 self.catalog_manager
670 .table(catalog, schema, table, None)
671 .await
672 .context(CatalogSnafu)?
673 .with_context(|| TableNotFoundSnafu {
674 table_name: table_ref.to_string(),
675 })
676 }
677
678 pub fn procedure_executor(&self) -> &ProcedureExecutorRef {
679 &self.procedure_executor
680 }
681
682 pub fn cache_invalidator(&self) -> &CacheInvalidatorRef {
683 &self.cache_invalidator
684 }
685
686 pub async fn convert_truncate_time_ranges(
689 &self,
690 table_name: &TableName,
691 sql_values_time_range: &[(sqlparser::ast::Value, sqlparser::ast::Value)],
692 query_ctx: &QueryContextRef,
693 ) -> Result<Vec<(Timestamp, Timestamp)>> {
694 if sql_values_time_range.is_empty() {
695 return Ok(vec![]);
696 }
697 let table = self.get_table(&table_name.table_ref()).await?;
698 let info = table.table_info();
699 let time_index_dt = info
700 .meta
701 .schema
702 .timestamp_column()
703 .context(UnexpectedSnafu {
704 violated: "Table must have a timestamp column",
705 })?;
706
707 let time_unit = time_index_dt
708 .data_type
709 .as_timestamp()
710 .with_context(|| UnexpectedSnafu {
711 violated: format!(
712 "Table {}'s time index column must be a timestamp type, found: {:?}",
713 table_name, time_index_dt
714 ),
715 })?
716 .unit();
717
718 let start_column = ColumnSchema::new(
719 "range_start",
720 ConcreteDataType::timestamp_datatype(time_unit),
721 false,
722 );
723 let end_column = ColumnSchema::new(
724 "range_end",
725 ConcreteDataType::timestamp_datatype(time_unit),
726 false,
727 );
728 let mut time_ranges = Vec::with_capacity(sql_values_time_range.len());
729 for (start, end) in sql_values_time_range {
730 let start = common_sql::convert::sql_value_to_value(
731 &start_column,
732 start,
733 Some(&query_ctx.timezone()),
734 None,
735 false,
736 )
737 .context(SqlCommonSnafu)
738 .and_then(|v| {
739 if let datatypes::value::Value::Timestamp(t) = v {
740 Ok(t)
741 } else {
742 error::InvalidSqlSnafu {
743 err_msg: format!("Expected a timestamp value, found {v:?}"),
744 }
745 .fail()
746 }
747 })?;
748
749 let end = common_sql::convert::sql_value_to_value(
750 &end_column,
751 end,
752 Some(&query_ctx.timezone()),
753 None,
754 false,
755 )
756 .context(SqlCommonSnafu)
757 .and_then(|v| {
758 if let datatypes::value::Value::Timestamp(t) = v {
759 Ok(t)
760 } else {
761 error::InvalidSqlSnafu {
762 err_msg: format!("Expected a timestamp value, found {v:?}"),
763 }
764 .fail()
765 }
766 })?;
767 time_ranges.push((start, end));
768 }
769 Ok(time_ranges)
770 }
771
772 pub(crate) fn inserter(&self) -> &InserterRef {
774 &self.inserter
775 }
776}
777
778fn to_copy_query_request(stmt: CopyQueryToArgument) -> Result<CopyQueryToRequest> {
779 let CopyQueryToArgument {
780 with,
781 connection,
782 location,
783 } = stmt;
784
785 Ok(CopyQueryToRequest {
786 location,
787 with: with.into_map(),
788 connection: connection.into_map(),
789 })
790}
791
792fn verify_time_related_format(with: &OptionMap) -> Result<()> {
794 let time_format = with.get(common_datasource::file_format::TIME_FORMAT);
795 let date_format = with.get(common_datasource::file_format::DATE_FORMAT);
796 let timestamp_format = with.get(common_datasource::file_format::TIMESTAMP_FORMAT);
797 let file_format = with.get(common_datasource::file_format::FORMAT_TYPE);
798
799 if !matches!(file_format, Some(f) if f.eq_ignore_ascii_case("csv")) {
800 ensure!(
801 time_format.is_none() && date_format.is_none() && timestamp_format.is_none(),
802 error::TimestampFormatNotSupportedSnafu {
803 format: "<unknown>".to_string(),
804 file_format: file_format.unwrap_or_default(),
805 }
806 );
807 }
808
809 for (key, format_opt) in [
810 (common_datasource::file_format::TIME_FORMAT, time_format),
811 (common_datasource::file_format::DATE_FORMAT, date_format),
812 (
813 common_datasource::file_format::TIMESTAMP_FORMAT,
814 timestamp_format,
815 ),
816 ] {
817 if let Some(format) = format_opt {
818 chrono::format::strftime::StrftimeItems::new(format)
819 .parse()
820 .map_err(|_| error::InvalidCopyParameterSnafu { key, value: format }.build())?;
821 }
822 }
823
824 Ok(())
825}
826
827fn to_copy_table_request(stmt: CopyTable, query_ctx: QueryContextRef) -> Result<CopyTableRequest> {
828 let direction = match stmt {
829 CopyTable::To(_) => CopyDirection::Export,
830 CopyTable::From(_) => CopyDirection::Import,
831 };
832
833 let CopyTableArgument {
834 location,
835 connection,
836 with,
837 table_name,
838 limit,
839 ..
840 } = match stmt {
841 CopyTable::To(arg) => arg,
842 CopyTable::From(arg) => arg,
843 };
844 let (catalog_name, schema_name, table_name) =
845 table_idents_to_full_name(&table_name, &query_ctx)
846 .map_err(BoxedError::new)
847 .context(ExternalSnafu)?;
848
849 let timestamp_range = timestamp_range_from_option_map(&with, &query_ctx)?;
850
851 verify_time_related_format(&with)?;
852
853 let pattern = with
854 .get(common_datasource::file_format::FILE_PATTERN)
855 .map(|x| x.to_string());
856
857 Ok(CopyTableRequest {
858 catalog_name,
859 schema_name,
860 table_name,
861 location,
862 with: with.into_map(),
863 connection: connection.into_map(),
864 pattern,
865 direction,
866 timestamp_range,
867 limit,
868 })
869}
870
871fn to_copy_database_request(
874 arg: CopyDatabaseArgument,
875 query_ctx: &QueryContextRef,
876) -> Result<CopyDatabaseRequest> {
877 let (catalog_name, database_name) = idents_to_full_database_name(&arg.database_name, query_ctx)
878 .map_err(BoxedError::new)
879 .context(ExternalSnafu)?;
880 let time_range = timestamp_range_from_option_map(&arg.with, query_ctx)?;
881
882 Ok(CopyDatabaseRequest {
883 catalog_name,
884 schema_name: database_name,
885 location: arg.location,
886 with: arg.with.into_map(),
887 connection: arg.connection.into_map(),
888 time_range,
889 })
890}
891
892fn timestamp_range_from_option_map(
896 options: &OptionMap,
897 query_ctx: &QueryContextRef,
898) -> Result<Option<TimestampRange>> {
899 let start_timestamp = extract_timestamp(options, COPY_DATABASE_TIME_START_KEY, query_ctx)?;
900 let end_timestamp = extract_timestamp(options, COPY_DATABASE_TIME_END_KEY, query_ctx)?;
901 let time_range = match (start_timestamp, end_timestamp) {
902 (Some(start), Some(end)) => Some(TimestampRange::new(start, end).with_context(|| {
903 error::InvalidTimestampRangeSnafu {
904 start: start.to_iso8601_string(),
905 end: end.to_iso8601_string(),
906 }
907 })?),
908 (Some(start), None) => Some(TimestampRange::from_start(start)),
909 (None, Some(end)) => Some(TimestampRange::until_end(end, false)), (None, None) => None,
911 };
912 Ok(time_range)
913}
914
915fn extract_timestamp(
917 map: &OptionMap,
918 key: &str,
919 query_ctx: &QueryContextRef,
920) -> Result<Option<Timestamp>> {
921 map.get(key)
922 .map(|v| {
923 Timestamp::from_str(v, Some(&query_ctx.timezone()))
924 .map_err(|_| error::InvalidCopyParameterSnafu { key, value: v }.build())
925 })
926 .transpose()
927}
928
929fn idents_to_full_database_name(
930 obj_name: &ObjectName,
931 query_ctx: &QueryContextRef,
932) -> Result<(String, String)> {
933 match &obj_name.0[..] {
934 [database] => Ok((
935 query_ctx.current_catalog().to_owned(),
936 database.to_string_unquoted(),
937 )),
938 [catalog, database] => Ok((catalog.to_string_unquoted(), database.to_string_unquoted())),
939 _ => InvalidSqlSnafu {
940 err_msg: format!(
941 "expect database name to be <catalog>.<database>, <database>, found: {obj_name}",
942 ),
943 }
944 .fail(),
945 }
946}
947
948pub struct InserterImpl {
950 statement_executor: StatementExecutorRef,
951 options: Option<InsertOptions>,
952}
953
954impl InserterImpl {
955 pub fn new(statement_executor: StatementExecutorRef, options: Option<InsertOptions>) -> Self {
956 Self {
957 statement_executor,
958 options,
959 }
960 }
961}
962
963#[async_trait::async_trait]
964impl Inserter for InserterImpl {
965 async fn insert_rows(
966 &self,
967 context: &client::inserter::Context<'_>,
968 requests: RowInsertRequests,
969 ) -> ClientResult<()> {
970 let mut ctx_builder = QueryContextBuilder::default()
971 .current_catalog(context.catalog.to_string())
972 .current_schema(context.schema.to_string());
973 if let Some(options) = self.options.as_ref() {
974 ctx_builder = ctx_builder
975 .set_extension(
976 TTL_KEY.to_string(),
977 format_duration(options.ttl).to_string(),
978 )
979 .set_extension(APPEND_MODE_KEY.to_string(), options.append_mode.to_string());
980 }
981 let query_ctx = ctx_builder.build().into();
982
983 self.statement_executor
984 .inserter()
985 .handle_row_inserts(
986 requests,
987 query_ctx,
988 self.statement_executor.as_ref(),
989 false,
990 false,
991 )
992 .await
993 .map_err(BoxedError::new)
994 .context(ClientExternalSnafu)
995 .map(|_| ())
996 }
997
998 fn set_options(&mut self, options: &InsertOptions) {
999 self.options = Some(*options);
1000 }
1001}
1002
1003#[cfg(test)]
1004mod tests {
1005 use std::assert_matches;
1006 use std::collections::HashMap;
1007
1008 use common_time::range::TimestampRange;
1009 use common_time::{Timestamp, Timezone};
1010 use session::context::QueryContextBuilder;
1011 use sql::statements::OptionMap;
1012
1013 use crate::error;
1014 use crate::statement::copy_database::{
1015 COPY_DATABASE_TIME_END_KEY, COPY_DATABASE_TIME_START_KEY,
1016 };
1017 use crate::statement::{timestamp_range_from_option_map, verify_time_related_format};
1018
1019 fn check_timestamp_range((start, end): (&str, &str)) -> error::Result<Option<TimestampRange>> {
1020 let query_ctx = QueryContextBuilder::default()
1021 .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
1022 .build()
1023 .into();
1024 let map = OptionMap::from(
1025 [
1026 (COPY_DATABASE_TIME_START_KEY.to_string(), start.to_string()),
1027 (COPY_DATABASE_TIME_END_KEY.to_string(), end.to_string()),
1028 ]
1029 .into_iter()
1030 .collect::<HashMap<_, _>>(),
1031 );
1032 timestamp_range_from_option_map(&map, &query_ctx)
1033 }
1034
1035 #[test]
1036 fn test_timestamp_range_from_option_map() {
1037 assert_eq!(
1038 Some(
1039 TimestampRange::new(
1040 Timestamp::new_second(1649635200),
1041 Timestamp::new_second(1649664000),
1042 )
1043 .unwrap(),
1044 ),
1045 check_timestamp_range(("2022-04-11 08:00:00", "2022-04-11 16:00:00"),).unwrap()
1046 );
1047
1048 assert_matches!(
1049 check_timestamp_range(("2022-04-11 08:00:00", "2022-04-11 07:00:00")).unwrap_err(),
1050 error::Error::InvalidTimestampRange { .. }
1051 );
1052 }
1053
1054 #[test]
1055 fn test_verify_timestamp_format() {
1056 let map = OptionMap::from(
1057 [
1058 (
1059 common_datasource::file_format::TIMESTAMP_FORMAT.to_string(),
1060 "%Y-%m-%d %H:%M:%S".to_string(),
1061 ),
1062 (
1063 common_datasource::file_format::FORMAT_TYPE.to_string(),
1064 "csv".to_string(),
1065 ),
1066 ]
1067 .into_iter()
1068 .collect::<HashMap<_, _>>(),
1069 );
1070 assert!(verify_time_related_format(&map).is_ok());
1071
1072 let map = OptionMap::from(
1073 [
1074 (
1075 common_datasource::file_format::TIMESTAMP_FORMAT.to_string(),
1076 "%Y-%m-%d %H:%M:%S".to_string(),
1077 ),
1078 (
1079 common_datasource::file_format::FORMAT_TYPE.to_string(),
1080 "json".to_string(),
1081 ),
1082 ]
1083 .into_iter()
1084 .collect::<HashMap<_, _>>(),
1085 );
1086
1087 assert_matches!(
1088 verify_time_related_format(&map).unwrap_err(),
1089 error::Error::TimestampFormatNotSupported { .. }
1090 );
1091 let map = OptionMap::from(
1092 [
1093 (
1094 common_datasource::file_format::TIMESTAMP_FORMAT.to_string(),
1095 "%111112".to_string(),
1096 ),
1097 (
1098 common_datasource::file_format::FORMAT_TYPE.to_string(),
1099 "csv".to_string(),
1100 ),
1101 ]
1102 .into_iter()
1103 .collect::<HashMap<_, _>>(),
1104 );
1105
1106 assert_matches!(
1107 verify_time_related_format(&map).unwrap_err(),
1108 error::Error::InvalidCopyParameter { .. }
1109 );
1110 }
1111}