1use std::any::Any;
16use std::borrow::Cow;
17use std::collections::{HashMap, HashSet};
18use std::str::FromStr;
19use std::sync::Arc;
20
21use arrow_schema::DataType;
22use async_trait::async_trait;
23use catalog::table_source::DfTableSourceProvider;
24use common_error::ext::BoxedError;
25use common_query::promql_annotations::promql_annotation_collector;
26use common_telemetry::tracing;
27use datafusion::common::{DFSchema, plan_err};
28use datafusion::execution::SessionStateBuilder;
29use datafusion::execution::context::SessionState;
30use datafusion::sql::planner::PlannerContext;
31use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
32use datafusion_common::{ScalarValue, ToDFSchema};
33use datafusion_expr::expr::{Exists, InSubquery};
34use datafusion_expr::{
35 Analyze, Explain, ExplainFormat, Expr as DfExpr, LogicalPlan, LogicalPlanBuilder, PlanType,
36 ToStringifiedPlan, col,
37};
38use datafusion_sql::planner::{ParserOptions, SqlToRel};
39use log_query::LogQuery;
40use promql_parser::parser::EvalStmt;
41use session::context::QueryContextRef;
42use snafu::{ResultExt, ensure};
43use sql::CteContent;
44use sql::ast::Expr as SqlExpr;
45use sql::statements::explain::ExplainStatement;
46use sql::statements::query::Query;
47use sql::statements::statement::Statement;
48use sql::statements::tql::Tql;
49
50use crate::error::{
51 CteColumnSchemaMismatchSnafu, PlanSqlSnafu, QueryPlanSnafu, Result, SqlSnafu,
52 UnimplementedSnafu,
53};
54use crate::log_query::planner::LogQueryPlanner;
55use crate::parser::{DEFAULT_LOOKBACK_STRING, PromQuery, QueryLanguageParser, QueryStatement};
56use crate::promql::planner::PromPlanner;
57use crate::query_engine::{DefaultPlanDecoder, QueryEngineState};
58use crate::range_select::plan_rewrite::RangePlanRewriter;
59use crate::{DfContextProviderAdapter, QueryEngineContext};
60
61#[async_trait]
62pub trait LogicalPlanner: Send + Sync {
63 async fn plan(&self, stmt: &QueryStatement, query_ctx: QueryContextRef) -> Result<LogicalPlan>;
64
65 async fn plan_logs_query(
66 &self,
67 query: LogQuery,
68 query_ctx: QueryContextRef,
69 ) -> Result<LogicalPlan>;
70
71 fn optimize(&self, plan: LogicalPlan) -> Result<LogicalPlan>;
72
73 fn as_any(&self) -> &dyn Any;
74}
75
76pub struct DfLogicalPlanner {
77 engine_state: Arc<QueryEngineState>,
78 session_state: SessionState,
79}
80
81impl DfLogicalPlanner {
82 pub fn new(engine_state: Arc<QueryEngineState>) -> Self {
83 let session_state = engine_state.session_state();
84 Self {
85 engine_state,
86 session_state,
87 }
88 }
89
90 fn derive_session_state_with_scheduled_time(
94 &self,
95 query_ctx: &QueryContextRef,
96 ) -> Result<SessionState> {
97 let extensions = query_ctx.extensions();
98 match crate::options::parse_scheduled_time_datetime(&extensions)? {
99 Some(dt) => {
100 let execution_props = self
101 .session_state
102 .execution_props()
103 .clone()
104 .with_query_execution_start_time(dt);
105 Ok(
106 SessionStateBuilder::new_from_existing(self.session_state.clone())
107 .with_execution_props(execution_props)
108 .build(),
109 )
110 }
111 None => Ok(self.session_state.clone()),
112 }
113 }
114
115 async fn explain_to_plan(
118 &self,
119 explain: &ExplainStatement,
120 query_ctx: QueryContextRef,
121 ) -> Result<LogicalPlan> {
122 let plan = self.plan_sql(&explain.statement, query_ctx).await?;
123 if matches!(plan, LogicalPlan::Explain(_)) {
124 return plan_err!("Nested EXPLAINs are not supported").context(PlanSqlSnafu);
125 }
126
127 let verbose = explain.verbose;
128 let analyze = explain.analyze;
129 let format = explain.format.map(|f| f.to_string());
130
131 let plan = Arc::new(plan);
132 let schema = LogicalPlan::explain_schema();
133 let schema = ToDFSchema::to_dfschema_ref(schema)?;
134
135 if verbose && format.is_some() {
136 return plan_err!("EXPLAIN VERBOSE with FORMAT is not supported").context(PlanSqlSnafu);
137 }
138
139 if analyze {
140 Ok(LogicalPlan::Analyze(Analyze {
142 verbose,
143 input: plan,
144 schema,
145 }))
146 } else {
147 let stringified_plans = vec![plan.to_stringified(PlanType::InitialLogicalPlan)];
148
149 let options = self.session_state.config().options();
151 let format = format
152 .map(|x| ExplainFormat::from_str(&x))
153 .transpose()?
154 .unwrap_or_else(|| options.explain.format.clone());
155
156 Ok(LogicalPlan::Explain(Explain {
157 verbose,
158 explain_format: format,
159 plan,
160 stringified_plans,
161 schema,
162 logical_optimization_succeeded: false,
163 }))
164 }
165 }
166
167 #[tracing::instrument(skip_all)]
168 #[async_recursion::async_recursion]
169 async fn plan_sql(&self, stmt: &Statement, query_ctx: QueryContextRef) -> Result<LogicalPlan> {
170 let mut planner_context = PlannerContext::new();
171 let mut stmt = Cow::Borrowed(stmt);
172 let mut is_tql_cte = false;
173
174 if let Statement::Explain(explain) = stmt.as_ref() {
176 return self.explain_to_plan(explain, query_ctx).await;
177 }
178
179 if self.has_hybrid_ctes(stmt.as_ref()) {
181 let stmt_owned = stmt.into_owned();
182 let mut query = match stmt_owned {
183 Statement::Query(query) => query.as_ref().clone(),
184 _ => unreachable!("has_hybrid_ctes should only return true for Query statements"),
185 };
186 self.plan_query_with_hybrid_ctes(&query, query_ctx.clone(), &mut planner_context)
187 .await?;
188
189 query.hybrid_cte = None;
191 stmt = Cow::Owned(Statement::Query(Box::new(query)));
192 is_tql_cte = true;
193 }
194
195 let mut df_stmt = stmt.as_ref().try_into().context(SqlSnafu)?;
196
197 if let datafusion::sql::parser::Statement::Statement(
199 box datafusion::sql::sqlparser::ast::Statement::Explain { .. },
200 ) = &mut df_stmt
201 {
202 UnimplementedSnafu {
203 operation: "EXPLAIN with FORMAT using raw datafusion planner",
204 }
205 .fail()?;
206 }
207
208 let scheduled_state = self.derive_session_state_with_scheduled_time(&query_ctx)?;
209 let table_provider = DfTableSourceProvider::new(
210 self.engine_state.catalog_manager().clone(),
211 self.engine_state.disallow_cross_catalog_query(),
212 query_ctx.clone(),
213 Arc::new(DefaultPlanDecoder::new(
214 scheduled_state.clone(),
215 &query_ctx,
216 )?),
217 scheduled_state
218 .config_options()
219 .sql_parser
220 .enable_ident_normalization,
221 );
222
223 let context_provider = DfContextProviderAdapter::try_new(
224 self.engine_state.clone(),
225 scheduled_state.clone(),
226 Some(&df_stmt),
227 query_ctx.clone(),
228 )
229 .await?;
230
231 let config_options = self.session_state.config().options();
232 let parser_options = &config_options.sql_parser;
233 let parser_options = ParserOptions {
234 map_string_types_to_utf8view: false,
235 ..parser_options.into()
236 };
237
238 let sql_to_rel = SqlToRel::new_with_options(&context_provider, parser_options);
239
240 let result = if is_tql_cte {
242 let Statement::Query(query) = stmt.into_owned() else {
243 unreachable!("is_tql_cte should only be true for Query statements");
244 };
245 let sqlparser_stmt = sqlparser::ast::Statement::Query(Box::new(query.inner));
246 sql_to_rel
247 .sql_statement_to_plan_with_context(sqlparser_stmt, &mut planner_context)
248 .context(PlanSqlSnafu)?
249 } else {
250 sql_to_rel
251 .statement_to_plan(df_stmt)
252 .context(PlanSqlSnafu)?
253 };
254
255 common_telemetry::debug!("Logical planner, statement to plan result: {result}");
256 let plan = RangePlanRewriter::new(table_provider, query_ctx.clone())
257 .rewrite(result)
258 .await?;
259
260 let context = QueryEngineContext::new(scheduled_state, query_ctx);
262 let plan = self
263 .engine_state
264 .optimize_by_extension_rules(plan, &context)?;
265 common_telemetry::debug!("Logical planner, optimize result: {plan}");
266
267 Ok(plan)
268 }
269
270 #[tracing::instrument(skip_all)]
272 pub(crate) async fn sql_to_expr(
273 &self,
274 sql: SqlExpr,
275 schema: &DFSchema,
276 normalize_ident: bool,
277 query_ctx: QueryContextRef,
278 ) -> Result<DfExpr> {
279 let scheduled_state = self.derive_session_state_with_scheduled_time(&query_ctx)?;
280 let context_provider = DfContextProviderAdapter::try_new(
281 self.engine_state.clone(),
282 scheduled_state,
283 None,
284 query_ctx,
285 )
286 .await?;
287
288 let config_options = self.session_state.config().options();
289 let parser_options = &config_options.sql_parser;
290 let parser_options: ParserOptions = ParserOptions {
291 map_string_types_to_utf8view: false,
292 enable_ident_normalization: normalize_ident,
293 ..parser_options.into()
294 };
295
296 let sql_to_rel = SqlToRel::new_with_options(&context_provider, parser_options);
297
298 Ok(sql_to_rel.sql_to_expr(sql, schema, &mut PlannerContext::new())?)
299 }
300
301 #[tracing::instrument(skip_all)]
302 async fn plan_pql(&self, stmt: &EvalStmt, query_ctx: QueryContextRef) -> Result<LogicalPlan> {
303 let mut scheduled_state = self.derive_session_state_with_scheduled_time(&query_ctx)?;
304 let promql_annotations = query_ctx.remote_query_id().map(promql_annotation_collector);
305 if let Some(collector) = &promql_annotations {
306 scheduled_state
307 .config_mut()
308 .options_mut()
309 .extensions
310 .insert(collector.clone());
311 }
312 let plan_decoder = Arc::new(DefaultPlanDecoder::new(
313 scheduled_state.clone(),
314 &query_ctx,
315 )?);
316 let table_provider = DfTableSourceProvider::new(
317 self.engine_state.catalog_manager().clone(),
318 self.engine_state.disallow_cross_catalog_query(),
319 query_ctx.clone(),
320 plan_decoder,
321 scheduled_state
322 .config_options()
323 .sql_parser
324 .enable_ident_normalization,
325 );
326 let plan = PromPlanner::stmt_to_plan_with_annotations(
327 table_provider,
328 stmt,
329 &self.engine_state,
330 promql_annotations,
331 )
332 .await
333 .map_err(BoxedError::new)
334 .context(QueryPlanSnafu)?;
335
336 let context = QueryEngineContext::new(scheduled_state, query_ctx);
337 Ok(self
338 .engine_state
339 .optimize_by_extension_rules(plan, &context)?)
340 }
341
342 #[tracing::instrument(skip_all)]
343 fn optimize_logical_plan(&self, plan: LogicalPlan) -> Result<LogicalPlan> {
344 Ok(self.engine_state.optimize_logical_plan(plan)?)
345 }
346
347 fn has_hybrid_ctes(&self, stmt: &Statement) -> bool {
349 if let Statement::Query(query) = stmt {
350 query
351 .hybrid_cte
352 .as_ref()
353 .map(|hybrid_cte| !hybrid_cte.cte_tables.is_empty())
354 .unwrap_or(false)
355 } else {
356 false
357 }
358 }
359
360 async fn plan_query_with_hybrid_ctes(
362 &self,
363 query: &Query,
364 query_ctx: QueryContextRef,
365 planner_context: &mut PlannerContext,
366 ) -> Result<()> {
367 let hybrid_cte = query.hybrid_cte.as_ref().unwrap();
368
369 for cte in &hybrid_cte.cte_tables {
370 match &cte.content {
371 CteContent::Tql(tql) => {
372 let mut logical_plan = self.tql_to_logical_plan(tql, query_ctx.clone()).await?;
374 if !cte.columns.is_empty() {
375 let schema = logical_plan.schema();
376 let schema_fields = schema.fields().to_vec();
377 ensure!(
378 schema_fields.len() == cte.columns.len(),
379 CteColumnSchemaMismatchSnafu {
380 cte_name: cte.name.value.clone(),
381 original: schema_fields
382 .iter()
383 .map(|field| field.name().clone())
384 .collect::<Vec<_>>(),
385 expected: cte
386 .columns
387 .iter()
388 .map(|column| column.to_string())
389 .collect::<Vec<_>>(),
390 }
391 );
392 let aliases = cte
393 .columns
394 .iter()
395 .zip(schema_fields.iter())
396 .map(|(column, field)| col(field.name()).alias(column.to_string()));
397 logical_plan = LogicalPlanBuilder::from(logical_plan)
398 .project(aliases)
399 .context(PlanSqlSnafu)?
400 .build()
401 .context(PlanSqlSnafu)?;
402 }
403
404 logical_plan = LogicalPlan::SubqueryAlias(
406 datafusion_expr::SubqueryAlias::try_new(
407 Arc::new(logical_plan),
408 cte.name.value.clone(),
409 )
410 .context(PlanSqlSnafu)?,
411 );
412
413 planner_context.insert_cte(&cte.name.value, logical_plan);
414 }
415 CteContent::Sql(_) => {
416 unreachable!("SQL CTEs should not be in hybrid_cte.cte_tables");
419 }
420 }
421 }
422
423 Ok(())
424 }
425
426 async fn tql_to_logical_plan(
428 &self,
429 tql: &Tql,
430 query_ctx: QueryContextRef,
431 ) -> Result<LogicalPlan> {
432 match tql {
433 Tql::Eval(eval) => {
434 let prom_query = PromQuery {
436 query: eval.query.clone(),
437 start: eval.start.clone(),
438 end: eval.end.clone(),
439 step: eval.step.clone(),
440 lookback: eval
441 .lookback
442 .clone()
443 .unwrap_or_else(|| DEFAULT_LOOKBACK_STRING.to_string()),
444 alias: eval.alias.clone(),
445 };
446 let stmt = QueryLanguageParser::parse_promql(&prom_query, &query_ctx)?;
447
448 self.plan(&stmt, query_ctx).await
449 }
450 Tql::Explain(_) => UnimplementedSnafu {
451 operation: "TQL EXPLAIN in CTEs",
452 }
453 .fail(),
454 Tql::Analyze(_) => UnimplementedSnafu {
455 operation: "TQL ANALYZE in CTEs",
456 }
457 .fail(),
458 }
459 }
460
461 fn extract_placeholder_cast_types(
471 plan: &LogicalPlan,
472 ) -> Result<HashMap<String, Option<DataType>>> {
473 let mut placeholder_types = HashMap::new();
474 let mut casted_placeholders = HashSet::new();
475
476 Self::extract_from_plan(plan, &mut placeholder_types, &mut casted_placeholders)?;
477
478 Ok(placeholder_types)
479 }
480
481 fn extract_from_plan(
482 plan: &LogicalPlan,
483 placeholder_types: &mut HashMap<String, Option<DataType>>,
484 casted_placeholders: &mut HashSet<String>,
485 ) -> Result<()> {
486 plan.apply(|node| {
487 for expr in node.expressions() {
488 let _ = expr.apply(|e| {
489 if let DfExpr::Cast(cast) = e
491 && let DfExpr::Placeholder(ph) = &*cast.expr
492 {
493 placeholder_types.insert(ph.id.clone(), Some(cast.data_type.clone()));
494 casted_placeholders.insert(ph.id.clone());
495 }
496
497 if let DfExpr::ScalarFunction(scalar_func) = e
499 && scalar_func.name() == "arrow_cast"
500 && scalar_func.args.len() == 2
501 && let DfExpr::Placeholder(ph) = &scalar_func.args[0]
502 && let DfExpr::Literal(ScalarValue::Utf8(Some(type_str)), _) =
503 &scalar_func.args[1]
504 && let Ok(data_type) = type_str.parse::<DataType>()
505 {
506 placeholder_types.insert(ph.id.clone(), Some(data_type));
507 casted_placeholders.insert(ph.id.clone());
508 }
509
510 if let DfExpr::Placeholder(ph) = e
512 && !casted_placeholders.contains(&ph.id)
513 && !placeholder_types.contains_key(&ph.id)
514 {
515 placeholder_types.insert(ph.id.clone(), None);
516 }
517
518 match e {
520 DfExpr::Exists(Exists { subquery, .. })
521 | DfExpr::InSubquery(InSubquery { subquery, .. })
522 | DfExpr::ScalarSubquery(subquery) => {
523 Self::extract_from_plan(
524 &subquery.subquery,
525 placeholder_types,
526 casted_placeholders,
527 )?;
528 }
529 _ => {}
530 }
531
532 Ok(TreeNodeRecursion::Continue)
533 });
534 }
535 Ok(TreeNodeRecursion::Continue)
536 })?;
537 Ok(())
538 }
539
540 fn infer_limit_placeholder_types(
541 plan: &LogicalPlan,
542 placeholder_types: &mut HashMap<String, Option<DataType>>,
543 ) -> Result<()> {
544 plan.apply(|node| {
545 if let LogicalPlan::Limit(limit) = node {
546 for expr in limit.skip.iter().chain(limit.fetch.iter()) {
547 expr.apply(|e| {
548 if let DfExpr::Placeholder(ph) = e {
549 placeholder_types
550 .entry(ph.id.clone())
551 .and_modify(|existing| {
552 if existing.is_none() {
553 *existing = Some(DataType::Int64);
554 }
555 })
556 .or_insert(Some(DataType::Int64));
557 }
558
559 Ok(TreeNodeRecursion::Continue)
560 })?;
561 }
562 }
563
564 Ok(TreeNodeRecursion::Continue)
565 })?;
566
567 Ok(())
568 }
569
570 pub fn get_inferred_parameter_types(
585 plan: &LogicalPlan,
586 ) -> Result<HashMap<String, Option<DataType>>> {
587 let mut param_types = plan.get_parameter_types().context(PlanSqlSnafu)?;
588
589 let has_none = param_types.values().any(|v| v.is_none());
590
591 if has_none {
592 let cast_types = Self::extract_placeholder_cast_types(plan)?;
593
594 for (id, opt_type) in cast_types {
595 param_types
596 .entry(id)
597 .and_modify(|existing| {
598 if existing.is_none() {
599 *existing = opt_type.clone();
600 }
601 })
602 .or_insert(opt_type);
603 }
604
605 Self::infer_limit_placeholder_types(plan, &mut param_types)?;
606 }
607
608 Ok(param_types)
609 }
610}
611
612#[async_trait]
613impl LogicalPlanner for DfLogicalPlanner {
614 #[tracing::instrument(skip_all)]
615 async fn plan(&self, stmt: &QueryStatement, query_ctx: QueryContextRef) -> Result<LogicalPlan> {
616 match stmt {
617 QueryStatement::Sql(stmt) => self.plan_sql(stmt, query_ctx).await,
618 QueryStatement::Promql(stmt, _alias) => self.plan_pql(stmt, query_ctx).await,
619 }
620 }
621
622 async fn plan_logs_query(
623 &self,
624 query: LogQuery,
625 query_ctx: QueryContextRef,
626 ) -> Result<LogicalPlan> {
627 let plan_decoder = Arc::new(DefaultPlanDecoder::new(
628 self.session_state.clone(),
629 &query_ctx,
630 )?);
631 let table_provider = DfTableSourceProvider::new(
632 self.engine_state.catalog_manager().clone(),
633 self.engine_state.disallow_cross_catalog_query(),
634 query_ctx,
635 plan_decoder,
636 self.session_state
637 .config_options()
638 .sql_parser
639 .enable_ident_normalization,
640 );
641
642 let mut planner = LogQueryPlanner::new(table_provider, self.session_state.clone());
643 planner
644 .query_to_plan(query)
645 .await
646 .map_err(BoxedError::new)
647 .context(QueryPlanSnafu)
648 }
649
650 fn optimize(&self, plan: LogicalPlan) -> Result<LogicalPlan> {
651 self.optimize_logical_plan(plan)
652 }
653
654 fn as_any(&self) -> &dyn Any {
655 self
656 }
657}
658
659#[cfg(test)]
660mod tests {
661 use std::sync::Arc;
662
663 use arrow_schema::DataType;
664 use catalog::RegisterTableRequest;
665 use catalog::memory::MemoryCatalogManager;
666 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
667 use common_time::Timezone;
668 use datatypes::prelude::ConcreteDataType;
669 use datatypes::schema::{ColumnSchema, Schema};
670 use session::context::{QueryContext, QueryContextBuilder};
671 use store_api::metric_engine_consts::{
672 DATA_SCHEMA_TABLE_ID_COLUMN_NAME, DATA_SCHEMA_TSID_COLUMN_NAME, LOGICAL_TABLE_METADATA_KEY,
673 METRIC_ENGINE_NAME,
674 };
675 use table::metadata::{TableInfoBuilder, TableMetaBuilder};
676 use table::test_util::EmptyTable;
677
678 use super::*;
679 use crate::parser::{PromQuery, QueryLanguageParser};
680 use crate::{QueryEngineFactory, QueryEngineRef};
681
682 async fn create_test_engine() -> QueryEngineRef {
683 let columns = vec![
684 ColumnSchema::new("id", ConcreteDataType::int32_datatype(), false),
685 ColumnSchema::new("name", ConcreteDataType::string_datatype(), true),
686 ];
687 let schema = Arc::new(Schema::new(columns));
688 let table_meta = TableMetaBuilder::empty()
689 .schema(schema)
690 .primary_key_indices(vec![0])
691 .value_indices(vec![1])
692 .next_column_id(1024)
693 .build()
694 .unwrap();
695 let table_info = TableInfoBuilder::new("test", table_meta).build().unwrap();
696 let table = EmptyTable::from_table_info(&table_info);
697
698 crate::tests::new_query_engine_with_table(table)
699 }
700
701 async fn create_timestamp_test_engine() -> QueryEngineRef {
702 let columns = vec![
703 ColumnSchema::new(
704 "ts",
705 ConcreteDataType::timestamp_millisecond_datatype(),
706 false,
707 )
708 .with_time_index(true),
709 ColumnSchema::new(
710 "st",
711 ConcreteDataType::timestamp_millisecond_datatype(),
712 false,
713 ),
714 ColumnSchema::new("note", ConcreteDataType::string_datatype(), true),
715 ColumnSchema::new(
716 "ts_ns",
717 ConcreteDataType::timestamp_nanosecond_datatype(),
718 true,
719 ),
720 ];
721 let schema = Arc::new(Schema::new(columns));
722 let table_meta = TableMetaBuilder::empty()
723 .schema(schema)
724 .primary_key_indices(vec![])
725 .value_indices(vec![0, 1, 2, 3])
726 .next_column_id(1024)
727 .build()
728 .unwrap();
729 let table_info = TableInfoBuilder::new("timestamps", table_meta)
730 .build()
731 .unwrap();
732 let table = EmptyTable::from_table_info(&table_info);
733
734 crate::tests::new_query_engine_with_table(table)
735 }
736
737 fn create_promql_test_engine() -> QueryEngineRef {
738 let catalog_manager = MemoryCatalogManager::with_default_setup();
739 let physical_table_name = "phy";
740 let physical_table_id = 999u32;
741
742 let physical_schema = Arc::new(Schema::new(vec![
743 ColumnSchema::new(
744 DATA_SCHEMA_TABLE_ID_COLUMN_NAME.to_string(),
745 ConcreteDataType::uint32_datatype(),
746 false,
747 ),
748 ColumnSchema::new(
749 DATA_SCHEMA_TSID_COLUMN_NAME.to_string(),
750 ConcreteDataType::uint64_datatype(),
751 false,
752 ),
753 ColumnSchema::new("tag_0", ConcreteDataType::string_datatype(), false),
754 ColumnSchema::new("tag_1", ConcreteDataType::string_datatype(), false),
755 ColumnSchema::new(
756 "timestamp",
757 ConcreteDataType::timestamp_millisecond_datatype(),
758 false,
759 )
760 .with_time_index(true),
761 ColumnSchema::new("field_0", ConcreteDataType::float64_datatype(), true),
762 ]));
763 let physical_meta = TableMetaBuilder::empty()
764 .schema(physical_schema)
765 .primary_key_indices(vec![0, 1, 2, 3])
766 .value_indices(vec![4, 5])
767 .engine(METRIC_ENGINE_NAME.to_string())
768 .next_column_id(1024)
769 .build()
770 .unwrap();
771 let physical_info = TableInfoBuilder::default()
772 .table_id(physical_table_id)
773 .name(physical_table_name)
774 .meta(physical_meta)
775 .build()
776 .unwrap();
777 catalog_manager
778 .register_table_sync(RegisterTableRequest {
779 catalog: DEFAULT_CATALOG_NAME.to_string(),
780 schema: DEFAULT_SCHEMA_NAME.to_string(),
781 table_name: physical_table_name.to_string(),
782 table_id: physical_table_id,
783 table: EmptyTable::from_table_info(&physical_info),
784 })
785 .unwrap();
786
787 let mut options = table::requests::TableOptions::default();
788 options.extra_options.insert(
789 LOGICAL_TABLE_METADATA_KEY.to_string(),
790 physical_table_name.to_string(),
791 );
792 let logical_schema = Arc::new(Schema::new(vec![
793 ColumnSchema::new("tag_0", ConcreteDataType::string_datatype(), false),
794 ColumnSchema::new("tag_1", ConcreteDataType::string_datatype(), false),
795 ColumnSchema::new(
796 "timestamp",
797 ConcreteDataType::timestamp_millisecond_datatype(),
798 false,
799 )
800 .with_time_index(true),
801 ColumnSchema::new("field_0", ConcreteDataType::float64_datatype(), true),
802 ]));
803 let logical_meta = TableMetaBuilder::empty()
804 .schema(logical_schema)
805 .primary_key_indices(vec![0, 1])
806 .value_indices(vec![3])
807 .engine(METRIC_ENGINE_NAME.to_string())
808 .options(options)
809 .next_column_id(1024)
810 .build()
811 .unwrap();
812 let logical_info = TableInfoBuilder::default()
813 .table_id(1024)
814 .name("some_metric")
815 .meta(logical_meta)
816 .build()
817 .unwrap();
818 catalog_manager
819 .register_table_sync(RegisterTableRequest {
820 catalog: DEFAULT_CATALOG_NAME.to_string(),
821 schema: DEFAULT_SCHEMA_NAME.to_string(),
822 table_name: "some_metric".to_string(),
823 table_id: 1024,
824 table: EmptyTable::from_table_info(&logical_info),
825 })
826 .unwrap();
827
828 QueryEngineFactory::new(
829 catalog_manager,
830 None,
831 None,
832 None,
833 None,
834 false,
835 crate::options::QueryOptions::default(),
836 )
837 .query_engine()
838 }
839
840 async fn parse_sql_to_plan(sql: &str) -> LogicalPlan {
841 let stmt = QueryLanguageParser::parse_sql(sql, &QueryContext::arc()).unwrap();
842 let engine = create_test_engine().await;
843 engine
844 .planner()
845 .plan(&stmt, QueryContext::arc())
846 .await
847 .unwrap()
848 }
849
850 async fn parse_promql_to_plan(query: &str) -> LogicalPlan {
851 let engine = create_promql_test_engine();
852 let query_ctx = QueryContext::arc();
853 let stmt = QueryLanguageParser::parse_promql(
854 &PromQuery {
855 query: query.to_string(),
856 start: "0".to_string(),
857 end: "10".to_string(),
858 step: "5s".to_string(),
859 lookback: "300s".to_string(),
860 alias: None,
861 },
862 &query_ctx,
863 )
864 .unwrap();
865
866 engine.planner().plan(&stmt, query_ctx).await.unwrap()
867 }
868
869 #[tokio::test]
870 async fn test_insert_timestamp_literals_use_query_timezone() {
871 let query_ctx = Arc::new(
872 QueryContextBuilder::default()
873 .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
874 .build(),
875 );
876 let engine = create_timestamp_test_engine().await;
877
878 for (sql, expected_timestamps) in [
879 (
880 "INSERT INTO timestamps (ts, st) \
881 VALUES ('2026-08-02 12:00:00.001', now())",
882 &[1_785_643_200_001_i64][..],
883 ),
884 (
885 "INSERT INTO timestamps (ts, st) \
886 SELECT '2026-08-03 12:00:00.001', now()",
887 &[1_785_729_600_001_i64][..],
888 ),
889 (
890 "INSERT INTO timestamps (ts, st) \
891 SELECT '2026-08-04 12:00:00.001', now() LIMIT 1",
892 &[1_785_816_000_001_i64][..],
893 ),
894 (
895 "INSERT INTO timestamps (ts, st) \
896 SELECT * FROM (\
897 SELECT '2026-08-05 12:00:00.001', now()\
898 ) AS source",
899 &[1_785_902_400_001_i64][..],
900 ),
901 (
902 "INSERT INTO timestamps (ts, st) \
903 SELECT '2026-08-06 12:00:00.001', now() \
904 UNION ALL \
905 SELECT '2026-08-07 12:00:00.001', now()",
906 &[1_785_988_800_001_i64, 1_786_075_200_001_i64][..],
907 ),
908 (
909 "INSERT INTO timestamps (ts, st) \
910 SELECT '2026-08-16 12:00:00.001', now() \
911 UNION \
912 SELECT '2026-08-17 12:00:00.001', now()",
913 &[1_786_852_800_001_i64, 1_786_939_200_001_i64][..],
914 ),
915 (
916 "INSERT INTO timestamps (ts, st) \
917 SELECT '2026-08-18 12:00:00.001', max(st) \
918 FROM timestamps GROUP BY note",
919 &[1_787_025_600_001_i64][..],
920 ),
921 (
922 "INSERT INTO timestamps (ts, st) \
923 SELECT c, s FROM (\
924 SELECT '2026-08-13 12:00:00.001' AS c, now() AS s\
925 ) AS t WHERE c > '2026-01-01'",
926 &[1_786_593_600_001_i64][..],
927 ),
928 (
929 "INSERT INTO timestamps (ts, st) \
930 SELECT c, s FROM (\
931 SELECT '2026-08-14 12:00:00.001' AS c, now() AS s\
932 ) AS t ORDER BY c",
933 &[1_786_680_000_001_i64][..],
934 ),
935 (
936 "INSERT INTO timestamps (ts, st) \
937 SELECT DISTINCT c, s FROM (\
938 SELECT '2026-08-15 12:00:00.001' AS c, now() AS s\
939 ) AS t",
940 &[1_786_766_400_001_i64][..],
941 ),
942 ] {
943 let stmt = QueryLanguageParser::parse_sql(sql, &query_ctx).unwrap();
944 let plan = engine
945 .planner()
946 .plan(&stmt, query_ctx.clone())
947 .await
948 .unwrap()
949 .display_indent()
950 .to_string();
951
952 for expected_timestamp in expected_timestamps {
953 assert!(
954 plan.contains(&format!("TimestampMillisecond({expected_timestamp}, None)")),
955 "{plan}"
956 );
957 }
958 }
959 }
960
961 #[tokio::test]
962 async fn test_insert_explicit_timestamp_cast_keeps_datafusion_semantics() {
963 let query_ctx = Arc::new(
964 QueryContextBuilder::default()
965 .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
966 .build(),
967 );
968 let engine = create_timestamp_test_engine().await;
969 let sql = "INSERT INTO timestamps (ts, st) \
970 VALUES (CAST('2026-08-08 12:00:00.001' AS TIMESTAMP), now())";
971 let stmt = QueryLanguageParser::parse_sql(sql, &query_ctx).unwrap();
972 let plan = engine
973 .planner()
974 .plan(&stmt, query_ctx)
975 .await
976 .unwrap()
977 .display_indent()
978 .to_string();
979
980 assert!(
981 plan.contains("arrow_cast(Utf8(\"2026-08-08 12:00:00.001\")"),
982 "{plan}"
983 );
984 }
985
986 #[tokio::test]
987 async fn test_insert_converts_source_literal_shared_by_several_columns() {
988 let query_ctx = Arc::new(
989 QueryContextBuilder::default()
990 .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
991 .build(),
992 );
993 let engine = create_timestamp_test_engine().await;
994
995 for (sql, expected) in [
997 (
998 "INSERT INTO timestamps (ts, note) SELECT a, b FROM (\
999 SELECT c AS a, c AS b FROM (\
1000 SELECT '2026-08-12 12:00:00.001' AS c\
1001 ) AS t1\
1002 ) AS t2",
1003 &[
1004 "TimestampMillisecond(1786507200001, None)",
1005 "[a:Utf8, b:Utf8]",
1006 ][..],
1007 ),
1008 (
1009 "INSERT INTO timestamps (ts, ts_ns) SELECT a, b FROM (\
1010 SELECT c AS a, c AS b FROM (\
1011 SELECT '2026-08-12 12:00:00.123456789' AS c\
1012 ) AS t1\
1013 ) AS t2",
1014 &[
1015 "TimestampMillisecond(1786507200123, None)",
1016 "TimestampNanosecond(1786507200123456789, None)",
1017 ][..],
1018 ),
1019 ] {
1020 let stmt = QueryLanguageParser::parse_sql(sql, &query_ctx).unwrap();
1021 let plan = engine
1022 .planner()
1023 .plan(&stmt, query_ctx.clone())
1024 .await
1025 .unwrap()
1026 .display_indent_schema()
1027 .to_string();
1028
1029 for expected in expected {
1030 assert!(plan.contains(expected), "{plan}");
1031 }
1032 }
1033 }
1034
1035 #[tokio::test]
1036 async fn test_insert_union_tolerates_uncoerced_untouched_column() {
1037 let query_ctx = Arc::new(
1038 QueryContextBuilder::default()
1039 .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
1040 .build(),
1041 );
1042 let engine = create_timestamp_test_engine().await;
1043 let sql = "INSERT INTO timestamps (ts, st) \
1045 SELECT '2026-08-06 12:00:00.001', now() \
1046 UNION ALL \
1047 SELECT '2026-08-07 12:00:00.001', NULL";
1048 let stmt = QueryLanguageParser::parse_sql(sql, &query_ctx).unwrap();
1049 let plan = engine
1050 .planner()
1051 .plan(&stmt, query_ctx)
1052 .await
1053 .unwrap()
1054 .display_indent()
1055 .to_string();
1056
1057 for expected_timestamp in [1_785_988_800_001_i64, 1_786_075_200_001_i64] {
1058 assert!(
1059 plan.contains(&format!("TimestampMillisecond({expected_timestamp}, None)")),
1060 "{plan}"
1061 );
1062 }
1063 }
1064
1065 #[tokio::test]
1066 async fn test_insert_mixed_union_keeps_source_coercion() {
1067 let query_ctx = Arc::new(
1068 QueryContextBuilder::default()
1069 .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
1070 .build(),
1071 );
1072 let engine = create_timestamp_test_engine().await;
1073 let sql = "INSERT INTO timestamps (ts, st) \
1074 SELECT '2026-08-10 12:00:00.001', now() \
1075 UNION ALL \
1076 SELECT CAST('2026-08-11 12:00:00.001' AS TIMESTAMP), now()";
1077 let stmt = QueryLanguageParser::parse_sql(sql, &query_ctx).unwrap();
1078 let plan = engine
1079 .planner()
1080 .plan(&stmt, query_ctx)
1081 .await
1082 .unwrap()
1083 .display_indent_schema()
1084 .to_string();
1085 assert!(
1086 !plan.contains("TimestampMillisecond(1786334400001, None)"),
1087 "{plan}"
1088 );
1089 assert!(
1090 plan.contains("arrow_cast(Utf8(\"2026-08-11 12:00:00.001\")"),
1091 "{plan}"
1092 );
1093 }
1094
1095 #[tokio::test]
1096 async fn test_extract_placeholder_cast_types_multiple() {
1097 let plan = parse_sql_to_plan(
1098 "SELECT $1::INT, $2::TEXT, $3, $4::INTEGER FROM test WHERE $5::FLOAT > 0",
1099 )
1100 .await;
1101 let types = DfLogicalPlanner::extract_placeholder_cast_types(&plan).unwrap();
1102
1103 assert_eq!(types.len(), 5);
1104 assert_eq!(types.get("$1"), Some(&Some(DataType::Int32)));
1105 assert_eq!(types.get("$2"), Some(&Some(DataType::Utf8)));
1106 assert_eq!(types.get("$3"), Some(&None));
1107 assert_eq!(types.get("$4"), Some(&Some(DataType::Int32)));
1108 assert_eq!(types.get("$5"), Some(&Some(DataType::Float32)));
1109 }
1110
1111 #[tokio::test]
1112 async fn test_get_inferred_parameter_types_fallback_for_udf_args() {
1113 let plan = parse_sql_to_plan(
1115 "SELECT parse_ident($1), parse_ident($2::TEXT) FROM test WHERE id > $3",
1116 )
1117 .await;
1118 let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
1119
1120 assert_eq!(types.len(), 3);
1121
1122 let type_1 = types.get("$1").unwrap();
1123 let type_2 = types.get("$2").unwrap();
1124 let type_3 = types.get("$3").unwrap();
1125
1126 assert!(type_1.is_none(), "Expected $1 to be None");
1127 assert_eq!(type_2, &Some(DataType::Utf8));
1128 assert_eq!(type_3, &Some(DataType::Int32));
1129 }
1130
1131 #[tokio::test]
1132 async fn test_get_inferred_parameter_types_limit_offset() {
1133 let plan = parse_sql_to_plan("SELECT id FROM test LIMIT $1 OFFSET $2").await;
1134 let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
1135
1136 assert_eq!(types.get("$1"), Some(&Some(DataType::Int64)));
1137 assert_eq!(types.get("$2"), Some(&Some(DataType::Int64)));
1138 }
1139
1140 #[tokio::test]
1141 async fn test_plan_pql_applies_extension_rules() {
1142 for inner_agg in ["count", "sum", "avg", "min", "max", "stddev", "stdvar"] {
1143 let plan = parse_promql_to_plan(&format!(
1144 "sum(irate(some_metric[1h])) / scalar(count({inner_agg}(some_metric) by (tag_0)))"
1145 ))
1146 .await;
1147 let plan_str = plan.display_indent_schema().to_string();
1148 assert!(plan_str.contains("Distinct:"), "{inner_agg}: {plan_str}");
1149 }
1150 }
1151
1152 #[tokio::test]
1153 async fn test_plan_pql_filters_null_only_groups_for_non_count_inner_aggs() {
1154 let count_plan = parse_promql_to_plan("scalar(count(count(some_metric) by (tag_0)))").await;
1155 let count_plan_str = count_plan.display_indent_schema().to_string();
1156 assert!(
1157 !count_plan_str.contains("field_0 IS NOT NULL"),
1158 "{count_plan_str}"
1159 );
1160
1161 for inner_agg in ["sum", "avg", "min", "max", "stddev", "stdvar"] {
1162 let plan = parse_promql_to_plan(&format!(
1163 "scalar(count({inner_agg}(some_metric) by (tag_0)))"
1164 ))
1165 .await;
1166 let plan_str = plan.display_indent_schema().to_string();
1167 assert!(
1168 plan_str.contains("field_0 IS NOT NULL"),
1169 "{inner_agg}: {plan_str}"
1170 );
1171 }
1172 }
1173
1174 #[tokio::test]
1175 async fn test_plan_pql_skips_extension_rules_for_non_direct_or_unsupported_inner_agg() {
1176 for query in [
1177 "sum(irate(some_metric[1h])) / scalar(count(sum(irate(some_metric[1h])) by (tag_0)))",
1178 "sum(irate(some_metric[1h])) / scalar(count(group(some_metric) by (tag_0)))",
1179 ] {
1180 let plan = parse_promql_to_plan(query).await;
1181 let plan_str = plan.display_indent_schema().to_string();
1182 assert!(!plan_str.contains("Distinct:"), "{query}: {plan_str}");
1183 }
1184 }
1185
1186 #[tokio::test]
1187 async fn test_plan_sql_does_not_apply_nested_count_rule() {
1188 let plan = parse_sql_to_plan(
1189 "SELECT id, count(inner_count) \
1190 FROM ( \
1191 SELECT id, count(name) AS inner_count \
1192 FROM test \
1193 GROUP BY id \
1194 ORDER BY id \
1195 LIMIT 1000000 \
1196 ) t \
1197 GROUP BY id \
1198 ORDER BY id",
1199 )
1200 .await;
1201
1202 let plan_str = plan.display_indent_schema().to_string();
1203 assert!(!plan_str.contains("Distinct:"), "{plan_str}");
1204 }
1205
1206 #[tokio::test]
1207 async fn test_get_inferred_parameter_types_subquery() {
1208 let plan = parse_sql_to_plan(
1209 r#"SELECT * FROM test WHERE id = (SELECT id FROM test CROSS JOIN (SELECT parse_ident($1::TEXT) AS parts) p LIMIT 1)"#,
1210 ).await;
1211 let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
1212
1213 assert_eq!(types.len(), 1);
1214 let type_1 = types.get("$1").unwrap();
1215 assert_eq!(type_1, &Some(DataType::Utf8));
1216 }
1217
1218 #[tokio::test]
1219 async fn test_get_inferred_parameter_types_insert() {
1220 let plan = parse_sql_to_plan("INSERT INTO test (id, name) VALUES ($1, $2), ($3, $4)").await;
1221 let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
1222
1223 assert_eq!(types.len(), 4);
1224 assert_eq!(types.get("$1"), Some(&Some(DataType::Int32)));
1225 assert_eq!(types.get("$2"), Some(&Some(DataType::Utf8)));
1226 assert_eq!(types.get("$3"), Some(&Some(DataType::Int32)));
1227 assert_eq!(types.get("$4"), Some(&Some(DataType::Utf8)));
1228 }
1229
1230 #[tokio::test]
1231 async fn test_get_inferred_parameter_types_arrow_cast() {
1232 let plan = parse_sql_to_plan("SELECT $1::INT64, $2::FLOAT64, $3::INT16, $4::INT32, $5::UINT8, $6::UINT16, $7::UINT32").await;
1233 let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
1234
1235 assert_eq!(types.get("$1"), Some(&Some(DataType::Int64)));
1236 assert_eq!(types.get("$2"), Some(&Some(DataType::Float64)));
1237 assert_eq!(types.get("$3"), Some(&Some(DataType::Int16)));
1238 assert_eq!(types.get("$4"), Some(&Some(DataType::Int32)));
1239 assert_eq!(types.get("$5"), Some(&Some(DataType::UInt8)));
1240 assert_eq!(types.get("$6"), Some(&Some(DataType::UInt16)));
1241 assert_eq!(types.get("$7"), Some(&Some(DataType::UInt32)));
1242
1243 let plan = parse_sql_to_plan("SELECT $1::INT8, $2::FLOAT8, $3::INT2, $4::INT8").await;
1244 let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
1245
1246 assert_eq!(types.get("$1"), Some(&Some(DataType::Int64)));
1247 assert_eq!(types.get("$2"), Some(&Some(DataType::Float64)));
1248 assert_eq!(types.get("$3"), Some(&Some(DataType::Int16)));
1249 }
1250}