1#![warn(unused_imports)]
16
17use std::sync::Arc;
18
19use common_error::ext::BoxedError;
20use common_telemetry::debug;
21use datafusion::execution::SessionStateBuilder;
22use datafusion::functions::all_default_functions;
23use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
24use datafusion_physical_expr::PhysicalExpr;
25use datafusion_substrait::logical_plan::consumer::DefaultSubstraitConsumer;
26use datatypes::data_type::ConcreteDataType as CDT;
27use snafu::{OptionExt, ResultExt, ensure};
28use substrait_proto::proto::Expression;
29use substrait_proto::proto::expression::field_reference::ReferenceType::DirectReference;
30use substrait_proto::proto::expression::reference_segment::ReferenceType::StructField;
31use substrait_proto::proto::expression::{IfThen, RexType, ScalarFunction};
32use substrait_proto::proto::function_argument::ArgType;
33
34use crate::error::{
35 DatafusionSnafu, DatatypesSnafu, Error, EvalSnafu, ExternalSnafu, InvalidQuerySnafu,
36 NotImplementedSnafu, PlanSnafu, UnexpectedSnafu,
37};
38use crate::expr::{
39 BinaryFunc, DfScalarFunction, RawDfScalarFn, ScalarExpr, TUMBLE_END, TUMBLE_START, TypedExpr,
40 UnaryFunc, UnmaterializableFunc, VariadicFunc,
41};
42use crate::repr::{ColumnType, RelationDesc, RelationType};
43use crate::transform::literal::{
44 from_substrait_literal, from_substrait_type, to_substrait_literal,
45};
46use crate::transform::{FunctionExtensions, substrait_proto};
47
48fn typename_to_cdt(name: &str) -> Result<CDT, Error> {
51 let ret = match name {
52 "Int8" => CDT::int8_datatype(),
53 "Int16" => CDT::int16_datatype(),
54 "Int32" => CDT::int32_datatype(),
55 "Int64" => CDT::int64_datatype(),
56 "UInt8" => CDT::uint8_datatype(),
57 "UInt16" => CDT::uint16_datatype(),
58 "UInt32" => CDT::uint32_datatype(),
59 "UInt64" => CDT::uint64_datatype(),
60 "Float32" => CDT::float32_datatype(),
61 "Float64" => CDT::float64_datatype(),
62 "Boolean" => CDT::boolean_datatype(),
63 "String" => CDT::string_datatype(),
64 "Date" | "Date32" | "Date64" => CDT::date_datatype(),
65 "Timestamp" => CDT::timestamp_second_datatype(),
66 "Timestamp(Second, None)" => CDT::timestamp_second_datatype(),
67 "Timestamp(Millisecond, None)" => CDT::timestamp_millisecond_datatype(),
68 "Timestamp(Microsecond, None)" => CDT::timestamp_microsecond_datatype(),
69 "Timestamp(Nanosecond, None)" => CDT::timestamp_nanosecond_datatype(),
70 "Time32(Second)" | "Time64(Second)" => CDT::time_second_datatype(),
71 "Time32(Millisecond)" | "Time64(Millisecond)" => CDT::time_millisecond_datatype(),
72 "Time32(Microsecond)" | "Time64(Microsecond)" => CDT::time_microsecond_datatype(),
73 "Time32(Nanosecond)" | "Time64(Nanosecond)" => CDT::time_nanosecond_datatype(),
74 _ => NotImplementedSnafu {
75 reason: format!("Unrecognized typename: {}", name),
76 }
77 .fail()?,
78 };
79 Ok(ret)
80}
81
82pub(crate) async fn from_scalar_fn_to_df_fn_impl(
84 f: &ScalarFunction,
85 input_schema: &RelationDesc,
86 extensions: &FunctionExtensions,
87) -> Result<Arc<dyn PhysicalExpr>, Error> {
88 let e = Expression {
89 rex_type: Some(RexType::ScalarFunction(f.clone())),
90 };
91 let schema = input_schema.to_df_schema()?;
92
93 let extensions = extensions.to_extensions();
94 let session_state = SessionStateBuilder::new()
95 .with_scalar_functions(all_default_functions())
96 .build();
97 let consumer = DefaultSubstraitConsumer::new(&extensions, &session_state);
98 let df_expr =
99 substrait::df_logical_plan::consumer::from_substrait_rex(&consumer, &e, &schema).await;
100 let expr = df_expr.context({
101 DatafusionSnafu {
102 context: "Failed to convert substrait scalar function to datafusion scalar function",
103 }
104 })?;
105 let phy_expr = datafusion::physical_expr::create_physical_expr(
106 &expr,
107 &schema,
108 &Default::default(),
109 &PhysicalPlanningContext::default(),
110 )
111 .context(DatafusionSnafu {
112 context: "Failed to create physical expression from logical expression",
113 })?;
114 Ok(phy_expr)
115}
116
117pub(crate) fn proto_col(i: usize) -> substrait_proto::proto::FunctionArgument {
119 use substrait_proto::proto::expression;
120 let expr = Expression {
121 rex_type: Some(expression::RexType::Selection(Box::new(
122 expression::FieldReference {
123 reference_type: Some(expression::field_reference::ReferenceType::DirectReference(
124 expression::ReferenceSegment {
125 reference_type: Some(
126 expression::reference_segment::ReferenceType::StructField(Box::new(
127 expression::reference_segment::StructField {
128 field: i as i32,
129 child: None,
130 },
131 )),
132 ),
133 },
134 )),
135 root_type: None,
136 },
137 ))),
138 };
139 substrait_proto::proto::FunctionArgument {
140 arg_type: Some(substrait_proto::proto::function_argument::ArgType::Value(
141 expr,
142 )),
143 }
144}
145
146fn is_proto_literal(arg: &substrait_proto::proto::FunctionArgument) -> bool {
147 use substrait_proto::proto::expression;
148 matches!(
149 arg.arg_type.as_ref().unwrap(),
150 ArgType::Value(Expression {
151 rex_type: Some(expression::RexType::Literal(_)),
152 })
153 )
154}
155
156fn build_proto_lit(
157 lit: substrait_proto::proto::expression::Literal,
158) -> substrait_proto::proto::FunctionArgument {
159 use substrait_proto::proto;
160 proto::FunctionArgument {
161 arg_type: Some(ArgType::Value(Expression {
162 rex_type: Some(proto::expression::RexType::Literal(lit)),
163 })),
164 }
165}
166
167fn rewrite_scalar_function(
171 f: &ScalarFunction,
172 arg_typed_exprs: &[TypedExpr],
173) -> Result<ScalarFunction, Error> {
174 let mut f_rewrite = f.clone();
175 ensure!(
176 f_rewrite.arguments.len() == arg_typed_exprs.len(),
177 crate::error::InternalSnafu {
178 reason: format!(
179 "Expect `f_rewrite` and `arg_typed_expr` to be same length, found {} and {}",
180 f_rewrite.arguments.len(),
181 arg_typed_exprs.len()
182 )
183 }
184 );
185 for (idx, raw_expr) in f_rewrite.arguments.iter_mut().enumerate() {
186 match (
190 is_proto_literal(raw_expr),
191 arg_typed_exprs[idx].expr.is_literal(),
192 ) {
193 (false, false) => *raw_expr = proto_col(idx),
194 (true, _) => (),
195 (false, true) => {
196 if let ScalarExpr::Literal(val, ty) = &arg_typed_exprs[idx].expr {
197 let df_val = val
198 .try_to_scalar_value(ty)
199 .map_err(BoxedError::new)
200 .context(ExternalSnafu)?;
201 let lit_sub = to_substrait_literal(&df_val)?;
202 *raw_expr = build_proto_lit(lit_sub);
204 } else {
205 UnexpectedSnafu {
206 reason: format!(
207 "Expect value to be literal, but found {:?}",
208 arg_typed_exprs[idx].expr
209 ),
210 }
211 .fail()?
212 }
213 }
214 }
215 }
216 Ok(f_rewrite)
217}
218
219impl TypedExpr {
220 pub async fn from_substrait_to_datafusion_scalar_func(
221 f: &ScalarFunction,
222 arg_typed_exprs: Vec<TypedExpr>,
223 extensions: &FunctionExtensions,
224 ) -> Result<TypedExpr, Error> {
225 let (arg_exprs, arg_types): (Vec<_>, Vec<_>) = arg_typed_exprs
226 .clone()
227 .into_iter()
228 .map(|e| (e.expr, e.typ))
229 .unzip();
230 debug!("Before rewrite: {:?}", f);
231 let f_rewrite = rewrite_scalar_function(f, &arg_typed_exprs)?;
232 debug!("After rewrite: {:?}", f_rewrite);
233 let input_schema = RelationType::new(arg_types).into_unnamed();
234 let raw_fn =
235 RawDfScalarFn::from_proto(&f_rewrite, input_schema.clone(), extensions.clone())?;
236
237 let df_func = DfScalarFunction::try_from_raw_fn(raw_fn).await?;
238 let expr = ScalarExpr::CallDf {
239 df_scalar_fn: df_func,
240 exprs: arg_exprs,
241 };
242 let ret_type = expr.typ(&[])?;
244 Ok(TypedExpr::new(expr, ret_type))
245 }
246
247 pub async fn from_substrait_scalar_func(
249 f: &ScalarFunction,
250 input_schema: &RelationDesc,
251 extensions: &FunctionExtensions,
252 ) -> Result<TypedExpr, Error> {
253 let fn_name =
254 extensions
255 .get(&f.function_reference)
256 .with_context(|| NotImplementedSnafu {
257 reason: format!(
258 "Aggregated function not found: function reference = {:?}",
259 f.function_reference
260 ),
261 })?;
262 let arg_len = f.arguments.len();
263 let arg_typed_exprs: Vec<TypedExpr> = {
264 let mut rets = Vec::new();
265 for arg in f.arguments.iter() {
266 let ret = match &arg.arg_type {
267 Some(ArgType::Value(e)) => {
268 TypedExpr::from_substrait_rex(e, input_schema, extensions).await
269 }
270 _ => not_impl_err!("Aggregated function argument non-Value type not supported"),
271 }?;
272 rets.push(ret);
273 }
274 rets
275 };
276
277 let (arg_exprs, arg_types): (Vec<_>, Vec<_>) = arg_typed_exprs
279 .clone()
280 .into_iter()
281 .map(
282 |TypedExpr {
283 expr: arg_val,
284 typ: arg_type,
285 }| {
286 if arg_val.is_literal() {
287 (arg_val, None)
288 } else {
289 (arg_val, Some(arg_type.scalar_type))
290 }
291 },
292 )
293 .unzip();
294
295 match arg_len {
296 1 if UnaryFunc::is_valid_func_name(fn_name) => {
297 let func = UnaryFunc::from_str_and_type(fn_name, None)?;
298 let arg = arg_exprs[0].clone();
299 let ret_type = ColumnType::new_nullable(func.signature().output.clone());
300
301 Ok(TypedExpr::new(arg.call_unary(func), ret_type))
302 }
303 2 if fn_name == "arrow_cast" => {
304 let cast_to = arg_exprs[1]
305 .clone()
306 .as_literal()
307 .and_then(|lit| lit.as_string())
308 .with_context(|| InvalidQuerySnafu {
309 reason: "array_cast's second argument must be a literal string",
310 })?;
311 let cast_to = typename_to_cdt(&cast_to)?;
312 let func = UnaryFunc::Cast(cast_to.clone());
313 let arg = arg_exprs[0].clone();
314 if arg.is_literal() {
316 let res = func.eval(&[], &arg).context(EvalSnafu)?;
317 Ok(TypedExpr::new(
318 ScalarExpr::Literal(res, cast_to.clone()),
319 ColumnType::new_nullable(cast_to),
320 ))
321 } else {
322 let ret_type = ColumnType::new_nullable(func.signature().output.clone());
323
324 Ok(TypedExpr::new(arg.call_unary(func), ret_type))
325 }
326 }
327 2 if BinaryFunc::is_valid_func_name(fn_name) => {
328 let (func, signature) =
329 BinaryFunc::from_str_expr_and_type(fn_name, &arg_exprs, &arg_types[0..2])?;
330
331 let is_all_literal = arg_exprs.iter().all(|arg| arg.is_literal());
333 if is_all_literal {
334 let res = func
335 .eval(&[], &arg_exprs[0], &arg_exprs[1])
336 .context(EvalSnafu)?;
337
338 let con_typ = signature.output.clone();
340 let typ = ColumnType::new_nullable(con_typ.clone());
341 return Ok(TypedExpr::new(ScalarExpr::Literal(res, con_typ), typ));
342 }
343
344 let mut arg_exprs = arg_exprs;
345 for (idx, arg_expr) in arg_exprs.iter_mut().enumerate() {
346 if let ScalarExpr::Literal(val, typ) = arg_expr {
347 let dest_type = signature.input[idx].clone();
348
349 let dest_val = if !dest_type.is_null() {
351 datatypes::types::cast(val.clone(), &dest_type)
352 .with_context(|_|
353 DatatypesSnafu{
354 extra: format!("Failed to implicitly cast literal {val:?} to type {dest_type:?}")
355 })?
356 } else {
357 val.clone()
358 };
359 *val = dest_val;
360 *typ = dest_type;
361 }
362 }
363
364 let ret_type = ColumnType::new_nullable(func.signature().output.clone());
365 let ret_expr = arg_exprs[0].clone().call_binary(arg_exprs[1].clone(), func);
366 Ok(TypedExpr::new(ret_expr, ret_type))
367 }
368 _var => {
369 if fn_name == TUMBLE_START || fn_name == TUMBLE_END {
370 let (func, arg) = UnaryFunc::from_tumble_func(fn_name, &arg_typed_exprs)?;
371
372 let ret_type = ColumnType::new_nullable(func.signature().output.clone());
373
374 Ok(TypedExpr::new(arg.expr.call_unary(func), ret_type))
375 } else if VariadicFunc::is_valid_func_name(fn_name) {
376 let func = VariadicFunc::from_str_and_types(fn_name, &arg_types)?;
377 let ret_type = ColumnType::new_nullable(func.signature().output.clone());
378 let mut expr = ScalarExpr::CallVariadic {
379 func,
380 exprs: arg_exprs,
381 };
382 expr.optimize();
383 Ok(TypedExpr::new(expr, ret_type))
384 } else if UnmaterializableFunc::is_valid_func_name(fn_name) {
385 let func = UnmaterializableFunc::from_str_args(fn_name, arg_typed_exprs)?;
386 let ret_type = ColumnType::new_nullable(func.signature().output.clone());
387 Ok(TypedExpr::new(
388 ScalarExpr::CallUnmaterializable(func),
389 ret_type,
390 ))
391 } else {
392 let try_as_df = Self::from_substrait_to_datafusion_scalar_func(
393 f,
394 arg_typed_exprs,
395 extensions,
396 )
397 .await?;
398 Ok(try_as_df)
399 }
400 }
401 }
402 }
403
404 pub async fn from_substrait_ifthen_rex(
406 if_then: &IfThen,
407 input_schema: &RelationDesc,
408 extensions: &FunctionExtensions,
409 ) -> Result<TypedExpr, Error> {
410 let ifs: Vec<_> = {
411 let mut ifs = Vec::new();
412 for if_clause in if_then.ifs.iter() {
413 let proto_if = if_clause.r#if.as_ref().with_context(|| InvalidQuerySnafu {
414 reason: "IfThen clause without if",
415 })?;
416 let proto_then = if_clause.then.as_ref().with_context(|| InvalidQuerySnafu {
417 reason: "IfThen clause without then",
418 })?;
419 let cond =
420 TypedExpr::from_substrait_rex(proto_if, input_schema, extensions).await?;
421 let then =
422 TypedExpr::from_substrait_rex(proto_then, input_schema, extensions).await?;
423 ifs.push((cond, then));
424 }
425 ifs
426 };
427 let els = match if_then
429 .r#else
430 .as_ref()
431 .map(|e| TypedExpr::from_substrait_rex(e, input_schema, extensions))
432 {
433 Some(fut) => Some(fut.await),
434 None => None,
435 }
436 .transpose()?
437 .unwrap_or_else(|| {
438 TypedExpr::new(
439 ScalarExpr::literal_null(),
440 ColumnType::new_nullable(CDT::null_datatype()),
441 )
442 });
443
444 fn build_if_then_recur(
445 mut next_if_then: impl Iterator<Item = (TypedExpr, TypedExpr)>,
446 els: TypedExpr,
447 ) -> TypedExpr {
448 if let Some((cond, then)) = next_if_then.next() {
449 TypedExpr::new(
451 ScalarExpr::If {
452 cond: Box::new(cond.expr),
453 then: Box::new(then.expr),
454 els: Box::new(build_if_then_recur(next_if_then, els).expr),
455 },
456 then.typ,
457 )
458 } else {
459 els
460 }
461 }
462 let expr_if = build_if_then_recur(ifs.into_iter(), els);
463 Ok(expr_if)
464 }
465 #[async_recursion::async_recursion]
467 pub async fn from_substrait_rex(
468 e: &Expression,
469 input_schema: &RelationDesc,
470 extensions: &FunctionExtensions,
471 ) -> Result<TypedExpr, Error> {
472 match &e.rex_type {
473 Some(RexType::Literal(lit)) => {
474 let lit = from_substrait_literal(lit)?;
475 Ok(TypedExpr::new(
476 ScalarExpr::Literal(lit.0, lit.1.clone()),
477 ColumnType::new_nullable(lit.1),
478 ))
479 }
480 Some(RexType::SingularOrList(s)) => {
481 let substrait_expr = s.value.as_ref().with_context(|| InvalidQuerySnafu {
482 reason: "SingularOrList expression without value",
483 })?;
484 let typed_expr =
485 TypedExpr::from_substrait_rex(substrait_expr, input_schema, extensions).await?;
486 if !s.options.is_empty() {
488 let mut list = Vec::with_capacity(s.options.len());
489 for opt in s.options.iter() {
490 let opt_expr =
491 TypedExpr::from_substrait_rex(opt, input_schema, extensions).await?;
492 list.push(opt_expr.expr);
493 }
494 let in_list_expr = ScalarExpr::InList {
495 expr: Box::new(typed_expr.expr),
496 list,
497 };
498 Ok(TypedExpr::new(
499 in_list_expr,
500 ColumnType::new_nullable(CDT::boolean_datatype()),
501 ))
502 } else {
503 Ok(typed_expr)
504 }
505 }
506 Some(RexType::Selection(field_ref)) => match &field_ref.reference_type {
507 Some(DirectReference(direct)) => match &direct.reference_type.as_ref() {
508 Some(StructField(x)) => match &x.child.as_ref() {
509 Some(_) => {
510 not_impl_err!(
511 "Direct reference StructField with child is not supported"
512 )
513 }
514 None => {
515 let column = x.field as usize;
516 let column_type = input_schema.typ().column_types[column].clone();
517 Ok(TypedExpr::new(ScalarExpr::Column(column), column_type))
518 }
519 },
520 _ => not_impl_err!(
521 "Direct reference with types other than StructField is not supported"
522 ),
523 },
524 _ => not_impl_err!("unsupported field ref type"),
525 },
526 Some(RexType::ScalarFunction(f)) => {
527 TypedExpr::from_substrait_scalar_func(f, input_schema, extensions).await
528 }
529 Some(RexType::IfThen(if_then)) => {
530 TypedExpr::from_substrait_ifthen_rex(if_then, input_schema, extensions).await
531 }
532 Some(RexType::Cast(cast)) => {
533 let input = cast.input.as_ref().with_context(|| InvalidQuerySnafu {
534 reason: "Cast expression without input",
535 })?;
536 let input = TypedExpr::from_substrait_rex(input, input_schema, extensions).await?;
537 let cast_type = from_substrait_type(cast.r#type.as_ref().with_context(|| {
538 InvalidQuerySnafu {
539 reason: "Cast expression without type",
540 }
541 })?)?;
542 let func = UnaryFunc::from_str_and_type("cast", Some(cast_type.clone()))?;
543 Ok(TypedExpr::new(
544 input.expr.call_unary(func),
545 ColumnType::new_nullable(cast_type),
546 ))
547 }
548 Some(RexType::WindowFunction(_)) => PlanSnafu {
549 reason:
550 "Window function is not supported yet. Please use aggregation function instead."
551 .to_string(),
552 }
553 .fail(),
554 _ => not_impl_err!("unsupported rex_type"),
555 }
556 }
557}
558
559#[cfg(test)]
560mod test {
561 use datatypes::prelude::ConcreteDataType;
562 use datatypes::value::Value;
563 use pretty_assertions::assert_eq;
564
565 use super::*;
566 use crate::expr::{GlobalId, MapFilterProject};
567 use crate::plan::{Plan, TypedPlan};
568 use crate::repr::{self, ColumnType, RelationType};
569 use crate::transform::test::{create_test_ctx, create_test_query_engine, sql_to_substrait};
570
571 #[tokio::test]
573 async fn test_where_and() {
574 let engine = create_test_query_engine();
575 let sql =
576 "SELECT number FROM numbers_with_ts WHERE number >= 1 AND number <= 3 AND number!=2";
577 let plan = sql_to_substrait(engine.clone(), sql).await;
578
579 let mut ctx = create_test_ctx();
580 let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).await;
581
582 let filter = ScalarExpr::CallVariadic {
584 func: VariadicFunc::And,
585 exprs: vec![
586 ScalarExpr::Column(2).call_binary(
587 ScalarExpr::Literal(Value::from(1i64), CDT::int64_datatype()),
588 BinaryFunc::Gte,
589 ),
590 ScalarExpr::Column(2).call_binary(
591 ScalarExpr::Literal(Value::from(3i64), CDT::int64_datatype()),
592 BinaryFunc::Lte,
593 ),
594 ScalarExpr::Column(2).call_binary(
595 ScalarExpr::Literal(Value::from(2i64), CDT::int64_datatype()),
596 BinaryFunc::NotEq,
597 ),
598 ],
599 };
600 let expected = TypedPlan {
601 schema: RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), false)])
602 .into_named(vec![Some("number".to_string())]),
603 plan: Plan::Mfp {
604 input: Box::new(
605 Plan::Get {
606 id: crate::expr::Id::Global(GlobalId::User(1)),
607 }
608 .with_types(
609 RelationType::new(vec![
610 ColumnType::new(ConcreteDataType::uint32_datatype(), false),
611 ColumnType::new(
612 ConcreteDataType::timestamp_millisecond_datatype(),
613 false,
614 ),
615 ])
616 .into_named(vec![Some("number".to_string()), Some("ts".to_string())]),
617 ),
618 ),
619 mfp: MapFilterProject::new(2)
620 .map(vec![
621 ScalarExpr::CallUnary {
622 func: UnaryFunc::Cast(CDT::int64_datatype()),
623 expr: Box::new(ScalarExpr::Column(0)),
624 },
625 ScalarExpr::Column(0),
626 ScalarExpr::Column(3),
627 ])
628 .unwrap()
629 .filter(vec![filter])
630 .unwrap()
631 .project(vec![4])
632 .unwrap(),
633 },
634 };
635 assert_eq!(flow_plan.unwrap(), expected);
636 }
637
638 #[tokio::test]
640 async fn test_binary_func_and_constant_folding() {
641 let engine = create_test_query_engine();
642 let sql = "SELECT 1+1*2-1/1+1%2==3 FROM numbers";
643 let plan = sql_to_substrait(engine.clone(), sql).await;
644
645 let mut ctx = create_test_ctx();
646 let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).await;
647
648 let expected = TypedPlan {
649 schema: RelationType::new(vec![ColumnType::new(CDT::boolean_datatype(), true)])
650 .into_named(vec![Some("Int64(1) + Int64(1) * Int64(2) - Int64(1) / Int64(1) + Int64(1) % Int64(2) = Int64(3)".to_string())]),
651 plan: Plan::Constant {
652 rows: vec![(
653 repr::Row::new(vec![Value::from(true)]),
654 repr::Timestamp::MIN,
655 1,
656 )],
657 },
658 };
659
660 assert_eq!(flow_plan.unwrap(), expected);
661 }
662
663 #[tokio::test]
665 async fn test_implicitly_cast() {
666 let engine = create_test_query_engine();
667 let sql = "SELECT number+1 FROM numbers";
668 let plan = sql_to_substrait(engine.clone(), sql).await;
669
670 let mut ctx = create_test_ctx();
671 let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).await;
672
673 let expected = TypedPlan {
674 schema: RelationType::new(vec![ColumnType::new(CDT::int64_datatype(), true)])
675 .into_named(vec![Some("numbers.number + Int64(1)".to_string())]),
676 plan: Plan::Mfp {
677 input: Box::new(
678 Plan::Get {
679 id: crate::expr::Id::Global(GlobalId::User(0)),
680 }
681 .with_types(
682 RelationType::new(vec![ColumnType::new(
683 ConcreteDataType::uint32_datatype(),
684 false,
685 )])
686 .into_named(vec![Some("number".to_string())]),
687 ),
688 ),
689 mfp: MapFilterProject::new(1)
690 .map(vec![
691 ScalarExpr::Column(0)
692 .call_unary(UnaryFunc::Cast(CDT::int64_datatype()))
693 .call_binary(
694 ScalarExpr::Literal(Value::from(1i64), CDT::int64_datatype()),
695 BinaryFunc::AddInt64,
696 ),
697 ])
698 .unwrap()
699 .project(vec![1])
700 .unwrap(),
701 },
702 };
703 assert_eq!(flow_plan.unwrap(), expected);
704 }
705
706 #[tokio::test]
707 async fn test_cast() {
708 let engine = create_test_query_engine();
709 let sql = "SELECT CAST(1 AS INT16) FROM numbers";
710 let plan = sql_to_substrait(engine.clone(), sql).await;
711
712 let mut ctx = create_test_ctx();
713 let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).await;
714
715 let expected = TypedPlan {
716 schema: RelationType::new(vec![ColumnType::new(CDT::int16_datatype(), true)])
717 .into_named(vec![Some(
718 "arrow_cast(Int64(1),Utf8(\"Int16\"))".to_string(),
719 )]),
720 plan: Plan::Constant {
721 rows: vec![(repr::Row::new(vec![Value::from(1i16)]), i64::MIN, 1)],
723 },
724 };
725 assert_eq!(flow_plan.unwrap(), expected);
726 }
727
728 #[tokio::test]
729 async fn test_select_add() {
730 let engine = create_test_query_engine();
731 let sql = "SELECT number+number FROM numbers";
732 let plan = sql_to_substrait(engine.clone(), sql).await;
733
734 let mut ctx = create_test_ctx();
735 let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).await;
736
737 let expected = TypedPlan {
738 schema: RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), true)])
739 .into_named(vec![Some("numbers.number + numbers.number".to_string())]),
740 plan: Plan::Mfp {
741 input: Box::new(
742 Plan::Get {
743 id: crate::expr::Id::Global(GlobalId::User(0)),
744 }
745 .with_types(
746 RelationType::new(vec![ColumnType::new(
747 ConcreteDataType::uint32_datatype(),
748 false,
749 )])
750 .into_named(vec![Some("number".to_string())]),
751 ),
752 ),
753 mfp: MapFilterProject::new(1)
754 .map(vec![
755 ScalarExpr::Column(0)
756 .call_binary(ScalarExpr::Column(0), BinaryFunc::AddUInt32),
757 ])
758 .unwrap()
759 .project(vec![1])
760 .unwrap(),
761 },
762 };
763
764 assert_eq!(flow_plan.unwrap(), expected);
765 }
766
767 #[tokio::test]
768 async fn test_func_sig() {
769 fn lit(v: impl ToString) -> substrait_proto::proto::FunctionArgument {
770 use substrait_proto::proto::expression;
771 let expr = Expression {
772 rex_type: Some(expression::RexType::Literal(expression::Literal {
773 nullable: false,
774 type_variation_reference: 0,
775 literal_type: Some(expression::literal::LiteralType::String(v.to_string())),
776 })),
777 };
778 substrait_proto::proto::FunctionArgument {
779 arg_type: Some(substrait_proto::proto::function_argument::ArgType::Value(
780 expr,
781 )),
782 }
783 }
784
785 let f = substrait_proto::proto::expression::ScalarFunction {
786 function_reference: 0,
787 arguments: vec![proto_col(0)],
788 options: vec![],
789 output_type: None,
790 ..Default::default()
791 };
792 let input_schema =
793 RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), false)]).into_unnamed();
794 let extensions = FunctionExtensions::from_iter([(0, "is_null".to_string())]);
795 let res = TypedExpr::from_substrait_scalar_func(&f, &input_schema, &extensions)
796 .await
797 .unwrap();
798
799 assert_eq!(
800 res,
801 TypedExpr {
802 expr: ScalarExpr::Column(0).call_unary(UnaryFunc::IsNull),
803 typ: ColumnType {
804 scalar_type: CDT::boolean_datatype(),
805 nullable: true,
806 },
807 }
808 );
809
810 let f = substrait_proto::proto::expression::ScalarFunction {
811 function_reference: 0,
812 arguments: vec![proto_col(0), proto_col(1)],
813 options: vec![],
814 output_type: None,
815 ..Default::default()
816 };
817 let input_schema = RelationType::new(vec![
818 ColumnType::new(CDT::uint32_datatype(), false),
819 ColumnType::new(CDT::uint32_datatype(), false),
820 ])
821 .into_unnamed();
822 let extensions = FunctionExtensions::from_iter([(0, "add".to_string())]);
823 let res = TypedExpr::from_substrait_scalar_func(&f, &input_schema, &extensions)
824 .await
825 .unwrap();
826
827 assert_eq!(
828 res,
829 TypedExpr {
830 expr: ScalarExpr::Column(0)
831 .call_binary(ScalarExpr::Column(1), BinaryFunc::AddUInt32,),
832 typ: ColumnType {
833 scalar_type: CDT::uint32_datatype(),
834 nullable: true,
835 },
836 }
837 );
838 }
839}