Skip to main content

flow/transform/
expr.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![warn(unused_imports)]
16
17use std::sync::Arc;
18
19use common_error::ext::BoxedError;
20use common_telemetry::debug;
21use datafusion::execution::SessionStateBuilder;
22use datafusion::functions::all_default_functions;
23use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
24use datafusion_physical_expr::PhysicalExpr;
25use datafusion_substrait::logical_plan::consumer::DefaultSubstraitConsumer;
26use datatypes::data_type::ConcreteDataType as CDT;
27use snafu::{OptionExt, ResultExt, ensure};
28use substrait_proto::proto::Expression;
29use substrait_proto::proto::expression::field_reference::ReferenceType::DirectReference;
30use substrait_proto::proto::expression::reference_segment::ReferenceType::StructField;
31use substrait_proto::proto::expression::{IfThen, RexType, ScalarFunction};
32use substrait_proto::proto::function_argument::ArgType;
33
34use crate::error::{
35    DatafusionSnafu, DatatypesSnafu, Error, EvalSnafu, ExternalSnafu, InvalidQuerySnafu,
36    NotImplementedSnafu, PlanSnafu, UnexpectedSnafu,
37};
38use crate::expr::{
39    BinaryFunc, DfScalarFunction, RawDfScalarFn, ScalarExpr, TUMBLE_END, TUMBLE_START, TypedExpr,
40    UnaryFunc, UnmaterializableFunc, VariadicFunc,
41};
42use crate::repr::{ColumnType, RelationDesc, RelationType};
43use crate::transform::literal::{
44    from_substrait_literal, from_substrait_type, to_substrait_literal,
45};
46use crate::transform::{FunctionExtensions, substrait_proto};
47
48// TODO(discord9): refactor plan to substrait convert of `arrow_cast` function thus remove this function
49/// ref to `arrow_schema::datatype` for type name
50fn typename_to_cdt(name: &str) -> Result<CDT, Error> {
51    let ret = match name {
52        "Int8" => CDT::int8_datatype(),
53        "Int16" => CDT::int16_datatype(),
54        "Int32" => CDT::int32_datatype(),
55        "Int64" => CDT::int64_datatype(),
56        "UInt8" => CDT::uint8_datatype(),
57        "UInt16" => CDT::uint16_datatype(),
58        "UInt32" => CDT::uint32_datatype(),
59        "UInt64" => CDT::uint64_datatype(),
60        "Float32" => CDT::float32_datatype(),
61        "Float64" => CDT::float64_datatype(),
62        "Boolean" => CDT::boolean_datatype(),
63        "String" => CDT::string_datatype(),
64        "Date" | "Date32" | "Date64" => CDT::date_datatype(),
65        "Timestamp" => CDT::timestamp_second_datatype(),
66        "Timestamp(Second, None)" => CDT::timestamp_second_datatype(),
67        "Timestamp(Millisecond, None)" => CDT::timestamp_millisecond_datatype(),
68        "Timestamp(Microsecond, None)" => CDT::timestamp_microsecond_datatype(),
69        "Timestamp(Nanosecond, None)" => CDT::timestamp_nanosecond_datatype(),
70        "Time32(Second)" | "Time64(Second)" => CDT::time_second_datatype(),
71        "Time32(Millisecond)" | "Time64(Millisecond)" => CDT::time_millisecond_datatype(),
72        "Time32(Microsecond)" | "Time64(Microsecond)" => CDT::time_microsecond_datatype(),
73        "Time32(Nanosecond)" | "Time64(Nanosecond)" => CDT::time_nanosecond_datatype(),
74        _ => NotImplementedSnafu {
75            reason: format!("Unrecognized typename: {}", name),
76        }
77        .fail()?,
78    };
79    Ok(ret)
80}
81
82/// Convert [`ScalarFunction`] to corresponding Datafusion's [`PhysicalExpr`]
83pub(crate) async fn from_scalar_fn_to_df_fn_impl(
84    f: &ScalarFunction,
85    input_schema: &RelationDesc,
86    extensions: &FunctionExtensions,
87) -> Result<Arc<dyn PhysicalExpr>, Error> {
88    let e = Expression {
89        rex_type: Some(RexType::ScalarFunction(f.clone())),
90    };
91    let schema = input_schema.to_df_schema()?;
92
93    let extensions = extensions.to_extensions();
94    let session_state = SessionStateBuilder::new()
95        .with_scalar_functions(all_default_functions())
96        .build();
97    let consumer = DefaultSubstraitConsumer::new(&extensions, &session_state);
98    let df_expr =
99        substrait::df_logical_plan::consumer::from_substrait_rex(&consumer, &e, &schema).await;
100    let expr = df_expr.context({
101        DatafusionSnafu {
102            context: "Failed to convert substrait scalar function to datafusion scalar function",
103        }
104    })?;
105    let phy_expr = datafusion::physical_expr::create_physical_expr(
106        &expr,
107        &schema,
108        &Default::default(),
109        &PhysicalPlanningContext::default(),
110    )
111    .context(DatafusionSnafu {
112        context: "Failed to create physical expression from logical expression",
113    })?;
114    Ok(phy_expr)
115}
116
117/// Return an [`Expression`](wrapped in a [`FunctionArgument`]) that references the i-th column of the input relation
118pub(crate) fn proto_col(i: usize) -> substrait_proto::proto::FunctionArgument {
119    use substrait_proto::proto::expression;
120    let expr = Expression {
121        rex_type: Some(expression::RexType::Selection(Box::new(
122            expression::FieldReference {
123                reference_type: Some(expression::field_reference::ReferenceType::DirectReference(
124                    expression::ReferenceSegment {
125                        reference_type: Some(
126                            expression::reference_segment::ReferenceType::StructField(Box::new(
127                                expression::reference_segment::StructField {
128                                    field: i as i32,
129                                    child: None,
130                                },
131                            )),
132                        ),
133                    },
134                )),
135                root_type: None,
136            },
137        ))),
138    };
139    substrait_proto::proto::FunctionArgument {
140        arg_type: Some(substrait_proto::proto::function_argument::ArgType::Value(
141            expr,
142        )),
143    }
144}
145
146fn is_proto_literal(arg: &substrait_proto::proto::FunctionArgument) -> bool {
147    use substrait_proto::proto::expression;
148    matches!(
149        arg.arg_type.as_ref().unwrap(),
150        ArgType::Value(Expression {
151            rex_type: Some(expression::RexType::Literal(_)),
152        })
153    )
154}
155
156fn build_proto_lit(
157    lit: substrait_proto::proto::expression::Literal,
158) -> substrait_proto::proto::FunctionArgument {
159    use substrait_proto::proto;
160    proto::FunctionArgument {
161        arg_type: Some(ArgType::Value(Expression {
162            rex_type: Some(proto::expression::RexType::Literal(lit)),
163        })),
164    }
165}
166
167/// rewrite ScalarFunction's arguments to Columns 0..n so nested exprs are still handled by us instead of datafusion
168///
169/// specially, if a argument is a literal, the replacement will not happen
170fn rewrite_scalar_function(
171    f: &ScalarFunction,
172    arg_typed_exprs: &[TypedExpr],
173) -> Result<ScalarFunction, Error> {
174    let mut f_rewrite = f.clone();
175    ensure!(
176        f_rewrite.arguments.len() == arg_typed_exprs.len(),
177        crate::error::InternalSnafu {
178            reason: format!(
179                "Expect `f_rewrite` and `arg_typed_expr` to be same length, found {} and {}",
180                f_rewrite.arguments.len(),
181                arg_typed_exprs.len()
182            )
183        }
184    );
185    for (idx, raw_expr) in f_rewrite.arguments.iter_mut().enumerate() {
186        // only replace it with col(idx) if it is not literal
187        // will try best to determine if it is literal, i.e. for function like `cast(<literal>)` will try
188        // in both world to understand if it results in a literal
189        match (
190            is_proto_literal(raw_expr),
191            arg_typed_exprs[idx].expr.is_literal(),
192        ) {
193            (false, false) => *raw_expr = proto_col(idx),
194            (true, _) => (),
195            (false, true) => {
196                if let ScalarExpr::Literal(val, ty) = &arg_typed_exprs[idx].expr {
197                    let df_val = val
198                        .try_to_scalar_value(ty)
199                        .map_err(BoxedError::new)
200                        .context(ExternalSnafu)?;
201                    let lit_sub = to_substrait_literal(&df_val)?;
202                    // put const-folded literal back to df to simplify stuff
203                    *raw_expr = build_proto_lit(lit_sub);
204                } else {
205                    UnexpectedSnafu {
206                        reason: format!(
207                            "Expect value to be literal, but found {:?}",
208                            arg_typed_exprs[idx].expr
209                        ),
210                    }
211                    .fail()?
212                }
213            }
214        }
215    }
216    Ok(f_rewrite)
217}
218
219impl TypedExpr {
220    pub async fn from_substrait_to_datafusion_scalar_func(
221        f: &ScalarFunction,
222        arg_typed_exprs: Vec<TypedExpr>,
223        extensions: &FunctionExtensions,
224    ) -> Result<TypedExpr, Error> {
225        let (arg_exprs, arg_types): (Vec<_>, Vec<_>) = arg_typed_exprs
226            .clone()
227            .into_iter()
228            .map(|e| (e.expr, e.typ))
229            .unzip();
230        debug!("Before rewrite: {:?}", f);
231        let f_rewrite = rewrite_scalar_function(f, &arg_typed_exprs)?;
232        debug!("After rewrite: {:?}", f_rewrite);
233        let input_schema = RelationType::new(arg_types).into_unnamed();
234        let raw_fn =
235            RawDfScalarFn::from_proto(&f_rewrite, input_schema.clone(), extensions.clone())?;
236
237        let df_func = DfScalarFunction::try_from_raw_fn(raw_fn).await?;
238        let expr = ScalarExpr::CallDf {
239            df_scalar_fn: df_func,
240            exprs: arg_exprs,
241        };
242        // df already know it's own schema, so not providing here
243        let ret_type = expr.typ(&[])?;
244        Ok(TypedExpr::new(expr, ret_type))
245    }
246
247    /// Convert ScalarFunction into Flow's ScalarExpr
248    pub async fn from_substrait_scalar_func(
249        f: &ScalarFunction,
250        input_schema: &RelationDesc,
251        extensions: &FunctionExtensions,
252    ) -> Result<TypedExpr, Error> {
253        let fn_name =
254            extensions
255                .get(&f.function_reference)
256                .with_context(|| NotImplementedSnafu {
257                    reason: format!(
258                        "Aggregated function not found: function reference = {:?}",
259                        f.function_reference
260                    ),
261                })?;
262        let arg_len = f.arguments.len();
263        let arg_typed_exprs: Vec<TypedExpr> = {
264            let mut rets = Vec::new();
265            for arg in f.arguments.iter() {
266                let ret = match &arg.arg_type {
267                    Some(ArgType::Value(e)) => {
268                        TypedExpr::from_substrait_rex(e, input_schema, extensions).await
269                    }
270                    _ => not_impl_err!("Aggregated function argument non-Value type not supported"),
271                }?;
272                rets.push(ret);
273            }
274            rets
275        };
276
277        // literal's type is determined by the function and type of other args
278        let (arg_exprs, arg_types): (Vec<_>, Vec<_>) = arg_typed_exprs
279            .clone()
280            .into_iter()
281            .map(
282                |TypedExpr {
283                     expr: arg_val,
284                     typ: arg_type,
285                 }| {
286                    if arg_val.is_literal() {
287                        (arg_val, None)
288                    } else {
289                        (arg_val, Some(arg_type.scalar_type))
290                    }
291                },
292            )
293            .unzip();
294
295        match arg_len {
296            1 if UnaryFunc::is_valid_func_name(fn_name) => {
297                let func = UnaryFunc::from_str_and_type(fn_name, None)?;
298                let arg = arg_exprs[0].clone();
299                let ret_type = ColumnType::new_nullable(func.signature().output.clone());
300
301                Ok(TypedExpr::new(arg.call_unary(func), ret_type))
302            }
303            2 if fn_name == "arrow_cast" => {
304                let cast_to = arg_exprs[1]
305                    .clone()
306                    .as_literal()
307                    .and_then(|lit| lit.as_string())
308                    .with_context(|| InvalidQuerySnafu {
309                        reason: "array_cast's second argument must be a literal string",
310                    })?;
311                let cast_to = typename_to_cdt(&cast_to)?;
312                let func = UnaryFunc::Cast(cast_to.clone());
313                let arg = arg_exprs[0].clone();
314                // constant folding here since some datafusion function require it for constant arg(i.e. `DATE_BIN`)
315                if arg.is_literal() {
316                    let res = func.eval(&[], &arg).context(EvalSnafu)?;
317                    Ok(TypedExpr::new(
318                        ScalarExpr::Literal(res, cast_to.clone()),
319                        ColumnType::new_nullable(cast_to),
320                    ))
321                } else {
322                    let ret_type = ColumnType::new_nullable(func.signature().output.clone());
323
324                    Ok(TypedExpr::new(arg.call_unary(func), ret_type))
325                }
326            }
327            2 if BinaryFunc::is_valid_func_name(fn_name) => {
328                let (func, signature) =
329                    BinaryFunc::from_str_expr_and_type(fn_name, &arg_exprs, &arg_types[0..2])?;
330
331                // constant folding here
332                let is_all_literal = arg_exprs.iter().all(|arg| arg.is_literal());
333                if is_all_literal {
334                    let res = func
335                        .eval(&[], &arg_exprs[0], &arg_exprs[1])
336                        .context(EvalSnafu)?;
337
338                    // if output type is null, it should be inferred from the input types
339                    let con_typ = signature.output.clone();
340                    let typ = ColumnType::new_nullable(con_typ.clone());
341                    return Ok(TypedExpr::new(ScalarExpr::Literal(res, con_typ), typ));
342                }
343
344                let mut arg_exprs = arg_exprs;
345                for (idx, arg_expr) in arg_exprs.iter_mut().enumerate() {
346                    if let ScalarExpr::Literal(val, typ) = arg_expr {
347                        let dest_type = signature.input[idx].clone();
348
349                        // cast val to target_type
350                        let dest_val = if !dest_type.is_null() {
351                            datatypes::types::cast(val.clone(), &dest_type)
352                        .with_context(|_|
353                            DatatypesSnafu{
354                                extra: format!("Failed to implicitly cast literal {val:?} to type {dest_type:?}")
355                            })?
356                        } else {
357                            val.clone()
358                        };
359                        *val = dest_val;
360                        *typ = dest_type;
361                    }
362                }
363
364                let ret_type = ColumnType::new_nullable(func.signature().output.clone());
365                let ret_expr = arg_exprs[0].clone().call_binary(arg_exprs[1].clone(), func);
366                Ok(TypedExpr::new(ret_expr, ret_type))
367            }
368            _var => {
369                if fn_name == TUMBLE_START || fn_name == TUMBLE_END {
370                    let (func, arg) = UnaryFunc::from_tumble_func(fn_name, &arg_typed_exprs)?;
371
372                    let ret_type = ColumnType::new_nullable(func.signature().output.clone());
373
374                    Ok(TypedExpr::new(arg.expr.call_unary(func), ret_type))
375                } else if VariadicFunc::is_valid_func_name(fn_name) {
376                    let func = VariadicFunc::from_str_and_types(fn_name, &arg_types)?;
377                    let ret_type = ColumnType::new_nullable(func.signature().output.clone());
378                    let mut expr = ScalarExpr::CallVariadic {
379                        func,
380                        exprs: arg_exprs,
381                    };
382                    expr.optimize();
383                    Ok(TypedExpr::new(expr, ret_type))
384                } else if UnmaterializableFunc::is_valid_func_name(fn_name) {
385                    let func = UnmaterializableFunc::from_str_args(fn_name, arg_typed_exprs)?;
386                    let ret_type = ColumnType::new_nullable(func.signature().output.clone());
387                    Ok(TypedExpr::new(
388                        ScalarExpr::CallUnmaterializable(func),
389                        ret_type,
390                    ))
391                } else {
392                    let try_as_df = Self::from_substrait_to_datafusion_scalar_func(
393                        f,
394                        arg_typed_exprs,
395                        extensions,
396                    )
397                    .await?;
398                    Ok(try_as_df)
399                }
400            }
401        }
402    }
403
404    /// Convert IfThen into Flow's ScalarExpr
405    pub async fn from_substrait_ifthen_rex(
406        if_then: &IfThen,
407        input_schema: &RelationDesc,
408        extensions: &FunctionExtensions,
409    ) -> Result<TypedExpr, Error> {
410        let ifs: Vec<_> = {
411            let mut ifs = Vec::new();
412            for if_clause in if_then.ifs.iter() {
413                let proto_if = if_clause.r#if.as_ref().with_context(|| InvalidQuerySnafu {
414                    reason: "IfThen clause without if",
415                })?;
416                let proto_then = if_clause.then.as_ref().with_context(|| InvalidQuerySnafu {
417                    reason: "IfThen clause without then",
418                })?;
419                let cond =
420                    TypedExpr::from_substrait_rex(proto_if, input_schema, extensions).await?;
421                let then =
422                    TypedExpr::from_substrait_rex(proto_then, input_schema, extensions).await?;
423                ifs.push((cond, then));
424            }
425            ifs
426        };
427        // if no else is presented
428        let els = match if_then
429            .r#else
430            .as_ref()
431            .map(|e| TypedExpr::from_substrait_rex(e, input_schema, extensions))
432        {
433            Some(fut) => Some(fut.await),
434            None => None,
435        }
436        .transpose()?
437        .unwrap_or_else(|| {
438            TypedExpr::new(
439                ScalarExpr::literal_null(),
440                ColumnType::new_nullable(CDT::null_datatype()),
441            )
442        });
443
444        fn build_if_then_recur(
445            mut next_if_then: impl Iterator<Item = (TypedExpr, TypedExpr)>,
446            els: TypedExpr,
447        ) -> TypedExpr {
448            if let Some((cond, then)) = next_if_then.next() {
449                // always assume the type of `if`` expr is the same with the `then`` expr
450                TypedExpr::new(
451                    ScalarExpr::If {
452                        cond: Box::new(cond.expr),
453                        then: Box::new(then.expr),
454                        els: Box::new(build_if_then_recur(next_if_then, els).expr),
455                    },
456                    then.typ,
457                )
458            } else {
459                els
460            }
461        }
462        let expr_if = build_if_then_recur(ifs.into_iter(), els);
463        Ok(expr_if)
464    }
465    /// Convert Substrait Rex into Flow's ScalarExpr
466    #[async_recursion::async_recursion]
467    pub async fn from_substrait_rex(
468        e: &Expression,
469        input_schema: &RelationDesc,
470        extensions: &FunctionExtensions,
471    ) -> Result<TypedExpr, Error> {
472        match &e.rex_type {
473            Some(RexType::Literal(lit)) => {
474                let lit = from_substrait_literal(lit)?;
475                Ok(TypedExpr::new(
476                    ScalarExpr::Literal(lit.0, lit.1.clone()),
477                    ColumnType::new_nullable(lit.1),
478                ))
479            }
480            Some(RexType::SingularOrList(s)) => {
481                let substrait_expr = s.value.as_ref().with_context(|| InvalidQuerySnafu {
482                    reason: "SingularOrList expression without value",
483                })?;
484                let typed_expr =
485                    TypedExpr::from_substrait_rex(substrait_expr, input_schema, extensions).await?;
486                // Note that we didn't impl support to in list expr
487                if !s.options.is_empty() {
488                    let mut list = Vec::with_capacity(s.options.len());
489                    for opt in s.options.iter() {
490                        let opt_expr =
491                            TypedExpr::from_substrait_rex(opt, input_schema, extensions).await?;
492                        list.push(opt_expr.expr);
493                    }
494                    let in_list_expr = ScalarExpr::InList {
495                        expr: Box::new(typed_expr.expr),
496                        list,
497                    };
498                    Ok(TypedExpr::new(
499                        in_list_expr,
500                        ColumnType::new_nullable(CDT::boolean_datatype()),
501                    ))
502                } else {
503                    Ok(typed_expr)
504                }
505            }
506            Some(RexType::Selection(field_ref)) => match &field_ref.reference_type {
507                Some(DirectReference(direct)) => match &direct.reference_type.as_ref() {
508                    Some(StructField(x)) => match &x.child.as_ref() {
509                        Some(_) => {
510                            not_impl_err!(
511                                "Direct reference StructField with child is not supported"
512                            )
513                        }
514                        None => {
515                            let column = x.field as usize;
516                            let column_type = input_schema.typ().column_types[column].clone();
517                            Ok(TypedExpr::new(ScalarExpr::Column(column), column_type))
518                        }
519                    },
520                    _ => not_impl_err!(
521                        "Direct reference with types other than StructField is not supported"
522                    ),
523                },
524                _ => not_impl_err!("unsupported field ref type"),
525            },
526            Some(RexType::ScalarFunction(f)) => {
527                TypedExpr::from_substrait_scalar_func(f, input_schema, extensions).await
528            }
529            Some(RexType::IfThen(if_then)) => {
530                TypedExpr::from_substrait_ifthen_rex(if_then, input_schema, extensions).await
531            }
532            Some(RexType::Cast(cast)) => {
533                let input = cast.input.as_ref().with_context(|| InvalidQuerySnafu {
534                    reason: "Cast expression without input",
535                })?;
536                let input = TypedExpr::from_substrait_rex(input, input_schema, extensions).await?;
537                let cast_type = from_substrait_type(cast.r#type.as_ref().with_context(|| {
538                    InvalidQuerySnafu {
539                        reason: "Cast expression without type",
540                    }
541                })?)?;
542                let func = UnaryFunc::from_str_and_type("cast", Some(cast_type.clone()))?;
543                Ok(TypedExpr::new(
544                    input.expr.call_unary(func),
545                    ColumnType::new_nullable(cast_type),
546                ))
547            }
548            Some(RexType::WindowFunction(_)) => PlanSnafu {
549                reason:
550                    "Window function is not supported yet. Please use aggregation function instead."
551                        .to_string(),
552            }
553            .fail(),
554            _ => not_impl_err!("unsupported rex_type"),
555        }
556    }
557}
558
559#[cfg(test)]
560mod test {
561    use datatypes::prelude::ConcreteDataType;
562    use datatypes::value::Value;
563    use pretty_assertions::assert_eq;
564
565    use super::*;
566    use crate::expr::{GlobalId, MapFilterProject};
567    use crate::plan::{Plan, TypedPlan};
568    use crate::repr::{self, ColumnType, RelationType};
569    use crate::transform::test::{create_test_ctx, create_test_query_engine, sql_to_substrait};
570
571    /// test if `WHERE` condition can be converted to Flow's ScalarExpr in mfp's filter
572    #[tokio::test]
573    async fn test_where_and() {
574        let engine = create_test_query_engine();
575        let sql =
576            "SELECT number FROM numbers_with_ts WHERE number >= 1 AND number <= 3 AND number!=2";
577        let plan = sql_to_substrait(engine.clone(), sql).await;
578
579        let mut ctx = create_test_ctx();
580        let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).await;
581
582        // optimize binary and to variadic and
583        let filter = ScalarExpr::CallVariadic {
584            func: VariadicFunc::And,
585            exprs: vec![
586                ScalarExpr::Column(2).call_binary(
587                    ScalarExpr::Literal(Value::from(1i64), CDT::int64_datatype()),
588                    BinaryFunc::Gte,
589                ),
590                ScalarExpr::Column(2).call_binary(
591                    ScalarExpr::Literal(Value::from(3i64), CDT::int64_datatype()),
592                    BinaryFunc::Lte,
593                ),
594                ScalarExpr::Column(2).call_binary(
595                    ScalarExpr::Literal(Value::from(2i64), CDT::int64_datatype()),
596                    BinaryFunc::NotEq,
597                ),
598            ],
599        };
600        let expected = TypedPlan {
601            schema: RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), false)])
602                .into_named(vec![Some("number".to_string())]),
603            plan: Plan::Mfp {
604                input: Box::new(
605                    Plan::Get {
606                        id: crate::expr::Id::Global(GlobalId::User(1)),
607                    }
608                    .with_types(
609                        RelationType::new(vec![
610                            ColumnType::new(ConcreteDataType::uint32_datatype(), false),
611                            ColumnType::new(
612                                ConcreteDataType::timestamp_millisecond_datatype(),
613                                false,
614                            ),
615                        ])
616                        .into_named(vec![Some("number".to_string()), Some("ts".to_string())]),
617                    ),
618                ),
619                mfp: MapFilterProject::new(2)
620                    .map(vec![
621                        ScalarExpr::CallUnary {
622                            func: UnaryFunc::Cast(CDT::int64_datatype()),
623                            expr: Box::new(ScalarExpr::Column(0)),
624                        },
625                        ScalarExpr::Column(0),
626                        ScalarExpr::Column(3),
627                    ])
628                    .unwrap()
629                    .filter(vec![filter])
630                    .unwrap()
631                    .project(vec![4])
632                    .unwrap(),
633            },
634        };
635        assert_eq!(flow_plan.unwrap(), expected);
636    }
637
638    /// case: binary functions&constant folding can happen in converting substrait plan
639    #[tokio::test]
640    async fn test_binary_func_and_constant_folding() {
641        let engine = create_test_query_engine();
642        let sql = "SELECT 1+1*2-1/1+1%2==3 FROM numbers";
643        let plan = sql_to_substrait(engine.clone(), sql).await;
644
645        let mut ctx = create_test_ctx();
646        let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).await;
647
648        let expected = TypedPlan {
649            schema: RelationType::new(vec![ColumnType::new(CDT::boolean_datatype(), true)])
650                .into_named(vec![Some("Int64(1) + Int64(1) * Int64(2) - Int64(1) / Int64(1) + Int64(1) % Int64(2) = Int64(3)".to_string())]),
651            plan: Plan::Constant {
652                rows: vec![(
653                    repr::Row::new(vec![Value::from(true)]),
654                    repr::Timestamp::MIN,
655                    1,
656                )],
657            },
658        };
659
660        assert_eq!(flow_plan.unwrap(), expected);
661    }
662
663    /// test if the type of the literal is correctly inferred, i.e. in here literal is decoded to be int64, but need to be uint32,
664    #[tokio::test]
665    async fn test_implicitly_cast() {
666        let engine = create_test_query_engine();
667        let sql = "SELECT number+1 FROM numbers";
668        let plan = sql_to_substrait(engine.clone(), sql).await;
669
670        let mut ctx = create_test_ctx();
671        let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).await;
672
673        let expected = TypedPlan {
674            schema: RelationType::new(vec![ColumnType::new(CDT::int64_datatype(), true)])
675                .into_named(vec![Some("numbers.number + Int64(1)".to_string())]),
676            plan: Plan::Mfp {
677                input: Box::new(
678                    Plan::Get {
679                        id: crate::expr::Id::Global(GlobalId::User(0)),
680                    }
681                    .with_types(
682                        RelationType::new(vec![ColumnType::new(
683                            ConcreteDataType::uint32_datatype(),
684                            false,
685                        )])
686                        .into_named(vec![Some("number".to_string())]),
687                    ),
688                ),
689                mfp: MapFilterProject::new(1)
690                    .map(vec![
691                        ScalarExpr::Column(0)
692                            .call_unary(UnaryFunc::Cast(CDT::int64_datatype()))
693                            .call_binary(
694                                ScalarExpr::Literal(Value::from(1i64), CDT::int64_datatype()),
695                                BinaryFunc::AddInt64,
696                            ),
697                    ])
698                    .unwrap()
699                    .project(vec![1])
700                    .unwrap(),
701            },
702        };
703        assert_eq!(flow_plan.unwrap(), expected);
704    }
705
706    #[tokio::test]
707    async fn test_cast() {
708        let engine = create_test_query_engine();
709        let sql = "SELECT CAST(1 AS INT16) FROM numbers";
710        let plan = sql_to_substrait(engine.clone(), sql).await;
711
712        let mut ctx = create_test_ctx();
713        let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).await;
714
715        let expected = TypedPlan {
716            schema: RelationType::new(vec![ColumnType::new(CDT::int16_datatype(), true)])
717                .into_named(vec![Some(
718                    "arrow_cast(Int64(1),Utf8(\"Int16\"))".to_string(),
719                )]),
720            plan: Plan::Constant {
721                // cast of literal is constant folded
722                rows: vec![(repr::Row::new(vec![Value::from(1i16)]), i64::MIN, 1)],
723            },
724        };
725        assert_eq!(flow_plan.unwrap(), expected);
726    }
727
728    #[tokio::test]
729    async fn test_select_add() {
730        let engine = create_test_query_engine();
731        let sql = "SELECT number+number FROM numbers";
732        let plan = sql_to_substrait(engine.clone(), sql).await;
733
734        let mut ctx = create_test_ctx();
735        let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan).await;
736
737        let expected = TypedPlan {
738            schema: RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), true)])
739                .into_named(vec![Some("numbers.number + numbers.number".to_string())]),
740            plan: Plan::Mfp {
741                input: Box::new(
742                    Plan::Get {
743                        id: crate::expr::Id::Global(GlobalId::User(0)),
744                    }
745                    .with_types(
746                        RelationType::new(vec![ColumnType::new(
747                            ConcreteDataType::uint32_datatype(),
748                            false,
749                        )])
750                        .into_named(vec![Some("number".to_string())]),
751                    ),
752                ),
753                mfp: MapFilterProject::new(1)
754                    .map(vec![
755                        ScalarExpr::Column(0)
756                            .call_binary(ScalarExpr::Column(0), BinaryFunc::AddUInt32),
757                    ])
758                    .unwrap()
759                    .project(vec![1])
760                    .unwrap(),
761            },
762        };
763
764        assert_eq!(flow_plan.unwrap(), expected);
765    }
766
767    #[tokio::test]
768    async fn test_func_sig() {
769        fn lit(v: impl ToString) -> substrait_proto::proto::FunctionArgument {
770            use substrait_proto::proto::expression;
771            let expr = Expression {
772                rex_type: Some(expression::RexType::Literal(expression::Literal {
773                    nullable: false,
774                    type_variation_reference: 0,
775                    literal_type: Some(expression::literal::LiteralType::String(v.to_string())),
776                })),
777            };
778            substrait_proto::proto::FunctionArgument {
779                arg_type: Some(substrait_proto::proto::function_argument::ArgType::Value(
780                    expr,
781                )),
782            }
783        }
784
785        let f = substrait_proto::proto::expression::ScalarFunction {
786            function_reference: 0,
787            arguments: vec![proto_col(0)],
788            options: vec![],
789            output_type: None,
790            ..Default::default()
791        };
792        let input_schema =
793            RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), false)]).into_unnamed();
794        let extensions = FunctionExtensions::from_iter([(0, "is_null".to_string())]);
795        let res = TypedExpr::from_substrait_scalar_func(&f, &input_schema, &extensions)
796            .await
797            .unwrap();
798
799        assert_eq!(
800            res,
801            TypedExpr {
802                expr: ScalarExpr::Column(0).call_unary(UnaryFunc::IsNull),
803                typ: ColumnType {
804                    scalar_type: CDT::boolean_datatype(),
805                    nullable: true,
806                },
807            }
808        );
809
810        let f = substrait_proto::proto::expression::ScalarFunction {
811            function_reference: 0,
812            arguments: vec![proto_col(0), proto_col(1)],
813            options: vec![],
814            output_type: None,
815            ..Default::default()
816        };
817        let input_schema = RelationType::new(vec![
818            ColumnType::new(CDT::uint32_datatype(), false),
819            ColumnType::new(CDT::uint32_datatype(), false),
820        ])
821        .into_unnamed();
822        let extensions = FunctionExtensions::from_iter([(0, "add".to_string())]);
823        let res = TypedExpr::from_substrait_scalar_func(&f, &input_schema, &extensions)
824            .await
825            .unwrap();
826
827        assert_eq!(
828            res,
829            TypedExpr {
830                expr: ScalarExpr::Column(0)
831                    .call_binary(ScalarExpr::Column(1), BinaryFunc::AddUInt32,),
832                typ: ColumnType {
833                    scalar_type: CDT::uint32_datatype(),
834                    nullable: true,
835                },
836            }
837        );
838    }
839}