chore: update datafusion family (#6675)

* chore: update datafusion family

Signed-off-by: luofucong <luofc@foxmail.com>

* fix ci

Signed-off-by: luofucong <luofc@foxmail.com>

* use official otel-arrow-rust

Signed-off-by: luofucong <luofc@foxmail.com>

* rebase

Signed-off-by: luofucong <luofc@foxmail.com>

* use the official orc-rust

Signed-off-by: luofucong <luofc@foxmail.com>

* resolve PR comments

Signed-off-by: luofucong <luofc@foxmail.com>

* remove the empty lines

Signed-off-by: luofucong <luofc@foxmail.com>

* try following PR comments

Signed-off-by: luofucong <luofc@foxmail.com>

---------

Signed-off-by: luofucong <luofc@foxmail.com>

src/table/src/predicate.rs

@@ -19,7 +19,8 @@ use common_time::range::TimestampRange;
 use common_time::timestamp::TimeUnit;
 use common_time::Timestamp;
 use datafusion::common::ScalarValue;
-use datafusion::physical_optimizer::pruning::{PruningPredicate, PruningStatistics};
+use datafusion::physical_optimizer::pruning::PruningPredicate;
+use datafusion_common::pruning::PruningStatistics;
 use datafusion_common::ToDFSchema;
 use datafusion_expr::expr::{Expr, InList};
 use datafusion_expr::{Between, BinaryExpr, Operator};
@@ -260,8 +261,8 @@ fn extract_from_binary_expr(
 
 fn get_timestamp_filter(ts_col_name: &str, left: &Expr, right: &Expr) -> Option<(Timestamp, bool)> {
     let (col, lit, reverse) = match (left, right) {
-        (Expr::Column(column), Expr::Literal(scalar)) => (column, scalar, false),
-        (Expr::Literal(scalar), Expr::Column(column)) => (column, scalar, true),
+        (Expr::Column(column), Expr::Literal(scalar, _)) => (column, scalar, false),
+        (Expr::Literal(scalar, _), Expr::Column(column)) => (column, scalar, true),
         _ => {
             return None;
         }
@@ -294,7 +295,7 @@ fn extract_from_between_expr(
     }
 
     match (low, high) {
-        (Expr::Literal(low), Expr::Literal(high)) => {
+        (Expr::Literal(low, _), Expr::Literal(high, _)) => {
             return_none_if_utf8!(low);
             return_none_if_utf8!(high);
 
@@ -330,7 +331,7 @@ fn extract_from_in_list_expr(
     }
     let mut init_range = TimestampRange::empty();
     for expr in list {
-        if let Expr::Literal(scalar) = expr {
+        if let Expr::Literal(scalar, _) = expr {
             return_none_if_utf8!(scalar);
             if let Some(timestamp) = scalar_value_to_timestamp(scalar, None) {
                 init_range = init_range.or(&TimestampRange::single(timestamp))
@@ -590,9 +591,7 @@ mod tests {
         vec![datafusion_expr::Expr::BinaryExpr(BinaryExpr {
             left: Box::new(datafusion_expr::Expr::Column(Column::from_name("cnt"))),
             op,
-            right: Box::new(datafusion_expr::Expr::Literal(ScalarValue::Int32(Some(
-                max_val,
-            )))),
+            right: Box::new(max_val.lit()),
         })]
     }
 

src/table/src/predicate/stats.rs

@@ -17,7 +17,7 @@ use std::sync::Arc;
 
 use datafusion::parquet::file::metadata::RowGroupMetaData;
 use datafusion::parquet::file::statistics::Statistics as ParquetStats;
-use datafusion::physical_optimizer::pruning::PruningStatistics;
+use datafusion_common::pruning::PruningStatistics;
 use datafusion_common::{Column, ScalarValue};
 use datatypes::arrow::array::{ArrayRef, BooleanArray, UInt64Array};
 use datatypes::arrow::datatypes::DataType;

src/table/src/table.rs

@@ -195,9 +195,10 @@ fn default_constraint_to_expr(
     target_type: &ConcreteDataType,
 ) -> Option<Expr> {
     match default_constraint {
-        ColumnDefaultConstraint::Value(v) => {
-            v.try_to_scalar_value(target_type).ok().map(Expr::Literal)
-        }
+        ColumnDefaultConstraint::Value(v) => v
+            .try_to_scalar_value(target_type)
+            .ok()
+            .map(|x| Expr::Literal(x, None)),
 
         ColumnDefaultConstraint::Function(name)
             if matches!(
@@ -245,7 +246,7 @@ mod tests {
 
         assert!(!column_defaults.contains_key("col1"));
         assert!(matches!(column_defaults.get("col2").unwrap(),
-                         Expr::Literal(ScalarValue::Utf8(Some(s))) if s == "test"));
+                         Expr::Literal(ScalarValue::Utf8(Some(s)), _) if s == "test"));
         assert!(matches!(
             column_defaults.get("ts").unwrap(),
             Expr::Cast(Cast {

src/table/src/table/scan.rs

@@ -34,9 +34,7 @@ use datafusion::physical_plan::{
 use datafusion_common::stats::Precision;
 use datafusion_common::{ColumnStatistics, DataFusionError, Statistics};
 use datafusion_physical_expr::expressions::Column;
-use datafusion_physical_expr::{
-    EquivalenceProperties, LexOrdering, Partitioning, PhysicalSortExpr,
-};
+use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalSortExpr};
 use datatypes::arrow::datatypes::SchemaRef as ArrowSchemaRef;
 use datatypes::compute::SortOptions;
 use futures::{Stream, StreamExt};
@@ -126,7 +124,7 @@ impl RegionScanExec {
                 }
                 EquivalenceProperties::new_with_orderings(
                     arrow_schema.clone(),
-                    &[LexOrdering::new(pk_sort_columns)],
+                    vec![pk_sort_columns],
                 )
             }
             Some(TimeSeriesDistribution::TimeWindowed) => {
@@ -135,7 +133,7 @@ impl RegionScanExec {
                 }
                 EquivalenceProperties::new_with_orderings(
                     arrow_schema.clone(),
-                    &[LexOrdering::new(pk_sort_columns)],
+                    vec![pk_sort_columns],
                 )
             }
             None => EquivalenceProperties::new(arrow_schema.clone()),
@@ -332,7 +330,11 @@ impl ExecutionPlan for RegionScanExec {
         Some(self.metric.clone_inner())
     }
 
-    fn statistics(&self) -> DfResult<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> DfResult<Statistics> {
+        if partition.is_some() {
+            return Ok(Statistics::new_unknown(self.schema().as_ref()));
+        }
+
         let statistics = if self.append_mode && !self.scanner.lock().unwrap().has_predicate() {
             let column_statistics = self
                 .arrow_schema