feat!: upgrade DataFusion to 55 (#8555)

* feat!: upgrade DataFusion dependencies to 55 Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * refactor: migrate DataFusion 55 APIs Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: preserve table function planning behavior Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: preserve PostgreSQL query compatibility Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: preserve distributed execution plan behavior Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: cover DataFusion 55 behavior regressions Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: update DataFusion 55 SQLness expectations Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: complete DataFusion 55 test API migration Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: address DataFusion 55 CI regressions Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: address remaining DataFusion 55 regressions Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: adapt latest base code to DataFusion 55 Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: normalize environment-specific DataFusion 55 plans Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: align final DataFusion 55 expectations Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: isolate DataFusion 55 regression cases Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: preserve empty result schema in timestamp widening Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: preserve JSON source column order Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * chore: use released DataFusion 55 integrations Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: adapt latest execution plan mock to DataFusion 55 Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: pin DataFusion recursive schema and date repairs Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix(promql): align dictionary temporality match keys Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: retain Greptime DataFusion fork behaviors on version 55 Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: restore ordinary function error expectations Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: refresh distributed count compatibility plan Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix(query): adapt last-row cast hint to DataFusion 55 Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: refresh instant last-row empty results for Arrow 59 Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * style: simplify DataFusion expression visitor imports Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: restore sorting and PostgreSQL column-order assertions Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix(function): restore primitive numeric coercion signatures Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * refactor(function): share geo integer signature types Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: cover timestamp widening overflow boundaries Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: fix decimal coercion regression imports Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix(function): preserve scalar count_hash NULL state semantics Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: simplify decimal clamp case type inference Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: retain historical count_hash wrapper result Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: restore timestamp widening equality and IN pruning Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: carry upstream aggregate dynamic filter correctness fix Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: carry upstream null and predicate simplification fixes Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: restore baseline JSON ordering expectations Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: restore histogram JSON ordering expectations Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: refresh empty PromQL range result schemas Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: align native timestamp plan with DF55 decimal display Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: refresh native timestamp SQLness results for DF55 Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: regenerate NULL sample empty result headers for DF55 Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: use DF55 child replacement API in timestamp regressions Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: expose pushed scan dynamic filters to DF55 producers Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: encode string-backed PostgreSQL OID aliases in binary results Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: verify REGPROC binary and text over PostgreSQL protocol Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: register real PostgreSQL catalogs in server fixtures Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: complete DF55 expression inventories for custom query plans Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: correct RangeSelect expression fixture and column identities Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * ci: wait for Kafka WAL helper deployment rollout Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * test: update custom storage empty result headers Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: require exact row counts in scan statistics Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> * fix: suppress deprecated partition_statistics warning in test Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> --------- Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> Co-authored-by: Ning Sun <sunng@protonmail.com>
2026-10-04 11:05:35 +00:00 · 2026-09-15 11:42:38 +00:00
parent c65e8d0962
commit 94d7e2c7fc
284 changed files with 5948 additions and 2687 deletions
@@ -107,7 +107,7 @@ mod tests {
            connect_by: vec![],
            select_token: AttachedToken::empty(),
            flavor: SelectFlavor::Standard,
-            optimizer_hint: None,
+            optimizer_hints: vec![],
        };

        let sp_query = Box::new(
@@ -30,8 +30,10 @@ impl ParserContext<'_> {
            .context(error::SyntaxSnafu)?;

        match spstatement {
-            SpStatement::Insert { .. } => {
-                Ok(Statement::Insert(Box::new(Insert { inner: spstatement })))
+            insert_stmt @ SpStatement::Insert { .. } => {
+                let insert = Insert::try_from(insert_stmt)
+                    .map_err(|e| error::InvalidSqlSnafu { msg: e.to_string() }.build())?;
+                Ok(Statement::Insert(Box::new(insert)))
            }
            unexp => error::UnsupportedSnafu {
                keyword: unexp.to_string(),
@@ -50,9 +52,9 @@ impl ParserContext<'_> {
        match spstatement {
            SpStatement::Insert(mut insert_stmt) => {
                insert_stmt.replace_into = true;
-                Ok(Statement::Insert(Box::new(Insert {
-                    inner: SpStatement::Insert(insert_stmt),
-                })))
+                let insert = Insert::try_from(SpStatement::Insert(insert_stmt))
+                    .map_err(|e| error::InvalidSqlSnafu { msg: e.to_string() }.build())?;
+                Ok(Statement::Insert(Box::new(insert)))
            }
            unexp => error::UnsupportedSnafu {
                keyword: unexp.to_string(),
@@ -24,8 +24,7 @@ use crate::statements::statement::Statement;
 /// SET variables statement parser implementation
 impl ParserContext<'_> {
    pub(crate) fn parse_set_variables(&mut self) -> Result<Statement> {
-        let _ = self.parser.next_token();
-        let spstatement = self.parser.parse_set().context(error::SyntaxSnafu)?;
+        let spstatement = self.parser.parse_statement().context(error::SyntaxSnafu)?;
        match spstatement {
            SpStatement::Set(set) => match set {
                Set::SingleAssignment {
@@ -144,4 +143,14 @@ mod tests {
        let sql = "SET STATEMENT_TIMEOUT TO 5000";
        assert_pg_parse_result(sql, "STATEMENT_TIMEOUT", expected_query_timeout_expr);
    }
+
+    #[test]
+    fn test_unsupported_set_variant_remains_rejected() {
+        let result = ParserContext::create_with_dialect(
+            "SET ROLE admin",
+            &GreptimeDbDialect {},
+            ParseOptions::default(),
+        );
+        assert!(result.is_err());
+    }
 }
@@ -22,10 +22,9 @@ use datafusion::execution::SessionStateBuilder;
 use datafusion::execution::context::SessionState;
 use datafusion::optimizer::simplify_expressions::ExprSimplifier;
 use datafusion_common::tree_node::{TreeNode, TreeNodeVisitor};
-use datafusion_common::{DFSchema, ScalarValue};
+use datafusion_common::{DFSchema, ScalarValue, TableReference};
 use datafusion_expr::simplify::SimplifyContext;
-use datafusion_expr::{AggregateUDF, Expr, ScalarUDF, TableSource, WindowUDF};
-use datafusion_sql::TableReference;
+use datafusion_expr::{AggregateUDF, Expr, HigherOrderUDF, ScalarUDF, TableSource, WindowUDF};
 use datafusion_sql::planner::{ContextProvider, SqlToRel};
 use datatypes::arrow::datatypes::DataType;
 use datatypes::schema::{
@@ -265,8 +264,10 @@ pub fn parser_expr_to_scalar_value_literal_at(

    // 2. simplify logical expr — use scheduled time if provided, else wall-clock
    let info = match scheduled_time {
-        Some(dt) => SimplifyContext::default().with_query_execution_start_time(Some(dt)),
-        None => SimplifyContext::default().with_current_time(),
+        Some(dt) => SimplifyContext::builder()
+            .with_query_execution_start_time(Some(dt))
+            .build(),
+        None => SimplifyContext::builder().with_current_time().build(),
    };
    let simplifier = ExprSimplifier::new(info);

@@ -316,6 +317,10 @@ impl ContextProvider for StubContextProvider {
        self.state.scalar_functions().get(name).cloned()
    }

+    fn get_higher_order_meta(&self, name: &str) -> Option<Arc<HigherOrderUDF>> {
+        self.state.higher_order_functions().get(name).cloned()
+    }
+
    fn get_aggregate_meta(&self, name: &str) -> Option<Arc<AggregateUDF>> {
        self.state.aggregate_functions().get(name).cloned()
    }
@@ -336,6 +341,14 @@ impl ContextProvider for StubContextProvider {
        self.state.scalar_functions().keys().cloned().collect()
    }

+    fn higher_order_function_names(&self) -> Vec<String> {
+        self.state
+            .higher_order_functions()
+            .keys()
+            .cloned()
+            .collect()
+    }
+
    fn udaf_names(&self) -> Vec<String> {
        self.state.aggregate_functions().keys().cloned().collect()
    }
@@ -553,7 +566,9 @@ SELECT * FROM tql_cte WHERE ts > 0
            ),
        ];

-        let info = SimplifyContext::default().with_query_execution_start_time(Some(now_time));
+        let info = SimplifyContext::builder()
+            .with_query_execution_start_time(Some(now_time))
+            .build();
        let simplifier = ExprSimplifier::new(info);
        for (expr, expected) in testcases {
            let expr_name = expr.schema_name().to_string();
@@ -129,6 +129,7 @@ impl ParserContext<'_> {
                                data_type: None,
                            })
                            .collect(),
+                        at: None,
                    },
                    query: body,
                    from: None,
@@ -14,8 +14,8 @@

 use serde::Serialize;
 use sqlparser::ast::{
-    Insert as SpInsert, ObjectName, Query, SetExpr, Statement, TableObject, UnaryOperator,
-    ValueWithSpan, Values,
+    Insert as SpInsert, ObjectName, ObjectNamePart, Parens, Query, SetExpr, Statement, TableObject,
+    UnaryOperator, ValueWithSpan, Values,
 };
 use sqlparser::parser::ParserError;
 use sqlparser_derive::{Visit, VisitMut};
@@ -57,7 +57,12 @@ impl Insert {

    pub fn columns(&self) -> Vec<&String> {
        match &self.inner {
-            Statement::Insert(insert) => insert.columns.iter().map(|ident| &ident.value).collect(),
+            Statement::Insert(insert) => insert
+                .columns
+                .iter()
+                .filter_map(single_part_column_ident)
+                .map(|ident| &ident.value)
+                .collect(),
            _ => unreachable!(),
        }
    }
@@ -137,7 +142,7 @@ impl Insert {
    }
 }

-fn sql_exprs_to_values(exprs: &[Vec<Expr>]) -> Result<Vec<Vec<Value>>> {
+fn sql_exprs_to_values(exprs: &[Parens<Vec<Expr>>]) -> Result<Vec<Vec<Value>>> {
    let mut values = Vec::with_capacity(exprs.len());
    for es in exprs.iter() {
        let mut vs = Vec::with_capacity(es.len());
@@ -188,16 +193,34 @@ fn sql_exprs_to_values(exprs: &[Vec<Expr>]) -> Result<Vec<Vec<Value>>> {
    Ok(values)
 }

+fn single_part_column_ident(name: &ObjectName) -> Option<&sqlparser::ast::Ident> {
+    let [ObjectNamePart::Identifier(ident)] = name.0.as_slice() else {
+        return None;
+    };
+    Some(ident)
+}
+
 impl TryFrom<Statement> for Insert {
    type Error = ParserError;

    fn try_from(value: Statement) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Statement::Insert { .. } => Ok(Insert { inner: value }),
-            unexp => Err(ParserError::ParserError(format!(
-                "Not expected to be {unexp}"
-            ))),
+        let Statement::Insert(insert) = &value else {
+            return Err(ParserError::ParserError(format!(
+                "Not expected to be {value}"
+            )));
+        };
+
+        if let Some(column) = insert
+            .columns
+            .iter()
+            .find(|column| single_part_column_ident(column).is_none())
+        {
+            return Err(ParserError::ParserError(format!(
+                "Expected a single-part insert column name, found {column}"
+            )));
        }
+
+        Ok(Insert { inner: value })
    }
 }

@@ -239,6 +262,32 @@ mod tests {
        }
    }

+    #[test]
+    fn test_insert_column_names_are_single_identifiers() {
+        let stmt = ParserContext::create_with_dialect(
+            "INSERT INTO my_table (host, \"value\") VALUES (1, 2)",
+            &GreptimeDbDialect {},
+            ParseOptions::default(),
+        )
+        .unwrap()
+        .remove(0);
+        let Statement::Insert(insert) = stmt else {
+            unreachable!()
+        };
+        assert_eq!(insert.columns(), vec!["host", "value"]);
+
+        let result = ParserContext::create_with_dialect(
+            "INSERT INTO my_table (metric.host) VALUES (1)",
+            &GreptimeDbDialect {},
+            ParseOptions::default(),
+        );
+        let error = result.unwrap_err().to_string();
+        assert!(
+            error.contains("Expected a single-part insert column name, found metric.host"),
+            "unexpected error: {error}"
+        );
+    }
+
    #[test]
    fn test_insert_value_with_default() {
        // insert "default"