From dbd080b8403ce1de032e18d2379bbb1fede6b6f9 Mon Sep 17 00:00:00 2001 From: dayan02dev Date: Mon, 16 Nov 2026 00:43:04 +0000 Subject: [PATCH] fix(recordbatch): treat -0.0 and +0.0 as equal in SimpleFilterEvaluator (#10443) * test(recordbatch): compare float zero predicates with DataFusion `SimpleFilterEvaluator` filters SST rows exactly, so it has to agree with the SQL predicate that DataFusion evaluates above the scan. DataFusion compares -0.0 and +0.0 as equal. Evaluate comparisons against both zeros, NaN and NULL in Float64 and Float32 columns, and check the selected rows against DataFusion's filter. On main this fails: `v = Float64(0)` selects only the +0.0 row, while DataFusion selects both zeros. Signed-off-by: dayan02dev * fix(recordbatch): compare -0.0 and +0.0 as equal in simple filters DataFusion 55 normalizes -0.0 to +0.0 before Arrow's comparison kernels, so SQL compares the two zeros as equal. SimpleFilterEvaluator called the kernels directly, and for floats they compare bit patterns: -0.0 != +0.0 and -0.0 < +0.0. Mito applies the evaluator to SST rows as an exact filter, so after a flush `v = 0.0`, `v >= 0` and `v IN (0.0)` dropped the rows holding -0.0, and `v = -0.0` dropped the rows holding +0.0. Rows still in a memtable matched. Normalize the literal when the predicate is built and the input before it is evaluated, with the helpers DataFusion uses for the same purpose. Signed-off-by: dayan02dev * test(sqlness): cover flushed float rows holding -0.0 Flush rows holding -0.0, 0.0 and 1.0, then filter with `= 0.0`, `>= 0`, `= -0.0` and `IN (0.0)`. Both zeros have to match every one of them, as they do while the rows are still in the memtable. Signed-off-by: dayan02dev --------- Signed-off-by: dayan02dev Co-authored-by: dennis zhuang --- src/common/recordbatch/src/filter.rs | 13 ++- .../recordbatch/src/filter/predicate.rs | 101 +++++++++++++++++- .../common/select/prune_field.result | 58 ++++++++++ .../standalone/common/select/prune_field.sql | 17 +++ 4 files changed, 185 insertions(+), 4 deletions(-) diff --git a/src/common/recordbatch/src/filter.rs b/src/common/recordbatch/src/filter.rs index 0893d2aa3ad..3d8d7da7051 100644 --- a/src/common/recordbatch/src/filter.rs +++ b/src/common/recordbatch/src/filter.rs @@ -25,7 +25,7 @@ use std::sync::Arc; use datafusion::error::Result as DfResult; use datafusion::logical_expr::{Expr, Literal, Operator, utils}; use datafusion::physical_plan::PhysicalExpr; -use datafusion_common::arrow::array::{ArrayRef, Datum}; +use datafusion_common::arrow::array::{ArrayRef, Datum, Scalar}; use datafusion_common::arrow::buffer::BooleanBuffer; use datafusion_common::cast::{as_boolean_array, as_null_array}; use datafusion_common::{DataFusionError, ScalarValue, internal_err}; @@ -79,6 +79,7 @@ impl MetadataFilter { /// Evaluates supported single-column predicates with precompiled regular expressions. /// Compound predicates preserve SQL null semantics until conversion to a scan mask. +/// Float comparisons treat `-0.0` and `+0.0` as equal, as DataFusion does. #[derive(Debug, Clone)] pub struct SimpleFilterEvaluator { column_name: String, @@ -104,7 +105,9 @@ impl SimpleFilterEvaluator { Some(Self { column_name, predicate: SimplePredicate::Comparison { - literal: value.to_scalar().ok()?, + literal: datafusion_common::utils::normalize_float_zero_scalar(value) + .to_scalar() + .ok()?, op, regex: None, regex_negative: false, @@ -224,11 +227,15 @@ impl SimpleFilterEvaluator { let input = input .to_scalar() .with_context(|_| ToArrowScalarSnafu { v: input.clone() })?; + let input = Scalar::new(datafusion_common::utils::normalize_float_zero( + &input.into_inner(), + )); Ok(self.evaluate_datum(&input, 1)?.value(0)) } pub fn evaluate_array(&self, input: &ArrayRef) -> Result { - self.evaluate_datum(input, input.len()) + let input = datafusion_common::utils::normalize_float_zero(input); + self.evaluate_datum(&input, input.len()) } pub fn evaluate_vector(&self, input: &VectorRef) -> Result { diff --git a/src/common/recordbatch/src/filter/predicate.rs b/src/common/recordbatch/src/filter/predicate.rs index e283633dc02..d9827004b53 100644 --- a/src/common/recordbatch/src/filter/predicate.rs +++ b/src/common/recordbatch/src/filter/predicate.rs @@ -98,7 +98,9 @@ impl SimplePredicate { }; let (regex, regex_negative) = Self::maybe_build_regex(op, literal).ok()?; Some(Self::Comparison { - literal: literal.to_scalar().ok()?, + literal: datafusion_common::utils::normalize_float_zero_scalar(literal.clone()) + .to_scalar() + .ok()?, op, regex, regex_negative, @@ -626,6 +628,103 @@ mod tests { } } + #[tokio::test] + async fn float_zero_predicates_match_datafusion() { + use datafusion::prelude::SessionContext; + use datatypes::arrow::array::{Float32Array, Float64Array, Int32Array}; + + let float64: ArrayRef = Arc::new(Float64Array::from(vec![ + Some(-0.0), + Some(0.0), + Some(1.0), + Some(f64::NAN), + None, + ])); + let float32: ArrayRef = Arc::new(Float32Array::from(vec![ + Some(-0.0), + Some(0.0), + Some(1.0), + Some(f32::NAN), + None, + ])); + for input in [float64, float32] { + let literal = |v: f64| { + if input.data_type() == &DataType::Float32 { + lit(v as f32) + } else { + lit(v) + } + }; + let ctx = SessionContext::new(); + let schema = Arc::new(Schema::new(vec![ + Field::new("v", input.data_type().clone(), true), + Field::new("id", DataType::Int32, false), + ])); + ctx.register_batch( + "floats", + RecordBatch::try_new( + schema, + vec![ + input.clone(), + Arc::new(Int32Array::from_iter_values(0..input.len() as i32)), + ], + ) + .unwrap(), + ) + .unwrap(); + for expr in [ + col("v").eq(literal(0.0)), + col("v").eq(literal(-0.0)), + col("v").not_eq(literal(-0.0)), + col("v").lt(literal(0.0)), + col("v").lt_eq(literal(-0.0)), + col("v").gt(literal(-0.0)), + col("v").gt_eq(literal(0.0)), + literal(0.0).lt_eq(col("v")), + col("v").eq(literal(0.0)).or(col("v").eq(literal(1.0))), + col("v").eq(literal(f64::NAN)), + ] { + let expected = ctx + .table("floats") + .await + .unwrap() + .filter(expr.clone()) + .unwrap() + .collect() + .await + .unwrap() + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect::>(); + let evaluator = SimpleFilterEvaluator::try_new(&expr).unwrap(); + let mask = evaluator.evaluate_array(&input).unwrap(); + let selected = mask + .iter() + .enumerate() + .filter_map(|(i, matched)| matched.then_some(i as i32)) + .collect::>(); + assert_eq!(selected, expected, "{expr}"); + for (i, matched) in mask.iter().enumerate() { + let scalar = ScalarValue::try_from_array(&input, i).unwrap(); + assert_eq!( + evaluator.evaluate_scalar(&scalar).unwrap(), + matched, + "{expr}, row={i}" + ); + } + } + } + } + #[test] fn supported_filter_op() { // equal diff --git a/tests/cases/standalone/common/select/prune_field.result b/tests/cases/standalone/common/select/prune_field.result index 6a19b67c21a..27aee9296ed 100644 --- a/tests/cases/standalone/common/select/prune_field.result +++ b/tests/cases/standalone/common/select/prune_field.result @@ -67,3 +67,61 @@ drop table prune_field; Affected Rows: 0 +-- -0.0 equals 0.0, so flushed rows holding either zero must keep matching. +CREATE TABLE prune_float_zero (ts TIMESTAMP TIME INDEX, v DOUBLE); + +Affected Rows: 0 + +insert into prune_float_zero values (1, -0.0), (2, 0.0), (3, 1.0); + +Affected Rows: 3 + +admin flush_table('prune_float_zero'); + ++---------------------------------------+ +| ADMIN flush_table('prune_float_zero') | ++---------------------------------------+ +| 0 | ++---------------------------------------+ + +select ts from prune_float_zero where v = 0.0 order by ts; + ++-------------------------+ +| ts | ++-------------------------+ +| 1970-01-01T00:00:00.001 | +| 1970-01-01T00:00:00.002 | ++-------------------------+ + +select ts from prune_float_zero where v >= 0 order by ts; + ++-------------------------+ +| ts | ++-------------------------+ +| 1970-01-01T00:00:00.001 | +| 1970-01-01T00:00:00.002 | +| 1970-01-01T00:00:00.003 | ++-------------------------+ + +select ts from prune_float_zero where v = -0.0 order by ts; + ++-------------------------+ +| ts | ++-------------------------+ +| 1970-01-01T00:00:00.001 | +| 1970-01-01T00:00:00.002 | ++-------------------------+ + +select ts from prune_float_zero where v in (0.0) order by ts; + ++-------------------------+ +| ts | ++-------------------------+ +| 1970-01-01T00:00:00.001 | +| 1970-01-01T00:00:00.002 | ++-------------------------+ + +drop table prune_float_zero; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/select/prune_field.sql b/tests/cases/standalone/common/select/prune_field.sql index a5cf255a0d6..03b4e5d89b4 100644 --- a/tests/cases/standalone/common/select/prune_field.sql +++ b/tests/cases/standalone/common/select/prune_field.sql @@ -22,3 +22,20 @@ select * from prune_field; select * from prune_field where a = 1 and b = 1; drop table prune_field; + +-- -0.0 equals 0.0, so flushed rows holding either zero must keep matching. +CREATE TABLE prune_float_zero (ts TIMESTAMP TIME INDEX, v DOUBLE); + +insert into prune_float_zero values (1, -0.0), (2, 0.0), (3, 1.0); + +admin flush_table('prune_float_zero'); + +select ts from prune_float_zero where v = 0.0 order by ts; + +select ts from prune_float_zero where v >= 0 order by ts; + +select ts from prune_float_zero where v = -0.0 order by ts; + +select ts from prune_float_zero where v in (0.0) order by ts; + +drop table prune_float_zero;