diff --git a/src/common/recordbatch/src/filter.rs b/src/common/recordbatch/src/filter.rs index 0893d2aa3ad..3d8d7da7051 100644 --- a/src/common/recordbatch/src/filter.rs +++ b/src/common/recordbatch/src/filter.rs @@ -25,7 +25,7 @@ use std::sync::Arc; use datafusion::error::Result as DfResult; use datafusion::logical_expr::{Expr, Literal, Operator, utils}; use datafusion::physical_plan::PhysicalExpr; -use datafusion_common::arrow::array::{ArrayRef, Datum}; +use datafusion_common::arrow::array::{ArrayRef, Datum, Scalar}; use datafusion_common::arrow::buffer::BooleanBuffer; use datafusion_common::cast::{as_boolean_array, as_null_array}; use datafusion_common::{DataFusionError, ScalarValue, internal_err}; @@ -79,6 +79,7 @@ impl MetadataFilter { /// Evaluates supported single-column predicates with precompiled regular expressions. /// Compound predicates preserve SQL null semantics until conversion to a scan mask. +/// Float comparisons treat `-0.0` and `+0.0` as equal, as DataFusion does. #[derive(Debug, Clone)] pub struct SimpleFilterEvaluator { column_name: String, @@ -104,7 +105,9 @@ impl SimpleFilterEvaluator { Some(Self { column_name, predicate: SimplePredicate::Comparison { - literal: value.to_scalar().ok()?, + literal: datafusion_common::utils::normalize_float_zero_scalar(value) + .to_scalar() + .ok()?, op, regex: None, regex_negative: false, @@ -224,11 +227,15 @@ impl SimpleFilterEvaluator { let input = input .to_scalar() .with_context(|_| ToArrowScalarSnafu { v: input.clone() })?; + let input = Scalar::new(datafusion_common::utils::normalize_float_zero( + &input.into_inner(), + )); Ok(self.evaluate_datum(&input, 1)?.value(0)) } pub fn evaluate_array(&self, input: &ArrayRef) -> Result { - self.evaluate_datum(input, input.len()) + let input = datafusion_common::utils::normalize_float_zero(input); + self.evaluate_datum(&input, input.len()) } pub fn evaluate_vector(&self, input: &VectorRef) -> Result { diff --git a/src/common/recordbatch/src/filter/predicate.rs b/src/common/recordbatch/src/filter/predicate.rs index e283633dc02..d9827004b53 100644 --- a/src/common/recordbatch/src/filter/predicate.rs +++ b/src/common/recordbatch/src/filter/predicate.rs @@ -98,7 +98,9 @@ impl SimplePredicate { }; let (regex, regex_negative) = Self::maybe_build_regex(op, literal).ok()?; Some(Self::Comparison { - literal: literal.to_scalar().ok()?, + literal: datafusion_common::utils::normalize_float_zero_scalar(literal.clone()) + .to_scalar() + .ok()?, op, regex, regex_negative, @@ -626,6 +628,103 @@ mod tests { } } + #[tokio::test] + async fn float_zero_predicates_match_datafusion() { + use datafusion::prelude::SessionContext; + use datatypes::arrow::array::{Float32Array, Float64Array, Int32Array}; + + let float64: ArrayRef = Arc::new(Float64Array::from(vec![ + Some(-0.0), + Some(0.0), + Some(1.0), + Some(f64::NAN), + None, + ])); + let float32: ArrayRef = Arc::new(Float32Array::from(vec![ + Some(-0.0), + Some(0.0), + Some(1.0), + Some(f32::NAN), + None, + ])); + for input in [float64, float32] { + let literal = |v: f64| { + if input.data_type() == &DataType::Float32 { + lit(v as f32) + } else { + lit(v) + } + }; + let ctx = SessionContext::new(); + let schema = Arc::new(Schema::new(vec![ + Field::new("v", input.data_type().clone(), true), + Field::new("id", DataType::Int32, false), + ])); + ctx.register_batch( + "floats", + RecordBatch::try_new( + schema, + vec![ + input.clone(), + Arc::new(Int32Array::from_iter_values(0..input.len() as i32)), + ], + ) + .unwrap(), + ) + .unwrap(); + for expr in [ + col("v").eq(literal(0.0)), + col("v").eq(literal(-0.0)), + col("v").not_eq(literal(-0.0)), + col("v").lt(literal(0.0)), + col("v").lt_eq(literal(-0.0)), + col("v").gt(literal(-0.0)), + col("v").gt_eq(literal(0.0)), + literal(0.0).lt_eq(col("v")), + col("v").eq(literal(0.0)).or(col("v").eq(literal(1.0))), + col("v").eq(literal(f64::NAN)), + ] { + let expected = ctx + .table("floats") + .await + .unwrap() + .filter(expr.clone()) + .unwrap() + .collect() + .await + .unwrap() + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect::>(); + let evaluator = SimpleFilterEvaluator::try_new(&expr).unwrap(); + let mask = evaluator.evaluate_array(&input).unwrap(); + let selected = mask + .iter() + .enumerate() + .filter_map(|(i, matched)| matched.then_some(i as i32)) + .collect::>(); + assert_eq!(selected, expected, "{expr}"); + for (i, matched) in mask.iter().enumerate() { + let scalar = ScalarValue::try_from_array(&input, i).unwrap(); + assert_eq!( + evaluator.evaluate_scalar(&scalar).unwrap(), + matched, + "{expr}, row={i}" + ); + } + } + } + } + #[test] fn supported_filter_op() { // equal diff --git a/tests/cases/standalone/common/select/prune_field.result b/tests/cases/standalone/common/select/prune_field.result index 6a19b67c21a..27aee9296ed 100644 --- a/tests/cases/standalone/common/select/prune_field.result +++ b/tests/cases/standalone/common/select/prune_field.result @@ -67,3 +67,61 @@ drop table prune_field; Affected Rows: 0 +-- -0.0 equals 0.0, so flushed rows holding either zero must keep matching. +CREATE TABLE prune_float_zero (ts TIMESTAMP TIME INDEX, v DOUBLE); + +Affected Rows: 0 + +insert into prune_float_zero values (1, -0.0), (2, 0.0), (3, 1.0); + +Affected Rows: 3 + +admin flush_table('prune_float_zero'); + ++---------------------------------------+ +| ADMIN flush_table('prune_float_zero') | ++---------------------------------------+ +| 0 | ++---------------------------------------+ + +select ts from prune_float_zero where v = 0.0 order by ts; + ++-------------------------+ +| ts | ++-------------------------+ +| 1970-01-01T00:00:00.001 | +| 1970-01-01T00:00:00.002 | ++-------------------------+ + +select ts from prune_float_zero where v >= 0 order by ts; + ++-------------------------+ +| ts | ++-------------------------+ +| 1970-01-01T00:00:00.001 | +| 1970-01-01T00:00:00.002 | +| 1970-01-01T00:00:00.003 | ++-------------------------+ + +select ts from prune_float_zero where v = -0.0 order by ts; + ++-------------------------+ +| ts | ++-------------------------+ +| 1970-01-01T00:00:00.001 | +| 1970-01-01T00:00:00.002 | ++-------------------------+ + +select ts from prune_float_zero where v in (0.0) order by ts; + ++-------------------------+ +| ts | ++-------------------------+ +| 1970-01-01T00:00:00.001 | +| 1970-01-01T00:00:00.002 | ++-------------------------+ + +drop table prune_float_zero; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/select/prune_field.sql b/tests/cases/standalone/common/select/prune_field.sql index a5cf255a0d6..03b4e5d89b4 100644 --- a/tests/cases/standalone/common/select/prune_field.sql +++ b/tests/cases/standalone/common/select/prune_field.sql @@ -22,3 +22,20 @@ select * from prune_field; select * from prune_field where a = 1 and b = 1; drop table prune_field; + +-- -0.0 equals 0.0, so flushed rows holding either zero must keep matching. +CREATE TABLE prune_float_zero (ts TIMESTAMP TIME INDEX, v DOUBLE); + +insert into prune_float_zero values (1, -0.0), (2, 0.0), (3, 1.0); + +admin flush_table('prune_float_zero'); + +select ts from prune_float_zero where v = 0.0 order by ts; + +select ts from prune_float_zero where v >= 0 order by ts; + +select ts from prune_float_zero where v = -0.0 order by ts; + +select ts from prune_float_zero where v in (0.0) order by ts; + +drop table prune_float_zero;