fix(recordbatch): treat -0.0 and +0.0 as equal in SimpleFilterEvaluator (#10443)

* test(recordbatch): compare float zero predicates with DataFusion

`SimpleFilterEvaluator` filters SST rows exactly, so it has to agree
with the SQL predicate that DataFusion evaluates above the scan.
DataFusion compares -0.0 and +0.0 as equal. Evaluate comparisons
against both zeros, NaN and NULL in Float64 and Float32 columns, and
check the selected rows against DataFusion's filter.

On main this fails: `v = Float64(0)` selects only the +0.0 row, while
DataFusion selects both zeros.

Signed-off-by: dayan02dev <pawarudayan92@gmail.com>

* fix(recordbatch): compare -0.0 and +0.0 as equal in simple filters

DataFusion 55 normalizes -0.0 to +0.0 before Arrow's comparison
kernels, so SQL compares the two zeros as equal. SimpleFilterEvaluator
called the kernels directly, and for floats they compare bit patterns:
-0.0 != +0.0 and -0.0 < +0.0. Mito applies the evaluator to SST rows as
an exact filter, so after a flush `v = 0.0`, `v >= 0` and `v IN (0.0)`
dropped the rows holding -0.0, and `v = -0.0` dropped the rows holding
+0.0. Rows still in a memtable matched.

Normalize the literal when the predicate is built and the input before
it is evaluated, with the helpers DataFusion uses for the same purpose.

Signed-off-by: dayan02dev <pawarudayan92@gmail.com>

* test(sqlness): cover flushed float rows holding -0.0

Flush rows holding -0.0, 0.0 and 1.0, then filter with `= 0.0`,
`>= 0`, `= -0.0` and `IN (0.0)`. Both zeros have to match every one of
them, as they do while the rows are still in the memtable.

Signed-off-by: dayan02dev <pawarudayan92@gmail.com>

---------

Signed-off-by: dayan02dev <pawarudayan92@gmail.com>
Co-authored-by: dennis zhuang <killme2008@gmail.com>
This commit is contained in:
dayan02dev
2026-11-16 00:43:04 +00:00
committed by GitHub
co-authored by dennis zhuang
parent 1dbc0cd80f
commit dbd080b840
4 changed files with 185 additions and 4 deletions
+10 -3
View File
@@ -25,7 +25,7 @@ use std::sync::Arc;
use datafusion::error::Result as DfResult;
use datafusion::logical_expr::{Expr, Literal, Operator, utils};
use datafusion::physical_plan::PhysicalExpr;
use datafusion_common::arrow::array::{ArrayRef, Datum};
use datafusion_common::arrow::array::{ArrayRef, Datum, Scalar};
use datafusion_common::arrow::buffer::BooleanBuffer;
use datafusion_common::cast::{as_boolean_array, as_null_array};
use datafusion_common::{DataFusionError, ScalarValue, internal_err};
@@ -79,6 +79,7 @@ impl MetadataFilter {
/// Evaluates supported single-column predicates with precompiled regular expressions.
/// Compound predicates preserve SQL null semantics until conversion to a scan mask.
/// Float comparisons treat `-0.0` and `+0.0` as equal, as DataFusion does.
#[derive(Debug, Clone)]
pub struct SimpleFilterEvaluator {
column_name: String,
@@ -104,7 +105,9 @@ impl SimpleFilterEvaluator {
Some(Self {
column_name,
predicate: SimplePredicate::Comparison {
literal: value.to_scalar().ok()?,
literal: datafusion_common::utils::normalize_float_zero_scalar(value)
.to_scalar()
.ok()?,
op,
regex: None,
regex_negative: false,
@@ -224,11 +227,15 @@ impl SimpleFilterEvaluator {
let input = input
.to_scalar()
.with_context(|_| ToArrowScalarSnafu { v: input.clone() })?;
let input = Scalar::new(datafusion_common::utils::normalize_float_zero(
&input.into_inner(),
));
Ok(self.evaluate_datum(&input, 1)?.value(0))
}
pub fn evaluate_array(&self, input: &ArrayRef) -> Result<BooleanBuffer> {
self.evaluate_datum(input, input.len())
let input = datafusion_common::utils::normalize_float_zero(input);
self.evaluate_datum(&input, input.len())
}
pub fn evaluate_vector(&self, input: &VectorRef) -> Result<BooleanBuffer> {
+100 -1
View File
@@ -98,7 +98,9 @@ impl SimplePredicate {
};
let (regex, regex_negative) = Self::maybe_build_regex(op, literal).ok()?;
Some(Self::Comparison {
literal: literal.to_scalar().ok()?,
literal: datafusion_common::utils::normalize_float_zero_scalar(literal.clone())
.to_scalar()
.ok()?,
op,
regex,
regex_negative,
@@ -626,6 +628,103 @@ mod tests {
}
}
#[tokio::test]
async fn float_zero_predicates_match_datafusion() {
use datafusion::prelude::SessionContext;
use datatypes::arrow::array::{Float32Array, Float64Array, Int32Array};
let float64: ArrayRef = Arc::new(Float64Array::from(vec![
Some(-0.0),
Some(0.0),
Some(1.0),
Some(f64::NAN),
None,
]));
let float32: ArrayRef = Arc::new(Float32Array::from(vec![
Some(-0.0),
Some(0.0),
Some(1.0),
Some(f32::NAN),
None,
]));
for input in [float64, float32] {
let literal = |v: f64| {
if input.data_type() == &DataType::Float32 {
lit(v as f32)
} else {
lit(v)
}
};
let ctx = SessionContext::new();
let schema = Arc::new(Schema::new(vec![
Field::new("v", input.data_type().clone(), true),
Field::new("id", DataType::Int32, false),
]));
ctx.register_batch(
"floats",
RecordBatch::try_new(
schema,
vec![
input.clone(),
Arc::new(Int32Array::from_iter_values(0..input.len() as i32)),
],
)
.unwrap(),
)
.unwrap();
for expr in [
col("v").eq(literal(0.0)),
col("v").eq(literal(-0.0)),
col("v").not_eq(literal(-0.0)),
col("v").lt(literal(0.0)),
col("v").lt_eq(literal(-0.0)),
col("v").gt(literal(-0.0)),
col("v").gt_eq(literal(0.0)),
literal(0.0).lt_eq(col("v")),
col("v").eq(literal(0.0)).or(col("v").eq(literal(1.0))),
col("v").eq(literal(f64::NAN)),
] {
let expected = ctx
.table("floats")
.await
.unwrap()
.filter(expr.clone())
.unwrap()
.collect()
.await
.unwrap()
.iter()
.flat_map(|batch| {
batch
.column_by_name("id")
.unwrap()
.as_any()
.downcast_ref::<Int32Array>()
.unwrap()
.values()
.to_vec()
})
.collect::<Vec<_>>();
let evaluator = SimpleFilterEvaluator::try_new(&expr).unwrap();
let mask = evaluator.evaluate_array(&input).unwrap();
let selected = mask
.iter()
.enumerate()
.filter_map(|(i, matched)| matched.then_some(i as i32))
.collect::<Vec<_>>();
assert_eq!(selected, expected, "{expr}");
for (i, matched) in mask.iter().enumerate() {
let scalar = ScalarValue::try_from_array(&input, i).unwrap();
assert_eq!(
evaluator.evaluate_scalar(&scalar).unwrap(),
matched,
"{expr}, row={i}"
);
}
}
}
}
#[test]
fn supported_filter_op() {
// equal
@@ -67,3 +67,61 @@ drop table prune_field;
Affected Rows: 0
-- -0.0 equals 0.0, so flushed rows holding either zero must keep matching.
CREATE TABLE prune_float_zero (ts TIMESTAMP TIME INDEX, v DOUBLE);
Affected Rows: 0
insert into prune_float_zero values (1, -0.0), (2, 0.0), (3, 1.0);
Affected Rows: 3
admin flush_table('prune_float_zero');
+---------------------------------------+
| ADMIN flush_table('prune_float_zero') |
+---------------------------------------+
| 0 |
+---------------------------------------+
select ts from prune_float_zero where v = 0.0 order by ts;
+-------------------------+
| ts |
+-------------------------+
| 1970-01-01T00:00:00.001 |
| 1970-01-01T00:00:00.002 |
+-------------------------+
select ts from prune_float_zero where v >= 0 order by ts;
+-------------------------+
| ts |
+-------------------------+
| 1970-01-01T00:00:00.001 |
| 1970-01-01T00:00:00.002 |
| 1970-01-01T00:00:00.003 |
+-------------------------+
select ts from prune_float_zero where v = -0.0 order by ts;
+-------------------------+
| ts |
+-------------------------+
| 1970-01-01T00:00:00.001 |
| 1970-01-01T00:00:00.002 |
+-------------------------+
select ts from prune_float_zero where v in (0.0) order by ts;
+-------------------------+
| ts |
+-------------------------+
| 1970-01-01T00:00:00.001 |
| 1970-01-01T00:00:00.002 |
+-------------------------+
drop table prune_float_zero;
Affected Rows: 0
@@ -22,3 +22,20 @@ select * from prune_field;
select * from prune_field where a = 1 and b = 1;
drop table prune_field;
-- -0.0 equals 0.0, so flushed rows holding either zero must keep matching.
CREATE TABLE prune_float_zero (ts TIMESTAMP TIME INDEX, v DOUBLE);
insert into prune_float_zero values (1, -0.0), (2, 0.0), (3, 1.0);
admin flush_table('prune_float_zero');
select ts from prune_float_zero where v = 0.0 order by ts;
select ts from prune_float_zero where v >= 0 order by ts;
select ts from prune_float_zero where v = -0.0 order by ts;
select ts from prune_float_zero where v in (0.0) order by ts;
drop table prune_float_zero;