mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-18 00:34:11 +00:00
fix(recordbatch): treat -0.0 and +0.0 as equal in SimpleFilterEvaluator (#10443)
* test(recordbatch): compare float zero predicates with DataFusion `SimpleFilterEvaluator` filters SST rows exactly, so it has to agree with the SQL predicate that DataFusion evaluates above the scan. DataFusion compares -0.0 and +0.0 as equal. Evaluate comparisons against both zeros, NaN and NULL in Float64 and Float32 columns, and check the selected rows against DataFusion's filter. On main this fails: `v = Float64(0)` selects only the +0.0 row, while DataFusion selects both zeros. Signed-off-by: dayan02dev <pawarudayan92@gmail.com> * fix(recordbatch): compare -0.0 and +0.0 as equal in simple filters DataFusion 55 normalizes -0.0 to +0.0 before Arrow's comparison kernels, so SQL compares the two zeros as equal. SimpleFilterEvaluator called the kernels directly, and for floats they compare bit patterns: -0.0 != +0.0 and -0.0 < +0.0. Mito applies the evaluator to SST rows as an exact filter, so after a flush `v = 0.0`, `v >= 0` and `v IN (0.0)` dropped the rows holding -0.0, and `v = -0.0` dropped the rows holding +0.0. Rows still in a memtable matched. Normalize the literal when the predicate is built and the input before it is evaluated, with the helpers DataFusion uses for the same purpose. Signed-off-by: dayan02dev <pawarudayan92@gmail.com> * test(sqlness): cover flushed float rows holding -0.0 Flush rows holding -0.0, 0.0 and 1.0, then filter with `= 0.0`, `>= 0`, `= -0.0` and `IN (0.0)`. Both zeros have to match every one of them, as they do while the rows are still in the memtable. Signed-off-by: dayan02dev <pawarudayan92@gmail.com> --------- Signed-off-by: dayan02dev <pawarudayan92@gmail.com> Co-authored-by: dennis zhuang <killme2008@gmail.com>
This commit is contained in:
co-authored by
dennis zhuang
parent
1dbc0cd80f
commit
dbd080b840
@@ -25,7 +25,7 @@ use std::sync::Arc;
|
||||
use datafusion::error::Result as DfResult;
|
||||
use datafusion::logical_expr::{Expr, Literal, Operator, utils};
|
||||
use datafusion::physical_plan::PhysicalExpr;
|
||||
use datafusion_common::arrow::array::{ArrayRef, Datum};
|
||||
use datafusion_common::arrow::array::{ArrayRef, Datum, Scalar};
|
||||
use datafusion_common::arrow::buffer::BooleanBuffer;
|
||||
use datafusion_common::cast::{as_boolean_array, as_null_array};
|
||||
use datafusion_common::{DataFusionError, ScalarValue, internal_err};
|
||||
@@ -79,6 +79,7 @@ impl MetadataFilter {
|
||||
|
||||
/// Evaluates supported single-column predicates with precompiled regular expressions.
|
||||
/// Compound predicates preserve SQL null semantics until conversion to a scan mask.
|
||||
/// Float comparisons treat `-0.0` and `+0.0` as equal, as DataFusion does.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SimpleFilterEvaluator {
|
||||
column_name: String,
|
||||
@@ -104,7 +105,9 @@ impl SimpleFilterEvaluator {
|
||||
Some(Self {
|
||||
column_name,
|
||||
predicate: SimplePredicate::Comparison {
|
||||
literal: value.to_scalar().ok()?,
|
||||
literal: datafusion_common::utils::normalize_float_zero_scalar(value)
|
||||
.to_scalar()
|
||||
.ok()?,
|
||||
op,
|
||||
regex: None,
|
||||
regex_negative: false,
|
||||
@@ -224,11 +227,15 @@ impl SimpleFilterEvaluator {
|
||||
let input = input
|
||||
.to_scalar()
|
||||
.with_context(|_| ToArrowScalarSnafu { v: input.clone() })?;
|
||||
let input = Scalar::new(datafusion_common::utils::normalize_float_zero(
|
||||
&input.into_inner(),
|
||||
));
|
||||
Ok(self.evaluate_datum(&input, 1)?.value(0))
|
||||
}
|
||||
|
||||
pub fn evaluate_array(&self, input: &ArrayRef) -> Result<BooleanBuffer> {
|
||||
self.evaluate_datum(input, input.len())
|
||||
let input = datafusion_common::utils::normalize_float_zero(input);
|
||||
self.evaluate_datum(&input, input.len())
|
||||
}
|
||||
|
||||
pub fn evaluate_vector(&self, input: &VectorRef) -> Result<BooleanBuffer> {
|
||||
|
||||
@@ -98,7 +98,9 @@ impl SimplePredicate {
|
||||
};
|
||||
let (regex, regex_negative) = Self::maybe_build_regex(op, literal).ok()?;
|
||||
Some(Self::Comparison {
|
||||
literal: literal.to_scalar().ok()?,
|
||||
literal: datafusion_common::utils::normalize_float_zero_scalar(literal.clone())
|
||||
.to_scalar()
|
||||
.ok()?,
|
||||
op,
|
||||
regex,
|
||||
regex_negative,
|
||||
@@ -626,6 +628,103 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn float_zero_predicates_match_datafusion() {
|
||||
use datafusion::prelude::SessionContext;
|
||||
use datatypes::arrow::array::{Float32Array, Float64Array, Int32Array};
|
||||
|
||||
let float64: ArrayRef = Arc::new(Float64Array::from(vec![
|
||||
Some(-0.0),
|
||||
Some(0.0),
|
||||
Some(1.0),
|
||||
Some(f64::NAN),
|
||||
None,
|
||||
]));
|
||||
let float32: ArrayRef = Arc::new(Float32Array::from(vec![
|
||||
Some(-0.0),
|
||||
Some(0.0),
|
||||
Some(1.0),
|
||||
Some(f32::NAN),
|
||||
None,
|
||||
]));
|
||||
for input in [float64, float32] {
|
||||
let literal = |v: f64| {
|
||||
if input.data_type() == &DataType::Float32 {
|
||||
lit(v as f32)
|
||||
} else {
|
||||
lit(v)
|
||||
}
|
||||
};
|
||||
let ctx = SessionContext::new();
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("v", input.data_type().clone(), true),
|
||||
Field::new("id", DataType::Int32, false),
|
||||
]));
|
||||
ctx.register_batch(
|
||||
"floats",
|
||||
RecordBatch::try_new(
|
||||
schema,
|
||||
vec![
|
||||
input.clone(),
|
||||
Arc::new(Int32Array::from_iter_values(0..input.len() as i32)),
|
||||
],
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
for expr in [
|
||||
col("v").eq(literal(0.0)),
|
||||
col("v").eq(literal(-0.0)),
|
||||
col("v").not_eq(literal(-0.0)),
|
||||
col("v").lt(literal(0.0)),
|
||||
col("v").lt_eq(literal(-0.0)),
|
||||
col("v").gt(literal(-0.0)),
|
||||
col("v").gt_eq(literal(0.0)),
|
||||
literal(0.0).lt_eq(col("v")),
|
||||
col("v").eq(literal(0.0)).or(col("v").eq(literal(1.0))),
|
||||
col("v").eq(literal(f64::NAN)),
|
||||
] {
|
||||
let expected = ctx
|
||||
.table("floats")
|
||||
.await
|
||||
.unwrap()
|
||||
.filter(expr.clone())
|
||||
.unwrap()
|
||||
.collect()
|
||||
.await
|
||||
.unwrap()
|
||||
.iter()
|
||||
.flat_map(|batch| {
|
||||
batch
|
||||
.column_by_name("id")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<Int32Array>()
|
||||
.unwrap()
|
||||
.values()
|
||||
.to_vec()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let evaluator = SimpleFilterEvaluator::try_new(&expr).unwrap();
|
||||
let mask = evaluator.evaluate_array(&input).unwrap();
|
||||
let selected = mask
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, matched)| matched.then_some(i as i32))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(selected, expected, "{expr}");
|
||||
for (i, matched) in mask.iter().enumerate() {
|
||||
let scalar = ScalarValue::try_from_array(&input, i).unwrap();
|
||||
assert_eq!(
|
||||
evaluator.evaluate_scalar(&scalar).unwrap(),
|
||||
matched,
|
||||
"{expr}, row={i}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn supported_filter_op() {
|
||||
// equal
|
||||
|
||||
@@ -67,3 +67,61 @@ drop table prune_field;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
-- -0.0 equals 0.0, so flushed rows holding either zero must keep matching.
|
||||
CREATE TABLE prune_float_zero (ts TIMESTAMP TIME INDEX, v DOUBLE);
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
insert into prune_float_zero values (1, -0.0), (2, 0.0), (3, 1.0);
|
||||
|
||||
Affected Rows: 3
|
||||
|
||||
admin flush_table('prune_float_zero');
|
||||
|
||||
+---------------------------------------+
|
||||
| ADMIN flush_table('prune_float_zero') |
|
||||
+---------------------------------------+
|
||||
| 0 |
|
||||
+---------------------------------------+
|
||||
|
||||
select ts from prune_float_zero where v = 0.0 order by ts;
|
||||
|
||||
+-------------------------+
|
||||
| ts |
|
||||
+-------------------------+
|
||||
| 1970-01-01T00:00:00.001 |
|
||||
| 1970-01-01T00:00:00.002 |
|
||||
+-------------------------+
|
||||
|
||||
select ts from prune_float_zero where v >= 0 order by ts;
|
||||
|
||||
+-------------------------+
|
||||
| ts |
|
||||
+-------------------------+
|
||||
| 1970-01-01T00:00:00.001 |
|
||||
| 1970-01-01T00:00:00.002 |
|
||||
| 1970-01-01T00:00:00.003 |
|
||||
+-------------------------+
|
||||
|
||||
select ts from prune_float_zero where v = -0.0 order by ts;
|
||||
|
||||
+-------------------------+
|
||||
| ts |
|
||||
+-------------------------+
|
||||
| 1970-01-01T00:00:00.001 |
|
||||
| 1970-01-01T00:00:00.002 |
|
||||
+-------------------------+
|
||||
|
||||
select ts from prune_float_zero where v in (0.0) order by ts;
|
||||
|
||||
+-------------------------+
|
||||
| ts |
|
||||
+-------------------------+
|
||||
| 1970-01-01T00:00:00.001 |
|
||||
| 1970-01-01T00:00:00.002 |
|
||||
+-------------------------+
|
||||
|
||||
drop table prune_float_zero;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
|
||||
@@ -22,3 +22,20 @@ select * from prune_field;
|
||||
select * from prune_field where a = 1 and b = 1;
|
||||
|
||||
drop table prune_field;
|
||||
|
||||
-- -0.0 equals 0.0, so flushed rows holding either zero must keep matching.
|
||||
CREATE TABLE prune_float_zero (ts TIMESTAMP TIME INDEX, v DOUBLE);
|
||||
|
||||
insert into prune_float_zero values (1, -0.0), (2, 0.0), (3, 1.0);
|
||||
|
||||
admin flush_table('prune_float_zero');
|
||||
|
||||
select ts from prune_float_zero where v = 0.0 order by ts;
|
||||
|
||||
select ts from prune_float_zero where v >= 0 order by ts;
|
||||
|
||||
select ts from prune_float_zero where v = -0.0 order by ts;
|
||||
|
||||
select ts from prune_float_zero where v in (0.0) order by ts;
|
||||
|
||||
drop table prune_float_zero;
|
||||
|
||||
Reference in New Issue
Block a user