From 7fea2ab8d8d63cc1ce41b88feb0923f09e25c2bb Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Wed, 14 Oct 2026 12:40:04 +0200 Subject: [PATCH] columnar: read datetime columns with the right type in sort_order (#3185) The datetime hash map stores `ColumnWriter` values, but `sort_order` read them as `NumericalColumnWriter`, a different and larger type. `ArenaHashMap::get` reads raw bytes, so this read garbage and could read past the end of the value. It is reachable through index sorting on a date field. Read a `ColumnWriter` and iterate its `NumericalValue` operations, as `serialize` already does. Co-authored-by: Paul Masurel --- columnar/src/columnar/writer/mod.rs | 115 ++++++++++++++++------------ 1 file changed, 68 insertions(+), 47 deletions(-) diff --git a/columnar/src/columnar/writer/mod.rs b/columnar/src/columnar/writer/mod.rs index 023f9785d..522e1d0d1 100644 --- a/columnar/src/columnar/writer/mod.rs +++ b/columnar/src/columnar/writer/mod.rs @@ -86,62 +86,55 @@ impl ColumnarWriter { /// the lowest possible score. /// /// The sort applied is stable. + /// + /// The sort_field has to be a schema field (not a json path): we rely on the fact that + /// all values have the same numerical type in `collect_numerical_sort_order`. pub fn sort_order(&self, sort_field: &str, num_docs: RowId, reversed: bool) -> Vec { - let Some(numerical_col_writer) = self + let sort_field_bytes = sort_field.as_bytes(); + let mut symbols_buffer = Vec::new(); + if let Some(numerical_col_writer) = self .numerical_field_hash_map - .get::(sort_field.as_bytes()) - .or_else(|| { - self.datetime_field_hash_map - .get::(sort_field.as_bytes()) - }) - else { - let str_or_bytes_column_opt = self - .str_field_hash_map - .get::(sort_field.as_bytes()) - .or_else(|| { - self.bytes_field_hash_map - .get::(sort_field.as_bytes()) - }); - let Some(str_or_bytes_column) = str_or_bytes_column_opt else { - return Vec::new(); - }; - - let dictionary_builder = &self.dictionaries[str_or_bytes_column.dictionary_id as usize]; - let term_id_mapping = dictionary_builder.build_term_id_mapping(&self.arena); - let mut symbols_buffer = Vec::new(); - - return collect_sort_order_from_ops( - str_or_bytes_column.operation_iterator(&self.arena, None, &mut symbols_buffer), + .get::(sort_field_bytes) + { + return collect_numerical_sort_order( + numerical_col_writer.operation_iterator(&self.arena, None, &mut symbols_buffer), num_docs, reversed, - |uid| Some(term_id_mapping.to_ord(uid).0), - None, - |a, b| a.cmp(b), ); + } + // Datetime columns store a plain `ColumnWriter` (not a `NumericalColumnWriter`), with + // `NumericalValue::I64` values. + if let Some(datetime_col_writer) = self + .datetime_field_hash_map + .get::(sort_field_bytes) + { + return collect_numerical_sort_order( + datetime_col_writer.operation_iterator::( + &self.arena, + None, + &mut symbols_buffer, + ), + num_docs, + reversed, + ); + } + let str_or_bytes_column_opt = self + .str_field_hash_map + .get::(sort_field_bytes) + .or_else(|| { + self.bytes_field_hash_map + .get::(sort_field_bytes) + }); + let Some(str_or_bytes_column) = str_or_bytes_column_opt else { + return Vec::new(); }; - let mut symbols_buffer = Vec::new(); + let dictionary_builder = &self.dictionaries[str_or_bytes_column.dictionary_id as usize]; + let term_id_mapping = dictionary_builder.build_term_id_mapping(&self.arena); collect_sort_order_from_ops( - numerical_col_writer.operation_iterator(&self.arena, None, &mut symbols_buffer), + str_or_bytes_column.operation_iterator(&self.arena, None, &mut symbols_buffer), num_docs, reversed, - // MonotonicallyMappableToU64 converts each value to u64 in an - // order-preserving way (u64: identity, i64: XOR sign bit, f64: bit - // manipulation). Converting once per document lets the comparator be - // a simple u64 cmp instead of unwrapping the NumericalValue variant - // on every comparison. - // - // For f64, NaN maps to a deterministic u64 via raw bit manipulation, - // so it sorts to a consistent position. Sorting only requires total - // ordering, not IEEE 754 equality semantics where NaN != NaN. - |nv| { - Some(match nv { - NumericalValue::U64(v) => v.to_u64(), - NumericalValue::I64(v) => v.to_u64(), - NumericalValue::F64(v) => v.to_u64(), - }) - }, - // None for missing values. Option sorts None < Some(_), - // placing nulls before non-null values. + |uid| Some(term_id_mapping.to_ord(uid).0), None, |a, b| a.cmp(b), ) @@ -498,6 +491,34 @@ impl ColumnarWriter { } } +/// Returns the sort order of a numerical or datetime column, given its operations. +/// +/// # Hidden contract +/// +/// For this to make sense, all numerical value emitted have to be of the same type. +fn collect_numerical_sort_order( + ops: impl Iterator>, + num_docs: RowId, + reversed: bool, +) -> Vec { + collect_sort_order_from_ops( + ops, + num_docs, + reversed, + |numerical_value| { + Some(match numerical_value { + NumericalValue::U64(val_u64) => val_u64.to_u64(), + NumericalValue::I64(val_i64) => val_i64.to_u64(), + NumericalValue::F64(val_f64) => val_f64.to_u64(), + }) + }, + // None for missing values. Option sorts None < Some(_), + // placing nulls before non-null values. + None, + |a, b| a.cmp(b), + ) +} + /// Shared sorting pattern for both numeric and Str/Bytes sort fields. /// /// Iterates column operations, fills gaps for missing docs with `default_key`, converts each value