From 224f16b60cda29ba0734a7af35ad5cc8f1595753 Mon Sep 17 00:00:00 2001 From: Luca Cominardi Date: Tue, 6 Oct 2026 17:08:22 +0200 Subject: [PATCH] Generate tie-breakers as i64 values --- columnar/src/column/serialize.rs | 24 ++++++++++++++++++++---- columnar/src/columnar/merge/tests.rs | 9 ++++----- columnar/src/columnar/reader/mod.rs | 9 +++++++-- columnar/src/columnar/writer/mod.rs | 4 ++-- src/fastfield/mod.rs | 4 +++- src/query/query_parser/query_parser.rs | 4 ++-- src/schema/field_type.rs | 6 +++--- src/schema/schema.rs | 6 +++--- 8 files changed, 44 insertions(+), 22 deletions(-) diff --git a/columnar/src/column/serialize.rs b/columnar/src/column/serialize.rs index f50605564..9edfedd4c 100644 --- a/columnar/src/column/serialize.rs +++ b/columnar/src/column/serialize.rs @@ -15,6 +15,10 @@ use crate::column_values::{ use crate::iterable::Iterable; use crate::{StrColumn, Version}; +fn shift_u32_to_i64(value: u32) -> i64 { + i64::from(value) + i64::from(i32::MIN) +} + pub fn serialize_column_mappable_to_u128( column_index: SerializableColumnIndex<'_>, iterable: &dyn Iterable, @@ -30,10 +34,10 @@ pub(crate) fn serialize_generated_tie_breaker_column( num_docs: u32, output: &mut impl Write, ) -> io::Result<()> { - // TODO: Lift this temporary u32 limit once downstream consumers support the full u64 range. - let max_start = (u32::MAX - num_docs) as u64; - let start: u64 = rand::rng().random_range(0..=max_start); - let end = start + num_docs as u64; + let max_start = u32::MAX - num_docs; + let unsigned_start: u32 = rand::rng().random_range(0..=max_start); + let start = shift_u32_to_i64(unsigned_start); + let end = start + i64::from(num_docs); let values = start..end; let column_index_num_bytes = serialize_column_index(SerializableColumnIndex::Full, output)?; serialize_u64_based_column_values( @@ -49,6 +53,18 @@ pub(crate) fn serialize_generated_tie_breaker_column( Ok(()) } +#[cfg(test)] +mod tests { + use super::shift_u32_to_i64; + + #[test] + fn test_shift_u32_to_i64_covers_i32_range() { + assert_eq!(shift_u32_to_i64(u32::MIN), i64::from(i32::MIN)); + assert_eq!(shift_u32_to_i64(1 << 31), 0); + assert_eq!(shift_u32_to_i64(u32::MAX), i64::from(i32::MAX)); + } +} + pub fn serialize_column_mappable_to_u64( column_index: SerializableColumnIndex<'_>, column_values: &impl Iterable, diff --git a/columnar/src/columnar/merge/tests.rs b/columnar/src/columnar/merge/tests.rs index 04c54fa3f..d1f4cc72e 100644 --- a/columnar/src/columnar/merge/tests.rs +++ b/columnar/src/columnar/merge/tests.rs @@ -91,7 +91,7 @@ fn test_merge_generated_tie_breaker_columns_stays_small() { let mut buffer = Vec::new(); merge_columnar( &columnar_refs, - &[("tie".to_string(), ColumnType::U64)], + &[("tie".to_string(), ColumnType::I64)], StackMergeOrder::stack(&columnar_refs).into(), &mut buffer, ) @@ -103,10 +103,9 @@ fn test_merge_generated_tie_breaker_columns_stays_small() { buffer.len() ); let merged = ColumnarReader::open(buffer).unwrap(); - let column = merged.read_columns("tie").unwrap()[0] - .open_u64_lenient() - .unwrap() - .unwrap(); + let DynamicColumn::I64(column) = merged.read_columns("tie").unwrap()[0].open().unwrap() else { + panic!("expected an i64 tie-breaker column"); + }; assert_eq!(merged.num_docs(), 1_000_000); assert_eq!(column.get_cardinality(), Cardinality::Full); for split_start in (0..1_000_000).step_by(250_000) { diff --git a/columnar/src/columnar/reader/mod.rs b/columnar/src/columnar/reader/mod.rs index 198f659de..6177cf4f3 100644 --- a/columnar/src/columnar/reader/mod.rs +++ b/columnar/src/columnar/reader/mod.rs @@ -218,7 +218,7 @@ impl ColumnarReader { mod tests { use common::json_path_writer::JSON_PATH_SEGMENT_SEP; - use crate::{ColumnType, ColumnarReader, ColumnarWriter}; + use crate::{ColumnType, ColumnarReader, ColumnarWriter, DynamicColumn}; #[test] fn test_list_columns() { @@ -245,8 +245,13 @@ mod tests { let columnar = ColumnarReader::open(buffer).unwrap(); let handles = columnar.read_columns("tie").unwrap(); - let column = handles[0].open_u64_lenient().unwrap().unwrap(); + assert_eq!(handles[0].column_type, ColumnType::I64); + let DynamicColumn::I64(column) = handles[0].open().unwrap() else { + panic!("expected an i64 tie-breaker column"); + }; assert_eq!(column.index.get_cardinality(), crate::Cardinality::Full); + assert!(column.first(0).unwrap() >= i64::from(i32::MIN)); + assert!(column.first(1_024).unwrap() <= i64::from(i32::MAX)); for doc in 1..1_025 { assert_eq!(column.first(doc), Some(column.first(doc - 1).unwrap() + 1)); } diff --git a/columnar/src/columnar/writer/mod.rs b/columnar/src/columnar/writer/mod.rs index 049a8a6f4..023f9785d 100644 --- a/columnar/src/columnar/writer/mod.rs +++ b/columnar/src/columnar/writer/mod.rs @@ -219,9 +219,9 @@ impl ColumnarWriter { } } - /// Registers a full `u64` column whose values are generated when this writer is serialized. + /// Registers a full `i64` column whose values are generated when this writer is serialized. pub fn record_tie_breaker_column(&mut self, column_name: &str) { - self.record_column_type(column_name, ColumnType::U64, false); + self.record_column_type(column_name, ColumnType::I64, false); self.generated_tie_breaker_columns .insert(column_name.as_bytes().to_vec()); } diff --git a/src/fastfield/mod.rs b/src/fastfield/mod.rs index 243f3571a..b8448afdd 100644 --- a/src/fastfield/mod.rs +++ b/src/fastfield/mod.rs @@ -173,7 +173,9 @@ mod tests { writer.serialize(&mut bytes, None).unwrap(); let readers = FastFieldReaders::open(bytes.into(), schema).unwrap(); - let values = readers.u64("tie").unwrap().first_or_default_col(0); + let values = readers.i64("tie").unwrap().first_or_default_col(0); + assert!(values.get_val(0) >= i64::from(i32::MIN)); + assert!(values.get_val(1_024) <= i64::from(i32::MAX)); for doc in 1..1_025 { assert_eq!(values.get_val(doc), values.get_val(doc - 1) + 1); } diff --git a/src/query/query_parser/query_parser.rs b/src/query/query_parser/query_parser.rs index 4d0185fdb..05ade3670 100644 --- a/src/query/query_parser/query_parser.rs +++ b/src/query/query_parser/query_parser.rs @@ -453,11 +453,11 @@ impl QueryParser { ))); } match *field_type { - FieldType::U64(_) | FieldType::TieBreaker => { + FieldType::U64(_) => { let val: u64 = u64::from_str(phrase)?; Ok(Term::from_field_u64(field, val)) } - FieldType::I64(_) => { + FieldType::TieBreaker | FieldType::I64(_) => { let val: i64 = i64::from_str(phrase)?; Ok(Term::from_field_i64(field, val)) } diff --git a/src/schema/field_type.rs b/src/schema/field_type.rs index f1297a3c2..0f500092d 100644 --- a/src/schema/field_type.rs +++ b/src/schema/field_type.rs @@ -190,7 +190,7 @@ pub enum FieldType { Str(TextOptions), /// Unsigned 64-bits integers field type configuration U64(NumericOptions), - /// Generated tie-breaker field, exposed as a `u64` fast field. + /// Generated tie-breaker field, exposed as an `i64` fast field. /// /// Values are almost always distinct, but not guaranteed to be unique across segments. TieBreaker, @@ -221,8 +221,8 @@ impl FieldType { pub fn value_type(&self) -> Type { match *self { FieldType::Str(_) => Type::Str, - FieldType::U64(_) | FieldType::TieBreaker => Type::U64, - FieldType::I64(_) => Type::I64, + FieldType::U64(_) => Type::U64, + FieldType::TieBreaker | FieldType::I64(_) => Type::I64, FieldType::F64(_) => Type::F64, FieldType::Bool(_) => Type::Bool, FieldType::Date(_) => Type::Date, diff --git a/src/schema/schema.rs b/src/schema/schema.rs index 745a1ec30..1b60d91ee 100644 --- a/src/schema/schema.rs +++ b/src/schema/schema.rs @@ -59,9 +59,9 @@ impl SchemaBuilder { /// Adds a generated tie-breaker fast field. /// - /// The field is exposed as a `u64` fast field. Values are consecutive within a segment, - /// starting at a random offset. Ranges of different segments may overlap, so values are - /// almost always distinct but not guaranteed to be unique. + /// The field is exposed as an `i64` fast field with values in the `i32` range. Values are + /// consecutive within a segment, starting at a random offset. Ranges of different segments + /// may overlap, so values are almost always distinct but not guaranteed to be unique. /// /// Values supplied by documents for this field are ignored. ///