From 679cea4ed5ce350eb0a3bed3c60147d02ef862fd Mon Sep 17 00:00:00 2001 From: Luca Cominardi Date: Tue, 6 Oct 2026 15:57:38 +0200 Subject: [PATCH] Cap generated tie-breaker values at i32::MAX Keep generated ranges inside signed 32-bit pagination cursors until downstream consumers can represent the full u64 domain. --- CHANGELOG.md | 2 +- columnar/src/column/serialize.rs | 5 +++-- columnar/src/columnar/reader/mod.rs | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1056470b4..54daab6c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Tantivy 0.27.0 - Added the `FieldType::TieBreaker` variant; exhaustive matches on `FieldType` need a new arm. ## Features/Improvements -- Add generated tie-breaker fast fields via `SchemaBuilder::add_tie_breaker_field`. Values are consecutive within a segment, start at a random offset, and are preserved across merges. +- Add generated tie-breaker fast fields via `SchemaBuilder::add_tie_breaker_field`. Values are consecutive within a segment, start at a random offset within `[0, i32::MAX]`, and are preserved across merges. Tantivy 0.26.2 diff --git a/columnar/src/column/serialize.rs b/columnar/src/column/serialize.rs index f50605564..cdc003c00 100644 --- a/columnar/src/column/serialize.rs +++ b/columnar/src/column/serialize.rs @@ -30,8 +30,9 @@ pub(crate) fn serialize_generated_tie_breaker_column( num_docs: u32, output: &mut impl Write, ) -> io::Result<()> { - // TODO: Lift this temporary u32 limit once downstream consumers support the full u64 range. - let max_start = (u32::MAX - num_docs) as u64; + // Temporary i32 cap: some downstream pagination cursors still store the tie-breaker as + // int32. Lift this once those consumers can represent the full u64 range. + let max_start = (i32::MAX as u64).saturating_sub(u64::from(num_docs.saturating_sub(1))); let start: u64 = rand::rng().random_range(0..=max_start); let end = start + num_docs as u64; let values = start..end; diff --git a/columnar/src/columnar/reader/mod.rs b/columnar/src/columnar/reader/mod.rs index 198f659de..6b4eb2e7a 100644 --- a/columnar/src/columnar/reader/mod.rs +++ b/columnar/src/columnar/reader/mod.rs @@ -247,6 +247,8 @@ mod tests { let handles = columnar.read_columns("tie").unwrap(); let column = handles[0].open_u64_lenient().unwrap().unwrap(); assert_eq!(column.index.get_cardinality(), crate::Cardinality::Full); + assert!(column.first(0).unwrap() <= i32::MAX as u64); + assert!(column.first(1_024).unwrap() <= i32::MAX as u64); for doc in 1..1_025 { assert_eq!(column.first(doc), Some(column.first(doc - 1).unwrap() + 1)); }