Merge pull request #3143 from quickwit-oss/mallets/tiebreaker

feat: add generated tie-breaker fast fields

CHANGELOG.md

@@ -3,6 +3,10 @@ Tantivy 0.27.0
 
 ## Breaking change
 - `.set_fast(..)` now takes a &str. The same behavior as `.set_fast(None)` can be obtained with .set_fast(tantivy::tokenizer::RAW_TOKENIZER_NAME).
+- Added the `FieldType::TieBreaker` variant; exhaustive matches on `FieldType` need a new arm.
+
+## Features/Improvements
+- Add generated tie-breaker fast fields via `SchemaBuilder::add_tie_breaker_field`. Values are consecutive within a segment, start at a random offset, and are preserved across merges.
 
 
 Tantivy 0.26.2

columnar/Cargo.toml

@@ -18,11 +18,11 @@ common = { version= "0.11", path = "../common", package = "tantivy-common" }
 tantivy-bitpacker = { version= "0.10", path = "../bitpacker/" }
 serde = "1.0.152"
 downcast-rs = "2.0.1"
+rand = "0.9"
 
 [dev-dependencies]
 proptest = "1"
 more-asserts = "0.3.1"
-rand = "0.9"
 binggan = "0.17.0"
 
 [[bench]]

columnar/src/column/mod.rs

@@ -8,6 +8,7 @@ use std::sync::Arc;
 
 use common::BinarySerializable;
 pub use dictionary_encoded::{BytesColumn, StrColumn};
+pub(crate) use serialize::serialize_generated_tie_breaker_column;
 pub use serialize::{
     open_column_bytes, open_column_str, open_column_u64, open_column_u128,
     open_column_u128_as_compact_u64, serialize_column_mappable_to_u64,

columnar/src/column/serialize.rs

@@ -3,6 +3,7 @@ use std::io::Write;
 use std::sync::Arc;
 
 use common::OwnedBytes;
+use rand::Rng;
 use sstable::Dictionary;
 
 use crate::column::{BytesColumn, Column};
@@ -25,6 +26,29 @@ pub fn serialize_column_mappable_to_u128<T: MonotonicallyMappableToU128>(
     Ok(())
 }
 
+pub(crate) fn serialize_generated_tie_breaker_column(
+    num_docs: u32,
+    output: &mut impl Write,
+) -> io::Result<()> {
+    // TODO: Lift this temporary u32 limit once downstream consumers support the full u64 range.
+    let max_start = (u32::MAX - num_docs) as u64;
+    let start: u64 = rand::rng().random_range(0..=max_start);
+    let end = start + num_docs as u64;
+    let values = start..end;
+    let column_index_num_bytes = serialize_column_index(SerializableColumnIndex::Full, output)?;
+    serialize_u64_based_column_values(
+        &values,
+        &[
+            CodecType::Bitpacked,
+            CodecType::Linear,
+            CodecType::BlockwiseLinear,
+        ],
+        output,
+    )?;
+    output.write_all(&column_index_num_bytes.to_le_bytes())?;
+    Ok(())
+}
+
 pub fn serialize_column_mappable_to_u64<T: MonotonicallyMappableToU64>(
     column_index: SerializableColumnIndex<'_>,
     column_values: &impl Iterable<T>,

columnar/src/columnar/merge/tests.rs

@@ -70,6 +70,52 @@ fn test_group_columns_with_required_column() {
     assert!(column_map.contains_key(&("numbers".to_string(), ColumnTypeCategory::Numerical)));
 }
 
+#[test]
+fn test_merge_generated_tie_breaker_columns_stays_small() {
+    fn make_tie_breaker_columnar(num_docs: u32) -> ColumnarReader {
+        let mut writer = ColumnarWriter::default();
+        writer.record_tie_breaker_column("tie");
+        let mut buffer = Vec::new();
+        writer.serialize(num_docs, None, &mut buffer).unwrap();
+        assert!(
+            buffer.len() <= 128,
+            "input columnar is {} bytes",
+            buffer.len()
+        );
+        ColumnarReader::open(buffer).unwrap()
+    }
+
+    let columnars: Vec<ColumnarReader> =
+        (0..4).map(|_| make_tie_breaker_columnar(250_000)).collect();
+    let columnar_refs: Vec<&ColumnarReader> = columnars.iter().collect();
+    let mut buffer = Vec::new();
+    merge_columnar(
+        &columnar_refs,
+        &[("tie".to_string(), ColumnType::U64)],
+        StackMergeOrder::stack(&columnar_refs).into(),
+        &mut buffer,
+    )
+    .unwrap();
+
+    assert!(
+        buffer.len() <= 45_000,
+        "merged columnar is {} bytes",
+        buffer.len()
+    );
+    let merged = ColumnarReader::open(buffer).unwrap();
+    let column = merged.read_columns("tie").unwrap()[0]
+        .open_u64_lenient()
+        .unwrap()
+        .unwrap();
+    assert_eq!(merged.num_docs(), 1_000_000);
+    assert_eq!(column.get_cardinality(), Cardinality::Full);
+    for split_start in (0..1_000_000).step_by(250_000) {
+        for doc in split_start + 1..split_start + 250_000 {
+            assert_eq!(column.first(doc), Some(column.first(doc - 1).unwrap() + 1));
+        }
+    }
+}
+
 #[test]
 fn test_group_columns_required_column_with_no_existing_columns() {
     let columnar1 = make_columnar("numbers", &[2u64]);

columnar/src/columnar/reader/mod.rs

@@ -236,6 +236,22 @@ mod tests {
         assert_eq!(columns[1].1.column_type(), ColumnType::U64);
     }
 
+    #[test]
+    fn test_generated_tie_breaker_column() {
+        let mut columnar_writer = ColumnarWriter::default();
+        columnar_writer.record_tie_breaker_column("tie");
+        let mut buffer = Vec::new();
+        columnar_writer.serialize(1_025, None, &mut buffer).unwrap();
+
+        let columnar = ColumnarReader::open(buffer).unwrap();
+        let handles = columnar.read_columns("tie").unwrap();
+        let column = handles[0].open_u64_lenient().unwrap().unwrap();
+        assert_eq!(column.index.get_cardinality(), crate::Cardinality::Full);
+        for doc in 1..1_025 {
+            assert_eq!(column.first(doc), Some(column.first(doc - 1).unwrap() + 1));
+        }
+    }
+
     #[test]
     fn test_list_columns_strict_typing_prevents_coercion() {
         let mut columnar_writer = ColumnarWriter::default();

columnar/src/columnar/writer/mod.rs

@@ -3,6 +3,7 @@ mod column_writers;
 mod serializer;
 mod value_index;
 
+use std::collections::HashSet;
 use std::io;
 use std::net::Ipv6Addr;
 
@@ -54,6 +55,7 @@ pub struct ColumnarWriter {
     ip_addr_field_hash_map: ArenaHashMap,
     bytes_field_hash_map: ArenaHashMap,
     str_field_hash_map: ArenaHashMap,
+    generated_tie_breaker_columns: HashSet<Vec<u8>>,
     arena: MemoryArena,
     // Dictionaries used to store dictionary-encoded values.
     dictionaries: Vec<DictionaryBuilder>,
@@ -217,6 +219,13 @@ impl ColumnarWriter {
         }
     }
 
+    /// Registers a full `u64` column whose values are generated when this writer is serialized.
+    pub fn record_tie_breaker_column(&mut self, column_name: &str) {
+        self.record_column_type(column_name, ColumnType::U64, false);
+        self.generated_tie_breaker_columns
+            .insert(column_name.as_bytes().to_vec());
+    }
+
     pub fn record_numerical<T: Into<NumericalValue> + Copy>(
         &mut self,
         doc: RowId,
@@ -436,24 +445,31 @@ impl ColumnarWriter {
                     column_serializer.finalize()?;
                 }
                 ColumnType::F64 | ColumnType::I64 | ColumnType::U64 => {
-                    let numerical_column_writer: NumericalColumnWriter =
-                        self.numerical_field_hash_map.read(addr);
-                    let cardinality = numerical_column_writer.cardinality(num_docs);
                     let mut column_serializer =
                         serializer.start_serialize_column(column_name, column_type);
-                    let numerical_type = column_type.numerical_type().unwrap();
-                    serialize_numerical_column(
-                        cardinality,
-                        num_docs,
-                        numerical_type,
-                        numerical_column_writer.operation_iterator(
-                            arena,
-                            old_to_new_row_ids,
-                            &mut symbol_byte_buffer,
-                        ),
-                        buffers,
-                        &mut column_serializer,
-                    )?;
+                    if self.generated_tie_breaker_columns.contains(column_name) {
+                        crate::column::serialize_generated_tie_breaker_column(
+                            num_docs,
+                            &mut column_serializer,
+                        )?;
+                    } else {
+                        let numerical_column_writer: NumericalColumnWriter =
+                            self.numerical_field_hash_map.read(addr);
+                        let cardinality = numerical_column_writer.cardinality(num_docs);
+                        let numerical_type = column_type.numerical_type().unwrap();
+                        serialize_numerical_column(
+                            cardinality,
+                            num_docs,
+                            numerical_type,
+                            numerical_column_writer.operation_iterator(
+                                arena,
+                                old_to_new_row_ids,
+                                &mut symbol_byte_buffer,
+                            ),
+                            buffers,
+                            &mut column_serializer,
+                        )?;
+                    }
                     column_serializer.finalize()?;
                 }
                 ColumnType::DateTime => {

src/fastfield/mod.rs

@@ -93,8 +93,8 @@ mod tests {
     use crate::index::SegmentId;
     use crate::merge_policy::NoMergePolicy;
     use crate::schema::{
-        DateOptions, Facet, FacetOptions, Field, JsonObjectOptions, Schema, SchemaBuilder,
-        TantivyDocument, TextOptions, FAST, INDEXED, STORED, STRING, TEXT,
+        DateOptions, Facet, FacetOptions, Field, FieldType, JsonObjectOptions, Schema,
+        SchemaBuilder, TantivyDocument, TextOptions, FAST, INDEXED, STORED, STRING, TEXT,
     };
     use crate::time::OffsetDateTime;
     use crate::tokenizer::{
@@ -148,6 +148,37 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_generated_tie_breaker_fast_field() {
+        let mut schema_builder = Schema::builder();
+        let tie = schema_builder.add_tie_breaker_field("tie");
+        let schema = schema_builder.build();
+        let entry = schema.get_field_entry(tie);
+        assert!(matches!(entry.field_type(), FieldType::TieBreaker));
+        assert!(entry.is_fast());
+        assert!(!entry.is_indexed());
+        assert!(!entry.is_stored());
+        let schema_json = serde_json::to_string(&schema).unwrap();
+        assert!(schema_json.contains(r#""type":"tie_breaker""#));
+        assert_eq!(
+            serde_json::from_str::<Schema>(&schema_json).unwrap(),
+            schema
+        );
+
+        let mut writer = FastFieldsWriter::from_schema(&schema).unwrap();
+        for _ in 0..1_025 {
+            writer.add_document(&TantivyDocument::default()).unwrap();
+        }
+        let mut bytes = Vec::new();
+        writer.serialize(&mut bytes, None).unwrap();
+
+        let readers = FastFieldReaders::open(bytes.into(), schema).unwrap();
+        let values = readers.u64("tie").unwrap().first_or_default_col(0);
+        for doc in 1..1_025 {
+            assert_eq!(values.get_val(doc), values.get_val(doc - 1) + 1);
+        }
+    }
+
     #[test]
     fn test_intfastfield_large() {
         let path = Path::new("test");

src/fastfield/writer.rs

@@ -52,6 +52,10 @@ impl FastFieldsWriter {
             if !field_entry.field_type().is_fast() {
                 continue;
             }
+            if matches!(field_entry.field_type(), FieldType::TieBreaker) {
+                columnar_writer.record_tie_breaker_column(field_entry.name());
+                continue;
+            }
             fast_field_names[field_id.field_id() as usize] = Some(field_entry.name().to_string());
             let value_type = field_entry.field_type().value_type();
             if let FieldType::Date(date_options) = field_entry.field_type() {

src/index/index.rs

@@ -320,6 +320,12 @@ impl IndexBuilder {
                     ))
                 })?;
                 let entry = schema.get_field_entry(schema_field);
+                if matches!(entry.field_type(), FieldType::TieBreaker) {
+                    return Err(TantivyError::InvalidArgument(format!(
+                        "Field {} is a tie-breaker field and cannot be used to sort an index",
+                        sort_by_field.field
+                    )));
+                }
                 if !entry.is_fast() {
                     return Err(TantivyError::InvalidArgument(format!(
                         "Field {} is no fast field. Field needs to be a single value fast field \

src/index/inverted_index_plugin.rs

@@ -391,9 +391,9 @@ impl InvertedIndexPluginWriter {
                         self.fieldnorms_writer.record(doc_id, field, num_vals);
                     }
                 }
-                // Custom fields are not indexed; the `is_indexed()` guard above skips them.
-                FieldType::Custom(_) => {
-                    unreachable!("the inverted index does not support custom field types")
+                // These fields are not indexed; the `is_indexed()` guard above skips them.
+                FieldType::TieBreaker | FieldType::Custom(_) => {
+                    unreachable!("the inverted index does not support this field type")
                 }
             }
         }

src/indexer/doc_id_mapping.rs

@@ -721,6 +721,29 @@ mod tests_indexsorting {
         assert!(matches!(error, TantivyError::InvalidArgument(_)));
     }
 
+    #[test]
+    fn test_index_builder_rejects_tie_breaker_sort_by_field() {
+        for order in [Order::Asc, Order::Desc] {
+            let mut schema_builder = Schema::builder();
+            schema_builder.add_tie_breaker_field("tie");
+            let schema = schema_builder.build();
+            let settings = IndexSettings {
+                sort_by_field: Some(IndexSortByField {
+                    field: "tie".to_string(),
+                    order,
+                }),
+                ..Default::default()
+            };
+
+            let error = Index::builder()
+                .schema(schema)
+                .settings(settings)
+                .create_in_ram()
+                .unwrap_err();
+            assert!(matches!(error, TantivyError::InvalidArgument(_)));
+        }
+    }
+
     #[test]
     fn test_doc_mapping() {
         let doc_mapping = DocIdMapping::from_new_id_to_old_id(vec![3, 2, 5]);

src/postings/per_field_postings_writer.rs

@@ -69,8 +69,10 @@ fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> Box<dyn Postings
                 JsonPostingsWriter::<DocIdRecorder>::default().into()
             }
         }
-        // Custom fields are never indexed, so this writer is never fed terms. It only needs to
+        // These fields are never indexed, so this writer is never fed terms. It only needs to
         // occupy the per-field slot (the vector is indexed by field id).
-        FieldType::Custom(_) => Box::<SpecializedPostingsWriter<DocIdRecorder>>::default(),
+        FieldType::TieBreaker | FieldType::Custom(_) => {
+            Box::<SpecializedPostingsWriter<DocIdRecorder>>::default()
+        }
     }
 }

src/query/query_parser/query_parser.rs

@@ -453,7 +453,7 @@ impl QueryParser {
             )));
         }
         match *field_type {
-            FieldType::U64(_) => {
+            FieldType::U64(_) | FieldType::TieBreaker => {
                 let val: u64 = u64::from_str(phrase)?;
                 Ok(Term::from_field_u64(field, val))
             }
@@ -635,10 +635,10 @@ impl QueryParser {
                 let term = Term::from_field_ip_addr(field, ip_v6);
                 Ok(vec![LogicalLiteral::Term(term)])
             }
-            // Custom fields are not indexed, so the `is_indexed()` guard above returns
+            // These fields are not indexed, so the `is_indexed()` guard above returns
             // `FieldNotIndexed` before this match.
-            FieldType::Custom(_) => {
-                unreachable!("the query parser does not support custom field types")
+            FieldType::TieBreaker | FieldType::Custom(_) => {
+                unreachable!("the query parser does not support this field type")
             }
         }
     }

src/schema/document/default_document.rs

@@ -12,7 +12,7 @@ use crate::schema::document::{
     DeserializeError, Document, DocumentDeserialize, DocumentDeserializer,
 };
 use crate::schema::field_type::ValueParsingError;
-use crate::schema::{Facet, Field, NamedFieldDocument, OwnedValue, Schema};
+use crate::schema::{Facet, Field, FieldType, NamedFieldDocument, OwnedValue, Schema};
 use crate::tokenizer::PreTokenizedString;
 
 #[repr(C, packed)]
@@ -219,6 +219,9 @@ impl CompactDoc {
             if let Ok(field) = schema.get_field(&field_name) {
                 let field_entry = schema.get_field_entry(field);
                 let field_type = field_entry.field_type();
+                if matches!(field_type, FieldType::TieBreaker) {
+                    continue;
+                }
                 match json_value {
                     serde_json::Value::Array(json_items) => {
                         for json_item in json_items {
@@ -752,6 +755,19 @@ mod tests {
         let _json = doc.to_named_doc(&schema);
     }
 
+    #[test]
+    fn test_parse_json_ignores_tie_breaker_values() {
+        let mut schema_builder = Schema::builder();
+        let title = schema_builder.add_text_field("title", TEXT);
+        let tie = schema_builder.add_tie_breaker_field("tie");
+        let schema = schema_builder.build();
+        let doc =
+            TantivyDocument::parse_json(&schema, r#"{"title": "hello", "tie": [1, "two", null]}"#)
+                .unwrap();
+        assert!(doc.get_first(title).is_some());
+        assert!(doc.get_first(tie).is_none());
+    }
+
     #[test]
     fn test_json_value() {
         let json_str = r#"{

src/schema/field_entry.rs

@@ -41,6 +41,11 @@ impl FieldEntry {
         Self::new(field_name, FieldType::U64(int_options))
     }
 
+    /// Creates a generated tie-breaker field entry.
+    pub fn new_tie_breaker(field_name: String) -> FieldEntry {
+        Self::new(field_name, FieldType::TieBreaker)
+    }
+
     /// Creates a new i64 field entry.
     pub fn new_i64(field_name: String, int_options: NumericOptions) -> FieldEntry {
         Self::new(field_name, FieldType::I64(int_options))
@@ -135,7 +140,7 @@ impl FieldEntry {
             FieldType::Bytes(ref options) => options.is_stored(),
             FieldType::JsonObject(ref options) => options.is_stored(),
             FieldType::IpAddr(ref options) => options.is_stored(),
-            FieldType::Custom(_) => false,
+            FieldType::TieBreaker | FieldType::Custom(_) => false,
         }
     }
 }

src/schema/field_type.rs

@@ -190,6 +190,10 @@ pub enum FieldType {
     Str(TextOptions),
     /// Unsigned 64-bits integers field type configuration
     U64(NumericOptions),
+    /// Generated tie-breaker field, exposed as a `u64` fast field.
+    ///
+    /// Values are almost always distinct, but not guaranteed to be unique across segments.
+    TieBreaker,
     /// Signed 64-bits integers 64 field type configuration
     I64(NumericOptions),
     /// 64-bits float 64 field type configuration
@@ -217,7 +221,7 @@ impl FieldType {
     pub fn value_type(&self) -> Type {
         match *self {
             FieldType::Str(_) => Type::Str,
-            FieldType::U64(_) => Type::U64,
+            FieldType::U64(_) | FieldType::TieBreaker => Type::U64,
             FieldType::I64(_) => Type::I64,
             FieldType::F64(_) => Type::F64,
             FieldType::Bool(_) => Type::Bool,
@@ -273,7 +277,7 @@ impl FieldType {
             FieldType::Bytes(ref bytes_options) => bytes_options.is_indexed(),
             FieldType::JsonObject(ref json_object_options) => json_object_options.is_indexed(),
             FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.is_indexed(),
-            FieldType::Custom(_) => false,
+            FieldType::TieBreaker | FieldType::Custom(_) => false,
         }
     }
 
@@ -311,6 +315,7 @@ impl FieldType {
             FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.is_fast(),
             FieldType::Facet(_) => true,
             FieldType::JsonObject(ref json_object_options) => json_object_options.is_fast(),
+            FieldType::TieBreaker => true,
             FieldType::Custom(_) => false,
         }
     }
@@ -331,7 +336,7 @@ impl FieldType {
             FieldType::Bytes(ref bytes_options) => bytes_options.fieldnorms(),
             FieldType::JsonObject(ref _json_object_options) => false,
             FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.fieldnorms(),
-            FieldType::Custom(_) => false,
+            FieldType::TieBreaker | FieldType::Custom(_) => false,
         }
     }
 
@@ -383,7 +388,7 @@ impl FieldType {
                     None
                 }
             }
-            FieldType::Custom(_) => None,
+            FieldType::TieBreaker | FieldType::Custom(_) => None,
         }
     }
 
@@ -486,6 +491,10 @@ impl FieldType {
 
                     Ok(OwnedValue::IpAddr(ip_addr.into_ipv6_addr()))
                 }
+                FieldType::TieBreaker => Err(ValueParsingError::TypeError {
+                    expected: "a generated tie-breaker field",
+                    json: JsonValue::String(field_text),
+                }),
                 FieldType::Custom(_) => Err(custom_not_json_error(JsonValue::String(field_text))),
             },
             JsonValue::Number(field_val_num) => match self {
@@ -545,6 +554,10 @@ impl FieldType {
                     expected: "a string with an ip addr",
                     json: JsonValue::Number(field_val_num),
                 }),
+                FieldType::TieBreaker => Err(ValueParsingError::TypeError {
+                    expected: "a generated tie-breaker field",
+                    json: JsonValue::Number(field_val_num),
+                }),
                 FieldType::Custom(_) => {
                     Err(custom_not_json_error(JsonValue::Number(field_val_num)))
                 }

src/schema/schema.rs

@@ -57,6 +57,21 @@ impl SchemaBuilder {
         self.add_field(field_entry)
     }
 
+    /// Adds a generated tie-breaker fast field.
+    ///
+    /// The field is exposed as a `u64` fast field. Values are consecutive within a segment,
+    /// starting at a random offset. Ranges of different segments may overlap, so values are
+    /// almost always distinct but not guaranteed to be unique.
+    ///
+    /// Values supplied by documents for this field are ignored.
+    ///
+    /// # Panics
+    ///
+    /// Panics when field already exists.
+    pub fn add_tie_breaker_field(&mut self, field_name_str: &str) -> Field {
+        self.add_field(FieldEntry::new_tie_breaker(field_name_str.to_string()))
+    }
+
     /// Adds a new i64 field.
     /// Returns the associated field handle
     ///