diff --git a/CHANGELOG.md b/CHANGELOG.md index 395dbc6b5..1056470b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ Tantivy 0.27.0 ## Breaking change - `.set_fast(..)` now takes a &str. The same behavior as `.set_fast(None)` can be obtained with .set_fast(tantivy::tokenizer::RAW_TOKENIZER_NAME). +- Added the `FieldType::TieBreaker` variant; exhaustive matches on `FieldType` need a new arm. + +## Features/Improvements +- Add generated tie-breaker fast fields via `SchemaBuilder::add_tie_breaker_field`. Values are consecutive within a segment, start at a random offset, and are preserved across merges. Tantivy 0.26.2 diff --git a/columnar/Cargo.toml b/columnar/Cargo.toml index a2e44978b..fb5e3912b 100644 --- a/columnar/Cargo.toml +++ b/columnar/Cargo.toml @@ -18,11 +18,11 @@ common = { version= "0.11", path = "../common", package = "tantivy-common" } tantivy-bitpacker = { version= "0.10", path = "../bitpacker/" } serde = "1.0.152" downcast-rs = "2.0.1" +rand = "0.9" [dev-dependencies] proptest = "1" more-asserts = "0.3.1" -rand = "0.9" binggan = "0.17.0" [[bench]] diff --git a/columnar/src/column/mod.rs b/columnar/src/column/mod.rs index 11c62bf5d..5bf7963cb 100644 --- a/columnar/src/column/mod.rs +++ b/columnar/src/column/mod.rs @@ -8,6 +8,7 @@ use std::sync::Arc; use common::BinarySerializable; pub use dictionary_encoded::{BytesColumn, StrColumn}; +pub(crate) use serialize::serialize_generated_tie_breaker_column; pub use serialize::{ open_column_bytes, open_column_str, open_column_u64, open_column_u128, open_column_u128_as_compact_u64, serialize_column_mappable_to_u64, diff --git a/columnar/src/column/serialize.rs b/columnar/src/column/serialize.rs index e2127933b..f50605564 100644 --- a/columnar/src/column/serialize.rs +++ b/columnar/src/column/serialize.rs @@ -3,6 +3,7 @@ use std::io::Write; use std::sync::Arc; use common::OwnedBytes; +use rand::Rng; use sstable::Dictionary; use crate::column::{BytesColumn, Column}; @@ -25,6 +26,29 @@ pub fn serialize_column_mappable_to_u128( Ok(()) } +pub(crate) fn serialize_generated_tie_breaker_column( + num_docs: u32, + output: &mut impl Write, +) -> io::Result<()> { + // TODO: Lift this temporary u32 limit once downstream consumers support the full u64 range. + let max_start = (u32::MAX - num_docs) as u64; + let start: u64 = rand::rng().random_range(0..=max_start); + let end = start + num_docs as u64; + let values = start..end; + let column_index_num_bytes = serialize_column_index(SerializableColumnIndex::Full, output)?; + serialize_u64_based_column_values( + &values, + &[ + CodecType::Bitpacked, + CodecType::Linear, + CodecType::BlockwiseLinear, + ], + output, + )?; + output.write_all(&column_index_num_bytes.to_le_bytes())?; + Ok(()) +} + pub fn serialize_column_mappable_to_u64( column_index: SerializableColumnIndex<'_>, column_values: &impl Iterable, diff --git a/columnar/src/columnar/merge/tests.rs b/columnar/src/columnar/merge/tests.rs index 8c812047a..04c54fa3f 100644 --- a/columnar/src/columnar/merge/tests.rs +++ b/columnar/src/columnar/merge/tests.rs @@ -70,6 +70,52 @@ fn test_group_columns_with_required_column() { assert!(column_map.contains_key(&("numbers".to_string(), ColumnTypeCategory::Numerical))); } +#[test] +fn test_merge_generated_tie_breaker_columns_stays_small() { + fn make_tie_breaker_columnar(num_docs: u32) -> ColumnarReader { + let mut writer = ColumnarWriter::default(); + writer.record_tie_breaker_column("tie"); + let mut buffer = Vec::new(); + writer.serialize(num_docs, None, &mut buffer).unwrap(); + assert!( + buffer.len() <= 128, + "input columnar is {} bytes", + buffer.len() + ); + ColumnarReader::open(buffer).unwrap() + } + + let columnars: Vec = + (0..4).map(|_| make_tie_breaker_columnar(250_000)).collect(); + let columnar_refs: Vec<&ColumnarReader> = columnars.iter().collect(); + let mut buffer = Vec::new(); + merge_columnar( + &columnar_refs, + &[("tie".to_string(), ColumnType::U64)], + StackMergeOrder::stack(&columnar_refs).into(), + &mut buffer, + ) + .unwrap(); + + assert!( + buffer.len() <= 45_000, + "merged columnar is {} bytes", + buffer.len() + ); + let merged = ColumnarReader::open(buffer).unwrap(); + let column = merged.read_columns("tie").unwrap()[0] + .open_u64_lenient() + .unwrap() + .unwrap(); + assert_eq!(merged.num_docs(), 1_000_000); + assert_eq!(column.get_cardinality(), Cardinality::Full); + for split_start in (0..1_000_000).step_by(250_000) { + for doc in split_start + 1..split_start + 250_000 { + assert_eq!(column.first(doc), Some(column.first(doc - 1).unwrap() + 1)); + } + } +} + #[test] fn test_group_columns_required_column_with_no_existing_columns() { let columnar1 = make_columnar("numbers", &[2u64]); diff --git a/columnar/src/columnar/reader/mod.rs b/columnar/src/columnar/reader/mod.rs index 8592b3a22..198f659de 100644 --- a/columnar/src/columnar/reader/mod.rs +++ b/columnar/src/columnar/reader/mod.rs @@ -236,6 +236,22 @@ mod tests { assert_eq!(columns[1].1.column_type(), ColumnType::U64); } + #[test] + fn test_generated_tie_breaker_column() { + let mut columnar_writer = ColumnarWriter::default(); + columnar_writer.record_tie_breaker_column("tie"); + let mut buffer = Vec::new(); + columnar_writer.serialize(1_025, None, &mut buffer).unwrap(); + + let columnar = ColumnarReader::open(buffer).unwrap(); + let handles = columnar.read_columns("tie").unwrap(); + let column = handles[0].open_u64_lenient().unwrap().unwrap(); + assert_eq!(column.index.get_cardinality(), crate::Cardinality::Full); + for doc in 1..1_025 { + assert_eq!(column.first(doc), Some(column.first(doc - 1).unwrap() + 1)); + } + } + #[test] fn test_list_columns_strict_typing_prevents_coercion() { let mut columnar_writer = ColumnarWriter::default(); diff --git a/columnar/src/columnar/writer/mod.rs b/columnar/src/columnar/writer/mod.rs index 999ccd058..049a8a6f4 100644 --- a/columnar/src/columnar/writer/mod.rs +++ b/columnar/src/columnar/writer/mod.rs @@ -3,6 +3,7 @@ mod column_writers; mod serializer; mod value_index; +use std::collections::HashSet; use std::io; use std::net::Ipv6Addr; @@ -54,6 +55,7 @@ pub struct ColumnarWriter { ip_addr_field_hash_map: ArenaHashMap, bytes_field_hash_map: ArenaHashMap, str_field_hash_map: ArenaHashMap, + generated_tie_breaker_columns: HashSet>, arena: MemoryArena, // Dictionaries used to store dictionary-encoded values. dictionaries: Vec, @@ -217,6 +219,13 @@ impl ColumnarWriter { } } + /// Registers a full `u64` column whose values are generated when this writer is serialized. + pub fn record_tie_breaker_column(&mut self, column_name: &str) { + self.record_column_type(column_name, ColumnType::U64, false); + self.generated_tie_breaker_columns + .insert(column_name.as_bytes().to_vec()); + } + pub fn record_numerical + Copy>( &mut self, doc: RowId, @@ -436,24 +445,31 @@ impl ColumnarWriter { column_serializer.finalize()?; } ColumnType::F64 | ColumnType::I64 | ColumnType::U64 => { - let numerical_column_writer: NumericalColumnWriter = - self.numerical_field_hash_map.read(addr); - let cardinality = numerical_column_writer.cardinality(num_docs); let mut column_serializer = serializer.start_serialize_column(column_name, column_type); - let numerical_type = column_type.numerical_type().unwrap(); - serialize_numerical_column( - cardinality, - num_docs, - numerical_type, - numerical_column_writer.operation_iterator( - arena, - old_to_new_row_ids, - &mut symbol_byte_buffer, - ), - buffers, - &mut column_serializer, - )?; + if self.generated_tie_breaker_columns.contains(column_name) { + crate::column::serialize_generated_tie_breaker_column( + num_docs, + &mut column_serializer, + )?; + } else { + let numerical_column_writer: NumericalColumnWriter = + self.numerical_field_hash_map.read(addr); + let cardinality = numerical_column_writer.cardinality(num_docs); + let numerical_type = column_type.numerical_type().unwrap(); + serialize_numerical_column( + cardinality, + num_docs, + numerical_type, + numerical_column_writer.operation_iterator( + arena, + old_to_new_row_ids, + &mut symbol_byte_buffer, + ), + buffers, + &mut column_serializer, + )?; + } column_serializer.finalize()?; } ColumnType::DateTime => { diff --git a/src/fastfield/mod.rs b/src/fastfield/mod.rs index d56dc27a8..243f3571a 100644 --- a/src/fastfield/mod.rs +++ b/src/fastfield/mod.rs @@ -93,8 +93,8 @@ mod tests { use crate::index::SegmentId; use crate::merge_policy::NoMergePolicy; use crate::schema::{ - DateOptions, Facet, FacetOptions, Field, JsonObjectOptions, Schema, SchemaBuilder, - TantivyDocument, TextOptions, FAST, INDEXED, STORED, STRING, TEXT, + DateOptions, Facet, FacetOptions, Field, FieldType, JsonObjectOptions, Schema, + SchemaBuilder, TantivyDocument, TextOptions, FAST, INDEXED, STORED, STRING, TEXT, }; use crate::time::OffsetDateTime; use crate::tokenizer::{ @@ -148,6 +148,37 @@ mod tests { Ok(()) } + #[test] + fn test_generated_tie_breaker_fast_field() { + let mut schema_builder = Schema::builder(); + let tie = schema_builder.add_tie_breaker_field("tie"); + let schema = schema_builder.build(); + let entry = schema.get_field_entry(tie); + assert!(matches!(entry.field_type(), FieldType::TieBreaker)); + assert!(entry.is_fast()); + assert!(!entry.is_indexed()); + assert!(!entry.is_stored()); + let schema_json = serde_json::to_string(&schema).unwrap(); + assert!(schema_json.contains(r#""type":"tie_breaker""#)); + assert_eq!( + serde_json::from_str::(&schema_json).unwrap(), + schema + ); + + let mut writer = FastFieldsWriter::from_schema(&schema).unwrap(); + for _ in 0..1_025 { + writer.add_document(&TantivyDocument::default()).unwrap(); + } + let mut bytes = Vec::new(); + writer.serialize(&mut bytes, None).unwrap(); + + let readers = FastFieldReaders::open(bytes.into(), schema).unwrap(); + let values = readers.u64("tie").unwrap().first_or_default_col(0); + for doc in 1..1_025 { + assert_eq!(values.get_val(doc), values.get_val(doc - 1) + 1); + } + } + #[test] fn test_intfastfield_large() { let path = Path::new("test"); diff --git a/src/fastfield/writer.rs b/src/fastfield/writer.rs index 9bca41357..e387d9541 100644 --- a/src/fastfield/writer.rs +++ b/src/fastfield/writer.rs @@ -52,6 +52,10 @@ impl FastFieldsWriter { if !field_entry.field_type().is_fast() { continue; } + if matches!(field_entry.field_type(), FieldType::TieBreaker) { + columnar_writer.record_tie_breaker_column(field_entry.name()); + continue; + } fast_field_names[field_id.field_id() as usize] = Some(field_entry.name().to_string()); let value_type = field_entry.field_type().value_type(); if let FieldType::Date(date_options) = field_entry.field_type() { diff --git a/src/index/index.rs b/src/index/index.rs index 0fc6993fb..250674775 100644 --- a/src/index/index.rs +++ b/src/index/index.rs @@ -320,6 +320,12 @@ impl IndexBuilder { )) })?; let entry = schema.get_field_entry(schema_field); + if matches!(entry.field_type(), FieldType::TieBreaker) { + return Err(TantivyError::InvalidArgument(format!( + "Field {} is a tie-breaker field and cannot be used to sort an index", + sort_by_field.field + ))); + } if !entry.is_fast() { return Err(TantivyError::InvalidArgument(format!( "Field {} is no fast field. Field needs to be a single value fast field \ diff --git a/src/index/inverted_index_plugin.rs b/src/index/inverted_index_plugin.rs index 500052439..19273c926 100644 --- a/src/index/inverted_index_plugin.rs +++ b/src/index/inverted_index_plugin.rs @@ -391,9 +391,9 @@ impl InvertedIndexPluginWriter { self.fieldnorms_writer.record(doc_id, field, num_vals); } } - // Custom fields are not indexed; the `is_indexed()` guard above skips them. - FieldType::Custom(_) => { - unreachable!("the inverted index does not support custom field types") + // These fields are not indexed; the `is_indexed()` guard above skips them. + FieldType::TieBreaker | FieldType::Custom(_) => { + unreachable!("the inverted index does not support this field type") } } } diff --git a/src/indexer/doc_id_mapping.rs b/src/indexer/doc_id_mapping.rs index 4819330da..dd5739785 100644 --- a/src/indexer/doc_id_mapping.rs +++ b/src/indexer/doc_id_mapping.rs @@ -721,6 +721,29 @@ mod tests_indexsorting { assert!(matches!(error, TantivyError::InvalidArgument(_))); } + #[test] + fn test_index_builder_rejects_tie_breaker_sort_by_field() { + for order in [Order::Asc, Order::Desc] { + let mut schema_builder = Schema::builder(); + schema_builder.add_tie_breaker_field("tie"); + let schema = schema_builder.build(); + let settings = IndexSettings { + sort_by_field: Some(IndexSortByField { + field: "tie".to_string(), + order, + }), + ..Default::default() + }; + + let error = Index::builder() + .schema(schema) + .settings(settings) + .create_in_ram() + .unwrap_err(); + assert!(matches!(error, TantivyError::InvalidArgument(_))); + } + } + #[test] fn test_doc_mapping() { let doc_mapping = DocIdMapping::from_new_id_to_old_id(vec![3, 2, 5]); diff --git a/src/postings/per_field_postings_writer.rs b/src/postings/per_field_postings_writer.rs index 5ec4e6a49..7147c356e 100644 --- a/src/postings/per_field_postings_writer.rs +++ b/src/postings/per_field_postings_writer.rs @@ -69,8 +69,10 @@ fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> Box::default().into() } } - // Custom fields are never indexed, so this writer is never fed terms. It only needs to + // These fields are never indexed, so this writer is never fed terms. It only needs to // occupy the per-field slot (the vector is indexed by field id). - FieldType::Custom(_) => Box::>::default(), + FieldType::TieBreaker | FieldType::Custom(_) => { + Box::>::default() + } } } diff --git a/src/query/query_parser/query_parser.rs b/src/query/query_parser/query_parser.rs index 05de1e776..4d0185fdb 100644 --- a/src/query/query_parser/query_parser.rs +++ b/src/query/query_parser/query_parser.rs @@ -453,7 +453,7 @@ impl QueryParser { ))); } match *field_type { - FieldType::U64(_) => { + FieldType::U64(_) | FieldType::TieBreaker => { let val: u64 = u64::from_str(phrase)?; Ok(Term::from_field_u64(field, val)) } @@ -635,10 +635,10 @@ impl QueryParser { let term = Term::from_field_ip_addr(field, ip_v6); Ok(vec![LogicalLiteral::Term(term)]) } - // Custom fields are not indexed, so the `is_indexed()` guard above returns + // These fields are not indexed, so the `is_indexed()` guard above returns // `FieldNotIndexed` before this match. - FieldType::Custom(_) => { - unreachable!("the query parser does not support custom field types") + FieldType::TieBreaker | FieldType::Custom(_) => { + unreachable!("the query parser does not support this field type") } } } diff --git a/src/schema/document/default_document.rs b/src/schema/document/default_document.rs index be66956e3..de4e0e824 100644 --- a/src/schema/document/default_document.rs +++ b/src/schema/document/default_document.rs @@ -12,7 +12,7 @@ use crate::schema::document::{ DeserializeError, Document, DocumentDeserialize, DocumentDeserializer, }; use crate::schema::field_type::ValueParsingError; -use crate::schema::{Facet, Field, NamedFieldDocument, OwnedValue, Schema}; +use crate::schema::{Facet, Field, FieldType, NamedFieldDocument, OwnedValue, Schema}; use crate::tokenizer::PreTokenizedString; #[repr(C, packed)] @@ -219,6 +219,9 @@ impl CompactDoc { if let Ok(field) = schema.get_field(&field_name) { let field_entry = schema.get_field_entry(field); let field_type = field_entry.field_type(); + if matches!(field_type, FieldType::TieBreaker) { + continue; + } match json_value { serde_json::Value::Array(json_items) => { for json_item in json_items { @@ -752,6 +755,19 @@ mod tests { let _json = doc.to_named_doc(&schema); } + #[test] + fn test_parse_json_ignores_tie_breaker_values() { + let mut schema_builder = Schema::builder(); + let title = schema_builder.add_text_field("title", TEXT); + let tie = schema_builder.add_tie_breaker_field("tie"); + let schema = schema_builder.build(); + let doc = + TantivyDocument::parse_json(&schema, r#"{"title": "hello", "tie": [1, "two", null]}"#) + .unwrap(); + assert!(doc.get_first(title).is_some()); + assert!(doc.get_first(tie).is_none()); + } + #[test] fn test_json_value() { let json_str = r#"{ diff --git a/src/schema/field_entry.rs b/src/schema/field_entry.rs index 5dddb5799..f7816b218 100644 --- a/src/schema/field_entry.rs +++ b/src/schema/field_entry.rs @@ -41,6 +41,11 @@ impl FieldEntry { Self::new(field_name, FieldType::U64(int_options)) } + /// Creates a generated tie-breaker field entry. + pub fn new_tie_breaker(field_name: String) -> FieldEntry { + Self::new(field_name, FieldType::TieBreaker) + } + /// Creates a new i64 field entry. pub fn new_i64(field_name: String, int_options: NumericOptions) -> FieldEntry { Self::new(field_name, FieldType::I64(int_options)) @@ -135,7 +140,7 @@ impl FieldEntry { FieldType::Bytes(ref options) => options.is_stored(), FieldType::JsonObject(ref options) => options.is_stored(), FieldType::IpAddr(ref options) => options.is_stored(), - FieldType::Custom(_) => false, + FieldType::TieBreaker | FieldType::Custom(_) => false, } } } diff --git a/src/schema/field_type.rs b/src/schema/field_type.rs index bdf8cdd60..f1297a3c2 100644 --- a/src/schema/field_type.rs +++ b/src/schema/field_type.rs @@ -190,6 +190,10 @@ pub enum FieldType { Str(TextOptions), /// Unsigned 64-bits integers field type configuration U64(NumericOptions), + /// Generated tie-breaker field, exposed as a `u64` fast field. + /// + /// Values are almost always distinct, but not guaranteed to be unique across segments. + TieBreaker, /// Signed 64-bits integers 64 field type configuration I64(NumericOptions), /// 64-bits float 64 field type configuration @@ -217,7 +221,7 @@ impl FieldType { pub fn value_type(&self) -> Type { match *self { FieldType::Str(_) => Type::Str, - FieldType::U64(_) => Type::U64, + FieldType::U64(_) | FieldType::TieBreaker => Type::U64, FieldType::I64(_) => Type::I64, FieldType::F64(_) => Type::F64, FieldType::Bool(_) => Type::Bool, @@ -273,7 +277,7 @@ impl FieldType { FieldType::Bytes(ref bytes_options) => bytes_options.is_indexed(), FieldType::JsonObject(ref json_object_options) => json_object_options.is_indexed(), FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.is_indexed(), - FieldType::Custom(_) => false, + FieldType::TieBreaker | FieldType::Custom(_) => false, } } @@ -311,6 +315,7 @@ impl FieldType { FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.is_fast(), FieldType::Facet(_) => true, FieldType::JsonObject(ref json_object_options) => json_object_options.is_fast(), + FieldType::TieBreaker => true, FieldType::Custom(_) => false, } } @@ -331,7 +336,7 @@ impl FieldType { FieldType::Bytes(ref bytes_options) => bytes_options.fieldnorms(), FieldType::JsonObject(ref _json_object_options) => false, FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.fieldnorms(), - FieldType::Custom(_) => false, + FieldType::TieBreaker | FieldType::Custom(_) => false, } } @@ -383,7 +388,7 @@ impl FieldType { None } } - FieldType::Custom(_) => None, + FieldType::TieBreaker | FieldType::Custom(_) => None, } } @@ -486,6 +491,10 @@ impl FieldType { Ok(OwnedValue::IpAddr(ip_addr.into_ipv6_addr())) } + FieldType::TieBreaker => Err(ValueParsingError::TypeError { + expected: "a generated tie-breaker field", + json: JsonValue::String(field_text), + }), FieldType::Custom(_) => Err(custom_not_json_error(JsonValue::String(field_text))), }, JsonValue::Number(field_val_num) => match self { @@ -545,6 +554,10 @@ impl FieldType { expected: "a string with an ip addr", json: JsonValue::Number(field_val_num), }), + FieldType::TieBreaker => Err(ValueParsingError::TypeError { + expected: "a generated tie-breaker field", + json: JsonValue::Number(field_val_num), + }), FieldType::Custom(_) => { Err(custom_not_json_error(JsonValue::Number(field_val_num))) } diff --git a/src/schema/schema.rs b/src/schema/schema.rs index 79414473e..745a1ec30 100644 --- a/src/schema/schema.rs +++ b/src/schema/schema.rs @@ -57,6 +57,21 @@ impl SchemaBuilder { self.add_field(field_entry) } + /// Adds a generated tie-breaker fast field. + /// + /// The field is exposed as a `u64` fast field. Values are consecutive within a segment, + /// starting at a random offset. Ranges of different segments may overlap, so values are + /// almost always distinct but not guaranteed to be unique. + /// + /// Values supplied by documents for this field are ignored. + /// + /// # Panics + /// + /// Panics when field already exists. + pub fn add_tie_breaker_field(&mut self, field_name_str: &str) -> Field { + self.add_field(FieldEntry::new_tie_breaker(field_name_str.to_string())) + } + /// Adds a new i64 field. /// Returns the associated field handle ///