From 7453df8db3e0b39df52d1d67ebbd9bf85c1258b1 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Tue, 13 Jan 2026 14:29:47 +0100 Subject: [PATCH] postings writer enum --- src/codec/mod.rs | 6 +- src/codec/standard/mod.rs | 5 +- src/index/index.rs | 1 - src/index/index_meta.rs | 3 +- src/index/segment.rs | 2 +- src/indexer/merger.rs | 10 +- src/indexer/segment_updater.rs | 5 +- src/indexer/segment_writer.rs | 7 +- src/lib.rs | 2 +- src/postings/json_postings_writer.rs | 6 - src/postings/per_field_postings_writer.rs | 19 ++- src/postings/postings_writer.rs | 148 ++++++++++++++++++++-- src/postings/serializer.rs | 4 +- 13 files changed, 180 insertions(+), 38 deletions(-) diff --git a/src/codec/mod.rs b/src/codec/mod.rs index 2aff31f3d..1259a82cf 100644 --- a/src/codec/mod.rs +++ b/src/codec/mod.rs @@ -32,7 +32,11 @@ impl CodecConfiguration { pub fn to_codec(&self) -> crate::Result { if self.name != C::NAME { - return Err(crate::TantivyError::InvalidArgument(format!("Codec name mismatch: expected {}, got {}", C::NAME, self.name))); + return Err(crate::TantivyError::InvalidArgument(format!( + "Codec name mismatch: expected {}, got {}", + C::NAME, + self.name + ))); } C::from_json_props(&self.props) } diff --git a/src/codec/standard/mod.rs b/src/codec/standard/mod.rs index f10fd0197..70e2e45e7 100644 --- a/src/codec/standard/mod.rs +++ b/src/codec/standard/mod.rs @@ -15,7 +15,10 @@ impl Codec for StandardCodec { fn from_json_props(json_value: &serde_json::Value) -> crate::Result { if !json_value.is_null() { - return Err(crate::TantivyError::InvalidArgument(format!("Codec property for the StandardCodec are unexpected. expected null, got {}", json_value.as_str().unwrap_or("null")))) + return Err(crate::TantivyError::InvalidArgument(format!( + "Codec property for the StandardCodec are unexpected. expected null, got {}", + json_value.as_str().unwrap_or("null") + ))); } Ok(StandardCodec) } diff --git a/src/index/index.rs b/src/index/index.rs index b42ea46a1..9b2f5cf32 100644 --- a/src/index/index.rs +++ b/src/index/index.rs @@ -406,7 +406,6 @@ impl Index { } } - impl Index { /// Returns a version of this index with the standard codec. /// This is useful when you need to pass the index to APIs that diff --git a/src/index/index_meta.rs b/src/index/index_meta.rs index e8ad1ec4c..5cd504ceb 100644 --- a/src/index/index_meta.rs +++ b/src/index/index_meta.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use serde::{Deserialize, Serialize}; use super::SegmentComponent; -use crate::codec::{Codec, CodecConfiguration, StandardCodec}; +use crate::codec::{Codec, CodecConfiguration}; use crate::index::SegmentId; use crate::schema::Schema; use crate::store::Compressor; @@ -324,7 +324,6 @@ pub struct IndexMeta { pub codec: CodecConfiguration, } - #[derive(Deserialize, Debug)] struct UntrackedIndexMeta { pub segments: Vec, diff --git a/src/index/segment.rs b/src/index/segment.rs index 175e138a2..19dbea15d 100644 --- a/src/index/segment.rs +++ b/src/index/segment.rs @@ -11,7 +11,7 @@ use crate::Opstamp; /// A segment is a piece of the index. #[derive(Clone)] -pub struct Segment { +pub struct Segment { index: Index, meta: SegmentMeta, } diff --git a/src/indexer/merger.rs b/src/indexer/merger.rs index 204a9a754..6f0ac5580 100644 --- a/src/indexer/merger.rs +++ b/src/indexer/merger.rs @@ -145,7 +145,10 @@ fn extract_fast_field_required_columns(schema: &Schema) -> Vec<(String, ColumnTy } impl IndexMerger { - pub fn open(schema: Schema, segments: &[Segment]) -> crate::Result { + pub fn open( + schema: Schema, + segments: &[Segment], + ) -> crate::Result { let alive_bitset = segments.iter().map(|_| None).collect_vec(); Self::open_with_custom_alive_set(schema, segments, alive_bitset) } @@ -525,7 +528,10 @@ impl IndexMerger { /// /// # Returns /// The number of documents in the resulting segment. - pub fn write(&self, mut serializer: SegmentSerializer) -> crate::Result { + pub fn write( + &self, + mut serializer: SegmentSerializer, + ) -> crate::Result { let doc_id_mapping = self.get_doc_id_from_concatenated_data()?; debug!("write-fieldnorms"); if let Some(fieldnorms_serializer) = serializer.extract_fieldnorms_serializer() { diff --git a/src/indexer/segment_updater.rs b/src/indexer/segment_updater.rs index 5395eeead..e226f16e9 100644 --- a/src/indexer/segment_updater.rs +++ b/src/indexer/segment_updater.rs @@ -949,7 +949,10 @@ mod tests { }; // mismatched schema index list - let result = merge_indices(&[first_index, second_index], Box::new(RamDirectory::default())); + let result = merge_indices( + &[first_index, second_index], + Box::new(RamDirectory::default()), + ); assert!(result.is_err()); Ok(()) diff --git a/src/indexer/segment_writer.rs b/src/indexer/segment_writer.rs index a6dda9506..60a7ff0e4 100644 --- a/src/indexer/segment_writer.rs +++ b/src/indexer/segment_writer.rs @@ -2,9 +2,9 @@ use columnar::MonotonicallyMappableToU64; use common::JsonPathWriter; use itertools::Itertools; use tokenizer_api::BoxTokenStream; -use crate::codec::Codec; use super::operation::AddOperation; +use crate::codec::Codec; use crate::fastfield::FastFieldsWriter; use crate::fieldnorm::{FieldNormReaders, FieldNormsWriter}; use crate::index::{Segment, SegmentComponent}; @@ -71,7 +71,10 @@ impl SegmentWriter { /// behavior as a memory limit. /// - segment: The segment being written /// - schema - pub fn for_segment(memory_budget_in_bytes: usize, segment: Segment) -> crate::Result { + pub fn for_segment( + memory_budget_in_bytes: usize, + segment: Segment, + ) -> crate::Result { let schema = segment.schema(); let tokenizer_manager = segment.index().tokenizers().clone(); let tokenizer_manager_fast_field = segment.index().fast_field_tokenizer().clone(); diff --git a/src/lib.rs b/src/lib.rs index e59f7a82e..bf5f15fa8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -166,8 +166,8 @@ mod functional_test; #[macro_use] mod macros; -mod future_result; pub mod codec; +mod future_result; // Re-exports pub use common::{ByteCount, DateTime}; diff --git a/src/postings/json_postings_writer.rs b/src/postings/json_postings_writer.rs index 99de17446..ca6c8248b 100644 --- a/src/postings/json_postings_writer.rs +++ b/src/postings/json_postings_writer.rs @@ -22,12 +22,6 @@ pub(crate) struct JsonPostingsWriter { non_str_posting_writer: SpecializedPostingsWriter, } -impl From> for Box { - fn from(json_postings_writer: JsonPostingsWriter) -> Box { - Box::new(json_postings_writer) - } -} - impl PostingsWriter for JsonPostingsWriter { #[inline] fn subscribe( diff --git a/src/postings/per_field_postings_writer.rs b/src/postings/per_field_postings_writer.rs index f3d6d6534..bf547cf5a 100644 --- a/src/postings/per_field_postings_writer.rs +++ b/src/postings/per_field_postings_writer.rs @@ -1,16 +1,15 @@ use crate::postings::json_postings_writer::JsonPostingsWriter; -use crate::postings::postings_writer::SpecializedPostingsWriter; +use crate::postings::postings_writer::{PostingsWriterEnum, SpecializedPostingsWriter}; use crate::postings::recorder::{DocIdRecorder, TermFrequencyRecorder, TfAndPositionRecorder}; -use crate::postings::PostingsWriter; use crate::schema::{Field, FieldEntry, FieldType, IndexRecordOption, Schema}; pub(crate) struct PerFieldPostingsWriter { - per_field_postings_writers: Vec>, + per_field_postings_writers: Vec, } impl PerFieldPostingsWriter { pub fn for_schema(schema: &Schema) -> Self { - let per_field_postings_writers = schema + let per_field_postings_writers: Vec = schema .fields() .map(|(_, field_entry)| posting_writer_from_field_entry(field_entry)) .collect(); @@ -19,16 +18,16 @@ impl PerFieldPostingsWriter { } } - pub(crate) fn get_for_field(&self, field: Field) -> &dyn PostingsWriter { - self.per_field_postings_writers[field.field_id() as usize].as_ref() + pub(crate) fn get_for_field(&self, field: Field) -> &PostingsWriterEnum { + &self.per_field_postings_writers[field.field_id() as usize] } - pub(crate) fn get_for_field_mut(&mut self, field: Field) -> &mut dyn PostingsWriter { - self.per_field_postings_writers[field.field_id() as usize].as_mut() + pub(crate) fn get_for_field_mut(&mut self, field: Field) -> &mut PostingsWriterEnum { + &mut self.per_field_postings_writers[field.field_id() as usize] } } -fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> Box { +fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> PostingsWriterEnum { match *field_entry.field_type() { FieldType::Str(ref text_options) => text_options .get_indexing_options() @@ -51,7 +50,7 @@ fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> Box Box::>::default(), + | FieldType::Facet(_) => >::default().into(), FieldType::JsonObject(ref json_object_options) => { if let Some(text_indexing_option) = json_object_options.get_text_indexing_options() { match text_indexing_option.index_option() { diff --git a/src/postings/postings_writer.rs b/src/postings/postings_writer.rs index c7a94ecef..816f5c184 100644 --- a/src/postings/postings_writer.rs +++ b/src/postings/postings_writer.rs @@ -7,7 +7,10 @@ use stacker::Addr; use crate::fieldnorm::FieldNormReaders; use crate::indexer::indexing_term::IndexingTerm; use crate::indexer::path_to_unordered_id::OrderedPathId; -use crate::postings::recorder::{BufferLender, Recorder}; +use crate::postings::json_postings_writer::JsonPostingsWriter; +use crate::postings::recorder::{ + BufferLender, DocIdRecorder, Recorder, TermFrequencyRecorder, TfAndPositionRecorder, +}; use crate::postings::{ FieldSerializer, IndexingContext, InvertedIndexSerializer, PerFieldPostingsWriter, }; @@ -100,6 +103,141 @@ pub(crate) struct IndexingPosition { pub end_position: u32, } +pub enum PostingsWriterEnum { + DocId(SpecializedPostingsWriter), + DocIdTf(SpecializedPostingsWriter), + DocTfAndPosition(SpecializedPostingsWriter), + JsonDocId(JsonPostingsWriter), + JsonDocIdTf(JsonPostingsWriter), + JsonDocTfAndPosition(JsonPostingsWriter), +} + +impl From> for PostingsWriterEnum { + fn from(doc_id_recorder_writer: SpecializedPostingsWriter) -> Self { + PostingsWriterEnum::DocId(doc_id_recorder_writer) + } +} + +impl From> for PostingsWriterEnum { + fn from(doc_id_tf_recorder_writer: SpecializedPostingsWriter) -> Self { + PostingsWriterEnum::DocIdTf(doc_id_tf_recorder_writer) + } +} + +impl From> for PostingsWriterEnum { + fn from( + doc_id_tf_and_positions_recorder_writer: SpecializedPostingsWriter, + ) -> Self { + PostingsWriterEnum::DocTfAndPosition(doc_id_tf_and_positions_recorder_writer) + } +} + +impl From> for PostingsWriterEnum { + fn from(doc_id_recorder_writer: JsonPostingsWriter) -> Self { + PostingsWriterEnum::JsonDocId(doc_id_recorder_writer) + } +} + +impl From> for PostingsWriterEnum { + fn from(doc_id_tf_recorder_writer: JsonPostingsWriter) -> Self { + PostingsWriterEnum::JsonDocIdTf(doc_id_tf_recorder_writer) + } +} + +impl From> for PostingsWriterEnum { + fn from( + doc_id_tf_and_positions_recorder_writer: JsonPostingsWriter, + ) -> Self { + PostingsWriterEnum::JsonDocTfAndPosition(doc_id_tf_and_positions_recorder_writer) + } +} + +impl PostingsWriter for PostingsWriterEnum { + fn subscribe(&mut self, doc: DocId, pos: u32, term: &IndexingTerm, ctx: &mut IndexingContext) { + match self { + PostingsWriterEnum::DocId(writer) => writer.subscribe(doc, pos, term, ctx), + PostingsWriterEnum::DocIdTf(writer) => writer.subscribe(doc, pos, term, ctx), + PostingsWriterEnum::DocTfAndPosition(writer) => writer.subscribe(doc, pos, term, ctx), + PostingsWriterEnum::JsonDocId(writer) => writer.subscribe(doc, pos, term, ctx), + PostingsWriterEnum::JsonDocIdTf(writer) => writer.subscribe(doc, pos, term, ctx), + PostingsWriterEnum::JsonDocTfAndPosition(writer) => { + writer.subscribe(doc, pos, term, ctx) + } + } + } + + fn serialize( + &self, + term_addrs: &[(Field, OrderedPathId, &[u8], Addr)], + ordered_id_to_path: &[&str], + ctx: &IndexingContext, + serializer: &mut FieldSerializer, + ) -> io::Result<()> { + match self { + PostingsWriterEnum::DocId(writer) => { + writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer) + } + PostingsWriterEnum::DocIdTf(writer) => { + writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer) + } + PostingsWriterEnum::DocTfAndPosition(writer) => { + writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer) + } + PostingsWriterEnum::JsonDocId(writer) => { + writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer) + } + PostingsWriterEnum::JsonDocIdTf(writer) => { + writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer) + } + PostingsWriterEnum::JsonDocTfAndPosition(writer) => { + writer.serialize(term_addrs, ordered_id_to_path, ctx, serializer) + } + } + } + + /// Tokenize a text and subscribe all of its token. + fn index_text( + &mut self, + doc_id: DocId, + token_stream: &mut dyn TokenStream, + term_buffer: &mut IndexingTerm, + ctx: &mut IndexingContext, + indexing_position: &mut IndexingPosition, + ) { + match self { + PostingsWriterEnum::DocId(writer) => { + writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position) + } + PostingsWriterEnum::DocIdTf(writer) => { + writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position) + } + PostingsWriterEnum::DocTfAndPosition(writer) => { + writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position) + } + PostingsWriterEnum::JsonDocId(writer) => { + writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position) + } + PostingsWriterEnum::JsonDocIdTf(writer) => { + writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position) + } + PostingsWriterEnum::JsonDocTfAndPosition(writer) => { + writer.index_text(doc_id, token_stream, term_buffer, ctx, indexing_position) + } + } + } + + fn total_num_tokens(&self) -> u64 { + match self { + PostingsWriterEnum::DocId(writer) => writer.total_num_tokens(), + PostingsWriterEnum::DocIdTf(writer) => writer.total_num_tokens(), + PostingsWriterEnum::DocTfAndPosition(writer) => writer.total_num_tokens(), + PostingsWriterEnum::JsonDocId(writer) => writer.total_num_tokens(), + PostingsWriterEnum::JsonDocIdTf(writer) => writer.total_num_tokens(), + PostingsWriterEnum::JsonDocTfAndPosition(writer) => writer.total_num_tokens(), + } + } +} + /// The `PostingsWriter` is in charge of receiving documenting /// and building a `Segment` in anonymous memory. /// @@ -171,14 +309,6 @@ pub(crate) struct SpecializedPostingsWriter { _recorder_type: PhantomData, } -impl From> for Box { - fn from( - specialized_postings_writer: SpecializedPostingsWriter, - ) -> Box { - Box::new(specialized_postings_writer) - } -} - impl SpecializedPostingsWriter { #[inline] pub(crate) fn serialize_one_term( diff --git a/src/postings/serializer.rs b/src/postings/serializer.rs index eb4fcd0d0..aa7b8a6c2 100644 --- a/src/postings/serializer.rs +++ b/src/postings/serializer.rs @@ -55,7 +55,9 @@ pub struct InvertedIndexSerializer { impl InvertedIndexSerializer { /// Open a new `InvertedIndexSerializer` for the given segment - pub fn open(segment: &mut Segment) -> crate::Result { + pub fn open( + segment: &mut Segment, + ) -> crate::Result { use crate::index::SegmentComponent::{Positions, Postings, Terms}; let inv_index_serializer = InvertedIndexSerializer { terms_write: CompositeWrite::wrap(segment.open_write(Terms)?),