Keep buffered union refill out of line

BufferedUnionScorer is the hot path for full union traversal, including (TopDocs, Count) where Count forces all matches to be visited. After the block-wand intersection changes, LLVM started inlining the refill helper into the advance path, which regressed TOP_100_COUNT union queries even though the union algorithm did not change.

Force the refill helper out of line so the advance loop stays small and stable while pruning collectors continue to use Block-WAND.

Benchmark on search-benchmark-game TOP_100_COUNT union query set (301 queries, sum of per-query medians):
- tantivy 0.26: 0.853646s
- main before: 0.918605s
- this change: 0.841659s

src/indexer/doc_id_mapping.rs

@@ -7,20 +7,16 @@ use super::SegmentWriter;
 use crate::schema::{Field, Schema};
 use crate::{DocAddress, DocId, IndexSortByField, TantivyError};
 
-/// Describes how the document ID mapping was produced during a merge.
 #[derive(Copy, Clone, Eq, PartialEq)]
 pub enum MappingType {
-    /// Segments are concatenated in order with no deletes; doc IDs are contiguous ranges.
     Stacked,
-    /// Segments are concatenated in order but some documents have been deleted and are skipped.
     StackedWithDeletes,
-    /// Documents have been reordered, for instance by a sort field or by caller-provided order.
     Shuffled,
 }
 
 /// Struct to provide mapping from new doc_id to old doc_id and segment.
 #[derive(Clone)]
-pub struct SegmentDocIdMapping {
+pub(crate) struct SegmentDocIdMapping {
     pub(crate) new_doc_id_to_old_doc_addr: Vec<DocAddress>,
     pub(crate) alive_bitsets: Vec<Option<ReadOnlyBitSet>>,
     mapping_type: MappingType,
@@ -39,24 +35,6 @@ impl SegmentDocIdMapping {
         }
     }
 
-    /// Build a `Shuffled` mapping from an explicit permutation of [`DocAddress`]es.
-    ///
-    /// `new_doc_id_to_old_doc_addr[new_id]` gives the source segment and doc id for
-    /// the document that should appear at position `new_id` in the merged segment.
-    /// `alive_bitsets` must contain one entry per source segment, in the same order
-    /// as the segments passed to [`IndexMerger::open_with_custom_alive_set`].
-    pub fn new_shuffled(
-        new_doc_id_to_old_doc_addr: Vec<DocAddress>,
-        alive_bitsets: Vec<Option<ReadOnlyBitSet>>,
-    ) -> Self {
-        Self {
-            new_doc_id_to_old_doc_addr,
-            alive_bitsets,
-            mapping_type: MappingType::Shuffled,
-        }
-    }
-
-    /// Returns the [`MappingType`] that describes how this mapping was constructed.
     pub fn mapping_type(&self) -> MappingType {
         self.mapping_type
     }
@@ -93,7 +71,6 @@ pub struct DocIdMapping {
 }
 
 impl DocIdMapping {
-    /// Constructs a [`DocIdMapping`] from a vector mapping each new doc ID to its old doc ID.
     pub fn from_new_id_to_old_id(new_doc_id_to_old: Vec<DocId>) -> Self {
         let max_doc = new_doc_id_to_old.len();
         let old_max_doc = new_doc_id_to_old
@@ -125,7 +102,6 @@ impl DocIdMapping {
         self.new_doc_id_to_old.iter().cloned()
     }
 
-    /// Returns a slice mapping each old doc ID to its corresponding new doc ID.
     pub fn old_to_new_ids(&self) -> &[DocId] {
         &self.old_doc_id_to_new[..]
     }
@@ -137,11 +113,9 @@ impl DocIdMapping {
             .map(|old_doc| els[*old_doc as usize])
             .collect()
     }
-    /// Returns the number of new doc IDs in this mapping.
     pub fn num_new_doc_ids(&self) -> usize {
         self.new_doc_id_to_old.len()
     }
-    /// Returns the number of old doc IDs covered by this mapping.
     pub fn num_old_doc_ids(&self) -> usize {
         self.old_doc_id_to_new.len()
     }

src/indexer/merger.rs

@@ -113,7 +113,6 @@ fn estimate_total_num_tokens(readers: &[SegmentReader], field: Field) -> crate::
     Ok(total_num_tokens)
 }
 
-/// Merges multiple index segments into one segment.
 pub struct IndexMerger {
     index_settings: IndexSettings,
     schema: Schema,
@@ -219,7 +218,6 @@ impl IndexMerger {
             .any(|doc_id| col.first(doc_id).is_none())
     }
 
-    /// Opens an [`IndexMerger`] over the given segments using their existing delete sets.
     pub fn open(
         schema: Schema,
         index_settings: IndexSettings,
@@ -241,9 +239,6 @@ impl IndexMerger {
     // This can be used to merge but also apply an additional filter.
     // One use case is demux, which is basically taking a list of
     // segments and partitions them e.g. by a value in a field.
-    /// Opens an [`IndexMerger`] with a custom alive set per segment.
-    ///
-    /// Each entry in `alive_bitset_opt` corresponds to the segment at the same ordinal.
     pub fn open_with_custom_alive_set(
         schema: Schema,
         index_settings: IndexSettings,
@@ -952,7 +947,7 @@ impl IndexMerger {
     ///
     /// # Returns
     /// The number of documents in the resulting segment.
-    pub fn write(&self, serializer: SegmentSerializer) -> crate::Result<u32> {
+    pub fn write(&self, mut serializer: SegmentSerializer) -> crate::Result<u32> {
         let doc_id_mapping = if let Some(sort_by_field) = self.index_settings.sort_by_field.as_ref()
         {
             if self.is_disjunct_and_sorted_on_sort_property(sort_by_field)? {
@@ -963,27 +958,6 @@ impl IndexMerger {
         } else {
             self.get_doc_id_from_concatenated_data()?
         };
-        self.write_with_mapping(serializer, doc_id_mapping)
-    }
-
-    /// Like [`IndexMerger::write`], but uses the caller-supplied `doc_id_mapping` instead of
-    /// deriving one from an index sort field.
-    ///
-    /// The mapping must cover all live documents across every segment passed to
-    /// [`IndexMerger::open_with_custom_alive_set`].
-    pub fn write_with_doc_id_mapping(
-        &self,
-        serializer: SegmentSerializer,
-        doc_id_mapping: SegmentDocIdMapping,
-    ) -> crate::Result<u32> {
-        self.write_with_mapping(serializer, doc_id_mapping)
-    }
-
-    fn write_with_mapping(
-        &self,
-        mut serializer: SegmentSerializer,
-        doc_id_mapping: SegmentDocIdMapping,
-    ) -> crate::Result<u32> {
         debug!("write-fieldnorms");
         if let Some(fieldnorms_serializer) = serializer.extract_fieldnorms_serializer() {
             self.write_fieldnorms(fieldnorms_serializer, &doc_id_mapping)?;

src/indexer/mod.rs

@@ -8,7 +8,7 @@
 pub(crate) mod delete_queue;
 pub(crate) mod path_to_unordered_id;
 
-pub mod doc_id_mapping;
+pub(crate) mod doc_id_mapping;
 mod doc_opstamp_mapping;
 mod flat_map_with_buffer;
 pub(crate) mod index_writer;
@@ -17,8 +17,7 @@ pub(crate) mod indexing_term;
 mod log_merge_policy;
 mod merge_operation;
 pub(crate) mod merge_policy;
-/// Segment merger APIs for combining multiple existing segments.
-pub mod merger;
+pub(crate) mod merger;
 mod merger_sorted_index_test;
 pub(crate) mod operation;
 pub(crate) mod prepared_commit;
@@ -34,19 +33,15 @@ mod stamper;
 use crossbeam_channel as channel;
 use smallvec::SmallVec;
 
-pub use self::doc_id_mapping::SegmentDocIdMapping;
 pub use self::index_writer::{advance_deletes, IndexWriter, IndexWriterOptions};
 pub use self::log_merge_policy::LogMergePolicy;
 pub use self::merge_operation::MergeOperation;
 pub use self::merge_policy::{MergeCandidate, MergePolicy, NoMergePolicy};
-pub use self::merger::IndexMerger;
 pub use self::operation::{AddOperation, DeleteOperation, UserOperation};
 pub use self::prepared_commit::PreparedCommit;
 pub use self::segment_entry::SegmentEntry;
 pub(crate) use self::segment_serializer::SegmentSerializer;
-pub use self::segment_updater::{
-    merge_filtered_segments, merge_indices, merge_segments_with_doc_id_mapping,
-};
+pub use self::segment_updater::{merge_filtered_segments, merge_indices};
 pub use self::segment_writer::SegmentWriter;
 pub use self::single_segment_index_writer::SingleSegmentIndexWriter;
 

src/indexer/segment_serializer.rs

@@ -11,7 +11,6 @@ use crate::store::StoreWriter;
 pub struct SegmentSerializer {
     segment: Segment,
     pub(crate) store_writer: StoreWriter,
-    store_is_temp: bool,
     fast_field_write: WritePtr,
     fieldnorms_serializer: Option<FieldNormsSerializer>,
     postings_serializer: InvertedIndexSerializer,
@@ -19,19 +18,14 @@ pub struct SegmentSerializer {
 
 impl SegmentSerializer {
     /// Creates a new `SegmentSerializer`.
-    pub fn for_segment(segment: Segment, is_in_merge: bool) -> crate::Result<SegmentSerializer> {
+    pub fn for_segment(
+        mut segment: Segment,
+        is_in_merge: bool,
+    ) -> crate::Result<SegmentSerializer> {
         // If the segment is going to be sorted, we stream the docs first to a temporary file.
         // In the merge case this is not necessary because we can kmerge the already sorted
         // segments
         let remapping_required = segment.index().settings().sort_by_field.is_some() && !is_in_merge;
-        Self::for_segment_with_remapping_required(segment, remapping_required)
-    }
-
-    /// Creates a new `SegmentSerializer` with an explicit remapping requirement.
-    pub fn for_segment_with_remapping_required(
-        mut segment: Segment,
-        remapping_required: bool,
-    ) -> crate::Result<SegmentSerializer> {
         let settings = segment.index().settings().clone();
         let store_writer = if remapping_required {
             let store_write = segment.open_write(SegmentComponent::TempStore)?;
@@ -63,7 +57,6 @@ impl SegmentSerializer {
         Ok(SegmentSerializer {
             segment,
             store_writer,
-            store_is_temp: remapping_required,
             fast_field_write,
             fieldnorms_serializer: Some(fieldnorms_serializer),
             postings_serializer,
@@ -83,10 +76,6 @@ impl SegmentSerializer {
         &mut self.segment
     }
 
-    pub fn store_is_temp(&self) -> bool {
-        self.store_is_temp
-    }
-
     /// Accessor to the `PostingsSerializer`.
     pub fn get_postings_serializer(&mut self) -> &mut InvertedIndexSerializer {
         &mut self.postings_serializer

src/indexer/segment_updater.rs

@@ -15,7 +15,6 @@ use crate::directory::{Directory, DirectoryClone, GarbageCollectionResult};
 use crate::fastfield::AliveBitSet;
 use crate::index::{Index, IndexMeta, IndexSettings, Segment, SegmentId, SegmentMeta};
 use crate::indexer::delete_queue::DeleteCursor;
-use crate::indexer::doc_id_mapping::SegmentDocIdMapping;
 use crate::indexer::index_writer::advance_deletes;
 use crate::indexer::merge_operation::MergeOperationInventory;
 use crate::indexer::merger::IndexMerger;
@@ -256,81 +255,6 @@ pub fn merge_filtered_segments<T: Into<Box<dyn Directory>>>(
     Ok(merged_index)
 }
 
-/// Like [`merge_filtered_segments`], but uses a caller-supplied [`SegmentDocIdMapping`]
-/// to control the final document order.
-///
-/// The mapping should be built from the same segments, in the same order, passed here.
-///
-/// # Warning
-/// Same caveats as [`merge_filtered_segments`]: no live `IndexWriter` is allowed.
-#[doc(hidden)]
-pub fn merge_segments_with_doc_id_mapping<T: Into<Box<dyn Directory>>>(
-    segments: &[Segment],
-    target_settings: IndexSettings,
-    filter_doc_ids: Vec<Option<AliveBitSet>>,
-    doc_id_mapping: SegmentDocIdMapping,
-    output_directory: T,
-) -> crate::Result<Index> {
-    if segments.is_empty() {
-        return Err(crate::TantivyError::InvalidArgument(
-            "No segments given to merge".to_string(),
-        ));
-    }
-
-    let target_schema = segments[0].schema();
-
-    if segments
-        .iter()
-        .skip(1)
-        .any(|segment| segment.schema() != target_schema)
-    {
-        return Err(crate::TantivyError::InvalidArgument(
-            "Attempt to merge different schema indices".to_string(),
-        ));
-    }
-
-    let mut merged_index = Index::create(
-        output_directory,
-        target_schema.clone(),
-        target_settings.clone(),
-    )?;
-    let merged_segment = merged_index.new_segment();
-    let merged_segment_id = merged_segment.id();
-    let merger = IndexMerger::open_with_custom_alive_set(
-        merged_index.schema(),
-        merged_index.settings().clone(),
-        segments,
-        filter_doc_ids,
-    )?;
-    let segment_serializer = SegmentSerializer::for_segment(merged_segment, true)?;
-    let num_docs = merger.write_with_doc_id_mapping(segment_serializer, doc_id_mapping)?;
-
-    let segment_meta = merged_index.new_segment_meta(merged_segment_id, num_docs);
-
-    let stats = format!(
-        "Segments Merge (external reordering): [{}]",
-        segments
-            .iter()
-            .fold(String::new(), |sum, current| format!(
-                "{sum}{} ",
-                current.meta().id().uuid_string()
-            ))
-            .trim_end()
-    );
-
-    let index_meta = IndexMeta {
-        index_settings: target_settings,
-        segments: vec![segment_meta],
-        schema: target_schema,
-        opstamp: 0u64,
-        payload: Some(stats),
-    };
-
-    save_metas(&index_meta, merged_index.directory_mut())?;
-
-    Ok(merged_index)
-}
-
 pub(crate) struct InnerSegmentUpdater {
     // we keep a copy of the current active IndexMeta to
     // avoid loading the file every time we need it in the

src/indexer/segment_writer.rs

@@ -18,9 +18,7 @@ use crate::postings::{
 use crate::schema::document::{Document, Value};
 use crate::schema::{FieldEntry, FieldType, Schema, DATE_TIME_PRECISION_INDEXED};
 use crate::store::{StoreReader, StoreWriter};
-use crate::tokenizer::{
-    FacetTokenizer, PreTokenizedStream, TextAnalyzer, Tokenizer, TokenizerManager,
-};
+use crate::tokenizer::{FacetTokenizer, PreTokenizedStream, TextAnalyzer, Tokenizer};
 use crate::{DocId, Opstamp, TantivyError};
 
 /// Computes the initial size of the hash table.
@@ -92,42 +90,8 @@ impl SegmentWriter {
         let schema = segment.schema();
         let tokenizer_manager = segment.index().tokenizers().clone();
         let tokenizer_manager_fast_field = segment.index().fast_field_tokenizer().clone();
-        let segment_serializer = SegmentSerializer::for_segment(segment, false)?;
-        Self::for_segment_serializer(
-            memory_budget_in_bytes,
-            schema,
-            tokenizer_manager,
-            tokenizer_manager_fast_field,
-            segment_serializer,
-        )
-    }
-
-    pub(crate) fn for_segment_with_provided_doc_id_mapping(
-        memory_budget_in_bytes: usize,
-        segment: Segment,
-    ) -> crate::Result<Self> {
-        let schema = segment.schema();
-        let tokenizer_manager = segment.index().tokenizers().clone();
-        let tokenizer_manager_fast_field = segment.index().fast_field_tokenizer().clone();
-        let segment_serializer =
-            SegmentSerializer::for_segment_with_remapping_required(segment, true)?;
-        Self::for_segment_serializer(
-            memory_budget_in_bytes,
-            schema,
-            tokenizer_manager,
-            tokenizer_manager_fast_field,
-            segment_serializer,
-        )
-    }
-
-    fn for_segment_serializer(
-        memory_budget_in_bytes: usize,
-        schema: Schema,
-        tokenizer_manager: TokenizerManager,
-        tokenizer_manager_fast_field: TokenizerManager,
-        segment_serializer: SegmentSerializer,
-    ) -> crate::Result<Self> {
         let table_size = compute_initial_table_size(memory_budget_in_bytes)?;
+        let segment_serializer = SegmentSerializer::for_segment(segment, false)?;
         let per_field_postings_writers = PerFieldPostingsWriter::for_schema(&schema);
         let per_field_text_analyzers = schema
             .fields()
@@ -185,31 +149,6 @@ impl SegmentWriter {
             .clone()
             .map(|sort_by_field| get_doc_id_mapping_from_field(sort_by_field, &self))
             .transpose()?;
-        self.finalize_with_mapping(mapping.as_ref())
-    }
-
-    /// Lay on disk the current content of the `SegmentWriter`, using a caller-provided document
-    /// order.
-    ///
-    /// `new_doc_id_to_old_doc_id[new_id]` is the old document id of the document that should be
-    /// serialized at `new_id`.
-    pub fn finalize_with_doc_id_mapping(
-        mut self,
-        new_doc_id_to_old_doc_id: Vec<DocId>,
-    ) -> crate::Result<Vec<u64>> {
-        if new_doc_id_to_old_doc_id.len() != self.max_doc as usize {
-            return Err(crate::TantivyError::InvalidArgument(format!(
-                "provided doc id mapping length {} does not match segment max_doc {}",
-                new_doc_id_to_old_doc_id.len(),
-                self.max_doc
-            )));
-        }
-        self.fieldnorms_writer.fill_up_to_max_doc(self.max_doc);
-        let mapping = DocIdMapping::from_new_id_to_old_id(new_doc_id_to_old_doc_id);
-        self.finalize_with_mapping(Some(&mapping))
-    }
-
-    fn finalize_with_mapping(self, mapping: Option<&DocIdMapping>) -> crate::Result<Vec<u64>> {
         remap_and_write(
             self.schema,
             &self.per_field_postings_writers,
@@ -217,10 +156,9 @@ impl SegmentWriter {
             self.fast_field_writers,
             &self.fieldnorms_writer,
             self.segment_serializer,
-            self.max_doc,
-            mapping,
+            mapping.as_ref(),
         )?;
-        let doc_opstamps = remap_doc_opstamps(self.doc_opstamps, mapping);
+        let doc_opstamps = remap_doc_opstamps(self.doc_opstamps, mapping.as_ref());
         Ok(doc_opstamps)
     }
 
@@ -482,7 +420,6 @@ fn remap_and_write(
     fast_field_writers: FastFieldsWriter,
     fieldnorms_writer: &FieldNormsWriter,
     mut serializer: SegmentSerializer,
-    max_doc: DocId,
     doc_id_map: Option<&DocIdMapping>,
 ) -> crate::Result<()> {
     debug!("remap-and-write");
@@ -504,10 +441,9 @@ fn remap_and_write(
     debug!("fastfield-serialize");
     fast_field_writers.serialize(serializer.get_fast_field_write(), doc_id_map)?;
 
-    // Finalize the temp docstore and create the final store. With a mapping, the final store
-    // reflects the new doc id order; without one, it preserves the insertion order.
-    if serializer.store_is_temp() {
-        debug!("rewrite-docstore");
+    // finalize temp docstore and create version, which reflects the doc_id_map
+    if let Some(doc_id_map) = doc_id_map {
+        debug!("resort-docstore");
         let store_write = serializer
             .segment_mut()
             .open_write(SegmentComponent::Store)?;
@@ -527,10 +463,7 @@ fn remap_and_write(
             1, /* The docstore is configured to have one doc per block, and each doc is
                 * accessed only once: we don't need caching. */
         )?;
-        let old_doc_ids = doc_id_map
-            .map(|doc_id_map| doc_id_map.iter_old_doc_ids().collect::<Vec<_>>())
-            .unwrap_or_else(|| (0..max_doc).collect::<Vec<_>>());
-        for old_doc_id in old_doc_ids {
+        for old_doc_id in doc_id_map.iter_old_doc_ids() {
             let doc_bytes = store_read.get_document_bytes(old_doc_id)?;
             serializer.get_store_writer().store_bytes(&doc_bytes)?;
         }

src/indexer/single_segment_index_writer.rs

@@ -4,7 +4,7 @@ use crate::indexer::operation::AddOperation;
 use crate::indexer::segment_updater::save_metas;
 use crate::indexer::SegmentWriter;
 use crate::schema::document::Document;
-use crate::{Directory, DocId, Index, IndexMeta, Opstamp, Segment, TantivyDocument};
+use crate::{Directory, Index, IndexMeta, Opstamp, Segment, TantivyDocument};
 
 #[doc(hidden)]
 pub struct SingleSegmentIndexWriter<D: Document = TantivyDocument> {
@@ -17,8 +17,7 @@ pub struct SingleSegmentIndexWriter<D: Document = TantivyDocument> {
 impl<D: Document> SingleSegmentIndexWriter<D> {
     pub fn new(index: Index, mem_budget: usize) -> crate::Result<Self> {
         let segment = index.new_segment();
-        let segment_writer =
-            SegmentWriter::for_segment_with_provided_doc_id_mapping(mem_budget, segment.clone())?;
+        let segment_writer = SegmentWriter::for_segment(mem_budget, segment.clone())?;
         Ok(Self {
             segment_writer,
             segment,
@@ -41,123 +40,17 @@ impl<D: Document> SingleSegmentIndexWriter<D> {
     pub fn finalize(self) -> crate::Result<Index> {
         let max_doc = self.segment_writer.max_doc();
         self.segment_writer.finalize()?;
-        finalize_segment(self.segment, max_doc)
-    }
-
-    /// Finalizes this single-segment index using a caller-provided document order.
-    ///
-    /// `new_doc_id_to_old_doc_id[new_id]` is the old insertion doc id of the document that should
-    /// be serialized at `new_id`.
-    pub fn finalize_with_doc_id_mapping(
-        self,
-        new_doc_id_to_old_doc_id: Vec<DocId>,
-    ) -> crate::Result<Index> {
-        let max_doc = self.segment_writer.max_doc();
-        self.segment_writer
-            .finalize_with_doc_id_mapping(new_doc_id_to_old_doc_id)?;
-        finalize_segment(self.segment, max_doc)
-    }
-}
-
-fn finalize_segment(segment: Segment, max_doc: DocId) -> crate::Result<Index> {
-    let segment: Segment = segment.with_max_doc(max_doc);
-    let index = segment.index();
-    let index_meta = IndexMeta {
-        index_settings: index.settings().clone(),
-        segments: vec![segment.meta().clone()],
-        schema: index.schema(),
-        opstamp: 0,
-        payload: None,
-    };
-    save_metas(&index_meta, index.directory())?;
-    index.directory().sync_directory()?;
-    Ok(segment.index().clone())
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::collector::TopDocs;
-    use crate::directory::RamDirectory;
-    use crate::query::QueryParser;
-    use crate::schema::{
-        IndexRecordOption, NumericOptions, Schema, TextFieldIndexing, TextOptions, Value, STORED,
-    };
-    use crate::{Index, ReloadPolicy, TantivyDocument};
-
-    #[test]
-    fn test_finalize_with_doc_id_mapping() -> crate::Result<()> {
-        let mut schema_builder = Schema::builder();
-        let id_field = schema_builder.add_u64_field("id", NumericOptions::default().set_fast());
-        let text_field = schema_builder.add_text_field(
-            "text",
-            TextOptions::default().set_stored().set_indexing_options(
-                TextFieldIndexing::default()
-                    .set_index_option(IndexRecordOption::WithFreqs)
-                    .set_fieldnorms(true),
-            ),
-        );
-        let stored_field = schema_builder.add_text_field("stored", STORED);
-        let schema = schema_builder.build();
-        let mut writer = Index::builder()
-            .schema(schema)
-            .single_segment_index_writer(RamDirectory::create(), 15_000_000)?;
-        writer.add_document(doc!(
-            id_field => 10u64,
-            text_field => "alpha beta",
-            stored_field => "old-0",
-        ))?;
-        writer.add_document(doc!(
-            id_field => 20u64,
-            text_field => "alpha",
-            stored_field => "old-1",
-        ))?;
-        writer.add_document(doc!(
-            id_field => 30u64,
-            text_field => "beta",
-            stored_field => "old-2",
-        ))?;
-
-        let index = writer.finalize_with_doc_id_mapping(vec![2, 0, 1])?;
-        let reader = index
-            .reader_builder()
-            .reload_policy(ReloadPolicy::Manual)
-            .try_into()?;
-        let searcher = reader.searcher();
-        let segment_reader = searcher.segment_reader(0);
-
-        let fast_field = segment_reader
-            .fast_fields()
-            .u64("id")?
-            .first_or_default_col(0);
-        assert_eq!(fast_field.get_val(0), 30u64);
-        assert_eq!(fast_field.get_val(1), 10u64);
-        assert_eq!(fast_field.get_val(2), 20u64);
-
-        let fieldnorm_reader = segment_reader.get_fieldnorms_reader(text_field)?;
-        assert_eq!(fieldnorm_reader.fieldnorm(0), 1);
-        assert_eq!(fieldnorm_reader.fieldnorm(1), 2);
-        assert_eq!(fieldnorm_reader.fieldnorm(2), 1);
-
-        let mut stored_values = Vec::new();
-        for doc_id in 0..segment_reader.max_doc() {
-            let doc: TantivyDocument = segment_reader.get_store_reader(1024)?.get(doc_id)?;
-            let stored_value = doc
-                .get_first(stored_field)
-                .and_then(|value| value.as_str())
-                .unwrap();
-            stored_values.push(stored_value.to_string());
-        }
-        assert_eq!(stored_values, ["old-2", "old-0", "old-1"]);
-
-        let query = QueryParser::for_index(&index, vec![text_field]).parse_query("beta")?;
-        let top_docs: Vec<(_, _)> =
-            searcher.search(&query, &TopDocs::with_limit(10).order_by_score())?;
-        let doc_ids = top_docs
-            .into_iter()
-            .map(|(_, doc_address)| doc_address.doc_id)
-            .collect::<Vec<_>>();
-        assert_eq!(doc_ids, [0, 1]);
-
-        Ok(())
+        let segment: Segment = self.segment.with_max_doc(max_doc);
+        let index = segment.index();
+        let index_meta = IndexMeta {
+            index_settings: index.settings().clone(),
+            segments: vec![segment.meta().clone()],
+            schema: index.schema(),
+            opstamp: 0,
+            payload: None,
+        };
+        save_metas(&index_meta, index.directory())?;
+        index.directory().sync_directory()?;
+        Ok(segment.index().clone())
     }
 }

src/lib.rs

@@ -229,10 +229,7 @@ pub use crate::index::{
     Index, IndexBuilder, IndexMeta, IndexSettings, IndexSortByField, InvertedIndexReader, Order,
     Segment, SegmentMeta, SegmentReader,
 };
-pub use crate::indexer::{
-    merge_segments_with_doc_id_mapping, IndexMerger, IndexWriter, SegmentDocIdMapping,
-    SingleSegmentIndexWriter,
-};
+pub use crate::indexer::{IndexWriter, SingleSegmentIndexWriter};
 pub use crate::schema::{Document, TantivyDocument, Term};
 
 /// Index format version.

src/query/boolean_query/boolean_weight.rs

@@ -91,10 +91,14 @@ fn into_box_scorer<TScoreCombiner: ScoreCombiner>(
     num_docs: u32,
 ) -> Box<dyn Scorer> {
     match scorer {
-        SpecializedScorer::TermUnion(term_scorers) => {
-            let union_scorer =
-                BufferedUnionScorer::build(term_scorers, score_combiner_fn, num_docs);
-            Box::new(union_scorer)
+        SpecializedScorer::TermUnion(mut term_scorers) => {
+            if term_scorers.len() == 1 {
+                Box::new(term_scorers.pop().unwrap())
+            } else {
+                let union_scorer =
+                    BufferedUnionScorer::build(term_scorers, score_combiner_fn, num_docs);
+                Box::new(union_scorer)
+            }
         }
         SpecializedScorer::TermIntersection(term_scorers) => {
             let boxed_scorers: Vec<Box<dyn Scorer>> = term_scorers
@@ -504,10 +508,15 @@ impl<TScoreCombiner: ScoreCombiner + Sync> Weight for BooleanWeight<TScoreCombin
         let scorer = self.complex_scorer(reader, 1.0, &self.score_combiner_fn)?;
         let num_docs = reader.num_docs();
         match scorer {
-            SpecializedScorer::TermUnion(term_scorers) => {
-                let mut union_scorer =
-                    BufferedUnionScorer::build(term_scorers, &self.score_combiner_fn, num_docs);
-                for_each_scorer(&mut union_scorer, callback);
+            SpecializedScorer::TermUnion(mut term_scorers) => {
+                if term_scorers.len() == 1 {
+                    let mut term_scorer = term_scorers.pop().unwrap();
+                    for_each_scorer(&mut term_scorer, callback);
+                } else {
+                    let mut union_scorer =
+                        BufferedUnionScorer::build(term_scorers, &self.score_combiner_fn, num_docs);
+                    for_each_scorer(&mut union_scorer, callback);
+                }
             }
             SpecializedScorer::TermIntersection(term_scorers) => {
                 let boxed_scorers: Vec<Box<dyn Scorer>> = term_scorers
@@ -534,10 +543,15 @@ impl<TScoreCombiner: ScoreCombiner + Sync> Weight for BooleanWeight<TScoreCombin
         let mut buffer = [0u32; COLLECT_BLOCK_BUFFER_LEN];
 
         match scorer {
-            SpecializedScorer::TermUnion(term_scorers) => {
-                let mut union_scorer =
-                    BufferedUnionScorer::build(term_scorers, &self.score_combiner_fn, num_docs);
-                for_each_docset_buffered(&mut union_scorer, &mut buffer, callback);
+            SpecializedScorer::TermUnion(mut term_scorers) => {
+                if term_scorers.len() == 1 {
+                    let mut term_scorer = term_scorers.pop().unwrap();
+                    for_each_docset_buffered(&mut term_scorer, &mut buffer, callback);
+                } else {
+                    let mut union_scorer =
+                        BufferedUnionScorer::build(term_scorers, &self.score_combiner_fn, num_docs);
+                    for_each_docset_buffered(&mut union_scorer, &mut buffer, callback);
+                }
             }
             SpecializedScorer::TermIntersection(term_scorers) => {
                 let boxed_scorers: Vec<Box<dyn Scorer>> = term_scorers

src/query/union/buffered_union.rs

@@ -55,6 +55,11 @@ pub struct BufferedUnionScorer<TScorer, TScoreCombiner = DoNothingCombiner> {
     num_docs: u32,
 }
 
+// Keep this helper out-of-line. When LLVM inlines it into
+// `BufferedUnionScorer::advance`, the full traversal path used by combined
+// collectors such as `(TopDocs, Count)` becomes sensitive to unrelated codegen
+// changes and regresses on large unions.
+#[inline(never)]
 fn refill<TScorer: Scorer, TScoreCombiner: ScoreCombiner>(
     scorers: &mut Vec<TScorer>,
     bitsets: &mut [TinySet; HORIZON_NUM_TINYBITSETS],