From 735c588f4f89b67eab6f05285cd93f8e14816574 Mon Sep 17 00:00:00 2001
From: PSeitz <PSeitz@users.noreply.github.com>
Date: Fri, 2 Jan 2026 19:06:51 +0800
Subject: [PATCH] fix union performance regression (#2790)

* add inlines

* fix union performance regression

Remove unwrap from hotpath generates better assembly.

closes #2788
---
 src/collector/sort_key/order.rs                       | 2 +-
 src/postings/mod.rs                                   | 1 +
 src/query/all_query.rs                                | 1 +
 src/query/boost_query.rs                              | 1 +
 src/query/const_score_query.rs                        | 1 +
 src/query/disjunction.rs                              | 2 ++
 src/query/empty_query.rs                              | 1 +
 src/query/exclude.rs                                  | 1 +
 src/query/intersection.rs                             | 3 +++
 src/query/phrase_prefix_query/phrase_prefix_scorer.rs | 2 ++
 src/query/phrase_query/phrase_scorer.rs               | 1 +
 src/query/reqopt_scorer.rs                            | 1 +
 src/query/score_combiner.rs                           | 3 +++
 src/query/scorer.rs                                   | 1 +
 src/query/term_query/term_scorer.rs                   | 1 +
 src/query/union/buffered_union.rs                     | 4 ++++
 16 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/collector/sort_key/order.rs b/src/collector/sort_key/order.rs
index c2f346901..3cac357ad 100644
--- a/src/collector/sort_key/order.rs
+++ b/src/collector/sort_key/order.rs
@@ -84,7 +84,7 @@ pub struct NaturalComparator;
 impl<T: PartialOrd> Comparator<T> for NaturalComparator {
     #[inline(always)]
     fn compare(&self, lhs: &T, rhs: &T) -> Ordering {
-        lhs.partial_cmp(rhs).unwrap()
+        lhs.partial_cmp(rhs).unwrap_or(Ordering::Equal)
     }
 }
 
diff --git a/src/postings/mod.rs b/src/postings/mod.rs
index efc0e069d..b9c400859 100644
--- a/src/postings/mod.rs
+++ b/src/postings/mod.rs
@@ -527,6 +527,7 @@ pub(crate) mod tests {
     }
 
     impl<TScorer: Scorer> Scorer for UnoptimizedDocSet<TScorer> {
+        #[inline]
         fn score(&mut self) -> Score {
             self.0.score()
         }
diff --git a/src/query/all_query.rs b/src/query/all_query.rs
index 612d7408c..5431a3a1b 100644
--- a/src/query/all_query.rs
+++ b/src/query/all_query.rs
@@ -105,6 +105,7 @@ impl DocSet for AllScorer {
 }
 
 impl Scorer for AllScorer {
+    #[inline]
     fn score(&mut self) -> Score {
         1.0
     }
diff --git a/src/query/boost_query.rs b/src/query/boost_query.rs
index ecbf3d8d6..cc4c10f7a 100644
--- a/src/query/boost_query.rs
+++ b/src/query/boost_query.rs
@@ -134,6 +134,7 @@ impl<S: Scorer> DocSet for BoostScorer<S> {
 }
 
 impl<S: Scorer> Scorer for BoostScorer<S> {
+    #[inline]
     fn score(&mut self) -> Score {
         self.underlying.score() * self.boost
     }
diff --git a/src/query/const_score_query.rs b/src/query/const_score_query.rs
index 570c7feca..d07e6a96f 100644
--- a/src/query/const_score_query.rs
+++ b/src/query/const_score_query.rs
@@ -137,6 +137,7 @@ impl<TDocSet: DocSet> DocSet for ConstScorer<TDocSet> {
 }
 
 impl<TDocSet: DocSet + 'static> Scorer for ConstScorer<TDocSet> {
+    #[inline]
     fn score(&mut self) -> Score {
         self.score
     }
diff --git a/src/query/disjunction.rs b/src/query/disjunction.rs
index b2f1080fc..ca7eab20d 100644
--- a/src/query/disjunction.rs
+++ b/src/query/disjunction.rs
@@ -173,6 +173,7 @@ impl<TScorer: Scorer, TScoreCombiner: ScoreCombiner> DocSet
 impl<TScorer: Scorer, TScoreCombiner: ScoreCombiner> Scorer
     for Disjunction<TScorer, TScoreCombiner>
 {
+    #[inline]
     fn score(&mut self) -> Score {
         self.current_score
     }
@@ -307,6 +308,7 @@ mod tests {
     }
 
     impl Scorer for DummyScorer {
+        #[inline]
         fn score(&mut self) -> Score {
             self.foo.get(self.cursor).map(|x| x.1).unwrap_or(0.0)
         }
diff --git a/src/query/empty_query.rs b/src/query/empty_query.rs
index 86ff84c08..2fa1772bd 100644
--- a/src/query/empty_query.rs
+++ b/src/query/empty_query.rs
@@ -55,6 +55,7 @@ impl DocSet for EmptyScorer {
 }
 
 impl Scorer for EmptyScorer {
+    #[inline]
     fn score(&mut self) -> Score {
         0.0
     }
diff --git a/src/query/exclude.rs b/src/query/exclude.rs
index 0b13e66e0..15e609c1e 100644
--- a/src/query/exclude.rs
+++ b/src/query/exclude.rs
@@ -84,6 +84,7 @@ where
     TScorer: Scorer,
     TDocSetExclude: DocSet + 'static,
 {
+    #[inline]
     fn score(&mut self) -> Score {
         self.underlying_docset.score()
     }
diff --git a/src/query/intersection.rs b/src/query/intersection.rs
index 3e8677d98..d536dcf05 100644
--- a/src/query/intersection.rs
+++ b/src/query/intersection.rs
@@ -105,6 +105,7 @@ impl<TDocSet: DocSet> Intersection<TDocSet, TDocSet> {
 }
 
 impl<TDocSet: DocSet, TOtherDocSet: DocSet> DocSet for Intersection<TDocSet, TOtherDocSet> {
+    #[inline]
     fn advance(&mut self) -> DocId {
         let (left, right) = (&mut self.left, &mut self.right);
         let mut candidate = left.advance();
@@ -174,6 +175,7 @@ impl<TDocSet: DocSet, TOtherDocSet: DocSet> DocSet for Intersection<TDocSet, TOt
                 .all(|docset| docset.seek_into_the_danger_zone(target))
     }
 
+    #[inline]
     fn doc(&self) -> DocId {
         self.left.doc()
     }
@@ -200,6 +202,7 @@ where
     TScorer: Scorer,
     TOtherScorer: Scorer,
 {
+    #[inline]
     fn score(&mut self) -> Score {
         self.left.score()
             + self.right.score()
diff --git a/src/query/phrase_prefix_query/phrase_prefix_scorer.rs b/src/query/phrase_prefix_query/phrase_prefix_scorer.rs
index cc7bb7886..8b03089fa 100644
--- a/src/query/phrase_prefix_query/phrase_prefix_scorer.rs
+++ b/src/query/phrase_prefix_query/phrase_prefix_scorer.rs
@@ -81,6 +81,7 @@ impl<TPostings: Postings> DocSet for PhraseKind<TPostings> {
 }
 
 impl<TPostings: Postings> Scorer for PhraseKind<TPostings> {
+    #[inline]
     fn score(&mut self) -> Score {
         match self {
             PhraseKind::SinglePrefix { positions, .. } => {
@@ -215,6 +216,7 @@ impl<TPostings: Postings> DocSet for PhrasePrefixScorer<TPostings> {
 }
 
 impl<TPostings: Postings> Scorer for PhrasePrefixScorer<TPostings> {
+    #[inline]
     fn score(&mut self) -> Score {
         // TODO modify score??
         self.phrase_scorer.score()
diff --git a/src/query/phrase_query/phrase_scorer.rs b/src/query/phrase_query/phrase_scorer.rs
index 4f8541cd2..108783b40 100644
--- a/src/query/phrase_query/phrase_scorer.rs
+++ b/src/query/phrase_query/phrase_scorer.rs
@@ -563,6 +563,7 @@ impl<TPostings: Postings> DocSet for PhraseScorer<TPostings> {
 }
 
 impl<TPostings: Postings> Scorer for PhraseScorer<TPostings> {
+    #[inline]
     fn score(&mut self) -> Score {
         let doc = self.doc();
         let fieldnorm_id = self.fieldnorm_reader.fieldnorm_id(doc);
diff --git a/src/query/reqopt_scorer.rs b/src/query/reqopt_scorer.rs
index 45857567c..bed99f5b7 100644
--- a/src/query/reqopt_scorer.rs
+++ b/src/query/reqopt_scorer.rs
@@ -81,6 +81,7 @@ where
     TOptScorer: Scorer,
     TScoreCombiner: ScoreCombiner,
 {
+    #[inline]
     fn score(&mut self) -> Score {
         if let Some(score) = self.score_cache {
             return score;
diff --git a/src/query/score_combiner.rs b/src/query/score_combiner.rs
index a49f8b104..2fe760c3d 100644
--- a/src/query/score_combiner.rs
+++ b/src/query/score_combiner.rs
@@ -29,6 +29,7 @@ impl ScoreCombiner for DoNothingCombiner {
 
     fn clear(&mut self) {}
 
+    #[inline]
     fn score(&self) -> Score {
         1.0
     }
@@ -49,6 +50,7 @@ impl ScoreCombiner for SumCombiner {
         self.score = 0.0;
     }
 
+    #[inline]
     fn score(&self) -> Score {
         self.score
     }
@@ -86,6 +88,7 @@ impl ScoreCombiner for DisjunctionMaxCombiner {
         self.sum = 0.0;
     }
 
+    #[inline]
     fn score(&self) -> Score {
         self.max + (self.sum - self.max) * self.tie_breaker
     }
diff --git a/src/query/scorer.rs b/src/query/scorer.rs
index 69448042f..e91fc2fbc 100644
--- a/src/query/scorer.rs
+++ b/src/query/scorer.rs
@@ -18,6 +18,7 @@ pub trait Scorer: downcast_rs::Downcast + DocSet + 'static {
 impl_downcast!(Scorer);
 
 impl Scorer for Box<dyn Scorer> {
+    #[inline]
     fn score(&mut self) -> Score {
         self.deref_mut().score()
     }
diff --git a/src/query/term_query/term_scorer.rs b/src/query/term_query/term_scorer.rs
index 293aa7871..00fb8ca0b 100644
--- a/src/query/term_query/term_scorer.rs
+++ b/src/query/term_query/term_scorer.rs
@@ -119,6 +119,7 @@ impl DocSet for TermScorer {
 }
 
 impl Scorer for TermScorer {
+    #[inline]
     fn score(&mut self) -> Score {
         let fieldnorm_id = self.fieldnorm_id();
         let term_freq = self.term_freq();
diff --git a/src/query/union/buffered_union.rs b/src/query/union/buffered_union.rs
index 70299ad6f..ee554e357 100644
--- a/src/query/union/buffered_union.rs
+++ b/src/query/union/buffered_union.rs
@@ -128,6 +128,7 @@ impl<TScorer: Scorer, TScoreCombiner: ScoreCombiner> BufferedUnionScorer<TScorer
         }
     }
 
+    #[inline]
     fn advance_buffered(&mut self) -> bool {
         while self.bucket_idx < HORIZON_NUM_TINYBITSETS {
             if let Some(val) = self.bitsets[self.bucket_idx].pop_lowest() {
@@ -156,6 +157,7 @@ where
     TScorer: Scorer,
     TScoreCombiner: ScoreCombiner,
 {
+    #[inline]
     fn advance(&mut self) -> DocId {
         if self.advance_buffered() {
             return self.doc;
@@ -245,6 +247,7 @@ where
         }
     }
 
+    #[inline]
     fn doc(&self) -> DocId {
         self.doc
     }
@@ -286,6 +289,7 @@ where
     TScoreCombiner: ScoreCombiner,
     TScorer: Scorer,
 {
+    #[inline]
     fn score(&mut self) -> Score {
         self.score
     }