Added seek_doc to intersections.

tantivy requires Scorer to be positioned on a DocId at all time.
This decision is not performance neutral.

When we have an intersection of a heavy DocSet with a lighter one
forcing the positioning of the first doc is needlessly expensive.

This PR fixes this by introducing a seek_doc parameter in the weight function.
Weights may skip over documents when they create the Scorer.
This commit is contained in:
Paul Masurel
2025-12-30 17:26:15 +01:00
parent 923f0508f2
commit 189882896f
27 changed files with 416 additions and 128 deletions

View File

@@ -446,7 +446,7 @@ impl DocumentQueryEvaluator {
let weight = query.weight(EnableScoring::disabled_from_schema(&schema))?;
// Get a scorer that iterates over matching documents
let mut scorer = weight.scorer(segment_reader, 1.0)?;
let mut scorer = weight.scorer(segment_reader, 1.0, 0)?;
// Create a BitSet to hold all matching documents
let mut bitset = BitSet::with_max_value(max_doc);

View File

@@ -42,6 +42,7 @@ pub trait DocSet: Send {
/// Calling `seek(TERMINATED)` is also legal and is the normal way to consume a `DocSet`.
///
/// `target` has to be larger or equal to `.doc()` when calling `seek`.
/// If `target` is equal to `.doc()` then the DocSet should not advance.
fn seek(&mut self, target: DocId) -> DocId {
let mut doc = self.doc();
debug_assert!(doc <= target);
@@ -164,6 +165,18 @@ pub trait DocSet: Send {
}
count
}
/// Consumes the `DocSet` and returns a Vec with all of the docs in the DocSet
/// including the current doc.
fn to_doc_vec(&mut self) -> Vec<DocId> {
let mut output = Vec::new();
let mut doc = self.doc();
while doc != TERMINATED {
output.push(doc);
doc = self.advance();
}
output
}
}
impl DocSet for &mut dyn DocSet {

View File

@@ -113,7 +113,7 @@ mod tests {
IndexRecordOption::WithFreqs,
);
let weight = query.weight(EnableScoring::enabled_from_searcher(&searcher))?;
let mut scorer = weight.scorer(searcher.segment_reader(0), 1.0f32)?;
let mut scorer = weight.scorer(searcher.segment_reader(0), 1.0f32, 0)?;
assert_eq!(scorer.doc(), 0);
assert!((scorer.score() - 0.22920431).abs() < 0.001f32);
assert_eq!(scorer.advance(), 1);
@@ -142,7 +142,7 @@ mod tests {
IndexRecordOption::WithFreqs,
);
let weight = query.weight(EnableScoring::enabled_from_searcher(&searcher))?;
let mut scorer = weight.scorer(searcher.segment_reader(0), 1.0f32)?;
let mut scorer = weight.scorer(searcher.segment_reader(0), 1.0f32, 0)?;
assert_eq!(scorer.doc(), 0);
assert!((scorer.score() - 0.22920431).abs() < 0.001f32);
assert_eq!(scorer.advance(), 1);

View File

@@ -14,6 +14,7 @@ use crate::positions::PositionReader;
use crate::postings::{BlockSegmentPostings, SegmentPostings, TermInfo};
use crate::schema::{IndexRecordOption, Term, Type};
use crate::termdict::TermDictionary;
use crate::DocId;
/// The inverted index reader is in charge of accessing
/// the inverted index associated with a specific field.
@@ -193,7 +194,7 @@ impl InvertedIndexReader {
option: IndexRecordOption,
) -> io::Result<Option<BlockSegmentPostings>> {
self.get_term_info(term)?
.map(move |term_info| self.read_block_postings_from_terminfo(&term_info, option))
.map(move |term_info| self.read_block_postings_from_terminfo(&term_info, option, 0))
.transpose()
}
@@ -205,6 +206,7 @@ impl InvertedIndexReader {
&self,
term_info: &TermInfo,
requested_option: IndexRecordOption,
seek_doc: DocId,
) -> io::Result<BlockSegmentPostings> {
let postings_data = self
.postings_file_slice
@@ -214,6 +216,7 @@ impl InvertedIndexReader {
postings_data,
self.record_option,
requested_option,
seek_doc,
)
}
@@ -225,10 +228,11 @@ impl InvertedIndexReader {
&self,
term_info: &TermInfo,
option: IndexRecordOption,
seek_doc: DocId,
) -> io::Result<SegmentPostings> {
let option = option.downgrade(self.record_option);
let block_postings = self.read_block_postings_from_terminfo(term_info, option)?;
let block_postings = self.read_block_postings_from_terminfo(term_info, option, seek_doc)?;
let position_reader = {
if option.has_positions() {
let positions_data = self
@@ -268,7 +272,7 @@ impl InvertedIndexReader {
option: IndexRecordOption,
) -> io::Result<Option<SegmentPostings>> {
self.get_term_info(term)?
.map(move |term_info| self.read_postings_from_terminfo(&term_info, option))
.map(move |term_info| self.read_postings_from_terminfo(&term_info, option, 0u32))
.transpose()
}

View File

@@ -256,7 +256,12 @@ mod tests {
struct DummyWeight;
impl Weight for DummyWeight {
fn scorer(&self, _reader: &SegmentReader, _boost: Score) -> crate::Result<Box<dyn Scorer>> {
fn scorer(
&self,
_reader: &SegmentReader,
_boost: Score,
_seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
Err(crate::TantivyError::InternalError("dummy impl".to_owned()))
}

View File

@@ -367,8 +367,11 @@ impl IndexMerger {
for (segment_ord, term_info) in merged_terms.current_segment_ords_and_term_infos() {
let segment_reader = &self.readers[segment_ord];
let inverted_index: &InvertedIndexReader = &field_readers[segment_ord];
let segment_postings = inverted_index
.read_postings_from_terminfo(&term_info, segment_postings_option)?;
let segment_postings = inverted_index.read_postings_from_terminfo(
&term_info,
segment_postings_option,
0u32,
)?;
let alive_bitset_opt = segment_reader.alive_bitset();
let doc_freq = if let Some(alive_bitset) = alive_bitset_opt {
segment_postings.doc_freq_given_deletes(alive_bitset)

View File

@@ -99,6 +99,7 @@ impl BlockSegmentPostings {
data: FileSlice,
mut record_option: IndexRecordOption,
requested_option: IndexRecordOption,
seek_doc: DocId,
) -> io::Result<BlockSegmentPostings> {
let bytes = data.read_bytes()?;
let (skip_data_opt, postings_data) = split_into_skips_and_postings(doc_freq, bytes)?;
@@ -135,6 +136,7 @@ impl BlockSegmentPostings {
data: postings_data,
skip_reader,
};
block_segment_postings.seek(seek_doc);
block_segment_postings.load_block();
Ok(block_segment_postings)
}
@@ -499,8 +501,11 @@ mod tests {
let inverted_index = segment_reader.inverted_index(int_field).unwrap();
let term = Term::from_field_u64(int_field, 0u64);
let term_info = inverted_index.get_term_info(&term)?.unwrap();
let block_postings = inverted_index
.read_block_postings_from_terminfo(&term_info, IndexRecordOption::Basic)?;
let block_postings = inverted_index.read_block_postings_from_terminfo(
&term_info,
IndexRecordOption::Basic,
0,
)?;
Ok(block_postings)
}
@@ -544,8 +549,11 @@ mod tests {
let term = Term::from_field_u64(int_field, 0u64);
let inverted_index = segment_reader.inverted_index(int_field)?;
let term_info = inverted_index.get_term_info(&term)?.unwrap();
block_segments = inverted_index
.read_block_postings_from_terminfo(&term_info, IndexRecordOption::Basic)?;
block_segments = inverted_index.read_block_postings_from_terminfo(
&term_info,
IndexRecordOption::Basic,
0,
)?;
}
assert_eq!(block_segments.docs(), &[0, 2, 4]);
{

View File

@@ -84,6 +84,7 @@ impl SegmentPostings {
FileSlice::from(buffer),
IndexRecordOption::Basic,
IndexRecordOption::Basic,
0u32,
)
.unwrap();
SegmentPostings::from_block_postings(block_segment_postings, None)
@@ -132,6 +133,7 @@ impl SegmentPostings {
FileSlice::from(buffer),
IndexRecordOption::WithFreqs,
IndexRecordOption::WithFreqs,
0u32,
)
.unwrap();
SegmentPostings::from_block_postings(block_segment_postings, None)

View File

@@ -21,7 +21,12 @@ impl Query for AllQuery {
pub struct AllWeight;
impl Weight for AllWeight {
fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
fn scorer(
&self,
reader: &SegmentReader,
boost: Score,
_seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
let all_scorer = AllScorer::new(reader.max_doc());
if boost != 1.0 {
Ok(Box::new(BoostScorer::new(all_scorer, boost)))
@@ -140,7 +145,7 @@ mod tests {
let weight = AllQuery.weight(EnableScoring::disabled_from_schema(&index.schema()))?;
{
let reader = searcher.segment_reader(0);
let mut scorer = weight.scorer(reader, 1.0)?;
let mut scorer = weight.scorer(reader, 1.0, 0)?;
assert_eq!(scorer.doc(), 0u32);
assert_eq!(scorer.advance(), 1u32);
assert_eq!(scorer.doc(), 1u32);
@@ -148,7 +153,7 @@ mod tests {
}
{
let reader = searcher.segment_reader(1);
let mut scorer = weight.scorer(reader, 1.0)?;
let mut scorer = weight.scorer(reader, 1.0, 0)?;
assert_eq!(scorer.doc(), 0u32);
assert_eq!(scorer.advance(), TERMINATED);
}
@@ -163,12 +168,12 @@ mod tests {
let weight = AllQuery.weight(EnableScoring::disabled_from_schema(searcher.schema()))?;
let reader = searcher.segment_reader(0);
{
let mut scorer = weight.scorer(reader, 2.0)?;
let mut scorer = weight.scorer(reader, 2.0, 0)?;
assert_eq!(scorer.doc(), 0u32);
assert_eq!(scorer.score(), 2.0);
}
{
let mut scorer = weight.scorer(reader, 1.5)?;
let mut scorer = weight.scorer(reader, 1.5, 0)?;
assert_eq!(scorer.doc(), 0u32);
assert_eq!(scorer.score(), 1.5);
}

View File

@@ -84,7 +84,12 @@ where
A: Automaton + Send + Sync + 'static,
A::State: Clone,
{
fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
fn scorer(
&self,
reader: &SegmentReader,
boost: Score,
seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
let max_doc = reader.max_doc();
let mut doc_bitset = BitSet::with_max_value(max_doc);
let inverted_index = reader.inverted_index(self.field)?;
@@ -92,8 +97,11 @@ where
let mut term_stream = self.automaton_stream(term_dict)?;
while term_stream.advance() {
let term_info = term_stream.value();
let mut block_segment_postings = inverted_index
.read_block_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
let mut block_segment_postings = inverted_index.read_block_postings_from_terminfo(
term_info,
IndexRecordOption::Basic,
seek_doc,
)?;
loop {
let docs = block_segment_postings.docs();
if docs.is_empty() {
@@ -111,7 +119,7 @@ where
}
fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
let mut scorer = self.scorer(reader, 1.0)?;
let mut scorer = self.scorer(reader, 1.0, 0)?;
if scorer.seek(doc) == doc {
Ok(Explanation::new("AutomatonScorer", 1.0))
} else {
@@ -186,7 +194,7 @@ mod tests {
let automaton_weight = AutomatonWeight::new(field, PrefixedByA);
let reader = index.reader()?;
let searcher = reader.searcher();
let mut scorer = automaton_weight.scorer(searcher.segment_reader(0u32), 1.0)?;
let mut scorer = automaton_weight.scorer(searcher.segment_reader(0u32), 1.0, 0)?;
assert_eq!(scorer.doc(), 0u32);
assert_eq!(scorer.score(), 1.0);
assert_eq!(scorer.advance(), 2u32);
@@ -203,7 +211,7 @@ mod tests {
let automaton_weight = AutomatonWeight::new(field, PrefixedByA);
let reader = index.reader()?;
let searcher = reader.searcher();
let mut scorer = automaton_weight.scorer(searcher.segment_reader(0u32), 1.32)?;
let mut scorer = automaton_weight.scorer(searcher.segment_reader(0u32), 1.32, 0)?;
assert_eq!(scorer.doc(), 0u32);
assert_eq!(scorer.score(), 1.32);
Ok(())

View File

@@ -12,7 +12,7 @@ use crate::query::{
intersect_scorers, AllScorer, BufferedUnionScorer, EmptyScorer, Exclude, Explanation, Occur,
RequiredOptionalScorer, Scorer, Weight,
};
use crate::{DocId, Score};
use crate::{DocId, Score, TERMINATED};
enum SpecializedScorer {
TermUnion(Vec<TermScorer>),
@@ -207,10 +207,28 @@ impl<TScoreCombiner: ScoreCombiner> BooleanWeight<TScoreCombiner> {
&self,
reader: &SegmentReader,
boost: Score,
mut seek_first_doc: DocId,
) -> crate::Result<HashMap<Occur, Vec<Box<dyn Scorer>>>> {
let mut per_occur_scorers: HashMap<Occur, Vec<Box<dyn Scorer>>> = HashMap::new();
for (occur, subweight) in &self.weights {
let sub_scorer: Box<dyn Scorer> = subweight.scorer(reader, boost)?;
let (mut must_weights, other_weights): (Vec<(Occur, _)>, Vec<(Occur, _)>) = self
.weights
.iter()
.map(|(occur, weight)| (*occur, weight))
.partition(|(occur, _weight)| *occur == Occur::Must);
// We start by must weights in order to get the best "seek_first_doc" so that we
// can skip the first few documents of the other scorers.
must_weights.sort_by_key(|weight| weight.1.intersection_priority());
for (_, must_sub_weight) in must_weights {
let sub_scorer: Box<dyn Scorer> =
must_sub_weight.scorer(reader, boost, seek_first_doc)?;
seek_first_doc = seek_first_doc.max(sub_scorer.doc());
per_occur_scorers
.entry(Occur::Must)
.or_default()
.push(sub_scorer);
}
for (occur, sub_weight) in &other_weights {
let sub_scorer: Box<dyn Scorer> = sub_weight.scorer(reader, boost, seek_first_doc)?;
per_occur_scorers
.entry(*occur)
.or_default()
@@ -224,9 +242,10 @@ impl<TScoreCombiner: ScoreCombiner> BooleanWeight<TScoreCombiner> {
reader: &SegmentReader,
boost: Score,
score_combiner_fn: impl Fn() -> TComplexScoreCombiner,
seek_doc: u32,
) -> crate::Result<SpecializedScorer> {
let num_docs = reader.num_docs();
let mut per_occur_scorers = self.per_occur_scorers(reader, boost)?;
let mut per_occur_scorers = self.per_occur_scorers(reader, boost, seek_doc)?;
// Indicate how should clauses are combined with must clauses.
let mut must_scorers: Vec<Box<dyn Scorer>> =
@@ -407,7 +426,7 @@ fn remove_and_count_all_and_empty_scorers(
if scorer.is::<AllScorer>() {
counts.num_all_scorers += 1;
false
} else if scorer.is::<EmptyScorer>() {
} else if scorer.doc() == TERMINATED {
counts.num_empty_scorers += 1;
false
} else {
@@ -418,7 +437,12 @@ fn remove_and_count_all_and_empty_scorers(
}
impl<TScoreCombiner: ScoreCombiner + Sync> Weight for BooleanWeight<TScoreCombiner> {
fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
fn scorer(
&self,
reader: &SegmentReader,
boost: Score,
seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
let num_docs = reader.num_docs();
if self.weights.is_empty() {
Ok(Box::new(EmptyScorer))
@@ -427,15 +451,15 @@ impl<TScoreCombiner: ScoreCombiner + Sync> Weight for BooleanWeight<TScoreCombin
if occur == Occur::MustNot {
Ok(Box::new(EmptyScorer))
} else {
weight.scorer(reader, boost)
weight.scorer(reader, boost, seek_doc)
}
} else if self.scoring_enabled {
self.complex_scorer(reader, boost, &self.score_combiner_fn)
self.complex_scorer(reader, boost, &self.score_combiner_fn, seek_doc)
.map(|specialized_scorer| {
into_box_scorer(specialized_scorer, &self.score_combiner_fn, num_docs)
})
} else {
self.complex_scorer(reader, boost, DoNothingCombiner::default)
self.complex_scorer(reader, boost, DoNothingCombiner::default, seek_doc)
.map(|specialized_scorer| {
into_box_scorer(specialized_scorer, DoNothingCombiner::default, num_docs)
})
@@ -443,7 +467,7 @@ impl<TScoreCombiner: ScoreCombiner + Sync> Weight for BooleanWeight<TScoreCombin
}
fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
let mut scorer = self.scorer(reader, 1.0)?;
let mut scorer = self.scorer(reader, 1.0, 0)?;
if scorer.seek(doc) != doc {
return Err(does_not_match(doc));
}
@@ -467,7 +491,7 @@ impl<TScoreCombiner: ScoreCombiner + Sync> Weight for BooleanWeight<TScoreCombin
reader: &SegmentReader,
callback: &mut dyn FnMut(DocId, Score),
) -> crate::Result<()> {
let scorer = self.complex_scorer(reader, 1.0, &self.score_combiner_fn)?;
let scorer = self.complex_scorer(reader, 1.0, &self.score_combiner_fn, 0)?;
match scorer {
SpecializedScorer::TermUnion(term_scorers) => {
let mut union_scorer = BufferedUnionScorer::build(
@@ -489,7 +513,7 @@ impl<TScoreCombiner: ScoreCombiner + Sync> Weight for BooleanWeight<TScoreCombin
reader: &SegmentReader,
callback: &mut dyn FnMut(&[DocId]),
) -> crate::Result<()> {
let scorer = self.complex_scorer(reader, 1.0, || DoNothingCombiner)?;
let scorer = self.complex_scorer(reader, 1.0, || DoNothingCombiner, 0u32)?;
let mut buffer = [0u32; COLLECT_BLOCK_BUFFER_LEN];
match scorer {
@@ -524,7 +548,7 @@ impl<TScoreCombiner: ScoreCombiner + Sync> Weight for BooleanWeight<TScoreCombin
reader: &SegmentReader,
callback: &mut dyn FnMut(DocId, Score) -> Score,
) -> crate::Result<()> {
let scorer = self.complex_scorer(reader, 1.0, &self.score_combiner_fn)?;
let scorer = self.complex_scorer(reader, 1.0, &self.score_combiner_fn, 0u32)?;
match scorer {
SpecializedScorer::TermUnion(term_scorers) => {
super::block_wand(term_scorers, threshold, callback);

View File

@@ -57,7 +57,7 @@ mod tests {
let query = query_parser.parse_query("+a")?;
let searcher = index.reader()?.searcher();
let weight = query.weight(EnableScoring::enabled_from_searcher(&searcher))?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0)?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0, 0)?;
assert!(scorer.is::<TermScorer>());
Ok(())
}
@@ -70,13 +70,13 @@ mod tests {
{
let query = query_parser.parse_query("+a +b +c")?;
let weight = query.weight(EnableScoring::enabled_from_searcher(&searcher))?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0)?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0, 0)?;
assert!(scorer.is::<Intersection<TermScorer>>());
}
{
let query = query_parser.parse_query("+a +(b c)")?;
let weight = query.weight(EnableScoring::enabled_from_searcher(&searcher))?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0)?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0, 0)?;
assert!(scorer.is::<Intersection<Box<dyn Scorer>>>());
}
Ok(())
@@ -90,14 +90,14 @@ mod tests {
{
let query = query_parser.parse_query("+a b")?;
let weight = query.weight(EnableScoring::enabled_from_searcher(&searcher))?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0)?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0, 0)?;
assert!(scorer
.is::<RequiredOptionalScorer<Box<dyn Scorer>, Box<dyn Scorer>, SumCombiner>>());
}
{
let query = query_parser.parse_query("+a b")?;
let weight = query.weight(EnableScoring::disabled_from_schema(searcher.schema()))?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0)?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0, 0)?;
assert!(scorer.is::<TermScorer>());
}
Ok(())
@@ -244,12 +244,14 @@ mod tests {
.weight(EnableScoring::enabled_from_searcher(&searcher))
.unwrap();
{
let mut boolean_scorer = boolean_weight.scorer(searcher.segment_reader(0u32), 1.0)?;
let mut boolean_scorer =
boolean_weight.scorer(searcher.segment_reader(0u32), 1.0, 0)?;
assert_eq!(boolean_scorer.doc(), 0u32);
assert_nearly_equals!(boolean_scorer.score(), 0.84163445);
}
{
let mut boolean_scorer = boolean_weight.scorer(searcher.segment_reader(0u32), 2.0)?;
let mut boolean_scorer =
boolean_weight.scorer(searcher.segment_reader(0u32), 2.0, 0)?;
assert_eq!(boolean_scorer.doc(), 0u32);
assert_nearly_equals!(boolean_scorer.score(), 1.6832689);
}
@@ -343,7 +345,7 @@ mod tests {
(Occur::Must, term_match_some.box_clone()),
]);
let weight = query.weight(EnableScoring::disabled_from_searcher(&searcher))?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0f32)?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0f32, 0)?;
assert!(scorer.is::<TermScorer>());
}
{
@@ -353,7 +355,7 @@ mod tests {
(Occur::Must, term_match_none.box_clone()),
]);
let weight = query.weight(EnableScoring::disabled_from_searcher(&searcher))?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0f32)?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0f32, 0)?;
assert!(scorer.is::<EmptyScorer>());
}
{
@@ -362,7 +364,7 @@ mod tests {
(Occur::Should, term_match_none.box_clone()),
]);
let weight = query.weight(EnableScoring::disabled_from_searcher(&searcher))?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0f32)?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0f32, 0)?;
assert!(scorer.is::<AllScorer>());
}
{
@@ -371,7 +373,7 @@ mod tests {
(Occur::Should, term_match_none.box_clone()),
]);
let weight = query.weight(EnableScoring::disabled_from_searcher(&searcher))?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0f32)?;
let scorer = weight.scorer(searcher.segment_reader(0u32), 1.0f32, 0)?;
assert!(scorer.is::<TermScorer>());
}
Ok(())

View File

@@ -67,8 +67,13 @@ impl BoostWeight {
}
impl Weight for BoostWeight {
fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
self.weight.scorer(reader, boost * self.boost)
fn scorer(
&self,
reader: &SegmentReader,
boost: Score,
seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
self.weight.scorer(reader, boost * self.boost, seek_doc)
}
fn explain(&self, reader: &SegmentReader, doc: u32) -> crate::Result<Explanation> {
@@ -83,6 +88,10 @@ impl Weight for BoostWeight {
fn count(&self, reader: &SegmentReader) -> crate::Result<u32> {
self.weight.count(reader)
}
fn intersection_priority(&self) -> u32 {
self.weight.intersection_priority()
}
}
pub(crate) struct BoostScorer<S: Scorer> {

View File

@@ -63,13 +63,18 @@ impl ConstWeight {
}
impl Weight for ConstWeight {
fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
let inner_scorer = self.weight.scorer(reader, boost)?;
fn scorer(
&self,
reader: &SegmentReader,
boost: Score,
seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
let inner_scorer = self.weight.scorer(reader, boost, seek_doc)?;
Ok(Box::new(ConstScorer::new(inner_scorer, boost * self.score)))
}
fn explain(&self, reader: &SegmentReader, doc: u32) -> crate::Result<Explanation> {
let mut scorer = self.scorer(reader, 1.0)?;
let mut scorer = self.scorer(reader, 1.0, 0)?;
if scorer.seek(doc) != doc {
return Err(TantivyError::InvalidArgument(format!(
"Document #({doc}) does not match"
@@ -84,6 +89,10 @@ impl Weight for ConstWeight {
fn count(&self, reader: &SegmentReader) -> crate::Result<u32> {
self.weight.count(reader)
}
fn intersection_priority(&self) -> u32 {
self.weight.intersection_priority()
}
}
/// Wraps a `DocSet` and simply returns a constant `Scorer`.

View File

@@ -26,7 +26,12 @@ impl Query for EmptyQuery {
/// It is useful for tests and handling edge cases.
pub struct EmptyWeight;
impl Weight for EmptyWeight {
fn scorer(&self, _reader: &SegmentReader, _boost: Score) -> crate::Result<Box<dyn Scorer>> {
fn scorer(
&self,
_reader: &SegmentReader,
_boost: Score,
_seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
Ok(Box::new(EmptyScorer))
}

View File

@@ -98,7 +98,12 @@ pub struct ExistsWeight {
}
impl Weight for ExistsWeight {
fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
fn scorer(
&self,
reader: &SegmentReader,
boost: Score,
_seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
let fast_field_reader = reader.fast_fields();
let mut column_handles = fast_field_reader.dynamic_column_handles(&self.field_name)?;
if self.field_type == Type::Json && self.json_subpaths {
@@ -166,7 +171,7 @@ impl Weight for ExistsWeight {
}
fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
let mut scorer = self.scorer(reader, 1.0)?;
let mut scorer = self.scorer(reader, 1.0, 0)?;
if scorer.seek(doc) != doc {
return Err(does_not_match(doc));
}

View File

@@ -42,10 +42,11 @@ impl PhrasePrefixWeight {
Ok(FieldNormReader::constant(reader.max_doc(), 1))
}
pub(crate) fn phrase_scorer(
pub(crate) fn prefix_phrase_scorer(
&self,
reader: &SegmentReader,
boost: Score,
seek_doc: DocId,
) -> crate::Result<Option<PhrasePrefixScorer<SegmentPostings>>> {
let similarity_weight_opt = self
.similarity_weight_opt
@@ -54,14 +55,16 @@ impl PhrasePrefixWeight {
let fieldnorm_reader = self.fieldnorm_reader(reader)?;
let mut term_postings_list = Vec::new();
for &(offset, ref term) in &self.phrase_terms {
if let Some(postings) = reader
.inverted_index(term.field())?
.read_postings(term, IndexRecordOption::WithFreqsAndPositions)?
{
term_postings_list.push((offset, postings));
} else {
let inverted_index = reader.inverted_index(term.field())?;
let Some(term_info) = inverted_index.get_term_info(term)? else {
return Ok(None);
}
};
let postings = inverted_index.read_postings_from_terminfo(
&term_info,
IndexRecordOption::WithFreqsAndPositions,
seek_doc,
)?;
term_postings_list.push((offset, postings));
}
let inv_index = reader.inverted_index(self.prefix.1.field())?;
@@ -114,8 +117,13 @@ impl PhrasePrefixWeight {
}
impl Weight for PhrasePrefixWeight {
fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
if let Some(scorer) = self.phrase_scorer(reader, boost)? {
fn scorer(
&self,
reader: &SegmentReader,
boost: Score,
seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
if let Some(scorer) = self.prefix_phrase_scorer(reader, boost, seek_doc)? {
Ok(Box::new(scorer))
} else {
Ok(Box::new(EmptyScorer))
@@ -123,7 +131,7 @@ impl Weight for PhrasePrefixWeight {
}
fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
let scorer_opt = self.phrase_scorer(reader, 1.0)?;
let scorer_opt = self.prefix_phrase_scorer(reader, 1.0, doc)?;
if scorer_opt.is_none() {
return Err(does_not_match(doc));
}
@@ -140,6 +148,10 @@ impl Weight for PhrasePrefixWeight {
}
Ok(explanation)
}
fn intersection_priority(&self) -> u32 {
30u32
}
}
#[cfg(test)]
@@ -187,7 +199,7 @@ mod tests {
.unwrap()
.unwrap();
let mut phrase_scorer = phrase_weight
.phrase_scorer(searcher.segment_reader(0u32), 1.0)?
.prefix_phrase_scorer(searcher.segment_reader(0u32), 1.0, 0u32)?
.unwrap();
assert_eq!(phrase_scorer.doc(), 1);
assert_eq!(phrase_scorer.phrase_count(), 2);
@@ -214,7 +226,7 @@ mod tests {
.unwrap()
.unwrap();
let mut phrase_scorer = phrase_weight
.phrase_scorer(searcher.segment_reader(0u32), 1.0)?
.prefix_phrase_scorer(searcher.segment_reader(0u32), 1.0, 0u32)?
.unwrap();
assert_eq!(phrase_scorer.doc(), 1);
assert_eq!(phrase_scorer.phrase_count(), 2);
@@ -238,7 +250,7 @@ mod tests {
.unwrap()
.is_none());
let weight = phrase_query.weight(enable_scoring).unwrap();
let mut phrase_scorer = weight.scorer(searcher.segment_reader(0u32), 1.0)?;
let mut phrase_scorer = weight.scorer(searcher.segment_reader(0u32), 1.0, 0)?;
assert_eq!(phrase_scorer.doc(), 1);
assert_eq!(phrase_scorer.advance(), 2);
assert_eq!(phrase_scorer.doc(), 2);
@@ -259,7 +271,7 @@ mod tests {
]);
let enable_scoring = EnableScoring::enabled_from_searcher(&searcher);
let weight = phrase_query.weight(enable_scoring).unwrap();
let mut phrase_scorer = weight.scorer(searcher.segment_reader(0u32), 1.0)?;
let mut phrase_scorer = weight.scorer(searcher.segment_reader(0u32), 1.0, 0)?;
assert_eq!(phrase_scorer.advance(), TERMINATED);
Ok(())
}

View File

@@ -84,7 +84,7 @@ pub(crate) mod tests {
let phrase_query = PhraseQuery::new(terms);
let phrase_weight =
phrase_query.phrase_weight(EnableScoring::disabled_from_schema(searcher.schema()))?;
let mut phrase_scorer = phrase_weight.scorer(searcher.segment_reader(0), 1.0)?;
let mut phrase_scorer = phrase_weight.scorer(searcher.segment_reader(0), 1.0, 0)?;
assert_eq!(phrase_scorer.doc(), 1);
assert_eq!(phrase_scorer.advance(), TERMINATED);
Ok(())
@@ -343,6 +343,43 @@ pub(crate) mod tests {
Ok(())
}
#[test]
pub fn test_phrase_weight_seek_doc() -> crate::Result<()> {
// Create an index with documents where the phrase "a b" appears in some of them.
// Documents: 0: "c d", 1: "a b", 2: "e f", 3: "a b c", 4: "g h", 5: "a b", 6: "i j"
let index = create_index(&["c d", "a b", "e f", "a b c", "g h", "a b", "i j"])?;
let text_field = index.schema().get_field("text").unwrap();
let searcher = index.reader()?.searcher();
let segment_reader = searcher.segment_reader(0);
let phrase_query = PhraseQuery::new(vec![
Term::from_field_text(text_field, "a"),
Term::from_field_text(text_field, "b"),
]);
let phrase_weight =
phrase_query.phrase_weight(EnableScoring::disabled_from_schema(searcher.schema()))?;
// Helper function to collect all docs from a scorer created with a given seek_doc
let docs_when_seeking_from = |seek_from: DocId| {
phrase_weight
.scorer(segment_reader, 1.0f32, seek_from)
.unwrap()
.to_doc_vec()
};
// Documents with "a b": 1, 3, 5
assert_eq!(docs_when_seeking_from(0), vec![1, 3, 5]);
assert_eq!(docs_when_seeking_from(1), vec![1, 3, 5]);
assert_eq!(docs_when_seeking_from(2), vec![3, 5]);
assert_eq!(docs_when_seeking_from(3), vec![3, 5]);
assert_eq!(docs_when_seeking_from(4), vec![5]);
assert_eq!(docs_when_seeking_from(5), vec![5]);
assert_eq!(docs_when_seeking_from(6), Vec::<DocId>::new());
assert_eq!(docs_when_seeking_from(7), Vec::<DocId>::new());
Ok(())
}
#[test]
pub fn test_phrase_query_on_json() -> crate::Result<()> {
let mut schema_builder = Schema::builder();
@@ -373,7 +410,7 @@ pub(crate) mod tests {
.weight(EnableScoring::disabled_from_schema(searcher.schema()))
.unwrap();
let mut phrase_scorer = phrase_weight
.scorer(searcher.segment_reader(0), 1.0f32)
.scorer(searcher.segment_reader(0), 1.0f32, 0)
.unwrap();
let mut docs = Vec::new();
loop {

View File

@@ -522,6 +522,8 @@ impl<TPostings: Postings> DocSet for PhraseScorer<TPostings> {
}
fn seek(&mut self, target: DocId) -> DocId {
dbg!(target);
dbg!(self.doc());
debug_assert!(target >= self.doc());
let doc = self.intersection_docset.seek(target);
if doc == TERMINATED || self.phrase_match() {

View File

@@ -43,6 +43,7 @@ impl PhraseWeight {
&self,
reader: &SegmentReader,
boost: Score,
seek_doc: DocId,
) -> crate::Result<Option<PhraseScorer<SegmentPostings>>> {
let similarity_weight_opt = self
.similarity_weight_opt
@@ -51,14 +52,16 @@ impl PhraseWeight {
let fieldnorm_reader = self.fieldnorm_reader(reader)?;
let mut term_postings_list = Vec::new();
for &(offset, ref term) in &self.phrase_terms {
if let Some(postings) = reader
.inverted_index(term.field())?
.read_postings(term, IndexRecordOption::WithFreqsAndPositions)?
{
term_postings_list.push((offset, postings));
} else {
let inverted_index = reader.inverted_index(term.field())?;
let Some(term_info) = inverted_index.get_term_info(term)? else {
return Ok(None);
}
};
let postings = inverted_index.read_postings_from_terminfo(
&term_info,
IndexRecordOption::WithFreqsAndPositions,
seek_doc,
)?;
term_postings_list.push((offset, postings));
}
Ok(Some(PhraseScorer::new(
term_postings_list,
@@ -74,8 +77,13 @@ impl PhraseWeight {
}
impl Weight for PhraseWeight {
fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
if let Some(scorer) = self.phrase_scorer(reader, boost)? {
fn scorer(
&self,
reader: &SegmentReader,
boost: Score,
seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
if let Some(scorer) = self.phrase_scorer(reader, boost, seek_doc)? {
Ok(Box::new(scorer))
} else {
Ok(Box::new(EmptyScorer))
@@ -83,12 +91,12 @@ impl Weight for PhraseWeight {
}
fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
let scorer_opt = self.phrase_scorer(reader, 1.0)?;
let scorer_opt = self.phrase_scorer(reader, 1.0, doc)?;
if scorer_opt.is_none() {
return Err(does_not_match(doc));
}
let mut scorer = scorer_opt.unwrap();
if scorer.seek(doc) != doc {
if scorer.doc() != doc {
return Err(does_not_match(doc));
}
let fieldnorm_reader = self.fieldnorm_reader(reader)?;
@@ -100,6 +108,10 @@ impl Weight for PhraseWeight {
}
Ok(explanation)
}
fn intersection_priority(&self) -> u32 {
10u32
}
}
#[cfg(test)]
@@ -122,7 +134,7 @@ mod tests {
let enable_scoring = EnableScoring::enabled_from_searcher(&searcher);
let phrase_weight = phrase_query.phrase_weight(enable_scoring).unwrap();
let mut phrase_scorer = phrase_weight
.phrase_scorer(searcher.segment_reader(0u32), 1.0)?
.phrase_scorer(searcher.segment_reader(0u32), 1.0, 0)?
.unwrap();
assert_eq!(phrase_scorer.doc(), 1);
assert_eq!(phrase_scorer.phrase_count(), 2);

View File

@@ -103,8 +103,11 @@ impl RegexPhraseWeight {
term_info: &TermInfo,
doc_bitset: &mut BitSet,
) -> crate::Result<()> {
let mut block_segment_postings = inverted_index
.read_block_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
let mut block_segment_postings = inverted_index.read_block_postings_from_terminfo(
term_info,
IndexRecordOption::Basic,
0u32,
)?;
loop {
let docs = block_segment_postings.docs();
if docs.is_empty() {
@@ -195,8 +198,11 @@ impl RegexPhraseWeight {
const SPARSE_TERM_DOC_THRESHOLD: u32 = 100;
for term_info in term_infos {
let mut term_posting = inverted_index
.read_postings_from_terminfo(term_info, IndexRecordOption::WithFreqsAndPositions)?;
let mut term_posting = inverted_index.read_postings_from_terminfo(
term_info,
IndexRecordOption::WithFreqsAndPositions,
0u32,
)?;
let num_docs = term_posting.doc_freq();
if num_docs < SPARSE_TERM_DOC_THRESHOLD {
@@ -269,7 +275,12 @@ impl RegexPhraseWeight {
}
impl Weight for RegexPhraseWeight {
fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
fn scorer(
&self,
reader: &SegmentReader,
boost: Score,
_seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
if let Some(scorer) = self.phrase_scorer(reader, boost)? {
Ok(Box::new(scorer))
} else {

View File

@@ -61,7 +61,11 @@ pub(crate) struct RangeDocSet<T> {
const DEFAULT_FETCH_HORIZON: u32 = 128;
impl<T: Send + Sync + PartialOrd + Copy + Debug + 'static> RangeDocSet<T> {
pub(crate) fn new(value_range: RangeInclusive<T>, column: Column<T>) -> Self {
pub(crate) fn new(
value_range: RangeInclusive<T>,
column: Column<T>,
seek_first_doc: DocId,
) -> Self {
if *value_range.start() > column.max_value() || *value_range.end() < column.min_value() {
return Self {
value_range,
@@ -77,7 +81,7 @@ impl<T: Send + Sync + PartialOrd + Copy + Debug + 'static> RangeDocSet<T> {
value_range,
column,
loaded_docs: VecCursor::new(),
next_fetch_start: 0,
next_fetch_start: seek_first_doc,
fetch_horizon: DEFAULT_FETCH_HORIZON,
last_seek_pos_opt: None,
};

View File

@@ -212,7 +212,12 @@ impl InvertedIndexRangeWeight {
}
impl Weight for InvertedIndexRangeWeight {
fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
fn scorer(
&self,
reader: &SegmentReader,
boost: Score,
seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
let max_doc = reader.max_doc();
let mut doc_bitset = BitSet::with_max_value(max_doc);
@@ -228,8 +233,11 @@ impl Weight for InvertedIndexRangeWeight {
}
processed_count += 1;
let term_info = term_range.value();
let mut block_segment_postings = inverted_index
.read_block_postings_from_terminfo(term_info, IndexRecordOption::Basic)?;
let mut block_segment_postings = inverted_index.read_block_postings_from_terminfo(
term_info,
IndexRecordOption::Basic,
seek_doc,
)?;
loop {
let docs = block_segment_postings.docs();
if docs.is_empty() {
@@ -246,7 +254,7 @@ impl Weight for InvertedIndexRangeWeight {
}
fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
let mut scorer = self.scorer(reader, 1.0)?;
let mut scorer = self.scorer(reader, 1.0, 0)?;
if scorer.seek(doc) != doc {
return Err(does_not_match(doc));
}
@@ -686,7 +694,7 @@ mod tests {
.weight(EnableScoring::disabled_from_schema(&schema))
.unwrap();
let range_scorer = range_weight
.scorer(&searcher.segment_readers()[0], 1.0f32)
.scorer(&searcher.segment_readers()[0], 1.0f32, 0)
.unwrap();
range_scorer
};

View File

@@ -52,7 +52,12 @@ impl FastFieldRangeWeight {
}
impl Weight for FastFieldRangeWeight {
fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
fn scorer(
&self,
reader: &SegmentReader,
boost: Score,
seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
// Check if both bounds are Bound::Unbounded
if self.bounds.is_unbounded() {
return Ok(Box::new(AllScorer::new(reader.max_doc())));
@@ -109,11 +114,21 @@ impl Weight for FastFieldRangeWeight {
else {
return Ok(Box::new(EmptyScorer));
};
search_on_u64_ff(column, boost, BoundsRange::new(lower_bound, upper_bound))
}
Type::U64 | Type::I64 | Type::F64 => {
search_on_json_numerical_field(reader, &field_name, typ, bounds, boost)
search_on_u64_ff(
column,
boost,
BoundsRange::new(lower_bound, upper_bound),
seek_doc,
)
}
Type::U64 | Type::I64 | Type::F64 => search_on_json_numerical_field(
reader,
&field_name,
typ,
bounds,
boost,
seek_doc,
),
Type::Date => {
let fast_field_reader = reader.fast_fields();
let Some((column, _col_type)) = fast_field_reader
@@ -126,6 +141,7 @@ impl Weight for FastFieldRangeWeight {
column,
boost,
BoundsRange::new(bounds.lower_bound, bounds.upper_bound),
seek_doc,
)
}
Type::Bool | Type::Facet | Type::Bytes | Type::Json | Type::IpAddr => {
@@ -154,7 +170,7 @@ impl Weight for FastFieldRangeWeight {
ip_addr_column.min_value(),
ip_addr_column.max_value(),
);
let docset = RangeDocSet::new(value_range, ip_addr_column);
let docset = RangeDocSet::new(value_range, ip_addr_column, seek_doc);
Ok(Box::new(ConstScorer::new(docset, boost)))
} else if field_type.is_str() {
let Some(str_dict_column): Option<StrColumn> = reader.fast_fields().str(&field_name)?
@@ -173,7 +189,12 @@ impl Weight for FastFieldRangeWeight {
else {
return Ok(Box::new(EmptyScorer));
};
search_on_u64_ff(column, boost, BoundsRange::new(lower_bound, upper_bound))
search_on_u64_ff(
column,
boost,
BoundsRange::new(lower_bound, upper_bound),
seek_doc,
)
} else {
assert!(
maps_to_u64_fastfield(field_type.value_type()),
@@ -215,12 +236,13 @@ impl Weight for FastFieldRangeWeight {
column,
boost,
BoundsRange::new(bounds.lower_bound, bounds.upper_bound),
seek_doc,
)
}
}
fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
let mut scorer = self.scorer(reader, 1.0)?;
let mut scorer = self.scorer(reader, 1.0, 0)?;
if scorer.seek(doc) != doc {
return Err(TantivyError::InvalidArgument(format!(
"Document #({doc}) does not match"
@@ -230,6 +252,10 @@ impl Weight for FastFieldRangeWeight {
Ok(explanation)
}
fn intersection_priority(&self) -> u32 {
20u32
}
}
/// On numerical fields the column type may not match the user provided one.
@@ -241,6 +267,7 @@ fn search_on_json_numerical_field(
typ: Type,
bounds: BoundsRange<ValueBytes<Vec<u8>>>,
boost: Score,
seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
// Since we don't know which type was interpolated for the internal column we
// have to check for all numeric types (only one exists)
@@ -318,6 +345,7 @@ fn search_on_json_numerical_field(
column,
boost,
BoundsRange::new(bounds.lower_bound, bounds.upper_bound),
seek_doc,
)
}
@@ -396,6 +424,7 @@ fn search_on_u64_ff(
column: Column<u64>,
boost: Score,
bounds: BoundsRange<u64>,
seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
let col_min_value = column.min_value();
let col_max_value = column.max_value();
@@ -426,7 +455,7 @@ fn search_on_u64_ff(
}
}
let docset = RangeDocSet::new(value_range, column);
let docset = RangeDocSet::new(value_range, column, seek_doc);
Ok(Box::new(ConstScorer::new(docset, boost)))
}
@@ -504,7 +533,7 @@ mod tests {
DateOptions, Field, NumericOptions, Schema, SchemaBuilder, FAST, INDEXED, STORED, STRING,
TEXT,
};
use crate::{Index, IndexWriter, TantivyDocument, Term, TERMINATED};
use crate::{DocId, Index, IndexWriter, TantivyDocument, Term, TERMINATED};
#[test]
fn test_text_field_ff_range_query() -> crate::Result<()> {
@@ -1142,11 +1171,52 @@ mod tests {
Bound::Included(Term::from_field_u64(field, 50_002)),
));
let scorer = range_query
.scorer(searcher.segment_reader(0), 1.0f32)
.scorer(searcher.segment_reader(0), 1.0f32, 0)
.unwrap();
assert_eq!(scorer.doc(), TERMINATED);
}
#[test]
fn test_fastfield_range_weight_seek_doc() {
let mut schema_builder = SchemaBuilder::new();
let field = schema_builder.add_u64_field("value", FAST);
let schema = schema_builder.build();
let index = Index::create_in_ram(schema);
let mut writer: IndexWriter = index.writer_for_tests().unwrap();
// Create 20 documents with values
// 0, 10, 20, ..., 90
// and then 50 again.
for i in 0..10 {
writer.add_document(doc!(field => (i * 10) as u64)).unwrap();
}
writer.add_document(doc!(field => 50u64)).unwrap();
writer.commit().unwrap();
let searcher = index.reader().unwrap().searcher();
let segment_reader = searcher.segment_reader(0);
let range_weight = FastFieldRangeWeight::new(BoundsRange::new(
Bound::Included(Term::from_field_u64(field, 30)),
Bound::Included(Term::from_field_u64(field, 70)),
));
let doc_when_seeking_from = |seek_from: DocId| {
range_weight
.scorer(segment_reader, 1.0f32, seek_from)
.unwrap()
.to_doc_vec()
};
assert_eq!(doc_when_seeking_from(0), vec![3, 4, 5, 6, 7, 10]);
assert_eq!(doc_when_seeking_from(1), vec![3, 4, 5, 6, 7, 10]);
assert_eq!(doc_when_seeking_from(3), vec![3, 4, 5, 6, 7, 10]);
assert_eq!(doc_when_seeking_from(7), vec![7, 10]);
assert_eq!(doc_when_seeking_from(8), vec![10]);
assert_eq!(doc_when_seeking_from(10), vec![10]);
assert_eq!(doc_when_seeking_from(11), Vec::<DocId>::new());
}
#[test]
fn range_regression3_test() {
let ops = vec![doc_from_id_1(1), doc_from_id_1(2), doc_from_id_1(3)];

View File

@@ -37,7 +37,7 @@ mod tests {
);
let term_weight = term_query.weight(EnableScoring::enabled_from_searcher(&searcher))?;
let segment_reader = searcher.segment_reader(0);
let mut term_scorer = term_weight.scorer(segment_reader, 1.0)?;
let mut term_scorer = term_weight.scorer(segment_reader, 1.0, 0)?;
assert_eq!(term_scorer.doc(), 0);
assert_nearly_equals!(term_scorer.score(), 0.28768212);
Ok(())
@@ -65,7 +65,7 @@ mod tests {
);
let term_weight = term_query.weight(EnableScoring::enabled_from_searcher(&searcher))?;
let segment_reader = searcher.segment_reader(0);
let mut term_scorer = term_weight.scorer(segment_reader, 1.0)?;
let mut term_scorer = term_weight.scorer(segment_reader, 1.0, 0)?;
for i in 0u32..COMPRESSION_BLOCK_SIZE as u32 {
assert_eq!(term_scorer.doc(), i);
if i == COMPRESSION_BLOCK_SIZE as u32 - 1u32 {
@@ -162,7 +162,7 @@ mod tests {
let searcher = index.reader()?.searcher();
let term_weight =
term_query.weight(EnableScoring::disabled_from_schema(searcher.schema()))?;
let mut term_scorer = term_weight.scorer(searcher.segment_reader(0u32), 1.0)?;
let mut term_scorer = term_weight.scorer(searcher.segment_reader(0u32), 1.0, 0)?;
assert_eq!(term_scorer.doc(), 0u32);
term_scorer.seek(1u32);
assert_eq!(term_scorer.doc(), 1u32);
@@ -470,7 +470,7 @@ mod tests {
.weight(EnableScoring::disabled_from_schema(&schema))
.unwrap();
term_weight
.scorer(searcher.segment_reader(0u32), 1.0f32)
.scorer(searcher.segment_reader(0u32), 1.0f32, 0)
.unwrap()
};
// Should be an allscorer
@@ -509,7 +509,7 @@ mod tests {
.weight(EnableScoring::enabled_from_searcher(&searcher))
.unwrap();
term_weight
.scorer(searcher.segment_reader(0u32), 1.0f32)
.scorer(searcher.segment_reader(0u32), 1.0f32, 0)
.unwrap()
};
// Should be an allscorer

View File

@@ -34,12 +34,19 @@ impl TermOrEmptyOrAllScorer {
}
impl Weight for TermWeight {
fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
Ok(self.specialized_scorer(reader, boost)?.into_boxed_scorer())
fn scorer(
&self,
reader: &SegmentReader,
boost: Score,
seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>> {
Ok(self
.specialized_scorer(reader, boost, seek_doc)?
.into_boxed_scorer())
}
fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
match self.specialized_scorer(reader, 1.0)? {
match self.specialized_scorer(reader, 1.0, doc)? {
TermOrEmptyOrAllScorer::TermScorer(mut term_scorer) => {
if term_scorer.doc() > doc || term_scorer.seek(doc) != doc {
return Err(does_not_match(doc));
@@ -55,7 +62,7 @@ impl Weight for TermWeight {
fn count(&self, reader: &SegmentReader) -> crate::Result<u32> {
if let Some(alive_bitset) = reader.alive_bitset() {
Ok(self.scorer(reader, 1.0)?.count(alive_bitset))
Ok(self.scorer(reader, 1.0, 0)?.count(alive_bitset))
} else {
let field = self.term.field();
let inv_index = reader.inverted_index(field)?;
@@ -71,7 +78,7 @@ impl Weight for TermWeight {
reader: &SegmentReader,
callback: &mut dyn FnMut(DocId, Score),
) -> crate::Result<()> {
match self.specialized_scorer(reader, 1.0)? {
match self.specialized_scorer(reader, 1.0, 0u32)? {
TermOrEmptyOrAllScorer::TermScorer(mut term_scorer) => {
for_each_scorer(&mut *term_scorer, callback);
}
@@ -90,7 +97,7 @@ impl Weight for TermWeight {
reader: &SegmentReader,
callback: &mut dyn FnMut(&[DocId]),
) -> crate::Result<()> {
match self.specialized_scorer(reader, 1.0)? {
match self.specialized_scorer(reader, 1.0, 0u32)? {
TermOrEmptyOrAllScorer::TermScorer(mut term_scorer) => {
let mut buffer = [0u32; COLLECT_BLOCK_BUFFER_LEN];
for_each_docset_buffered(&mut term_scorer, &mut buffer, callback);
@@ -121,7 +128,7 @@ impl Weight for TermWeight {
reader: &SegmentReader,
callback: &mut dyn FnMut(DocId, Score) -> Score,
) -> crate::Result<()> {
let specialized_scorer = self.specialized_scorer(reader, 1.0)?;
let specialized_scorer = self.specialized_scorer(reader, 1.0, 0u32)?;
match specialized_scorer {
TermOrEmptyOrAllScorer::TermScorer(term_scorer) => {
crate::query::boolean_query::block_wand_single_scorer(
@@ -169,7 +176,7 @@ impl TermWeight {
reader: &SegmentReader,
boost: Score,
) -> crate::Result<Option<TermScorer>> {
let scorer = self.specialized_scorer(reader, boost)?;
let scorer = self.specialized_scorer(reader, boost, 0u32)?;
Ok(match scorer {
TermOrEmptyOrAllScorer::TermScorer(scorer) => Some(*scorer),
_ => None,
@@ -180,6 +187,7 @@ impl TermWeight {
&self,
reader: &SegmentReader,
boost: Score,
seek_doc: DocId,
) -> crate::Result<TermOrEmptyOrAllScorer> {
let field = self.term.field();
let inverted_index = reader.inverted_index(field)?;
@@ -196,8 +204,11 @@ impl TermWeight {
)));
}
let segment_postings: SegmentPostings =
inverted_index.read_postings_from_terminfo(&term_info, self.index_record_option)?;
let segment_postings: SegmentPostings = inverted_index.read_postings_from_terminfo(
&term_info,
self.index_record_option,
seek_doc,
)?;
let fieldnorm_reader = self.fieldnorm_reader(reader)?;
let similarity_weight = self.similarity_weight.boost_by(boost);

View File

@@ -68,15 +68,28 @@ pub trait Weight: Send + Sync + 'static {
///
/// `boost` is a multiplier to apply to the score.
///
/// As an optimization, the scorer can be positioned on any document below `seek_doc`
/// matching the request.
/// If there are no such document, it should match the first document matching the request;
/// (or TERMINATED if no documents match).
///
/// Entirely ignoring that parameter and positionning the Scorer on the first document
/// is always correct.
///
/// See [`Query`](crate::query::Query).
fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>>;
fn scorer(
&self,
reader: &SegmentReader,
boost: Score,
seek_doc: DocId,
) -> crate::Result<Box<dyn Scorer>>;
/// Returns an [`Explanation`] for the given document.
fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation>;
/// Returns the number documents within the given [`SegmentReader`].
fn count(&self, reader: &SegmentReader) -> crate::Result<u32> {
let mut scorer = self.scorer(reader, 1.0)?;
let mut scorer = self.scorer(reader, 1.0, 0)?;
if let Some(alive_bitset) = reader.alive_bitset() {
Ok(scorer.count(alive_bitset))
} else {
@@ -91,7 +104,7 @@ pub trait Weight: Send + Sync + 'static {
reader: &SegmentReader,
callback: &mut dyn FnMut(DocId, Score),
) -> crate::Result<()> {
let mut scorer = self.scorer(reader, 1.0)?;
let mut scorer = self.scorer(reader, 1.0, 0)?;
for_each_scorer(scorer.as_mut(), callback);
Ok(())
}
@@ -103,7 +116,7 @@ pub trait Weight: Send + Sync + 'static {
reader: &SegmentReader,
callback: &mut dyn FnMut(&[DocId]),
) -> crate::Result<()> {
let mut docset = self.scorer(reader, 1.0)?;
let mut docset = self.scorer(reader, 1.0, 0)?;
let mut buffer = [0u32; COLLECT_BLOCK_BUFFER_LEN];
for_each_docset_buffered(&mut docset, &mut buffer, callback);
@@ -126,8 +139,14 @@ pub trait Weight: Send + Sync + 'static {
reader: &SegmentReader,
callback: &mut dyn FnMut(DocId, Score) -> Score,
) -> crate::Result<()> {
let mut scorer = self.scorer(reader, 1.0)?;
let mut scorer = self.scorer(reader, 1.0, 0)?;
for_each_pruning_scorer(scorer.as_mut(), threshold, callback);
Ok(())
}
/// Returns a priority number used to sort weights when running an
/// intersection.
fn intersection_priority(&self) -> u32 {
0u32
}
}