diff --git a/src/core/segment_reader.rs b/src/core/segment_reader.rs index ba05fb632..62a9347d8 100644 --- a/src/core/segment_reader.rs +++ b/src/core/segment_reader.rs @@ -19,7 +19,7 @@ use std::sync::Arc; use std::fmt; use schema::Field; use postings::SegmentPostingsOption; -use postings::SegmentPostings; +use postings::{SegmentPostings, BlockSegmentPostings}; use fastfield::{FastFieldsReader, FastFieldReader, U64FastFieldReader}; use schema::Schema; use schema::FieldType; @@ -219,6 +219,12 @@ impl SegmentReader { term_info: &TermInfo, option: SegmentPostingsOption) -> SegmentPostings { + let block_postings = self.read_block_postings_from_terminfo(term_info, option); + let delete_bitset = self.delete_bitset.clone(); + SegmentPostings::from_block_postings(block_postings, delete_bitset) + } + + pub fn read_block_postings_from_terminfo(&self, term_info: &TermInfo, option: SegmentPostingsOption) -> BlockSegmentPostings { let offset = term_info.postings_offset as usize; let postings_data = &self.postings_data[offset..]; let freq_handler = match option { @@ -230,34 +236,7 @@ impl SegmentReader { FreqHandler::new_with_freq_and_position(offseted_position_data) } }; - SegmentPostings::from_data(term_info.doc_freq, - postings_data, - &self.delete_bitset, - freq_handler) - } - - - /// Returns the posting list associated with a term. - /// - /// If the term is not found, return None. - /// Even when non-null, because of deletes, the posting object - /// returned by this method may contain no documents. - pub fn read_postings_all_info(&self, term: &Term) -> Option { - let field_entry = self.schema.get_field_entry(term.field()); - let segment_posting_option = match *field_entry.field_type() { - FieldType::Str(ref text_options) => { - match text_options.get_indexing_options() { - TextIndexingOptions::TokenizedWithFreq => SegmentPostingsOption::Freq, - TextIndexingOptions::TokenizedWithFreqAndPosition => { - SegmentPostingsOption::FreqAndPositions - } - _ => SegmentPostingsOption::NoFreq, - } - } - FieldType::U64(_) | - FieldType::I64(_) => SegmentPostingsOption::NoFreq, - }; - self.read_postings(term, segment_posting_option) + BlockSegmentPostings::from_data(term_info.doc_freq as usize, postings_data, freq_handler) } /// Returns the term info associated with the term. diff --git a/src/lib.rs b/src/lib.rs index bfd098a96..ea45de7c9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -207,6 +207,7 @@ mod tests { use schema::*; use DocSet; use IndexWriter; + use postings::SegmentPostingsOption::FreqAndPositions; use fastfield::{FastFieldReader, U64FastFieldReader, I64FastFieldReader}; use Postings; use rand::{XorShiftRng, Rng, SeedableRng}; @@ -338,6 +339,10 @@ mod tests { fn test_delete_postings1() { let mut schema_builder = SchemaBuilder::default(); let text_field = schema_builder.add_text_field("text", TEXT); + let term_abcd = Term::from_field_text(text_field, "abcd"); + let term_a = Term::from_field_text(text_field, "a"); + let term_b = Term::from_field_text(text_field, "b"); + let term_c = Term::from_field_text(text_field, "c"); let schema = schema_builder.build(); let index = Index::create_in_ram(schema); { @@ -386,11 +391,11 @@ mod tests { let searcher = index.searcher(); let reader = searcher.segment_reader(0); assert!(reader - .read_postings_all_info(&Term::from_field_text(text_field, "abcd")) - .is_none()); + .read_postings(&term_abcd, FreqAndPositions) + .is_none()); { let mut postings = reader - .read_postings_all_info(&Term::from_field_text(text_field, "a")) + .read_postings(&term_a, FreqAndPositions) .unwrap(); assert!(postings.advance()); assert_eq!(postings.doc(), 5); @@ -398,7 +403,7 @@ mod tests { } { let mut postings = reader - .read_postings_all_info(&Term::from_field_text(text_field, "b")) + .read_postings(&term_b, FreqAndPositions) .unwrap(); assert!(postings.advance()); assert_eq!(postings.doc(), 3); @@ -425,12 +430,13 @@ mod tests { index.load_searchers().unwrap(); let searcher = index.searcher(); let reader = searcher.segment_reader(0); + assert!(reader - .read_postings_all_info(&Term::from_field_text(text_field, "abcd")) - .is_none()); + .read_postings(&term_abcd, FreqAndPositions) + .is_none()); { let mut postings = reader - .read_postings_all_info(&Term::from_field_text(text_field, "a")) + .read_postings(&term_a, FreqAndPositions) .unwrap(); assert!(postings.advance()); assert_eq!(postings.doc(), 5); @@ -438,7 +444,7 @@ mod tests { } { let mut postings = reader - .read_postings_all_info(&Term::from_field_text(text_field, "b")) + .read_postings(&term_b, FreqAndPositions) .unwrap(); assert!(postings.advance()); assert_eq!(postings.doc(), 3); @@ -466,17 +472,17 @@ mod tests { let searcher = index.searcher(); let reader = searcher.segment_reader(0); assert!(reader - .read_postings_all_info(&Term::from_field_text(text_field, "abcd")) - .is_none()); + .read_postings(&term_abcd, FreqAndPositions) + .is_none()); { let mut postings = reader - .read_postings_all_info(&Term::from_field_text(text_field, "a")) + .read_postings(&term_a, FreqAndPositions) .unwrap(); assert!(!postings.advance()); } { let mut postings = reader - .read_postings_all_info(&Term::from_field_text(text_field, "b")) + .read_postings(&term_b, FreqAndPositions) .unwrap(); assert!(postings.advance()); assert_eq!(postings.doc(), 3); @@ -486,7 +492,7 @@ mod tests { } { let mut postings = reader - .read_postings_all_info(&Term::from_field_text(text_field, "c")) + .read_postings(&term_c, FreqAndPositions) .unwrap(); assert!(postings.advance()); assert_eq!(postings.doc(), 4); @@ -596,11 +602,13 @@ mod tests { index.load_searchers().unwrap(); let searcher = index.searcher(); let reader = searcher.segment_reader(0); + let term_abcd = Term::from_field_text(text_field, "abcd"); assert!(reader - .read_postings_all_info(&Term::from_field_text(text_field, "abcd")) - .is_none()); + .read_postings(&term_abcd, FreqAndPositions) + .is_none()); + let term_af = Term::from_field_text(text_field, "af"); let mut postings = reader - .read_postings_all_info(&Term::from_field_text(text_field, "af")) + .read_postings(&term_af, FreqAndPositions) .unwrap(); assert!(postings.advance()); assert_eq!(postings.doc(), 0); diff --git a/src/postings/mod.rs b/src/postings/mod.rs index 6dbb55ab0..483b7ed46 100644 --- a/src/postings/mod.rs +++ b/src/postings/mod.rs @@ -26,7 +26,7 @@ pub use self::postings::Postings; #[cfg(test)] pub use self::vec_postings::VecPostings; -pub use self::segment_postings::SegmentPostings; +pub use self::segment_postings::{SegmentPostings, BlockSegmentPostings}; pub use self::intersection::IntersectionDocSet; pub use self::freq_handler::FreqHandler; pub use self::segment_postings_option::SegmentPostingsOption; @@ -42,6 +42,7 @@ mod tests { use indexer::SegmentWriter; use core::SegmentReader; use core::Index; + use postings::SegmentPostingsOption::FreqAndPositions; use std::iter; use datastruct::stacker::Heap; use fastfield::FastFieldReader; @@ -129,11 +130,11 @@ mod tests { } { let term_a = Term::from_field_text(text_field, "abcdef"); - assert!(segment_reader.read_postings_all_info(&term_a).is_none()); + assert!(segment_reader.read_postings(&term_a, FreqAndPositions).is_none()); } { let term_a = Term::from_field_text(text_field, "a"); - let mut postings_a = segment_reader.read_postings_all_info(&term_a).unwrap(); + let mut postings_a = segment_reader.read_postings(&term_a, FreqAndPositions).unwrap(); assert_eq!(postings_a.len(), 1000); assert!(postings_a.advance()); assert_eq!(postings_a.doc(), 0); @@ -152,7 +153,7 @@ mod tests { } { let term_e = Term::from_field_text(text_field, "e"); - let mut postings_e = segment_reader.read_postings_all_info(&term_e).unwrap(); + let mut postings_e = segment_reader.read_postings(&term_e, FreqAndPositions).unwrap(); assert_eq!(postings_e.len(), 1000 - 2); for i in 2u32..1000u32 { assert!(postings_e.advance()); @@ -467,6 +468,7 @@ mod tests { }); } + fn bench_skip_next(p: f32, b: &mut Bencher) { let searcher = INDEX.searcher(); let segment_reader = searcher.segment_reader(0); @@ -475,6 +477,7 @@ mod tests { let mut segment_postings = segment_reader .read_postings(&*TERM_A, SegmentPostingsOption::NoFreq) .unwrap(); + let mut existing_docs = Vec::new(); for doc in &docs { if *doc >= segment_postings.doc() { @@ -490,6 +493,7 @@ mod tests { .read_postings(&*TERM_A, SegmentPostingsOption::NoFreq) .unwrap(); for doc in &existing_docs { + println!("doc {}", doc); if segment_postings.skip_next(*doc) == SkipResult::End { break; } diff --git a/src/postings/segment_postings.rs b/src/postings/segment_postings.rs index debc423c1..f917449b2 100644 --- a/src/postings/segment_postings.rs +++ b/src/postings/segment_postings.rs @@ -8,6 +8,8 @@ use fastfield::DeleteBitSet; const EMPTY_DATA: [u8; 0] = [0u8; 0]; + +/* /// `SegmentPostings` represents the inverted list or postings associated to /// a term in a `Segment`. /// @@ -85,11 +87,7 @@ impl<'a> SegmentPostings<'a> { } } - /// Index within a block is used as an address when - /// interacting with the `FreqHandler` - fn index_within_block(&self) -> usize { - self.cur.0 % NUM_DOCS_PER_BLOCK - } + /// Sets the current position to a location relative /// to the current block @@ -119,6 +117,107 @@ impl<'a> DocSet for SegmentPostings<'a> { } } + + + #[inline] + fn doc(&self) -> DocId { + self.block_decoder.output(self.index_within_block()) + } +} + +impl<'a> HasLen for SegmentPostings<'a> { + fn len(&self) -> usize { + self.len + } +} + +impl<'a> Postings for SegmentPostings<'a> { + fn term_freq(&self) -> u32 { + self.freq_handler.freq(self.index_within_block()) + } + + fn positions(&self) -> &[u32] { + self.freq_handler.positions(self.index_within_block()) + } +} + +*/ + + + +/// `SegmentPostings` represents the inverted list or postings associated to +/// a term in a `Segment`. +/// +/// As we iterate through the `SegmentPostings`, the frequencies are optionally decoded. +/// Positions on the other hand, are optionally entirely decoded upfront. +pub struct SegmentPostings<'a> { + len: usize, + cur: Wrapping, + block_cursor: BlockSegmentPostings<'a>, + cur_block_len: usize, + delete_bitset: DeleteBitSet, +} + +impl<'a> SegmentPostings<'a> { + + + /// Reads a Segment postings from an &[u8] + /// + /// * `len` - number of document in the posting lists. + /// * `data` - data array. The complete data is not necessarily used. + /// * `freq_handler` - the freq handler is in charge of decoding + /// frequencies and/or positions + pub fn from_block_postings( + segment_block_postings: BlockSegmentPostings<'a>, + delete_bitset: DeleteBitSet) -> SegmentPostings<'a> { + SegmentPostings { + len: segment_block_postings.len, + block_cursor: segment_block_postings, + cur: Wrapping(usize::max_value()), + cur_block_len: 0, + delete_bitset: delete_bitset, + } + } + + /// Returns an empty segment postings object + pub fn empty() -> SegmentPostings<'static> { + let empty_block_cursor = BlockSegmentPostings::empty(); + SegmentPostings { + len: 0, + block_cursor: empty_block_cursor, + delete_bitset: DeleteBitSet::empty(), + cur: Wrapping(usize::max_value()), + cur_block_len: 0, + } + } +} + + +impl<'a> DocSet for SegmentPostings<'a> { + // goes to the next element. + // next needs to be called a first time to point to the correct element. + #[inline] + fn advance(&mut self) -> bool { + loop { + self.cur += Wrapping(1); + assert!(self.cur.0 >= 0); + assert!(self.cur.0 <= self.cur_block_len); + if self.cur.0 == self.cur_block_len { + self.cur = Wrapping(0); + if !self.block_cursor.advance() { + self.cur_block_len = 0; + self.cur = Wrapping(usize::max_value()); + return false; + } + self.cur_block_len = self.block_cursor.docs().len(); + } + if !self.delete_bitset.is_deleted(self.doc()) { + return true; + } + } + } + + /* fn skip_next(&mut self, target: DocId) -> SkipResult { if !self.advance() { return SkipResult::End; @@ -197,10 +296,11 @@ impl<'a> DocSet for SegmentPostings<'a> { SkipResult::End } } - + */ + #[inline] fn doc(&self) -> DocId { - self.block_decoder.output(self.index_within_block()) + self.block_cursor.docs()[self.cur.0] } } @@ -212,10 +312,93 @@ impl<'a> HasLen for SegmentPostings<'a> { impl<'a> Postings for SegmentPostings<'a> { fn term_freq(&self) -> u32 { - self.freq_handler.freq(self.index_within_block()) + self.block_cursor.freq_handler().freq(self.cur.0) } fn positions(&self) -> &[u32] { - self.freq_handler.positions(self.index_within_block()) + self.block_cursor.freq_handler().positions(self.cur.0) } } + + + + +pub struct BlockSegmentPostings<'a> { + num_binpacked_blocks: usize, + num_vint_docs: usize, + block_decoder: BlockDecoder, + freq_handler: FreqHandler, + remaining_data: &'a [u8], + doc_offset: DocId, + len: usize, +} + +impl<'a> BlockSegmentPostings<'a> { + + pub fn from_data(len: usize, data: &'a [u8], freq_handler: FreqHandler) -> BlockSegmentPostings<'a> { + let num_binpacked_blocks: usize = (len as usize) / NUM_DOCS_PER_BLOCK; + let num_vint_docs = (len as usize) - NUM_DOCS_PER_BLOCK * num_binpacked_blocks; + BlockSegmentPostings { + num_binpacked_blocks: num_binpacked_blocks, + num_vint_docs: num_vint_docs, + block_decoder: BlockDecoder::new(), + freq_handler: freq_handler, + remaining_data: data, + doc_offset: 0, + len: len, + } + } + + pub fn reset(&mut self, len: usize, data: &'a [u8]) { + let num_binpacked_blocks: usize = (len as usize) / NUM_DOCS_PER_BLOCK; + let num_vint_docs = (len as usize) - NUM_DOCS_PER_BLOCK * num_binpacked_blocks; + self.num_binpacked_blocks = num_binpacked_blocks; + self.num_vint_docs = num_vint_docs; + self.remaining_data = data; + self.doc_offset = 0; + self.len = len; + } + + pub fn docs(&self) -> &[DocId] { + self.block_decoder.output_array() + } + + pub fn freq_handler(&self) -> &FreqHandler { + &self.freq_handler + } + + pub fn advance(&mut self) -> bool { + if self.num_binpacked_blocks > 0 { + self.remaining_data = self.block_decoder.uncompress_block_sorted(self.remaining_data, self.doc_offset); + self.remaining_data = self.freq_handler.read_freq_block(self.remaining_data); + self.doc_offset = self.block_decoder.output(NUM_DOCS_PER_BLOCK - 1); + self.num_binpacked_blocks -= 1; + true + } + else { + if self.num_vint_docs > 0 { + self.remaining_data = self.block_decoder.uncompress_vint_sorted(self.remaining_data, self.doc_offset, self.num_vint_docs); + self.freq_handler.read_freq_vint(self.remaining_data, self.num_vint_docs); + self.num_vint_docs = 0; + true + } + else { + false + } + } + } + + /// Returns an empty segment postings object + pub fn empty() -> BlockSegmentPostings<'static> { + BlockSegmentPostings { + num_binpacked_blocks: 0, + num_vint_docs: 0, + block_decoder: BlockDecoder::new(), + freq_handler: FreqHandler::new_without_freq(), + remaining_data: &EMPTY_DATA, + doc_offset: 0, + len: 0, + } + } + +}