issue/162 Create block postings

src/core/segment_reader.rs

@@ -19,7 +19,7 @@ use std::sync::Arc;
 use std::fmt;
 use schema::Field;
 use postings::SegmentPostingsOption;
-use postings::SegmentPostings;
+use postings::{SegmentPostings, BlockSegmentPostings};
 use fastfield::{FastFieldsReader, FastFieldReader, U64FastFieldReader};
 use schema::Schema;
 use schema::FieldType;
@@ -219,6 +219,12 @@ impl SegmentReader {
                                        term_info: &TermInfo,
                                        option: SegmentPostingsOption)
                                        -> SegmentPostings {
+        let block_postings = self.read_block_postings_from_terminfo(term_info, option);
+        let delete_bitset = self.delete_bitset.clone();
+        SegmentPostings::from_block_postings(block_postings, delete_bitset)
+    }
+
+    pub fn read_block_postings_from_terminfo(&self, term_info: &TermInfo, option: SegmentPostingsOption) -> BlockSegmentPostings {
         let offset = term_info.postings_offset as usize;
         let postings_data = &self.postings_data[offset..];
         let freq_handler = match option {
@@ -230,34 +236,7 @@ impl SegmentReader {
                 FreqHandler::new_with_freq_and_position(offseted_position_data)
             }
         };
-        SegmentPostings::from_data(term_info.doc_freq,
-                                   postings_data,
-                                   &self.delete_bitset,
-                                   freq_handler)
-    }
-
-
-    /// Returns the posting list associated with a term.
-    ///
-    /// If the term is not found, return None.
-    /// Even when non-null, because of deletes, the posting object
-    /// returned by this method may contain no documents.
-    pub fn read_postings_all_info(&self, term: &Term) -> Option<SegmentPostings> {
-        let field_entry = self.schema.get_field_entry(term.field());
-        let segment_posting_option = match *field_entry.field_type() {
-            FieldType::Str(ref text_options) => {
-                match text_options.get_indexing_options() {
-                    TextIndexingOptions::TokenizedWithFreq => SegmentPostingsOption::Freq,
-                    TextIndexingOptions::TokenizedWithFreqAndPosition => {
-                        SegmentPostingsOption::FreqAndPositions
-                    }
-                    _ => SegmentPostingsOption::NoFreq,
-                }
-            }
-            FieldType::U64(_) |
-            FieldType::I64(_) => SegmentPostingsOption::NoFreq,
-        };
-        self.read_postings(term, segment_posting_option)
+        BlockSegmentPostings::from_data(term_info.doc_freq as usize, postings_data, freq_handler)
     }
 
     /// Returns the term info associated with the term.

src/lib.rs

@@ -207,6 +207,7 @@ mod tests {
     use schema::*;
     use DocSet;
     use IndexWriter;
+    use postings::SegmentPostingsOption::FreqAndPositions;
     use fastfield::{FastFieldReader, U64FastFieldReader, I64FastFieldReader};
     use Postings;
     use rand::{XorShiftRng, Rng, SeedableRng};
@@ -338,6 +339,10 @@ mod tests {
     fn test_delete_postings1() {
         let mut schema_builder = SchemaBuilder::default();
         let text_field = schema_builder.add_text_field("text", TEXT);
+        let term_abcd = Term::from_field_text(text_field, "abcd");
+        let term_a = Term::from_field_text(text_field, "a");
+        let term_b = Term::from_field_text(text_field, "b");
+        let term_c = Term::from_field_text(text_field, "c");
         let schema = schema_builder.build();
         let index = Index::create_in_ram(schema);
         {
@@ -386,11 +391,11 @@ mod tests {
             let searcher = index.searcher();
             let reader = searcher.segment_reader(0);
             assert!(reader
-                        .read_postings_all_info(&Term::from_field_text(text_field, "abcd"))
-                        .is_none());
+                .read_postings(&term_abcd, FreqAndPositions)
+                .is_none());
             {
                 let mut postings = reader
-                    .read_postings_all_info(&Term::from_field_text(text_field, "a"))
+                    .read_postings(&term_a, FreqAndPositions)
                     .unwrap();
                 assert!(postings.advance());
                 assert_eq!(postings.doc(), 5);
@@ -398,7 +403,7 @@ mod tests {
             }
             {
                 let mut postings = reader
-                    .read_postings_all_info(&Term::from_field_text(text_field, "b"))
+                    .read_postings(&term_b, FreqAndPositions)
                     .unwrap();
                 assert!(postings.advance());
                 assert_eq!(postings.doc(), 3);
@@ -425,12 +430,13 @@ mod tests {
             index.load_searchers().unwrap();
             let searcher = index.searcher();
             let reader = searcher.segment_reader(0);
+            
             assert!(reader
-                        .read_postings_all_info(&Term::from_field_text(text_field, "abcd"))
-                        .is_none());
+                    .read_postings(&term_abcd, FreqAndPositions)
+                    .is_none());
             {
                 let mut postings = reader
-                    .read_postings_all_info(&Term::from_field_text(text_field, "a"))
+                    .read_postings(&term_a, FreqAndPositions)
                     .unwrap();
                 assert!(postings.advance());
                 assert_eq!(postings.doc(), 5);
@@ -438,7 +444,7 @@ mod tests {
             }
             {
                 let mut postings = reader
-                    .read_postings_all_info(&Term::from_field_text(text_field, "b"))
+                    .read_postings(&term_b, FreqAndPositions)
                     .unwrap();
                 assert!(postings.advance());
                 assert_eq!(postings.doc(), 3);
@@ -466,17 +472,17 @@ mod tests {
             let searcher = index.searcher();
             let reader = searcher.segment_reader(0);
             assert!(reader
-                        .read_postings_all_info(&Term::from_field_text(text_field, "abcd"))
-                        .is_none());
+                .read_postings(&term_abcd, FreqAndPositions)
+                .is_none());
             {
                 let mut postings = reader
-                    .read_postings_all_info(&Term::from_field_text(text_field, "a"))
+                    .read_postings(&term_a, FreqAndPositions)
                     .unwrap();
                 assert!(!postings.advance());
             }
             {
                 let mut postings = reader
-                    .read_postings_all_info(&Term::from_field_text(text_field, "b"))
+                    .read_postings(&term_b, FreqAndPositions)
                     .unwrap();
                 assert!(postings.advance());
                 assert_eq!(postings.doc(), 3);
@@ -486,7 +492,7 @@ mod tests {
             }
             {
                 let mut postings = reader
-                    .read_postings_all_info(&Term::from_field_text(text_field, "c"))
+                    .read_postings(&term_c, FreqAndPositions)
                     .unwrap();
                 assert!(postings.advance());
                 assert_eq!(postings.doc(), 4);
@@ -596,11 +602,13 @@ mod tests {
             index.load_searchers().unwrap();
             let searcher = index.searcher();
             let reader = searcher.segment_reader(0);
+            let term_abcd = Term::from_field_text(text_field, "abcd");
             assert!(reader
-                        .read_postings_all_info(&Term::from_field_text(text_field, "abcd"))
-                        .is_none());
+                    .read_postings(&term_abcd, FreqAndPositions)
+                    .is_none());
+            let term_af = Term::from_field_text(text_field, "af");
             let mut postings = reader
-                .read_postings_all_info(&Term::from_field_text(text_field, "af"))
+                .read_postings(&term_af, FreqAndPositions)
                 .unwrap();
             assert!(postings.advance());
             assert_eq!(postings.doc(), 0);

src/postings/mod.rs

@@ -26,7 +26,7 @@ pub use self::postings::Postings;
 #[cfg(test)]
 pub use self::vec_postings::VecPostings;
 
-pub use self::segment_postings::SegmentPostings;
+pub use self::segment_postings::{SegmentPostings, BlockSegmentPostings};
 pub use self::intersection::IntersectionDocSet;
 pub use self::freq_handler::FreqHandler;
 pub use self::segment_postings_option::SegmentPostingsOption;
@@ -42,6 +42,7 @@ mod tests {
     use indexer::SegmentWriter;
     use core::SegmentReader;
     use core::Index;
+    use postings::SegmentPostingsOption::FreqAndPositions;
     use std::iter;
     use datastruct::stacker::Heap;
     use fastfield::FastFieldReader;
@@ -129,11 +130,11 @@ mod tests {
             }
             {
                 let term_a = Term::from_field_text(text_field, "abcdef");
-                assert!(segment_reader.read_postings_all_info(&term_a).is_none());
+                assert!(segment_reader.read_postings(&term_a, FreqAndPositions).is_none());
             }
             {
                 let term_a = Term::from_field_text(text_field, "a");
-                let mut postings_a = segment_reader.read_postings_all_info(&term_a).unwrap();
+                let mut postings_a = segment_reader.read_postings(&term_a, FreqAndPositions).unwrap();
                 assert_eq!(postings_a.len(), 1000);
                 assert!(postings_a.advance());
                 assert_eq!(postings_a.doc(), 0);
@@ -152,7 +153,7 @@ mod tests {
             }
             {
                 let term_e = Term::from_field_text(text_field, "e");
-                let mut postings_e = segment_reader.read_postings_all_info(&term_e).unwrap();
+                let mut postings_e = segment_reader.read_postings(&term_e, FreqAndPositions).unwrap();
                 assert_eq!(postings_e.len(), 1000 - 2);
                 for i in 2u32..1000u32 {
                     assert!(postings_e.advance());
@@ -467,6 +468,7 @@ mod tests {
         });
     }
 
+
     fn bench_skip_next(p: f32, b: &mut Bencher) {
         let searcher = INDEX.searcher();
         let segment_reader = searcher.segment_reader(0);
@@ -475,6 +477,7 @@ mod tests {
         let mut segment_postings = segment_reader
             .read_postings(&*TERM_A, SegmentPostingsOption::NoFreq)
             .unwrap();
+        
         let mut existing_docs = Vec::new();
         for doc in &docs {
             if *doc >= segment_postings.doc() {
@@ -490,6 +493,7 @@ mod tests {
                 .read_postings(&*TERM_A, SegmentPostingsOption::NoFreq)
                 .unwrap();
             for doc in &existing_docs {
+                println!("doc {}", doc);
                 if segment_postings.skip_next(*doc) == SkipResult::End {
                     break;
                 }

src/postings/segment_postings.rs

@@ -8,6 +8,8 @@ use fastfield::DeleteBitSet;
 
 const EMPTY_DATA: [u8; 0] = [0u8; 0];
 
+
+/*
 /// `SegmentPostings` represents the inverted list or postings associated to
 /// a term in a `Segment`.
 ///
@@ -85,11 +87,7 @@ impl<'a> SegmentPostings<'a> {
         }
     }
 
-    /// Index within a block is used as an address when
-    /// interacting with the `FreqHandler`
-    fn index_within_block(&self) -> usize {
-        self.cur.0 % NUM_DOCS_PER_BLOCK
-    }
+
 
     /// Sets the current position to a location relative
     /// to the current block
@@ -119,6 +117,107 @@ impl<'a> DocSet for SegmentPostings<'a> {
         }
     }
 
+    
+
+    #[inline]
+    fn doc(&self) -> DocId {
+        self.block_decoder.output(self.index_within_block())
+    }
+}
+
+impl<'a> HasLen for SegmentPostings<'a> {
+    fn len(&self) -> usize {
+        self.len
+    }
+}
+
+impl<'a> Postings for SegmentPostings<'a> {
+    fn term_freq(&self) -> u32 {
+        self.freq_handler.freq(self.index_within_block())
+    }
+
+    fn positions(&self) -> &[u32] {
+        self.freq_handler.positions(self.index_within_block())
+    }
+}
+
+*/
+
+
+
+/// `SegmentPostings` represents the inverted list or postings associated to
+/// a term in a `Segment`.
+///
+/// As we iterate through the `SegmentPostings`, the frequencies are optionally decoded.
+/// Positions on the other hand, are optionally entirely decoded upfront.
+pub struct SegmentPostings<'a> {
+    len: usize,
+    cur: Wrapping<usize>,
+    block_cursor: BlockSegmentPostings<'a>,
+    cur_block_len: usize,
+    delete_bitset: DeleteBitSet,
+}
+
+impl<'a> SegmentPostings<'a> {
+
+
+    /// Reads a Segment postings from an &[u8]
+    ///
+    /// * `len` - number of document in the posting lists.
+    /// * `data` - data array. The complete data is not necessarily used.
+    /// * `freq_handler` - the freq handler is in charge of decoding
+    ///   frequencies and/or positions
+    pub fn from_block_postings(
+            segment_block_postings: BlockSegmentPostings<'a>,
+            delete_bitset: DeleteBitSet) -> SegmentPostings<'a> {
+        SegmentPostings {
+            len: segment_block_postings.len,
+            block_cursor: segment_block_postings,
+            cur: Wrapping(usize::max_value()),
+            cur_block_len: 0,
+            delete_bitset: delete_bitset,
+        }
+    }
+    
+    /// Returns an empty segment postings object
+    pub fn empty() -> SegmentPostings<'static> {
+        let empty_block_cursor = BlockSegmentPostings::empty();
+        SegmentPostings {
+            len: 0,
+            block_cursor: empty_block_cursor,
+            delete_bitset: DeleteBitSet::empty(),
+            cur: Wrapping(usize::max_value()),
+            cur_block_len: 0,
+        }
+    }
+}
+
+
+impl<'a> DocSet for SegmentPostings<'a> {
+    // goes to the next element.
+    // next needs to be called a first time to point to the correct element.
+    #[inline]
+    fn advance(&mut self) -> bool {
+        loop {   
+            self.cur += Wrapping(1);
+            assert!(self.cur.0 >= 0);
+            assert!(self.cur.0 <= self.cur_block_len);
+            if self.cur.0 == self.cur_block_len {
+                self.cur = Wrapping(0);
+                if !self.block_cursor.advance() {
+                    self.cur_block_len = 0;
+                    self.cur = Wrapping(usize::max_value());
+                    return false;
+                }
+                self.cur_block_len = self.block_cursor.docs().len();
+            }
+            if !self.delete_bitset.is_deleted(self.doc()) {
+                return true;
+            }
+        }
+    }
+
+    /*
     fn skip_next(&mut self, target: DocId) -> SkipResult {
         if !self.advance() {
             return SkipResult::End;
@@ -197,10 +296,11 @@ impl<'a> DocSet for SegmentPostings<'a> {
             SkipResult::End
         }
     }
-
+    */
+    
     #[inline]
     fn doc(&self) -> DocId {
-        self.block_decoder.output(self.index_within_block())
+        self.block_cursor.docs()[self.cur.0]
     }
 }
 
@@ -212,10 +312,93 @@ impl<'a> HasLen for SegmentPostings<'a> {
 
 impl<'a> Postings for SegmentPostings<'a> {
     fn term_freq(&self) -> u32 {
-        self.freq_handler.freq(self.index_within_block())
+        self.block_cursor.freq_handler().freq(self.cur.0)
     }
 
     fn positions(&self) -> &[u32] {
-        self.freq_handler.positions(self.index_within_block())
+        self.block_cursor.freq_handler().positions(self.cur.0)
     }
 }
+
+
+
+
+pub struct BlockSegmentPostings<'a> {
+    num_binpacked_blocks: usize,
+    num_vint_docs: usize,
+    block_decoder: BlockDecoder,
+    freq_handler: FreqHandler,
+    remaining_data: &'a [u8],
+    doc_offset: DocId,
+    len: usize,
+}
+
+impl<'a> BlockSegmentPostings<'a> {
+    
+    pub fn from_data(len: usize, data: &'a [u8], freq_handler: FreqHandler) -> BlockSegmentPostings<'a> {
+        let num_binpacked_blocks: usize = (len as usize) / NUM_DOCS_PER_BLOCK;
+        let num_vint_docs = (len as usize) - NUM_DOCS_PER_BLOCK * num_binpacked_blocks;
+        BlockSegmentPostings {
+            num_binpacked_blocks: num_binpacked_blocks,
+            num_vint_docs: num_vint_docs,
+            block_decoder: BlockDecoder::new(),
+            freq_handler: freq_handler,
+            remaining_data: data,
+            doc_offset: 0,
+            len: len,
+        }
+    }
+
+    pub fn reset(&mut self, len: usize, data: &'a [u8]) {
+        let num_binpacked_blocks: usize = (len as usize) / NUM_DOCS_PER_BLOCK;
+        let num_vint_docs = (len as usize) - NUM_DOCS_PER_BLOCK * num_binpacked_blocks;
+        self.num_binpacked_blocks = num_binpacked_blocks;
+        self.num_vint_docs = num_vint_docs;
+        self.remaining_data = data;
+        self.doc_offset = 0;
+        self.len = len;
+    }
+
+    pub fn docs(&self) -> &[DocId] {
+        self.block_decoder.output_array()
+    }
+    
+    pub fn freq_handler(&self) -> &FreqHandler {
+        &self.freq_handler
+    }
+    
+    pub fn advance(&mut self) -> bool {
+        if self.num_binpacked_blocks > 0 {
+            self.remaining_data = self.block_decoder.uncompress_block_sorted(self.remaining_data, self.doc_offset);
+            self.remaining_data = self.freq_handler.read_freq_block(self.remaining_data);
+            self.doc_offset = self.block_decoder.output(NUM_DOCS_PER_BLOCK - 1);
+            self.num_binpacked_blocks -= 1;
+            true
+        }
+        else {
+            if self.num_vint_docs > 0 {
+                self.remaining_data = self.block_decoder.uncompress_vint_sorted(self.remaining_data, self.doc_offset, self.num_vint_docs);
+                self.freq_handler.read_freq_vint(self.remaining_data, self.num_vint_docs);
+                self.num_vint_docs = 0;
+                true
+            }
+            else {
+                false
+            }
+        }
+    }
+    
+    /// Returns an empty segment postings object
+    pub fn empty() -> BlockSegmentPostings<'static> {
+        BlockSegmentPostings {
+            num_binpacked_blocks: 0,
+            num_vint_docs: 0,
+            block_decoder: BlockDecoder::new(),
+            freq_handler: FreqHandler::new_without_freq(),
+            remaining_data:  &EMPTY_DATA,
+            doc_offset: 0,
+            len: 0,
+        }
+    }
+    
+}