blop

src/cli/merge.rs

@@ -11,10 +11,12 @@ fn main() {
         let mut ap = ArgumentParser::new();
         ap.set_description("Merge a few segments together");
         ap.refer(&mut directory)
-            .add_option(&["-d", "--directory"], Store,
-            "Path to the tantivy index directory");
+          .add_option(&["-d", "--directory"],
+                      Store,
+                      "Path to the tantivy index directory");
         ap.parse_args_or_exit();
     }
+    
     println!("Directory : {:?}", directory);
     let index = Index::open(Path::new(&directory)).unwrap();
     let mut index_writer = index.writer().unwrap();

src/compression/block128.rs

@@ -68,7 +68,7 @@ impl Block128Decoder {
             output: [0u32; 128]
         }
     }
-
+    
     pub fn decode<'a, 'b>(
           &'b mut self,
           compressed_data: &'a [u32]) -> (&'a[u32], &'b[u32; 128]) {

src/compression/s4bp128.rs

@@ -13,21 +13,19 @@ extern {
 //-------------------------
 // s4-bp128-dm
 
-
 pub struct S4BP128Encoder {
     input_buffer: Vec<u32>,
     output_buffer: Vec<u32>,
 }
 
 impl S4BP128Encoder {
-
     pub fn new() -> S4BP128Encoder {
         S4BP128Encoder {
             input_buffer: Vec::new(),
             output_buffer: Vec::new(),
         }
     }
-
+    
     pub fn encode_sorted(&mut self, input: &[u32]) -> &[u32] {
         self.input_buffer.clear();
         let input_len = input.len();
@@ -50,7 +48,6 @@ impl S4BP128Encoder {
     }
 }
 
-
 pub struct S4BP128Decoder;
 
 impl S4BP128Decoder {

src/core/merger.rs

@@ -180,7 +180,8 @@ impl IndexMerger {
                 Some((term, doc_ids)) => {
                     try!(postings_serializer.new_term(&Term::from(&term), doc_ids.len() as DocId));
                     for doc_id in doc_ids.iter() {
-                        try!(postings_serializer.write_doc(doc_id.clone(), None));
+                        // TODO fix this
+                        // try!(postings_serializer.write_doc(doc_id.clone(), None));
                     }
                 }
                 None => { break; }

src/core/writer.rs

@@ -107,7 +107,7 @@ impl IndexWriter {
 		}
 		Ok(())
 	}
-
+ 
     pub fn add_document(&mut self, doc: Document) -> io::Result<()> {
         let arc_doc = ArcDoc::new(doc);
 		try!(
@@ -147,9 +147,10 @@ impl SegmentWriter {
 	// - the dictionary in an fst
 	// - the postings
 	// - the segment info
-	// The segment cannot be used after this.
-	fn finalize(	self,) -> io::Result<()> {
+	// The segment writer cannot be used after this.
+	fn finalize(mut self,) -> io::Result<()> {
 		let segment_info = self.segment_info();
+		self.postings_writer.close();
 		write(&self.postings_writer,
 			  &self.fast_field_writers,
 			  segment_info,

src/datastruct/skip/skiplist.rs

@@ -86,6 +86,7 @@ impl<'a, T: BinarySerializable> Layer<'a, T> {
     }
 }
 
+
 pub struct SkipList<'a, T: BinarySerializable> {
     data_layer: Layer<'a, T>,
     skip_layers: Vec<Layer<'a, u32>>,

src/datastruct/skip/skiplist_builder.rs

@@ -51,6 +51,7 @@ impl<T: BinarySerializable> LayerBuilder<T> {
     }
 }
 
+
 pub struct SkipListBuilder<T: BinarySerializable> {
     period: usize,
     data_layer: LayerBuilder<T>,

src/postings/serializer.rs

@@ -2,7 +2,6 @@ use datastruct::FstMapBuilder;
 use super::TermInfo;
 use schema::Term;
 use directory::WritePtr;
-use compression::NUM_DOCS_PER_BLOCK;
 use compression::{Block128Encoder, VIntsEncoder};
 use DocId;
 use core::index::Segment;
@@ -20,6 +19,8 @@ pub struct PostingsSerializer {
     block_encoder: Block128Encoder,
     vints_encoder: VIntsEncoder,
     doc_ids: Vec<DocId>,
+    term_freqs: Vec<u32>,
+    positions: Vec<u32>,
 }
 
 impl PostingsSerializer {
@@ -38,6 +39,8 @@ impl PostingsSerializer {
             block_encoder: Block128Encoder::new(),
             vints_encoder: VIntsEncoder::new(),
             doc_ids: Vec::new(),
+            term_freqs: Vec::new(),
+            positions: Vec::new(),
         })
     }
 
@@ -55,31 +58,53 @@ impl PostingsSerializer {
 
     pub fn close_term(&mut self,) -> io::Result<()> {
         if !self.doc_ids.is_empty() {
-            let block_encoded = self.vints_encoder.encode_sorted(&self.doc_ids[..]);
-            self.written_bytes_postings += try!(VInt(block_encoded.len() as u64).serialize(&mut self.postings_write));
-            for num in block_encoded {
-                self.written_bytes_postings += try!(num.serialize(&mut self.postings_write));
+            {
+                let block_encoded = self.vints_encoder.encode_sorted(&self.doc_ids[..]);
+                self.written_bytes_postings += try!(VInt(block_encoded.len() as u64).serialize(&mut self.postings_write));
+                for num in block_encoded {
+                    self.written_bytes_postings += try!(num.serialize(&mut self.postings_write));
+                }
             }
+            {
+                let block_encoded = self.vints_encoder.encode_sorted(&self.term_freqs[..]);
+                self.written_bytes_postings += try!(VInt(block_encoded.len() as u64).serialize(&mut self.postings_write));
+                for num in block_encoded {
+                    self.written_bytes_postings += try!(num.serialize(&mut self.postings_write));
+                }
+            }
+            self.doc_ids.clear();
+            self.term_freqs.clear();
         }
         Ok(())
     }
 
-    pub fn write_doc(&mut self, doc_id: DocId, positions: Option<&[u32]>) -> io::Result<()> {
+    pub fn write_doc(&mut self, doc_id: DocId, term_freq: u32, positions: &[u32]) -> io::Result<()> {
         self.doc_ids.push(doc_id);
-        if self.doc_ids.len() == 128 {
-            let block_encoded: &[u32] = self.block_encoder.encode_sorted(&self.doc_ids);
-            for num in block_encoded {
-                self.written_bytes_postings += try!(num.serialize(&mut self.postings_write));
+        self.term_freqs.push(term_freq as u32);
+        if self.doc_ids.len() == 128 { 
+            {
+                let block_encoded: &[u32] = self.block_encoder.encode_sorted(&self.doc_ids);
+                for num in block_encoded {
+                    self.written_bytes_postings += try!(num.serialize(&mut self.postings_write));
+                }
             }
+            {
+                let block_encoded: &[u32] = self.block_encoder.encode_sorted(&self.term_freqs);
+                for num in block_encoded {
+                    self.written_bytes_postings += try!(num.serialize(&mut self.postings_write));
+                }    
+            }
+            self.doc_ids.clear();
+            self.term_freqs.clear();
         }
         Ok(())
     }
-
-
+    
     pub fn close(mut self,) -> io::Result<()> {
         try!(self.close_term());
         try!(self.terms_fst_builder.finish());
         try!(self.postings_write.flush());
+        try!(self.positions_write.flush());
         Ok(())
     }
 }

src/postings/writer.rs

@@ -7,6 +7,8 @@ use std::io;
 pub trait U32sRecorder {
     fn new() -> Self;
     fn record(&mut self, val: u32);
+    fn get(&self, idx: usize) -> u32;
+    fn slice(&self, start: usize, stop: usize) -> &[u32];
 }
 
 pub struct VecRecorder(Vec<u32>);
@@ -15,11 +17,22 @@ impl U32sRecorder for VecRecorder {
     fn new() -> VecRecorder {
         VecRecorder(Vec::new())
     }
+    
     fn record(&mut self, val: u32) {
         self.0.push(val);
     }
+    
+    fn get(&self, idx: usize) -> u32 {
+        self.0[idx]
+    }
+    
+    fn slice(&self, start: usize, stop: usize) -> &[u32] {
+        &self.0[start..stop]
+    }
 }
 
+const EMPTY_ARRAY: [u32; 0] = [];
+
 pub struct ObliviousRecorder;
 
 impl U32sRecorder for ObliviousRecorder {
@@ -27,10 +40,18 @@ impl U32sRecorder for ObliviousRecorder {
         ObliviousRecorder
     }
     fn record(&mut self, _: u32) {
+        // do nothing here.
+    }
+     
+    fn get(&self, _idx: usize) -> u32 {
+        0u32
+    }
+    
+    fn slice(&self, _start: usize, _stop: usize) -> &[u32] {
+        &EMPTY_ARRAY[0..0]
     }
 }
 
-
 struct TermPostingsWriter<TermFreqsRec: U32sRecorder, PositionsRec: U32sRecorder> {
     doc_ids: Vec<DocId>,
     term_freqs: TermFreqsRec,
@@ -39,6 +60,7 @@ struct TermPostingsWriter<TermFreqsRec: U32sRecorder, PositionsRec: U32sRecorder
     current_freq: u32,
 }
 
+
 impl<TermFreqsRec: U32sRecorder, PositionsRec: U32sRecorder> TermPostingsWriter<TermFreqsRec, PositionsRec> {
     pub fn new() -> TermPostingsWriter<TermFreqsRec, PositionsRec> {
         TermPostingsWriter {
@@ -84,6 +106,17 @@ impl<TermFreqsRec: U32sRecorder, PositionsRec: U32sRecorder> TermPostingsWriter<
         self.positions.record(pos - self.current_position);
         self.current_position = pos;
     }
+    
+    pub fn serialize(&self, serializer: &mut PostingsSerializer) -> io::Result<()> {
+        let mut positions_idx = 0;
+        for (i, doc) in self.doc_ids.iter().enumerate() {
+            let term_freq: u32 = self.term_freqs.get(i);
+            let positions: &[u32] = self.positions.slice(positions_idx, positions_idx + term_freq as usize); 
+            try!(serializer.write_doc(doc.clone(), term_freq, positions));
+            positions_idx += term_freq as usize;
+        }
+        Ok(())
+    }       
 }
 
 
@@ -100,6 +133,12 @@ impl PostingsWriter {
             term_index: BTreeMap::new(),
         }
     }
+    
+    pub fn close(&mut self,) {
+        for term_postings_writer in self.postings.iter_mut() {
+            term_postings_writer.close();
+        }
+    }
 
     pub fn suscribe(&mut self, doc: DocId, pos: u32, term: Term) {
         let doc_ids: &mut TermPostingsWriter<ObliviousRecorder, ObliviousRecorder> = self.get_term_postings(term);
@@ -124,9 +163,7 @@ impl PostingsWriter {
             let term_postings_writer = &self.postings[postings_id.clone()];
             let term_docfreq = term_postings_writer.doc_freq();
             try!(serializer.new_term(&term, term_docfreq));
-            for doc in term_postings_writer.doc_ids.iter() {
-                try!(serializer.write_doc(doc.clone(), None));
-            }
+            try!(term_postings_writer.serialize(serializer));
         }
         Ok(())
     }