From 84e8919c1f7c23a8386e07cab551dca32fca3932 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Wed, 4 May 2016 16:04:46 +0900 Subject: [PATCH] blop --- src/cli/merge.rs | 6 ++- src/compression/block128.rs | 2 +- src/compression/s4bp128.rs | 5 +-- src/core/merger.rs | 3 +- src/core/writer.rs | 7 ++-- src/datastruct/skip/skiplist.rs | 1 + src/datastruct/skip/skiplist_builder.rs | 1 + src/postings/serializer.rs | 49 +++++++++++++++++++------ src/postings/writer.rs | 45 +++++++++++++++++++++-- 9 files changed, 92 insertions(+), 27 deletions(-) diff --git a/src/cli/merge.rs b/src/cli/merge.rs index d419967c0..c7cbe34ba 100644 --- a/src/cli/merge.rs +++ b/src/cli/merge.rs @@ -11,10 +11,12 @@ fn main() { let mut ap = ArgumentParser::new(); ap.set_description("Merge a few segments together"); ap.refer(&mut directory) - .add_option(&["-d", "--directory"], Store, - "Path to the tantivy index directory"); + .add_option(&["-d", "--directory"], + Store, + "Path to the tantivy index directory"); ap.parse_args_or_exit(); } + println!("Directory : {:?}", directory); let index = Index::open(Path::new(&directory)).unwrap(); let mut index_writer = index.writer().unwrap(); diff --git a/src/compression/block128.rs b/src/compression/block128.rs index 072271c77..9a4a9f59a 100644 --- a/src/compression/block128.rs +++ b/src/compression/block128.rs @@ -68,7 +68,7 @@ impl Block128Decoder { output: [0u32; 128] } } - + pub fn decode<'a, 'b>( &'b mut self, compressed_data: &'a [u32]) -> (&'a[u32], &'b[u32; 128]) { diff --git a/src/compression/s4bp128.rs b/src/compression/s4bp128.rs index f97be6572..2da125c76 100644 --- a/src/compression/s4bp128.rs +++ b/src/compression/s4bp128.rs @@ -13,21 +13,19 @@ extern { //------------------------- // s4-bp128-dm - pub struct S4BP128Encoder { input_buffer: Vec, output_buffer: Vec, } impl S4BP128Encoder { - pub fn new() -> S4BP128Encoder { S4BP128Encoder { input_buffer: Vec::new(), output_buffer: Vec::new(), } } - + pub fn encode_sorted(&mut self, input: &[u32]) -> &[u32] { self.input_buffer.clear(); let input_len = input.len(); @@ -50,7 +48,6 @@ impl S4BP128Encoder { } } - pub struct S4BP128Decoder; impl S4BP128Decoder { diff --git a/src/core/merger.rs b/src/core/merger.rs index 10a6723a1..ca712b94a 100644 --- a/src/core/merger.rs +++ b/src/core/merger.rs @@ -180,7 +180,8 @@ impl IndexMerger { Some((term, doc_ids)) => { try!(postings_serializer.new_term(&Term::from(&term), doc_ids.len() as DocId)); for doc_id in doc_ids.iter() { - try!(postings_serializer.write_doc(doc_id.clone(), None)); + // TODO fix this + // try!(postings_serializer.write_doc(doc_id.clone(), None)); } } None => { break; } diff --git a/src/core/writer.rs b/src/core/writer.rs index 159554428..d8eba23d5 100644 --- a/src/core/writer.rs +++ b/src/core/writer.rs @@ -107,7 +107,7 @@ impl IndexWriter { } Ok(()) } - + pub fn add_document(&mut self, doc: Document) -> io::Result<()> { let arc_doc = ArcDoc::new(doc); try!( @@ -147,9 +147,10 @@ impl SegmentWriter { // - the dictionary in an fst // - the postings // - the segment info - // The segment cannot be used after this. - fn finalize( self,) -> io::Result<()> { + // The segment writer cannot be used after this. + fn finalize(mut self,) -> io::Result<()> { let segment_info = self.segment_info(); + self.postings_writer.close(); write(&self.postings_writer, &self.fast_field_writers, segment_info, diff --git a/src/datastruct/skip/skiplist.rs b/src/datastruct/skip/skiplist.rs index 11d6f987b..f9562a294 100644 --- a/src/datastruct/skip/skiplist.rs +++ b/src/datastruct/skip/skiplist.rs @@ -86,6 +86,7 @@ impl<'a, T: BinarySerializable> Layer<'a, T> { } } + pub struct SkipList<'a, T: BinarySerializable> { data_layer: Layer<'a, T>, skip_layers: Vec>, diff --git a/src/datastruct/skip/skiplist_builder.rs b/src/datastruct/skip/skiplist_builder.rs index 6b0f70683..549b5f39d 100644 --- a/src/datastruct/skip/skiplist_builder.rs +++ b/src/datastruct/skip/skiplist_builder.rs @@ -51,6 +51,7 @@ impl LayerBuilder { } } + pub struct SkipListBuilder { period: usize, data_layer: LayerBuilder, diff --git a/src/postings/serializer.rs b/src/postings/serializer.rs index c8797ecbd..db18f42ed 100644 --- a/src/postings/serializer.rs +++ b/src/postings/serializer.rs @@ -2,7 +2,6 @@ use datastruct::FstMapBuilder; use super::TermInfo; use schema::Term; use directory::WritePtr; -use compression::NUM_DOCS_PER_BLOCK; use compression::{Block128Encoder, VIntsEncoder}; use DocId; use core::index::Segment; @@ -20,6 +19,8 @@ pub struct PostingsSerializer { block_encoder: Block128Encoder, vints_encoder: VIntsEncoder, doc_ids: Vec, + term_freqs: Vec, + positions: Vec, } impl PostingsSerializer { @@ -38,6 +39,8 @@ impl PostingsSerializer { block_encoder: Block128Encoder::new(), vints_encoder: VIntsEncoder::new(), doc_ids: Vec::new(), + term_freqs: Vec::new(), + positions: Vec::new(), }) } @@ -55,31 +58,53 @@ impl PostingsSerializer { pub fn close_term(&mut self,) -> io::Result<()> { if !self.doc_ids.is_empty() { - let block_encoded = self.vints_encoder.encode_sorted(&self.doc_ids[..]); - self.written_bytes_postings += try!(VInt(block_encoded.len() as u64).serialize(&mut self.postings_write)); - for num in block_encoded { - self.written_bytes_postings += try!(num.serialize(&mut self.postings_write)); + { + let block_encoded = self.vints_encoder.encode_sorted(&self.doc_ids[..]); + self.written_bytes_postings += try!(VInt(block_encoded.len() as u64).serialize(&mut self.postings_write)); + for num in block_encoded { + self.written_bytes_postings += try!(num.serialize(&mut self.postings_write)); + } } + { + let block_encoded = self.vints_encoder.encode_sorted(&self.term_freqs[..]); + self.written_bytes_postings += try!(VInt(block_encoded.len() as u64).serialize(&mut self.postings_write)); + for num in block_encoded { + self.written_bytes_postings += try!(num.serialize(&mut self.postings_write)); + } + } + self.doc_ids.clear(); + self.term_freqs.clear(); } Ok(()) } - pub fn write_doc(&mut self, doc_id: DocId, positions: Option<&[u32]>) -> io::Result<()> { + pub fn write_doc(&mut self, doc_id: DocId, term_freq: u32, positions: &[u32]) -> io::Result<()> { self.doc_ids.push(doc_id); - if self.doc_ids.len() == 128 { - let block_encoded: &[u32] = self.block_encoder.encode_sorted(&self.doc_ids); - for num in block_encoded { - self.written_bytes_postings += try!(num.serialize(&mut self.postings_write)); + self.term_freqs.push(term_freq as u32); + if self.doc_ids.len() == 128 { + { + let block_encoded: &[u32] = self.block_encoder.encode_sorted(&self.doc_ids); + for num in block_encoded { + self.written_bytes_postings += try!(num.serialize(&mut self.postings_write)); + } } + { + let block_encoded: &[u32] = self.block_encoder.encode_sorted(&self.term_freqs); + for num in block_encoded { + self.written_bytes_postings += try!(num.serialize(&mut self.postings_write)); + } + } + self.doc_ids.clear(); + self.term_freqs.clear(); } Ok(()) } - - + pub fn close(mut self,) -> io::Result<()> { try!(self.close_term()); try!(self.terms_fst_builder.finish()); try!(self.postings_write.flush()); + try!(self.positions_write.flush()); Ok(()) } } diff --git a/src/postings/writer.rs b/src/postings/writer.rs index ab3e31400..ce99de463 100644 --- a/src/postings/writer.rs +++ b/src/postings/writer.rs @@ -7,6 +7,8 @@ use std::io; pub trait U32sRecorder { fn new() -> Self; fn record(&mut self, val: u32); + fn get(&self, idx: usize) -> u32; + fn slice(&self, start: usize, stop: usize) -> &[u32]; } pub struct VecRecorder(Vec); @@ -15,11 +17,22 @@ impl U32sRecorder for VecRecorder { fn new() -> VecRecorder { VecRecorder(Vec::new()) } + fn record(&mut self, val: u32) { self.0.push(val); } + + fn get(&self, idx: usize) -> u32 { + self.0[idx] + } + + fn slice(&self, start: usize, stop: usize) -> &[u32] { + &self.0[start..stop] + } } +const EMPTY_ARRAY: [u32; 0] = []; + pub struct ObliviousRecorder; impl U32sRecorder for ObliviousRecorder { @@ -27,10 +40,18 @@ impl U32sRecorder for ObliviousRecorder { ObliviousRecorder } fn record(&mut self, _: u32) { + // do nothing here. + } + + fn get(&self, _idx: usize) -> u32 { + 0u32 + } + + fn slice(&self, _start: usize, _stop: usize) -> &[u32] { + &EMPTY_ARRAY[0..0] } } - struct TermPostingsWriter { doc_ids: Vec, term_freqs: TermFreqsRec, @@ -39,6 +60,7 @@ struct TermPostingsWriter TermPostingsWriter { pub fn new() -> TermPostingsWriter { TermPostingsWriter { @@ -84,6 +106,17 @@ impl TermPostingsWriter< self.positions.record(pos - self.current_position); self.current_position = pos; } + + pub fn serialize(&self, serializer: &mut PostingsSerializer) -> io::Result<()> { + let mut positions_idx = 0; + for (i, doc) in self.doc_ids.iter().enumerate() { + let term_freq: u32 = self.term_freqs.get(i); + let positions: &[u32] = self.positions.slice(positions_idx, positions_idx + term_freq as usize); + try!(serializer.write_doc(doc.clone(), term_freq, positions)); + positions_idx += term_freq as usize; + } + Ok(()) + } } @@ -100,6 +133,12 @@ impl PostingsWriter { term_index: BTreeMap::new(), } } + + pub fn close(&mut self,) { + for term_postings_writer in self.postings.iter_mut() { + term_postings_writer.close(); + } + } pub fn suscribe(&mut self, doc: DocId, pos: u32, term: Term) { let doc_ids: &mut TermPostingsWriter = self.get_term_postings(term); @@ -124,9 +163,7 @@ impl PostingsWriter { let term_postings_writer = &self.postings[postings_id.clone()]; let term_docfreq = term_postings_writer.doc_freq(); try!(serializer.new_term(&term, term_docfreq)); - for doc in term_postings_writer.doc_ids.iter() { - try!(serializer.write_doc(doc.clone(), None)); - } + try!(term_postings_writer.serialize(serializer)); } Ok(()) }