From b3cdd9bb37ed651b3c1c12a390fccfdbc51e252f Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Mon, 7 Mar 2016 09:35:19 +0900 Subject: [PATCH] blop --- src/core/fstmap.rs | 6 +++--- src/core/reader.rs | 29 ++++++++++++++++++++++++----- src/core/searcher.rs | 2 +- src/core/store.rs | 14 +++++++------- src/core/writer.rs | 2 +- src/lib.rs | 2 +- 6 files changed, 37 insertions(+), 18 deletions(-) diff --git a/src/core/fstmap.rs b/src/core/fstmap.rs index e0fcd3d58..14898dee2 100644 --- a/src/core/fstmap.rs +++ b/src/core/fstmap.rs @@ -34,7 +34,7 @@ impl FstMapBuilder { try!(self.fst_builder .insert(key, self.data.len() as u64) .map_err(convert_fst_error)); - value.serialize(&mut self.data); + try!(value.serialize(&mut self.data)); Ok(()) } @@ -44,8 +44,8 @@ impl FstMapBuilder { .into_inner() .map_err(convert_fst_error)); let footer_size = self.data.len() as u32; - file.write_all(&self.data); - (footer_size as u32).serialize(&mut file); + try!(file.write_all(&self.data)); + try!((footer_size as u32).serialize(&mut file)); file.flush(); Ok(file) } diff --git a/src/core/reader.rs b/src/core/reader.rs index 5640c1193..4fcce0024 100644 --- a/src/core/reader.rs +++ b/src/core/reader.rs @@ -11,14 +11,20 @@ use core::index::SegmentComponent; use core::postings::Postings; use core::simdcompression::Decoder; use std::io; +use std::str; use core::codec::TermInfo; use core::fstmap::FstMap; +use std::error; +use rustc_serialize::json; use core::serial::SegmentSerializer; use core::serial::SerializableSegment; +use std::str::Utf8Error; +use core::index::SegmentInfo; // TODO file structure should be in codec pub struct SegmentReader { + segment_info: SegmentInfo, segment: Segment, term_offsets: FstMap, postings_data: ReadOnlySource, @@ -93,22 +99,35 @@ impl Iterator for SegmentPostings { } +fn convert_to_ioerror(err: E) -> io::Error { + io::Error::new( + io::ErrorKind::InvalidData, + err + ) +} + + + impl SegmentReader { pub fn id(&self,) -> SegmentId { self.segment.id() } - pub fn num_docs(&self,) -> DocId { - self.store_reader.num_docs() + pub fn max_doc(&self,) -> DocId { + self.segment_info.max_doc } pub fn open(segment: Segment) -> io::Result { + let segment_info_reader = try!(segment.open_read(SegmentComponent::INFO)); + let segment_info_data = try!(str::from_utf8(&*segment_info_reader).map_err(convert_to_ioerror)); + let segment_info: SegmentInfo = try!(json::decode(&segment_info_data).map_err(convert_to_ioerror)); let source = try!(segment.open_read(SegmentComponent::TERMS)); let term_offsets = try!(FstMap::from_source(source)); let store_reader = StoreReader::new(try!(segment.open_read(SegmentComponent::STORE))); let postings_shared_mmap = try!(segment.open_read(SegmentComponent::POSTINGS)); Ok(SegmentReader { + segment_info: segment_info, postings_data: postings_shared_mmap, term_offsets: term_offsets, segment: segment, @@ -116,7 +135,7 @@ impl SegmentReader { }) } - pub fn get_doc(&self, doc_id: &DocId) -> Document { + pub fn get_doc(&self, doc_id: &DocId) -> io::Result { self.store_reader.get(doc_id) } @@ -166,8 +185,8 @@ impl SerializableSegment for SegmentReader { None => { break; } } } - for doc_id in 0..self.num_docs() { - let doc = self.store_reader.get(&doc_id); + for doc_id in 0..self.max_doc() { + let doc = try!(self.store_reader.get(&doc_id)); serializer.store_doc(&mut doc.fields()); } serializer.close() diff --git a/src/core/searcher.rs b/src/core/searcher.rs index 8e3d151e8..603dcd3de 100644 --- a/src/core/searcher.rs +++ b/src/core/searcher.rs @@ -20,7 +20,7 @@ pub struct DocAddress(pub SegmentId, pub DocId); impl Searcher { - pub fn get_doc(&self, doc_address: &DocAddress) -> Document { + pub fn get_doc(&self, doc_address: &DocAddress) -> io::Result { // TODO err let DocAddress(ref segment_id, ref doc_id) = *doc_address; let segment_ord = self.segments_idx.get(&segment_id).unwrap(); diff --git a/src/core/store.rs b/src/core/store.rs index e73a2731c..14d1ace6a 100644 --- a/src/core/store.rs +++ b/src/core/store.rs @@ -112,7 +112,7 @@ impl StoreReader { pub fn num_docs(&self,) -> DocId { self.offsets.len() as DocId } - + fn read_header(data: &ReadOnlySource) -> Vec { // todo err let mut cursor = Cursor::new(data.as_slice()); @@ -146,23 +146,23 @@ impl StoreReader { lz4_decoder.read_to_end(&mut current_block_mut); } - pub fn get(&self, doc_id: &DocId) -> Document { + pub fn get(&self, doc_id: &DocId) -> io::Result { let OffsetIndex(first_doc_id, block_offset) = self.block_offset(doc_id); self.read_block(block_offset as usize); let mut current_block_mut = self.current_block.borrow_mut(); let mut cursor = Cursor::new(&mut current_block_mut[..]); for _ in first_doc_id..*doc_id { - let block_length = u32::deserialize(&mut cursor).unwrap(); + let block_length = try!(u32::deserialize(&mut cursor)); cursor.seek(SeekFrom::Current(block_length as i64)); } - let block_length = u32::deserialize(&mut cursor).unwrap(); + try!(u32::deserialize(&mut cursor)); let mut field_values = Vec::new(); let num_fields = u32::deserialize(&mut cursor).unwrap(); for _ in 0..num_fields { - let field_value = FieldValue::deserialize(&mut cursor).unwrap(); + let field_value = try!(FieldValue::deserialize(&mut cursor)); field_values.push(field_value); } - Document::from(field_values) + Ok(Document::from(field_values)) } pub fn new(data: ReadOnlySource) -> StoreReader { @@ -230,7 +230,7 @@ mod tests { let store_source = directory.open_read(&path).unwrap(); let store = StoreReader::new(store_source); for i in (0..10).map(|i| i * 3 / 2) { - assert_eq!(*store.get(&i).get_one(&field_title).unwrap(), format!("Doc {}", i)); + assert_eq!(*store.get(&i).unwrap().get_one(&field_title).unwrap(), format!("Doc {}", i)); } } diff --git a/src/core/writer.rs b/src/core/writer.rs index da04b4d4b..3c1982dd7 100644 --- a/src/core/writer.rs +++ b/src/core/writer.rs @@ -124,7 +124,7 @@ impl SegmentWriter { self.segment_serializer.close() } - pub fn num_docs(&self,) -> DocId { + pub fn max_doc(&self,) -> DocId { self.max_doc } diff --git a/src/lib.rs b/src/lib.rs index 0943e0528..d9dbaf970 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -102,7 +102,7 @@ mod tests { let segment = commit_result.unwrap(); let segment_reader = SegmentReader::open(segment).unwrap(); - assert_eq!(segment_reader.num_docs(), 3); + assert_eq!(segment_reader.max_doc(), 3); let segment_str_after_reading = DebugSegmentSerializer::debug_string(&segment_reader); assert_eq!(segment_str_before_writing, segment_str_after_reading); }