From 4cb3a7f077c527eeecfa8efa68a0889d9e5b6ac8 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Sun, 1 May 2016 11:26:34 +0900 Subject: [PATCH] moved common --- src/common/mod.rs | 9 +++++ src/{core => common}/serialize.rs | 64 +++---------------------------- src/{core => common}/timer.rs | 0 src/common/vint.rs | 58 ++++++++++++++++++++++++++++ src/core/mod.rs | 2 - src/core/reader.rs | 4 +- src/core/searcher.rs | 2 +- src/datastruct/fstmap.rs | 2 +- src/fastfield/reader.rs | 10 +++-- src/fastfield/serializer.rs | 2 +- src/lib.rs | 2 + src/postings/serializer.rs | 2 +- src/postings/term_info.rs | 2 +- src/schema/term.rs | 2 +- src/schema/text_field.rs | 2 +- src/schema/u32_field.rs | 2 +- src/store/reader.rs | 2 +- src/store/writer.rs | 2 +- 18 files changed, 93 insertions(+), 76 deletions(-) create mode 100644 src/common/mod.rs rename src/{core => common}/serialize.rs (77%) rename src/{core => common}/timer.rs (100%) create mode 100644 src/common/vint.rs diff --git a/src/common/mod.rs b/src/common/mod.rs new file mode 100644 index 000000000..b7c0bfb5e --- /dev/null +++ b/src/common/mod.rs @@ -0,0 +1,9 @@ +mod serialize; +mod timer; +mod vint; + +pub use self::serialize::BinarySerializable; +pub use self::timer::Timing; +pub use self::timer::TimerTree; +pub use self::timer::OpenTimer; +pub use self::vint::VInt; diff --git a/src/core/serialize.rs b/src/common/serialize.rs similarity index 77% rename from src/core/serialize.rs rename to src/common/serialize.rs index 48705f7a0..1a8e32fbb 100644 --- a/src/core/serialize.rs +++ b/src/common/serialize.rs @@ -4,6 +4,7 @@ use std::fmt; use std::io::Write; use std::io::Read; use std::io; +use common::VInt; use byteorder; pub trait BinarySerializable : fmt::Debug + Sized { @@ -11,61 +12,6 @@ pub trait BinarySerializable : fmt::Debug + Sized { fn deserialize(reader: &mut Read) -> io::Result; } -#[derive(Debug, Eq, PartialEq)] -pub struct VInt(pub u64); - -impl VInt { - pub fn val(&self,) -> u64 { - self.0.clone() - } -} - -impl BinarySerializable for VInt { - fn serialize(&self, writer: &mut Write) -> io::Result { - let mut remaining = self.0.clone(); - let mut written: usize = 0; - let mut buffer = [0u8; 10]; - loop { - let mut next_byte: u8 = (remaining % 128u64) as u8; - remaining /= 128u64; - if remaining == 0u64 { - buffer[written] = next_byte; - written += 1; - break; - } - else { - next_byte |= 128u8; - buffer[written] = next_byte; - written += 1; - } - } - try!(writer.write_all(&buffer[0..written])); - Ok(written) - } - - fn deserialize(reader: &mut Read) -> io::Result { - let mut bytes = reader.bytes(); - let mut result = 0u64; - let mut shift = 0u64; - loop { - match bytes.next() { - Some(Ok(b)) => { - result += ((b % 128u8) as u64) << shift; - if b & 128 == 0u8 { - break; - } - shift += 7; - } - _ => { - return Err(io::Error::new(io::ErrorKind::InvalidData, "Reach end of buffer")) - } - } - } - Ok(VInt(result)) - } -} - - fn convert_byte_order_error(byteorder_error: byteorder::Error) -> io::Error { match byteorder_error { byteorder::Error::UnexpectedEOF => io::Error::new(io::ErrorKind::InvalidData, "Reached EOF unexpectedly"), @@ -171,6 +117,11 @@ impl BinarySerializable for String { #[cfg(test)] mod test { + + use std::io::Cursor; + use common::VInt; + use super::*; + fn serialize_test(v: T, num_bytes: usize) { let mut buffer: Vec = Vec::new(); assert_eq!(v.serialize(&mut buffer).unwrap(), num_bytes); @@ -180,9 +131,6 @@ mod test { assert_eq!(deser, v); } - use std::io::Cursor; - use super::*; - #[test] fn test_serialize_u8() { serialize_test(3u8, 1); diff --git a/src/core/timer.rs b/src/common/timer.rs similarity index 100% rename from src/core/timer.rs rename to src/common/timer.rs diff --git a/src/common/vint.rs b/src/common/vint.rs new file mode 100644 index 000000000..5c3eeabe9 --- /dev/null +++ b/src/common/vint.rs @@ -0,0 +1,58 @@ +use super::BinarySerializable; +use std::io; +use std::io::Write; +use std::io::Read; + +#[derive(Debug, Eq, PartialEq)] +pub struct VInt(pub u64); + +impl VInt { + pub fn val(&self,) -> u64 { + self.0.clone() + } +} + +impl BinarySerializable for VInt { + fn serialize(&self, writer: &mut Write) -> io::Result { + let mut remaining = self.0.clone(); + let mut written: usize = 0; + let mut buffer = [0u8; 10]; + loop { + let mut next_byte: u8 = (remaining % 128u64) as u8; + remaining /= 128u64; + if remaining == 0u64 { + buffer[written] = next_byte; + written += 1; + break; + } + else { + next_byte |= 128u8; + buffer[written] = next_byte; + written += 1; + } + } + try!(writer.write_all(&buffer[0..written])); + Ok(written) + } + + fn deserialize(reader: &mut Read) -> io::Result { + let mut bytes = reader.bytes(); + let mut result = 0u64; + let mut shift = 0u64; + loop { + match bytes.next() { + Some(Ok(b)) => { + result += ((b % 128u8) as u64) << shift; + if b & 128 == 0u8 { + break; + } + shift += 7; + } + _ => { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Reach end of buffer")) + } + } + } + Ok(VInt(result)) + } +} diff --git a/src/core/mod.rs b/src/core/mod.rs index f6661d3f6..d5fc86b7e 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -4,10 +4,8 @@ pub mod reader; pub mod codec; pub mod searcher; pub mod collector; -pub mod serialize; pub mod index; pub mod merger; -pub mod timer; use std::error; use std::io; diff --git a/src/core/reader.rs b/src/core/reader.rs index 410631453..59afdfe1f 100644 --- a/src/core/reader.rs +++ b/src/core/reader.rs @@ -13,10 +13,10 @@ use datastruct::FstMap; use std::fmt; use rustc_serialize::json; use core::index::SegmentInfo; -use core::timer::OpenTimer; +use common::OpenTimer; use schema::U32Field; use core::convert_to_ioerror; -use core::serialize::BinarySerializable; +use common::BinarySerializable; use fastfield::{U32FastFieldsReader, U32FastFieldReader}; use compression; use std::mem; diff --git a/src/core/searcher.rs b/src/core/searcher.rs index 6197fff3d..ddc836f8c 100644 --- a/src/core/searcher.rs +++ b/src/core/searcher.rs @@ -5,7 +5,7 @@ use DocId; use schema::{Document, Term}; use core::collector::Collector; use std::io; -use core::timer::TimerTree; +use common::TimerTree; #[derive(Debug)] pub struct Searcher { diff --git a/src/datastruct/fstmap.rs b/src/datastruct/fstmap.rs index 424bda684..e84923a11 100644 --- a/src/datastruct/fstmap.rs +++ b/src/datastruct/fstmap.rs @@ -7,7 +7,7 @@ use fst::raw::Fst; use fst::Streamer; use directory::ReadOnlySource; -use core::serialize::BinarySerializable; +use common::BinarySerializable; use std::marker::PhantomData; fn convert_fst_error(e: fst::Error) -> io::Error { diff --git a/src/fastfield/reader.rs b/src/fastfield/reader.rs index 4aa9c1e6d..05d1f6f94 100644 --- a/src/fastfield/reader.rs +++ b/src/fastfield/reader.rs @@ -1,12 +1,14 @@ use std::io; +use std::io::{SeekFrom, Seek}; +use std::collections::HashMap; +use std::ops::Deref; + use directory::ReadOnlySource; use fastfield::DividerU32; -use core::serialize::BinarySerializable; +use common::BinarySerializable; use DocId; -use std::collections::HashMap; use schema::U32Field; -use std::io::{SeekFrom, Seek}; -use std::ops::Deref; + use super::compute_num_bits; pub struct U32FastFieldReader { diff --git a/src/fastfield/serializer.rs b/src/fastfield/serializer.rs index a9abc4432..a30a3dc43 100644 --- a/src/fastfield/serializer.rs +++ b/src/fastfield/serializer.rs @@ -1,4 +1,4 @@ -use core::serialize::BinarySerializable; +use common::BinarySerializable; use directory::WritePtr; use schema::U32Field; use std::io; diff --git a/src/lib.rs b/src/lib.rs index f5f227b26..883b43e5e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,6 +34,8 @@ mod directory; mod compression; mod fastfield; mod store; +mod common; + pub mod schema; pub use directory::Directory; diff --git a/src/postings/serializer.rs b/src/postings/serializer.rs index b48191ca7..5b497650a 100644 --- a/src/postings/serializer.rs +++ b/src/postings/serializer.rs @@ -7,7 +7,7 @@ use DocId; use core::index::Segment; use std::io; use core::index::SegmentComponent; -use core::serialize::BinarySerializable; +use common::BinarySerializable; pub struct PostingsSerializer { terms_fst_builder: FstMapBuilder, // TODO find an alternative to work around the "move" diff --git a/src/postings/term_info.rs b/src/postings/term_info.rs index 8dc0728f6..a49f91b9d 100644 --- a/src/postings/term_info.rs +++ b/src/postings/term_info.rs @@ -1,4 +1,4 @@ -use core::serialize::BinarySerializable; +use common::BinarySerializable; use std::io; #[derive(Debug,Ord,PartialOrd,Eq,PartialEq,Clone)] diff --git a/src/schema/term.rs b/src/schema/term.rs index 5494eef40..1e9dbdcd2 100644 --- a/src/schema/term.rs +++ b/src/schema/term.rs @@ -1,7 +1,7 @@ use std::io::Write; use std::fmt; -use core::serialize::BinarySerializable; +use common::BinarySerializable; use super::U32Field; use super::TextField; diff --git a/src/schema/text_field.rs b/src/schema/text_field.rs index a72faaf9b..8915b7648 100644 --- a/src/schema/text_field.rs +++ b/src/schema/text_field.rs @@ -2,7 +2,7 @@ use std::io::Write; use std::io; use std::io::Read; -use core::serialize::BinarySerializable; +use common::BinarySerializable; use rustc_serialize::Encodable; use rustc_serialize::Decoder; use rustc_serialize::Encoder; diff --git a/src/schema/u32_field.rs b/src/schema/u32_field.rs index 21a1ff54d..06a214bb2 100644 --- a/src/schema/u32_field.rs +++ b/src/schema/u32_field.rs @@ -2,7 +2,7 @@ use std::io; use std::io::Write; use std::io::Read; -use core::serialize::BinarySerializable; +use common::BinarySerializable; use rustc_serialize::Encodable; use rustc_serialize::Decoder; use rustc_serialize::Encoder; diff --git a/src/store/reader.rs b/src/store/reader.rs index 9aab2034c..a5b240c51 100644 --- a/src/store/reader.rs +++ b/src/store/reader.rs @@ -3,7 +3,7 @@ use std::cell::RefCell; use DocId; use schema::Document; use schema::TextFieldValue; -use core::serialize::BinarySerializable; +use common::BinarySerializable; use std::io::Read; use std::io::Cursor; diff --git a/src/store/writer.rs b/src/store/writer.rs index 8ebcb5ec0..53f48a0ce 100644 --- a/src/store/writer.rs +++ b/src/store/writer.rs @@ -1,7 +1,7 @@ use directory::WritePtr; use DocId; use schema::TextFieldValue; -use core::serialize::BinarySerializable; +use common::BinarySerializable; use std::io::Write; use std::io::Read; use std::io;