diff --git a/Cargo.toml b/Cargo.toml index 32038fb53..394412207 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ bincode = "0.4.0" serde = "0.6.11" libc = "0.2.6" lz4 = "1.13.131" +time = "0.1.34" [build-dependencies] gcc = "0.3.24" diff --git a/src/core/simdcompression.rs b/src/core/simdcompression.rs index 31a4afef3..f14aabda1 100644 --- a/src/core/simdcompression.rs +++ b/src/core/simdcompression.rs @@ -74,13 +74,13 @@ impl Decoder { #[test] fn test_encode_big() { let mut encoder = Encoder::new(); - let input: Vec = (0..100000).into_iter().collect(); + let input: Vec = (0..10000).into_iter().collect(); let data = encoder.encode(&input); - assert_eq!(data.len(), 9578); + assert_eq!(data.len(), 962); let decoder = Decoder::new(); - let mut data_output: Vec = (0..100000).collect(); - assert_eq!(100000, decoder.decode(&data[0..9578], &mut data_output)); - for i in 0..100000 { + let mut data_output: Vec = (0..10000).collect(); + assert_eq!(10000, decoder.decode(&data[0..962], &mut data_output)); + for i in 0..10000 { assert_eq!(data_output[i], input[i]) ; } } diff --git a/src/core/store.rs b/src/core/store.rs index 36777c0ec..2bdb57b0f 100644 --- a/src/core/store.rs +++ b/src/core/store.rs @@ -1,3 +1,4 @@ +use time::PreciseTime; use std::io::BufWriter; use std::fs::File; use std::fmt; @@ -21,7 +22,7 @@ use tempfile; // TODO cache uncompressed pages -const BLOCK_SIZE: usize = 262144; +const BLOCK_SIZE: usize = 131072; pub struct StoreWriter { doc: DocId, @@ -124,7 +125,6 @@ impl StoreReader { fn block_offset(&self, doc_id: &DocId) -> OffsetIndex { let mut offset = OffsetIndex(0, 0); for &OffsetIndex(first_doc_id, block_offset) in self.offsets.iter() { - println!("First doc id {}", first_doc_id); if first_doc_id > *doc_id { break; } @@ -139,7 +139,7 @@ impl StoreReader { let mut current_block_mut = self.current_block.borrow_mut(); current_block_mut.clear(); let total_buffer = unsafe {self.data.as_slice()}; - let mut cursor = Cursor::new(total_buffer); + let mut cursor = Cursor::new(&total_buffer[block_offset..]); let block_length = u32::deserialize(&mut cursor).unwrap(); let block_array: &[u8] = &total_buffer[(block_offset + 4 as usize)..(block_offset + 4 + block_length as usize)]; let mut lz4_decoder = lz4::Decoder::new(Cursor::new(block_array)).unwrap(); @@ -151,7 +151,6 @@ impl StoreReader { self.read_block(block_offset as usize); let mut current_block_mut = self.current_block.borrow_mut(); let mut cursor = Cursor::new(&mut current_block_mut[..]); - println!("{} / {}", first_doc_id, doc_id); for _ in first_doc_id..*doc_id { let block_length = u32::deserialize(&mut cursor).unwrap(); cursor.seek(SeekFrom::Current(block_length as i64)); diff --git a/src/lib.rs b/src/lib.rs index fcdda6c14..7145e4acf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,7 @@ extern crate combine; extern crate atomicwrites; extern crate tempdir; extern crate bincode; +extern crate time; extern crate serde; extern crate libc; extern crate lz4;