smaller block size for store

Cargo.toml

@@ -23,6 +23,7 @@ bincode = "0.4.0"
 serde = "0.6.11"
 libc = "0.2.6"
 lz4 = "1.13.131"
+time = "0.1.34"
 
 [build-dependencies]
 gcc = "0.3.24"

src/core/simdcompression.rs

@@ -74,13 +74,13 @@ impl Decoder {
 #[test]
 fn test_encode_big() {
     let mut encoder = Encoder::new();
-    let input: Vec<u32> = (0..100000).into_iter().collect();
+    let input: Vec<u32> = (0..10000).into_iter().collect();
     let data = encoder.encode(&input);
-    assert_eq!(data.len(), 9578);
+    assert_eq!(data.len(), 962);
     let decoder = Decoder::new();
-    let mut data_output: Vec<u32> = (0..100000).collect();
-    assert_eq!(100000, decoder.decode(&data[0..9578], &mut data_output));
-    for i in 0..100000 {
+    let mut data_output: Vec<u32> = (0..10000).collect();
+    assert_eq!(10000, decoder.decode(&data[0..962], &mut data_output));
+    for i in 0..10000 {
         assert_eq!(data_output[i], input[i])    ;
     }
 }

src/core/store.rs

@@ -1,3 +1,4 @@
+use time::PreciseTime;
 use std::io::BufWriter;
 use std::fs::File;
 use std::fmt;
@@ -21,7 +22,7 @@ use tempfile;
 
 // TODO cache uncompressed pages
 
-const BLOCK_SIZE: usize = 262144;
+const BLOCK_SIZE: usize = 131072;
 
 pub struct StoreWriter {
     doc: DocId,
@@ -124,7 +125,6 @@ impl StoreReader {
     fn block_offset(&self, doc_id: &DocId) -> OffsetIndex {
         let mut offset = OffsetIndex(0, 0);
         for &OffsetIndex(first_doc_id, block_offset) in self.offsets.iter() {
-            println!("First doc id {}", first_doc_id);
             if first_doc_id > *doc_id {
                 break;
             }
@@ -139,7 +139,7 @@ impl StoreReader {
         let mut current_block_mut = self.current_block.borrow_mut();
         current_block_mut.clear();
         let total_buffer = unsafe {self.data.as_slice()};
-        let mut cursor = Cursor::new(total_buffer);
+        let mut cursor = Cursor::new(&total_buffer[block_offset..]);
         let block_length = u32::deserialize(&mut cursor).unwrap();
         let block_array: &[u8] = &total_buffer[(block_offset + 4 as usize)..(block_offset + 4 + block_length as usize)];
         let mut lz4_decoder = lz4::Decoder::new(Cursor::new(block_array)).unwrap();
@@ -151,7 +151,6 @@ impl StoreReader {
         self.read_block(block_offset as usize);
         let mut current_block_mut = self.current_block.borrow_mut();
         let mut cursor = Cursor::new(&mut current_block_mut[..]);
-        println!("{} / {}", first_doc_id, doc_id);
         for _ in first_doc_id..*doc_id  {
             let block_length = u32::deserialize(&mut cursor).unwrap();
             cursor.seek(SeekFrom::Current(block_length as i64));

src/lib.rs

@@ -19,6 +19,7 @@ extern crate combine;
 extern crate atomicwrites;
 extern crate tempdir;
 extern crate bincode;
+extern crate time;
 extern crate serde;
 extern crate libc;
 extern crate lz4;