test store

Cargo.toml

@@ -47,6 +47,7 @@ murmurhash32 = "0.2"
 chrono = "0.4"
 smallvec = "1"
 rayon = "1"
+env_logger = "0.8"
 lru = "0.6"
 
 [target.'cfg(windows)'.dependencies]
@@ -73,7 +74,7 @@ debug-assertions = true
 overflow-checks = true
 
 [features]
-default = ["mmap", "brotli"]
+default = ["mmap"]
 mmap = ["fs2", "tempfile", "memmap"]
 brotli-compression = ["brotli"]
 lz4-compression = ["lz4"]

src/core/index.rs

@@ -35,12 +35,18 @@ fn load_metas(
     inventory: &SegmentMetaInventory,
 ) -> crate::Result<IndexMeta> {
     let meta_data = directory.atomic_read(&META_FILEPATH)?;
-    let meta_string = String::from_utf8_lossy(&meta_data);
+    let meta_string = String::from_utf8(meta_data)
+        .map_err(|utf8_err| {
+            DataCorruption::new(
+                META_FILEPATH.to_path_buf(),
+                format!("Meta file is not valid utf-8. {:?}", utf8_err)
+            )
+        })?;
     IndexMeta::deserialize(&meta_string, &inventory)
         .map_err(|e| {
             DataCorruption::new(
                 META_FILEPATH.to_path_buf(),
-                format!("Meta file cannot be deserialized. {:?}.", e),
+                format!("Meta file cannot be deserialized. {:?}. content = {}", e, meta_string),
             )
         })
         .map_err(From::from)

src/functional_test.rs

@@ -20,6 +20,7 @@ fn check_index_content(searcher: &Searcher, vals: &[u64]) -> crate::Result<()> {
 #[test]
 #[ignore]
 fn test_functional_store() -> crate::Result<()> {
+    env_logger::init();
     let mut schema_builder = Schema::builder();
 
     let id_field = schema_builder.add_u64_field("id", INDEXED | STORED);
@@ -35,8 +36,7 @@ fn test_functional_store() -> crate::Result<()> {
     let mut doc_set: Vec<u64> = Vec::new();
 
     let mut doc_id = 0u64;
-    for iteration in 0..500 {
-        dbg!(iteration);
+    for iteration in 0.. {
         let num_docs: usize = rng.gen_range(0..4);
         if doc_set.len() >= 1 {
             let doc_to_remove_id = rng.gen_range(0..doc_set.len());
@@ -51,6 +51,7 @@ fn test_functional_store() -> crate::Result<()> {
         index_writer.commit()?;
         reader.reload()?;
         let searcher = reader.searcher();
+        println!("#{} - {}", iteration, searcher.segment_readers().len());
         check_index_content(&searcher, &doc_set)?;
     }
     Ok(())

src/store/index/mod.rs

@@ -1,4 +1,4 @@
-const CHECKPOINT_PERIOD: usize = 8;
+const CHECKPOINT_PERIOD: usize = 2;
 
 use std::fmt;
 mod block;

src/store/mod.rs

@@ -36,8 +36,6 @@ and should rely on either
 mod index;
 mod reader;
 mod writer;
-mod tests_store;
-
 pub use self::reader::StoreReader;
 pub use self::writer::StoreWriter;
 

src/store/reader.rs

@@ -50,22 +50,6 @@ impl StoreReader {
         self.skip_index.checkpoints()
     }
 
-    pub fn documents(&self) -> Vec<Document>  {
-        let mut documents = Vec::new();
-        for checkpoint in self.skip_index.checkpoints() {
-            println!("{:?}", checkpoint);
-            let block =  self.read_block(&checkpoint).unwrap();
-            let mut cursor = &block[..];
-            while cursor.len() > 0 {
-                let doc_length = VInt::deserialize(&mut cursor).unwrap().val() as usize;
-                let doc = Document::deserialize(&mut &cursor[..doc_length]).unwrap();
-                documents.push(doc);
-                cursor = &cursor[doc_length..];
-            }
-        }
-        documents
-    }
-
     fn block_checkpoint(&self, doc_id: DocId) -> Option<Checkpoint> {
         self.skip_index.seek(doc_id)
     }
@@ -120,6 +104,7 @@ impl StoreReader {
             let doc_length = VInt::deserialize(&mut cursor)?.val() as usize;
             cursor = &cursor[doc_length..];
         }
+
         let doc_length = VInt::deserialize(&mut cursor)?.val() as usize;
         cursor = &cursor[..doc_length];
         Ok(Document::deserialize(&mut cursor)?)

src/store/writer.rs

@@ -10,7 +10,7 @@ use crate::store::index::Checkpoint;
 use crate::DocId;
 use std::io::{self, Write};
 
-const BLOCK_SIZE: usize = 16_384;
+const BLOCK_SIZE: usize = 30;
 
 /// Write tantivy's [`Store`](./index.html)
 ///