Added README in tantivy-query-grammar

* WIP implemented is_compatible

hide Footer::from_bytes from public consumption - only found Footer::extract
used outside the module

Add a new error type for IncompatibleIndex
add a prototypical call to footer.is_compatible() in ManagedDirectory::open_read
to make sure we error before reading it further

* Make error handling more ergonomic

Add an error subtype for OpenReadError and converters to TantivyError

* Remove an unnecessary assert

it's follower by the same check that Errors instead of panicking

* Correct the compatibility check logic

Leave a defensive versioned footer check to make sure we add new logic handling
when we add possible footer versions

Restricted VersionedFooter::from_bytes to be used inside the crate only

remove a half-baked test

* WIP.

* Return an error if index incompatible - closes #662

Enrich the error type with incompatibility

Change return type to Result<bool, TantivyError>, instead of bool

Add an Incompatibility enum that enriches the IncompatibleIndex error variant
with information, which then allows us to generate a developer-friendly hint how
to upgrade library version or switch feature flags for a different compression
algorithm

Updated changelog

Change the signature of is_compatible

Added documentation to the Incompatibility
Added a conditional test on a Footer with lz4 erroring

CHANGELOG.md

@@ -9,6 +9,7 @@ Tantivy 0.11.0
 - API change around `Box<BoxableTokenizer>`. See detail in #629
 - Avoid rebuilding Regex automaton whenever a regex query is reused. #639 (@brainlock)
 - Add footer with some metadata to index files. #605 (@fdb-hiroshima)
+- Add a method to check the compatibility of the footer in the index with the running version of tantivy (@petr-tik)
 - TopDocs collector: ensure stable sorting on equal score. #671 (@brainlock)
 - Added handling of pre-tokenized text fields (#642), which will enable users to
   load tokens created outside tantivy. See usage in examples/pre_tokenized_text. (@kkoziara)
@@ -16,10 +17,11 @@ Tantivy 0.11.0
 
 ## How to update?
 
+- The index format is changed. You are required to reindex your data to use tantivy 0.11. 
 - `Box<dyn BoxableTokenizer>` has been replaced by a `BoxedTokenizer` struct.
 - Regex are now compiled when the `RegexQuery` instance is built. As a result, it can now return
 an error and handling the `Result` is required.
-
+- `tantivy::version()` now returns a `Version` object. This object implements `ToString()`
 
 Tantivy 0.10.2
 =====================

Cargo.toml

@@ -33,18 +33,16 @@ fs2={version="0.4", optional=true}
 itertools = "0.8"
 levenshtein_automata = {version="0.1", features=["fst_automaton"]}
 notify = {version="4", optional=true}
-bit-set = "0.5"
 uuid = { version = "0.8", features = ["v4", "serde"] }
 crossbeam = "0.7"
-futures = "0.1"
-futures-cpupool = "0.1"
+futures = {version = "0.3",  features=["thread-pool"] }
 owning_ref = "0.4"
 stable_deref_trait = "1.0.0"
-rust-stemmers = "1.1"
+rust-stemmers = "1.2"
 downcast-rs = { version="1.0" }
-tantivy-query-grammar = { path="./query-grammar" }
+tantivy-query-grammar = { version="0.11", path="./query-grammar" }
 bitpacking = {version="0.8", default-features = false, features=["bitpacker4x"]}
-census = "0.2"
+census = "0.4"
 fnv = "1.0.6"
 owned-read = "0.4"
 failure = "0.1"

examples/pre_tokenized_text.rs

@@ -65,11 +65,8 @@ fn main() -> tantivy::Result<()> {
         tokens: pre_tokenize_text(body_text),
     };
 
-    // Now lets create a document and add our `PreTokenizedString` using
-    // `add_pre_tokenized_text` method of `Document`
-    let mut old_man_doc = Document::default();
-    old_man_doc.add_pre_tokenized_text(title, &title_tok);
-    old_man_doc.add_pre_tokenized_text(body, &body_tok);
+    // Now lets create a document and add our `PreTokenizedString`
+    let old_man_doc = doc!(title => title_tok, body => body_tok);
 
     // ... now let's just add it to the IndexWriter
     index_writer.add_document(old_man_doc);
@@ -116,6 +113,9 @@ fn main() -> tantivy::Result<()> {
 
     assert_eq!(count, 2);
 
+    // Now let's print out the results.
+    // Note that the tokens are not stored along with the original text
+    // in the document store
     for (_score, doc_address) in top_docs {
         let retrieved_doc = searcher.doc(doc_address)?;
         println!("Document: {}", schema.to_json(&retrieved_doc));

query-grammar/README.md

@@ -0,0 +1,3 @@
+# Tantivy Query Grammar
+
+This crate is used by tantivy to parse queries.

src/collector/count_collector.rs

@@ -13,44 +13,29 @@ use crate::SegmentReader;
 /// use tantivy::collector::Count;
 /// use tantivy::query::QueryParser;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Result};
+/// use tantivy::{doc, Index};
 ///
-/// # fn main() { example().unwrap(); }
-/// fn example() -> Result<()> {
-///     let mut schema_builder = Schema::builder();
-///     let title = schema_builder.add_text_field("title", TEXT);
-///     let schema = schema_builder.build();
-///     let index = Index::create_in_ram(schema);
-///     {
-///         let mut index_writer = index.writer(3_000_000)?;
-///         index_writer.add_document(doc!(
-///             title => "The Name of the Wind",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of Muadib",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "A Dairy Cow",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of a Young Girl",
-///         ));
-///         index_writer.commit().unwrap();
-///     }
+/// let mut schema_builder = Schema::builder();
+/// let title = schema_builder.add_text_field("title", TEXT);
+/// let schema = schema_builder.build();
+/// let index = Index::create_in_ram(schema);
 ///
-///     let reader = index.reader()?;
-///     let searcher = reader.searcher();
+/// let mut index_writer = index.writer(3_000_000).unwrap();
+/// index_writer.add_document(doc!(title => "The Name of the Wind"));
+/// index_writer.add_document(doc!(title => "The Diary of Muadib"));
+/// index_writer.add_document(doc!(title => "A Dairy Cow"));
+/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"));
+/// assert!(index_writer.commit().is_ok());
 ///
-///     {
-///         let query_parser = QueryParser::for_index(&index, vec![title]);
-///         let query = query_parser.parse_query("diary")?;
-///         let count = searcher.search(&query, &Count).unwrap();
+/// let reader = index.reader().unwrap();
+/// let searcher = reader.searcher();
 ///
-///         assert_eq!(count, 2);
-///     }
+/// // Here comes the important part
+/// let query_parser = QueryParser::for_index(&index, vec![title]);
+/// let query = query_parser.parse_query("diary").unwrap();
+/// let count = searcher.search(&query, &Count).unwrap();
 ///
-///     Ok(())
-/// }
+/// assert_eq!(count, 2);
 /// ```
 pub struct Count;
 

src/collector/facet_collector.rs

@@ -86,7 +86,6 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 /// use tantivy::schema::{Facet, Schema, TEXT};
 /// use tantivy::{doc, Index, Result};
 ///
-/// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {
 ///     let mut schema_builder = Schema::builder();
 ///
@@ -127,7 +126,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///     let searcher = reader.searcher();
 ///
 ///     {
-///			let mut facet_collector = FacetCollector::for_field(facet);
+///         let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/lang");
 ///         facet_collector.add_facet("/category");
 ///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
@@ -143,7 +142,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///     }
 ///
 ///     {
-///			let mut facet_collector = FacetCollector::for_field(facet);
+///         let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/category/fiction");
 ///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
@@ -158,8 +157,8 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///         ]);
 ///     }
 ///
-///    {
-///			let mut facet_collector = FacetCollector::for_field(facet);
+///     {
+///         let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/category/fiction");
 ///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
@@ -172,6 +171,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///
 ///     Ok(())
 /// }
+/// # assert!(example().is_ok());
 /// ```
 pub struct FacetCollector {
     field: Field,

src/collector/multi_collector.rs

@@ -108,49 +108,35 @@ impl<TFruit: Fruit> FruitHandle<TFruit> {
 /// use tantivy::collector::{Count, TopDocs, MultiCollector};
 /// use tantivy::query::QueryParser;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Result};
+/// use tantivy::{doc, Index};
 ///
-/// # fn main() { example().unwrap(); }
-/// fn example() -> Result<()> {
-///     let mut schema_builder = Schema::builder();
-///     let title = schema_builder.add_text_field("title", TEXT);
-///     let schema = schema_builder.build();
-///     let index = Index::create_in_ram(schema);
-///     {
-///         let mut index_writer = index.writer(3_000_000)?;
-///         index_writer.add_document(doc!(
-///             title => "The Name of the Wind",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of Muadib",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "A Dairy Cow",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of a Young Girl",
-///         ));
-///         index_writer.commit().unwrap();
-///     }
+/// let mut schema_builder = Schema::builder();
+/// let title = schema_builder.add_text_field("title", TEXT);
+/// let schema = schema_builder.build();
+/// let index = Index::create_in_ram(schema);
 ///
-///     let reader = index.reader()?;
-///     let searcher = reader.searcher();
+/// let mut index_writer = index.writer(3_000_000).unwrap();
+/// index_writer.add_document(doc!(title => "The Name of the Wind"));
+/// index_writer.add_document(doc!(title => "The Diary of Muadib"));
+/// index_writer.add_document(doc!(title => "A Dairy Cow"));
+/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"));
+/// assert!(index_writer.commit().is_ok());
 ///
-///     let mut collectors = MultiCollector::new();
-///     let top_docs_handle = collectors.add_collector(TopDocs::with_limit(2));
-///     let count_handle = collectors.add_collector(Count);
-///     let query_parser = QueryParser::for_index(&index, vec![title]);
-///     let query = query_parser.parse_query("diary")?;
-///     let mut multi_fruit = searcher.search(&query, &collectors)?;
+/// let reader = index.reader().unwrap();
+/// let searcher = reader.searcher();
 ///
-///     let count = count_handle.extract(&mut multi_fruit);
-///     let top_docs = top_docs_handle.extract(&mut multi_fruit);
+/// let mut collectors = MultiCollector::new();
+/// let top_docs_handle = collectors.add_collector(TopDocs::with_limit(2));
+/// let count_handle = collectors.add_collector(Count);
+/// let query_parser = QueryParser::for_index(&index, vec![title]);
+/// let query = query_parser.parse_query("diary").unwrap();
+/// let mut multi_fruit = searcher.search(&query, &collectors).unwrap();
 ///
-///     # assert_eq!(count, 2);
-///     # assert_eq!(top_docs.len(), 2);
+/// let count = count_handle.extract(&mut multi_fruit);
+/// let top_docs = top_docs_handle.extract(&mut multi_fruit);
 ///
-///     Ok(())
-/// }
+/// assert_eq!(count, 2);
+/// assert_eq!(top_docs.len(), 2);
 /// ```
 #[allow(clippy::type_complexity)]
 #[derive(Default)]

src/collector/top_score_collector.rs

@@ -29,43 +29,29 @@ use std::fmt;
 /// use tantivy::collector::TopDocs;
 /// use tantivy::query::QueryParser;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, DocAddress, Index, Result};
+/// use tantivy::{doc, DocAddress, Index};
 ///
-/// # fn main() { example().unwrap(); }
-/// fn example() -> Result<()> {
-///     let mut schema_builder = Schema::builder();
-///     let title = schema_builder.add_text_field("title", TEXT);
-///     let schema = schema_builder.build();
-///     let index = Index::create_in_ram(schema);
-///     {
-///         let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
-///         index_writer.add_document(doc!(
-///             title => "The Name of the Wind",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of Muadib",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "A Dairy Cow",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of a Young Girl",
-///         ));
-///         index_writer.commit().unwrap();
-///     }
+/// let mut schema_builder = Schema::builder();
+/// let title = schema_builder.add_text_field("title", TEXT);
+/// let schema = schema_builder.build();
+/// let index = Index::create_in_ram(schema);
 ///
-///     let reader = index.reader()?;
-///     let searcher = reader.searcher();
+/// let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+/// index_writer.add_document(doc!(title => "The Name of the Wind"));
+/// index_writer.add_document(doc!(title => "The Diary of Muadib"));
+/// index_writer.add_document(doc!(title => "A Dairy Cow"));
+/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"));
+/// assert!(index_writer.commit().is_ok());
 ///
-///     let query_parser = QueryParser::for_index(&index, vec![title]);
-///     let query = query_parser.parse_query("diary")?;
-///     let top_docs = searcher.search(&query, &TopDocs::with_limit(2))?;
+/// let reader = index.reader().unwrap();
+/// let searcher = reader.searcher();
 ///
-///     assert_eq!(&top_docs[0], &(0.7261542, DocAddress(0, 1)));
-///     assert_eq!(&top_docs[1], &(0.6099695, DocAddress(0, 3)));
+/// let query_parser = QueryParser::for_index(&index, vec![title]);
+/// let query = query_parser.parse_query("diary").unwrap();
+/// let top_docs = searcher.search(&query, &TopDocs::with_limit(2)).unwrap();
 ///
-///     Ok(())
-/// }
+/// assert_eq!(&top_docs[0], &(0.7261542, DocAddress(0, 1)));
+/// assert_eq!(&top_docs[1], &(0.6099695, DocAddress(0, 3)));
 /// ```
 pub struct TopDocs(TopCollector<Score>);
 
@@ -102,15 +88,12 @@ impl TopDocs {
     /// #  
     /// #   let index = Index::create_in_ram(schema);
     /// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
-    /// #   index_writer.add_document(doc!(
-    /// #       title => "The Name of the Wind",
-    /// #       rating => 92u64,
-    /// #   ));
+    /// #   index_writer.add_document(doc!(title => "The Name of the Wind", rating => 92u64));
     /// #   index_writer.add_document(doc!(title => "The Diary of Muadib", rating => 97u64));
     /// #   index_writer.add_document(doc!(title => "A Dairy Cow", rating => 63u64));
     /// #   index_writer.add_document(doc!(title => "The Diary of a Young Girl", rating => 80u64));
-    /// #   index_writer.commit()?;
-    /// #   let reader = index.reader()?;
+    /// #   assert!(index_writer.commit().is_ok());
+    /// #   let reader = index.reader().unwrap();
     /// #   let query = QueryParser::for_index(&index, vec![title]).parse_query("diary")?;
     /// #   let top_docs = docs_sorted_by_rating(&reader.searcher(), &query, rating)?;
     /// #   assert_eq!(top_docs,
@@ -202,27 +185,33 @@ impl TopDocs {
     /// use tantivy::collector::TopDocs;
     /// use tantivy::schema::Field;
     ///
-    /// # fn create_schema() -> Schema {
-    /// #    let mut schema_builder = Schema::builder();
-    /// #    schema_builder.add_text_field("product_name", TEXT);
-    /// #    schema_builder.add_u64_field("popularity", FAST);
-    /// #    schema_builder.build()
-    /// # }
-    /// #
-    /// # fn main() -> tantivy::Result<()> {
-    /// #   let schema = create_schema();
-    /// #   let index = Index::create_in_ram(schema);
-    /// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
-    /// #   let product_name = index.schema().get_field("product_name").unwrap();
-    /// #   
+    /// fn create_schema() -> Schema {
+    ///    let mut schema_builder = Schema::builder();
+    ///    schema_builder.add_text_field("product_name", TEXT);
+    ///    schema_builder.add_u64_field("popularity", FAST);
+    ///    schema_builder.build()
+    /// }
+    ///
+    /// fn create_index() -> tantivy::Result<Index> {
+    ///   let schema = create_schema();
+    ///   let index = Index::create_in_ram(schema);
+    ///   let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
+    ///   let product_name = index.schema().get_field("product_name").unwrap();
+    ///   let popularity: Field = index.schema().get_field("popularity").unwrap();
+    ///   index_writer.add_document(doc!(product_name => "The Diary of Muadib", popularity => 1u64));
+    ///   index_writer.add_document(doc!(product_name => "A Dairy Cow", popularity => 10u64));
+    ///   index_writer.add_document(doc!(product_name => "The Diary of a Young Girl", popularity => 15u64));
+    ///   index_writer.commit()?;
+    ///   Ok(index)
+    /// }
+    ///
+    /// let index = create_index().unwrap();
+    /// let product_name = index.schema().get_field("product_name").unwrap();
     /// let popularity: Field = index.schema().get_field("popularity").unwrap();
-    /// #   index_writer.add_document(doc!(product_name => "The Diary of Muadib", popularity => 1u64));
-    /// #   index_writer.add_document(doc!(product_name => "A Dairy Cow", popularity => 10u64));
-    /// #   index_writer.add_document(doc!(product_name => "The Diary of a Young Girl", popularity => 15u64));
-    /// #   index_writer.commit()?;
-    /// // ...
-    /// # let user_query = "diary";
-    /// # let query = QueryParser::for_index(&index, vec![product_name]).parse_query(user_query)?;
+    ///
+    /// let user_query_str = "diary";
+    /// let query_parser = QueryParser::for_index(&index, vec![product_name]);
+    /// let query = query_parser.parse_query(user_query_str).unwrap();
     ///
     /// // This is where we build our collector with our custom score.
     /// let top_docs_by_custom_score = TopDocs
@@ -249,15 +238,12 @@ impl TopDocs {
     ///                 popularity_boost_score * original_score
     ///             }
     ///           });
-    /// # let reader = index.reader()?;
-    /// # let searcher = reader.searcher();
+    /// let reader = index.reader().unwrap();
+    /// let searcher = reader.searcher();
     /// // ... and here are our documents. Note this is a simple vec.
     /// // The `Score` in the pair is our tweaked score.
     /// let resulting_docs: Vec<(Score, DocAddress)> =
-    ///      searcher.search(&*query, &top_docs_by_custom_score)?;
-    ///
-    /// # Ok(())
-    /// # }
+    ///      searcher.search(&query, &top_docs_by_custom_score).unwrap();
     /// ```
     ///
     /// # See also

src/core/index.rs

@@ -104,23 +104,21 @@ impl Index {
         if Index::exists(&mmap_directory) {
             return Err(TantivyError::IndexAlreadyExists);
         }
-
         Index::create(mmap_directory, schema)
     }
 
     /// Opens or creates a new index in the provided directory
     pub fn open_or_create<Dir: Directory>(dir: Dir, schema: Schema) -> Result<Index> {
-        if Index::exists(&dir) {
-            let index = Index::open(dir)?;
-            if index.schema() == schema {
-                Ok(index)
-            } else {
-                Err(TantivyError::SchemaError(
-                    "An index exists but the schema does not match.".to_string(),
-                ))
-            }
+        if !Index::exists(&dir) {
+            return Index::create(dir, schema);
+        }
+        let index = Index::open(dir)?;
+        if index.schema() == schema {
+            Ok(index)
         } else {
-            Index::create(dir, schema)
+            Err(TantivyError::SchemaError(
+                "An index exists but the schema does not match.".to_string(),
+            ))
         }
     }
 
@@ -388,12 +386,9 @@ mod tests {
     use crate::directory::RAMDirectory;
     use crate::schema::Field;
     use crate::schema::{Schema, INDEXED, TEXT};
-    use crate::Index;
     use crate::IndexReader;
-    use crate::IndexWriter;
     use crate::ReloadPolicy;
-    use std::thread;
-    use std::time::Duration;
+    use crate::{Directory, Index};
 
     #[test]
     fn test_indexer_for_field() {
@@ -471,14 +466,14 @@ mod tests {
             .try_into()
             .unwrap();
         assert_eq!(reader.searcher().num_docs(), 0);
-        let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-        test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
+        test_index_on_commit_reload_policy_aux(field, &index, &reader);
     }
 
     #[cfg(feature = "mmap")]
     mod mmap_specific {
 
         use super::*;
+        use crate::Directory;
         use std::path::PathBuf;
         use tempfile::TempDir;
 
@@ -489,22 +484,20 @@ mod tests {
             let tempdir = TempDir::new().unwrap();
             let tempdir_path = PathBuf::from(tempdir.path());
             let index = Index::create_in_dir(&tempdir_path, schema).unwrap();
-            let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-            writer.commit().unwrap();
             let reader = index
                 .reader_builder()
                 .reload_policy(ReloadPolicy::OnCommit)
                 .try_into()
                 .unwrap();
             assert_eq!(reader.searcher().num_docs(), 0);
-            test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
+            test_index_on_commit_reload_policy_aux(field, &index, &reader);
         }
 
         #[test]
         fn test_index_manual_policy_mmap() {
             let schema = throw_away_schema();
             let field = schema.get_field("num_likes").unwrap();
-            let index = Index::create_from_tempdir(schema).unwrap();
+            let mut index = Index::create_from_tempdir(schema).unwrap();
             let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
             writer.commit().unwrap();
             let reader = index
@@ -514,8 +507,12 @@ mod tests {
                 .unwrap();
             assert_eq!(reader.searcher().num_docs(), 0);
             writer.add_document(doc!(field=>1u64));
+            let (sender, receiver) = crossbeam::channel::unbounded();
+            let _handle = index.directory_mut().watch(Box::new(move || {
+                let _ = sender.send(());
+            }));
             writer.commit().unwrap();
-            thread::sleep(Duration::from_millis(500));
+            assert!(receiver.recv().is_ok());
             assert_eq!(reader.searcher().num_docs(), 0);
             reader.reload().unwrap();
             assert_eq!(reader.searcher().num_docs(), 1);
@@ -535,39 +532,26 @@ mod tests {
                 .try_into()
                 .unwrap();
             assert_eq!(reader.searcher().num_docs(), 0);
-            let mut writer = write_index.writer_with_num_threads(1, 3_000_000).unwrap();
-            test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
+            test_index_on_commit_reload_policy_aux(field, &write_index, &reader);
         }
     }
 
-    fn test_index_on_commit_reload_policy_aux(
-        field: Field,
-        writer: &mut IndexWriter,
-        reader: &IndexReader,
-    ) {
+    fn test_index_on_commit_reload_policy_aux(field: Field, index: &Index, reader: &IndexReader) {
+        let mut reader_index = reader.index();
+        let (sender, receiver) = crossbeam::channel::unbounded();
+        let _watch_handle = reader_index.directory_mut().watch(Box::new(move || {
+            let _ = sender.send(());
+        }));
+        let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
         assert_eq!(reader.searcher().num_docs(), 0);
         writer.add_document(doc!(field=>1u64));
         writer.commit().unwrap();
-        let mut count = 0;
-        for _ in 0..100 {
-            count = reader.searcher().num_docs();
-            if count > 0 {
-                break;
-            }
-            thread::sleep(Duration::from_millis(100));
-        }
-        assert_eq!(count, 1);
+        assert!(receiver.recv().is_ok());
+        assert_eq!(reader.searcher().num_docs(), 1);
         writer.add_document(doc!(field=>2u64));
         writer.commit().unwrap();
-        let mut count = 0;
-        for _ in 0..10 {
-            count = reader.searcher().num_docs();
-            if count > 1 {
-                break;
-            }
-            thread::sleep(Duration::from_millis(100));
-        }
-        assert_eq!(count, 2);
+        assert!(receiver.recv().is_ok());
+        assert_eq!(reader.searcher().num_docs(), 2);
     }
 
     // This test will not pass on windows, because windows
@@ -584,9 +568,13 @@ mod tests {
         for i in 0u64..8_000u64 {
             writer.add_document(doc!(field => i));
         }
+        let (sender, receiver) = crossbeam::channel::unbounded();
+        let _handle = directory.watch(Box::new(move || {
+            let _ = sender.send(());
+        }));
         writer.commit().unwrap();
         let mem_right_after_commit = directory.total_mem_usage();
-        thread::sleep(Duration::from_millis(1_000));
+        assert!(receiver.recv().is_ok());
         let reader = index
             .reader_builder()
             .reload_policy(ReloadPolicy::Manual)
@@ -600,6 +588,11 @@ mod tests {
         reader.reload().unwrap();
         let searcher = reader.searcher();
         assert_eq!(searcher.num_docs(), 8_000);
-        assert!(mem_right_after_merge_finished < mem_right_after_commit);
+        assert!(
+            mem_right_after_merge_finished < mem_right_after_commit,
+            "(mem after merge){} is expected < (mem before merge){}",
+            mem_right_after_merge_finished,
+            mem_right_after_commit
+        );
     }
 }

src/core/index_meta.rs

@@ -300,6 +300,9 @@ mod tests {
             payload: None,
         };
         let json = serde_json::ser::to_string(&index_metas).expect("serialization failed");
-        assert_eq!(json, r#"{"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","tokenizer":"default"},"stored":false}}],"opstamp":0}"#);
+        assert_eq!(
+            json,
+            r#"{"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","tokenizer":"default"},"stored":false}}],"opstamp":0}"#
+        );
     }
 }

src/directory/error.rs

@@ -1,3 +1,4 @@
+use crate::Version;
 use std::error::Error as StdError;
 use std::fmt;
 use std::io;
@@ -156,6 +157,65 @@ impl StdError for OpenWriteError {
     }
 }
 
+/// Type of index incompatibility between the library and the index found on disk
+/// Used to catch and provide a hint to solve this incompatibility issue
+pub enum Incompatibility {
+    /// This library cannot decompress the index found on disk
+    CompressionMismatch {
+        /// Compression algorithm used by the current version of tantivy
+        library_compression_format: String,
+        /// Compression algorithm that was used to serialise the index
+        index_compression_format: String,
+    },
+    /// The index format found on disk isn't supported by this version of the library
+    IndexMismatch {
+        /// Version used by the library
+        library_version: Version,
+        /// Version the index was built with
+        index_version: Version,
+    },
+}
+
+impl fmt::Debug for Incompatibility {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
+        match self {
+            Incompatibility::CompressionMismatch {
+                library_compression_format,
+                index_compression_format,
+            } => {
+                let err = format!(
+                    "Library was compiled with {:?} compression, index was compressed with {:?}",
+                    library_compression_format, index_compression_format
+                );
+                let advice = format!(
+                    "Change the feature flag to {:?} and rebuild the library",
+                    index_compression_format
+                );
+                write!(f, "{}. {}", err, advice)?;
+            }
+            Incompatibility::IndexMismatch {
+                library_version,
+                index_version,
+            } => {
+                let err = format!(
+                    "Library version: {}, index version: {}",
+                    library_version.index_format_version, index_version.index_format_version
+                );
+                // TODO make a more useful error message
+                // include the version range that supports this index_format_version
+                let advice = format!(
+                    "Change tantivy to a version compatible with index format {} (e.g. {}.{}.x) \
+                     and rebuild your project.",
+                    index_version.index_format_version, index_version.major, index_version.minor
+                );
+                write!(f, "{}. {}", err, advice)?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
 /// Error that may occur when accessing a file read
 #[derive(Debug)]
 pub enum OpenReadError {
@@ -164,6 +224,8 @@ pub enum OpenReadError {
     /// Any kind of IO error that happens when
     /// interacting with the underlying IO device.
     IOError(IOError),
+    /// This library doesn't support the index version found on disk
+    IncompatibleIndex(Incompatibility),
 }
 
 impl From<IOError> for OpenReadError {
@@ -183,19 +245,9 @@ impl fmt::Display for OpenReadError {
                 "an io error occurred while opening a file for reading: '{}'",
                 err
             ),
-        }
-    }
-}
-
-impl StdError for OpenReadError {
-    fn description(&self) -> &str {
-        "error occurred while opening a file for reading"
-    }
-
-    fn cause(&self) -> Option<&dyn StdError> {
-        match *self {
-            OpenReadError::FileDoesNotExist(_) => None,
-            OpenReadError::IOError(ref err) => Some(err),
+            OpenReadError::IncompatibleIndex(ref footer) => {
+                write!(f, "Incompatible index format: {:?}", footer)
+            }
         }
     }
 }
@@ -216,6 +268,12 @@ impl From<IOError> for DeleteError {
     }
 }
 
+impl From<Incompatibility> for OpenReadError {
+    fn from(incompatibility: Incompatibility) -> Self {
+        OpenReadError::IncompatibleIndex(incompatibility)
+    }
+}
+
 impl fmt::Display for DeleteError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {

src/directory/footer.rs

@@ -1,159 +1,175 @@
+use crate::common::{BinarySerializable, CountingWriter, FixedSize, VInt};
+use crate::directory::error::Incompatibility;
 use crate::directory::read_only_source::ReadOnlySource;
 use crate::directory::{AntiCallToken, TerminatingWrite};
-use byteorder::{ByteOrder, LittleEndian};
+use crate::Version;
+use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
 use crc32fast::Hasher;
 use std::io;
 use std::io::Write;
 
-const COMMON_FOOTER_SIZE: usize = 4 * 5;
+type CrcHashU32 = u32;
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct Footer {
-    pub tantivy_version: (u32, u32, u32),
+    pub version: Version,
     pub meta: String,
     pub versioned_footer: VersionedFooter,
 }
 
+/// Serialises the footer to a byte-array
+/// - versioned_footer_len : 4 bytes
+///-  versioned_footer: variable bytes
+/// - meta_len: 4 bytes
+/// - meta: variable bytes
+/// - version_len: 4 bytes
+/// - version json: variable bytes
+impl BinarySerializable for Footer {
+    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
+        BinarySerializable::serialize(&self.versioned_footer, writer)?;
+        BinarySerializable::serialize(&self.meta, writer)?;
+        let version_string =
+            serde_json::to_string(&self.version).map_err(|_err| io::ErrorKind::InvalidInput)?;
+        BinarySerializable::serialize(&version_string, writer)?;
+        Ok(())
+    }
+
+    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
+        let versioned_footer = VersionedFooter::deserialize(reader)?;
+        let meta = String::deserialize(reader)?;
+        let version_json = String::deserialize(reader)?;
+        let version = serde_json::from_str(&version_json)?;
+        Ok(Footer {
+            version,
+            meta,
+            versioned_footer,
+        })
+    }
+}
+
 impl Footer {
     pub fn new(versioned_footer: VersionedFooter) -> Self {
-        let tantivy_version = (
-            env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap(),
-            env!("CARGO_PKG_VERSION_MINOR").parse().unwrap(),
-            env!("CARGO_PKG_VERSION_PATCH").parse().unwrap(),
-        );
+        let version = crate::VERSION.clone();
+        let meta = version.to_string();
         Footer {
-            tantivy_version,
-            meta: format!(
-                "tantivy {}.{}.{}, index v{}",
-                tantivy_version.0,
-                tantivy_version.1,
-                tantivy_version.2,
-                versioned_footer.version()
-            ),
+            version,
+            meta,
             versioned_footer,
         }
     }
 
-    pub fn to_bytes(&self) -> Vec<u8> {
-        let mut res = self.versioned_footer.to_bytes();
-        res.extend_from_slice(self.meta.as_bytes());
-        let len = res.len();
-        res.resize(len + COMMON_FOOTER_SIZE, 0);
-        let mut common_footer = &mut res[len..];
-        LittleEndian::write_u32(&mut common_footer, self.meta.len() as u32);
-        LittleEndian::write_u32(&mut common_footer[4..], self.tantivy_version.0);
-        LittleEndian::write_u32(&mut common_footer[8..], self.tantivy_version.1);
-        LittleEndian::write_u32(&mut common_footer[12..], self.tantivy_version.2);
-        LittleEndian::write_u32(&mut common_footer[16..], (len + COMMON_FOOTER_SIZE) as u32);
-        res
-    }
-
-    pub fn from_bytes(data: &[u8]) -> Result<Self, io::Error> {
-        let len = data.len();
-        if len < COMMON_FOOTER_SIZE + 4 {
-            // 4 bytes for index version, stored in versioned footer
-            return Err(io::Error::new(
-                    io::ErrorKind::UnexpectedEof,
-                    format!("File corrupted. The footer len must be over 24, while the entire file len is {}", len)
-                    )
-            );
-        }
-
-        let size = LittleEndian::read_u32(&data[len - 4..]) as usize;
-        if len < size as usize {
-            return Err(io::Error::new(
-                io::ErrorKind::UnexpectedEof,
-                format!(
-                    "File corrupted. The footer len is {}, while the entire file len is {}",
-                    size, len
-                ),
-            ));
-        }
-        let footer = &data[len - size as usize..];
-        let meta_len = LittleEndian::read_u32(&footer[size - 20..]) as usize;
-        let tantivy_major = LittleEndian::read_u32(&footer[size - 16..]);
-        let tantivy_minor = LittleEndian::read_u32(&footer[size - 12..]);
-        let tantivy_patch = LittleEndian::read_u32(&footer[size - 8..]);
-        Ok(Footer {
-            tantivy_version: (tantivy_major, tantivy_minor, tantivy_patch),
-            meta: String::from_utf8_lossy(&footer[size - meta_len - 20..size - 20]).into_owned(),
-            versioned_footer: VersionedFooter::from_bytes(&footer[..size - meta_len - 20])?,
-        })
+    pub fn append_footer<W: io::Write>(&self, mut write: &mut W) -> io::Result<()> {
+        let mut counting_write = CountingWriter::wrap(&mut write);
+        self.serialize(&mut counting_write)?;
+        let written_len = counting_write.written_bytes();
+        write.write_u32::<LittleEndian>(written_len as u32)?;
+        Ok(())
     }
 
     pub fn extract_footer(source: ReadOnlySource) -> Result<(Footer, ReadOnlySource), io::Error> {
-        let footer = Footer::from_bytes(source.as_slice())?;
-        let reader = source.slice_to(source.as_slice().len() - footer.size());
-        Ok((footer, reader))
-    }
-
-    pub fn size(&self) -> usize {
-        self.versioned_footer.size() as usize + self.meta.len() + 20
-    }
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub enum VersionedFooter {
-    UnknownVersion { version: u32, size: u32 },
-    V0(u32), // crc
-}
-
-impl VersionedFooter {
-    pub fn to_bytes(&self) -> Vec<u8> {
-        match self {
-            VersionedFooter::V0(crc) => {
-                let mut res = vec![0; 8];
-                LittleEndian::write_u32(&mut res, 0);
-                LittleEndian::write_u32(&mut res[4..], *crc);
-                res
-            }
-            VersionedFooter::UnknownVersion { .. } => {
-                panic!("Unsupported index should never get serialized");
-            }
+        if source.len() < 4 {
+            return Err(io::Error::new(
+                io::ErrorKind::UnexpectedEof,
+                format!(
+                    "File corrupted. The file is smaller than 4 bytes (len={}).",
+                    source.len()
+                ),
+            ));
         }
+        let (body_footer, footer_len_bytes) = source.split_from_end(u32::SIZE_IN_BYTES);
+        let footer_len = LittleEndian::read_u32(footer_len_bytes.as_slice()) as usize;
+        let body_len = body_footer.len() - footer_len;
+        let (body, footer_data) = body_footer.split(body_len);
+        let mut cursor = footer_data.as_slice();
+        let footer = Footer::deserialize(&mut cursor)?;
+        Ok((footer, body))
     }
 
-    pub fn from_bytes(footer: &[u8]) -> Result<Self, io::Error> {
-        assert!(footer.len() >= 4);
-        let version = LittleEndian::read_u32(footer);
-        match version {
-            0 => {
-                if footer.len() == 8 {
-                    Ok(VersionedFooter::V0(LittleEndian::read_u32(&footer[4..])))
-                } else {
-                    Err(io::Error::new(
-                        io::ErrorKind::UnexpectedEof,
-                        format!(
-                            "File corrupted. The versioned footer len is {}, while it should be 8",
-                            footer.len()
-                        ),
-                    ))
+    /// Confirms that the index will be read correctly by this version of tantivy
+    /// Has to be called after `extract_footer` to make sure it's not accessing uninitialised memory
+    pub fn is_compatible(&self) -> Result<(), Incompatibility> {
+        let library_version = crate::version();
+        match &self.versioned_footer {
+            VersionedFooter::V1 {
+                crc32: _crc,
+                store_compression: compression,
+            } => {
+                if &library_version.store_compression != compression {
+                    return Err(Incompatibility::CompressionMismatch {
+                        library_compression_format: library_version.store_compression.to_string(),
+                        index_compression_format: compression.to_string(),
+                    });
                 }
+                Ok(())
             }
-            version => Ok(VersionedFooter::UnknownVersion {
-                version,
-                size: footer.len() as u32,
+            VersionedFooter::UnknownVersion => Err(Incompatibility::IndexMismatch {
+                library_version: library_version.clone(),
+                index_version: self.version.clone(),
             }),
         }
     }
+}
 
-    pub fn size(&self) -> u32 {
+/// Footer that includes a crc32 hash that enables us to checksum files in the index
+#[derive(Debug, Clone, PartialEq)]
+pub enum VersionedFooter {
+    UnknownVersion,
+    V1 {
+        crc32: CrcHashU32,
+        store_compression: String,
+    },
+}
+
+impl BinarySerializable for VersionedFooter {
+    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
+        let mut buf = Vec::new();
         match self {
-            VersionedFooter::V0(_) => 8,
-            VersionedFooter::UnknownVersion { size, .. } => *size,
+            VersionedFooter::V1 {
+                crc32,
+                store_compression: compression,
+            } => {
+                // Serializes a valid `VersionedFooter` or panics if the version is unknown
+                // [   version    |   crc_hash  | compression_mode ]
+                // [    0..4      |     4..8    |     variable     ]
+                BinarySerializable::serialize(&1u32, &mut buf)?;
+                BinarySerializable::serialize(crc32, &mut buf)?;
+                BinarySerializable::serialize(compression, &mut buf)?;
+            }
+            VersionedFooter::UnknownVersion => {
+                return Err(io::Error::new(
+                    io::ErrorKind::InvalidInput,
+                    "Cannot serialize an unknown versioned footer ",
+                ));
+            }
         }
+        BinarySerializable::serialize(&VInt(buf.len() as u64), writer)?;
+        writer.write_all(&buf[..])?;
+        Ok(())
     }
 
-    pub fn version(&self) -> u32 {
-        match self {
-            VersionedFooter::V0(_) => 0,
-            VersionedFooter::UnknownVersion { version, .. } => *version,
+    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
+        let len = VInt::deserialize(reader)?.0 as usize;
+        let mut buf = vec![0u8; len];
+        reader.read_exact(&mut buf[..])?;
+        let mut cursor = &buf[..];
+        let version = u32::deserialize(&mut cursor)?;
+        if version == 1 {
+            let crc32 = u32::deserialize(&mut cursor)?;
+            let compression = String::deserialize(&mut cursor)?;
+            Ok(VersionedFooter::V1 {
+                crc32,
+                store_compression: compression,
+            })
+        } else {
+            Ok(VersionedFooter::UnknownVersion)
         }
     }
+}
 
-    pub fn crc(&self) -> Option<u32> {
+impl VersionedFooter {
+    pub fn crc(&self) -> Option<CrcHashU32> {
         match self {
-            VersionedFooter::V0(crc) => Some(*crc),
+            VersionedFooter::V1 { crc32, .. } => Some(*crc32),
             VersionedFooter::UnknownVersion { .. } => None,
         }
     }
@@ -189,25 +205,135 @@ impl<W: TerminatingWrite> Write for FooterProxy<W> {
 
 impl<W: TerminatingWrite> TerminatingWrite for FooterProxy<W> {
     fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()> {
-        let crc = self.hasher.take().unwrap().finalize();
-
-        let footer = Footer::new(VersionedFooter::V0(crc)).to_bytes();
+        let crc32 = self.hasher.take().unwrap().finalize();
+        let footer = Footer::new(VersionedFooter::V1 {
+            crc32,
+            store_compression: crate::store::COMPRESSION.to_string(),
+        });
         let mut writer = self.writer.take().unwrap();
-        writer.write_all(&footer)?;
+        footer.append_footer(&mut writer)?;
         writer.terminate()
     }
 }
 
 #[cfg(test)]
 mod tests {
+
+    use super::CrcHashU32;
+    use super::FooterProxy;
+    use crate::common::BinarySerializable;
     use crate::directory::footer::{Footer, VersionedFooter};
+    use crate::directory::TerminatingWrite;
+    use byteorder::{ByteOrder, LittleEndian};
+    use regex::Regex;
+
+    #[test]
+    fn test_versioned_footer() {
+        let mut vec = Vec::new();
+        let footer_proxy = FooterProxy::new(&mut vec);
+        assert!(footer_proxy.terminate().is_ok());
+        assert_eq!(vec.len(), 167);
+        let footer = Footer::deserialize(&mut &vec[..]).unwrap();
+        if let VersionedFooter::V1 {
+            crc32: _,
+            store_compression,
+        } = footer.versioned_footer
+        {
+            assert_eq!(store_compression, crate::store::COMPRESSION);
+        } else {
+            panic!("Versioned footer should be V1.");
+        }
+        assert_eq!(&footer.version, crate::version());
+    }
 
     #[test]
     fn test_serialize_deserialize_footer() {
-        let crc = 123456;
-        let footer = Footer::new(VersionedFooter::V0(crc));
-        let footer_bytes = footer.to_bytes();
+        let mut buffer = Vec::new();
+        let crc32 = 123456u32;
+        let footer: Footer = Footer::new(VersionedFooter::V1 {
+            crc32,
+            store_compression: "lz4".to_string(),
+        });
+        footer.serialize(&mut buffer).unwrap();
+        let footer_deser = Footer::deserialize(&mut &buffer[..]).unwrap();
+        assert_eq!(footer_deser, footer);
+    }
 
-        assert_eq!(Footer::from_bytes(&footer_bytes).unwrap(), footer);
+    #[test]
+    fn footer_length() {
+        let crc32 = 1111111u32;
+        let versioned_footer = VersionedFooter::V1 {
+            crc32,
+            store_compression: "lz4".to_string(),
+        };
+        let mut buf = Vec::new();
+        versioned_footer.serialize(&mut buf).unwrap();
+        assert_eq!(buf.len(), 13);
+        let footer = Footer::new(versioned_footer);
+        let regex_ptn = Regex::new(
+            "tantivy v[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.{0,10}, index_format v[0-9]{1,5}",
+        )
+        .unwrap();
+        assert!(regex_ptn.is_match(&footer.meta));
+    }
+
+    #[test]
+    fn versioned_footer_from_bytes() {
+        let v_footer_bytes = vec![
+            // versionned footer length
+            12 | 128,
+            // index format version
+            1,
+            0,
+            0,
+            0,
+            // crc 32
+            12,
+            35,
+            89,
+            18,
+            // compression format
+            3 | 128,
+            b'l',
+            b'z',
+            b'4',
+        ];
+        let mut cursor = &v_footer_bytes[..];
+        let versioned_footer = VersionedFooter::deserialize(&mut cursor).unwrap();
+        assert!(cursor.is_empty());
+        let expected_crc: u32 = LittleEndian::read_u32(&v_footer_bytes[5..9]) as CrcHashU32;
+        let expected_versioned_footer: VersionedFooter = VersionedFooter::V1 {
+            crc32: expected_crc,
+            store_compression: "lz4".to_string(),
+        };
+        assert_eq!(versioned_footer, expected_versioned_footer);
+        let mut buffer = Vec::new();
+        assert!(versioned_footer.serialize(&mut buffer).is_ok());
+        assert_eq!(&v_footer_bytes[..], &buffer[..]);
+    }
+
+    #[test]
+    fn versioned_footer_panic() {
+        let v_footer_bytes = vec![6u8 | 128u8, 3u8, 0u8, 0u8, 1u8, 0u8, 0u8];
+        let mut b = &v_footer_bytes[..];
+        let versioned_footer = VersionedFooter::deserialize(&mut b).unwrap();
+        assert!(b.is_empty());
+        let expected_versioned_footer = VersionedFooter::UnknownVersion;
+        assert_eq!(versioned_footer, expected_versioned_footer);
+        let mut buf = Vec::new();
+        assert!(versioned_footer.serialize(&mut buf).is_err());
+    }
+
+    #[test]
+    #[cfg(not(feature = "lz4"))]
+    fn compression_mismatch() {
+        let crc32 = 1111111u32;
+        let versioned_footer = VersionedFooter::V1 {
+            crc32,
+            store_compression: "lz4".to_string(),
+        };
+        let footer = Footer::new(versioned_footer);
+        let res = footer.is_compatible();
+        assert!(res.is_err());
     }
 }

src/directory/managed_directory.rs

@@ -2,13 +2,14 @@ use crate::core::MANAGED_FILEPATH;
 use crate::directory::error::{DeleteError, IOError, LockError, OpenReadError, OpenWriteError};
 use crate::directory::footer::{Footer, FooterProxy};
 use crate::directory::DirectoryLock;
+use crate::directory::GarbageCollectionResult;
 use crate::directory::Lock;
 use crate::directory::META_LOCK;
 use crate::directory::{ReadOnlySource, WritePtr};
 use crate::directory::{WatchCallback, WatchHandle};
 use crate::error::DataCorruption;
 use crate::Directory;
-use crate::Result;
+
 use crc32fast::Hasher;
 use serde_json;
 use std::collections::HashSet;
@@ -64,7 +65,7 @@ fn save_managed_paths(
 
 impl ManagedDirectory {
     /// Wraps a directory as managed directory.
-    pub fn wrap<Dir: Directory>(directory: Dir) -> Result<ManagedDirectory> {
+    pub fn wrap<Dir: Directory>(directory: Dir) -> crate::Result<ManagedDirectory> {
         match directory.atomic_read(&MANAGED_FILEPATH) {
             Ok(data) => {
                 let managed_files_json = String::from_utf8_lossy(&data);
@@ -87,6 +88,11 @@ impl ManagedDirectory {
                 meta_informations: Arc::default(),
             }),
             Err(OpenReadError::IOError(e)) => Err(From::from(e)),
+            Err(OpenReadError::IncompatibleIndex(incompatibility)) => {
+                // For the moment, this should never happen  `meta.json`
+                // do not have any footer and cannot detect incompatibility.
+                Err(crate::TantivyError::IncompatibleIndex(incompatibility))
+            }
         }
     }
 
@@ -104,7 +110,10 @@ impl ManagedDirectory {
     /// If a file cannot be deleted (for permission reasons for instance)
     /// an error is simply logged, and the file remains in the list of managed
     /// files.
-    pub fn garbage_collect<L: FnOnce() -> HashSet<PathBuf>>(&mut self, get_living_files: L) {
+    pub fn garbage_collect<L: FnOnce() -> HashSet<PathBuf>>(
+        &mut self,
+        get_living_files: L,
+    ) -> crate::Result<GarbageCollectionResult> {
         info!("Garbage collect");
         let mut files_to_delete = vec![];
 
@@ -130,19 +139,25 @@ impl ManagedDirectory {
             // 2) writer change meta.json (for instance after a merge or a commit)
             // 3) gc kicks in.
             // 4) gc removes a file that was useful for process B, before process B opened it.
-            if let Ok(_meta_lock) = self.acquire_lock(&META_LOCK) {
-                let living_files = get_living_files();
-                for managed_path in &meta_informations_rlock.managed_paths {
-                    if !living_files.contains(managed_path) {
-                        files_to_delete.push(managed_path.clone());
+            match self.acquire_lock(&META_LOCK) {
+                Ok(_meta_lock) => {
+                    let living_files = get_living_files();
+                    for managed_path in &meta_informations_rlock.managed_paths {
+                        if !living_files.contains(managed_path) {
+                            files_to_delete.push(managed_path.clone());
+                        }
                     }
                 }
-            } else {
-                error!("Failed to acquire lock for GC");
+                Err(err) => {
+                    error!("Failed to acquire lock for GC");
+                    return Err(crate::Error::from(err));
+                }
             }
         }
 
+        let mut failed_to_delete_files = vec![];
         let mut deleted_files = vec![];
+
         for file_to_delete in files_to_delete {
             match self.delete(&file_to_delete) {
                 Ok(_) => {
@@ -152,9 +167,10 @@ impl ManagedDirectory {
                 Err(file_error) => {
                     match file_error {
                         DeleteError::FileDoesNotExist(_) => {
-                            deleted_files.push(file_to_delete);
+                            deleted_files.push(file_to_delete.clone());
                         }
                         DeleteError::IOError(_) => {
+                            failed_to_delete_files.push(file_to_delete.clone());
                             if !cfg!(target_os = "windows") {
                                 // On windows, delete is expected to fail if the file
                                 // is mmapped.
@@ -177,10 +193,13 @@ impl ManagedDirectory {
             for delete_file in &deleted_files {
                 managed_paths_write.remove(delete_file);
             }
-            if save_managed_paths(self.directory.as_mut(), &meta_informations_wlock).is_err() {
-                error!("Failed to save the list of managed files.");
-            }
+            save_managed_paths(self.directory.as_mut(), &meta_informations_wlock)?;
         }
+
+        Ok(GarbageCollectionResult {
+            deleted_files,
+            failed_to_delete_files,
+        })
     }
 
     /// Registers a file as managed
@@ -247,8 +266,9 @@ impl ManagedDirectory {
 impl Directory for ManagedDirectory {
     fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
         let read_only_source = self.directory.open_read(path)?;
-        let (_footer, reader) = Footer::extract_footer(read_only_source)
+        let (footer, reader) = Footer::extract_footer(read_only_source)
             .map_err(|err| IOError::with_path(path.to_path_buf(), err))?;
+        footer.is_compatible()?;
         Ok(reader)
     }
 
@@ -328,7 +348,7 @@ mod tests_mmap_specific {
             assert!(managed_directory.exists(test_path1));
             assert!(managed_directory.exists(test_path2));
             let living_files: HashSet<PathBuf> = [test_path1.to_owned()].iter().cloned().collect();
-            managed_directory.garbage_collect(|| living_files);
+            assert!(managed_directory.garbage_collect(|| living_files).is_ok());
             assert!(managed_directory.exists(test_path1));
             assert!(!managed_directory.exists(test_path2));
         }
@@ -338,7 +358,7 @@ mod tests_mmap_specific {
             assert!(managed_directory.exists(test_path1));
             assert!(!managed_directory.exists(test_path2));
             let living_files: HashSet<PathBuf> = HashSet::new();
-            managed_directory.garbage_collect(|| living_files);
+            assert!(managed_directory.garbage_collect(|| living_files).is_ok());
             assert!(!managed_directory.exists(test_path1));
             assert!(!managed_directory.exists(test_path2));
         }
@@ -360,7 +380,9 @@ mod tests_mmap_specific {
         assert!(managed_directory.exists(test_path1));
 
         let _mmap_read = managed_directory.open_read(test_path1).unwrap();
-        managed_directory.garbage_collect(|| living_files.clone());
+        assert!(managed_directory
+            .garbage_collect(|| living_files.clone())
+            .is_ok());
         if cfg!(target_os = "windows") {
             // On Windows, gc should try and fail the file as it is mmapped.
             assert!(managed_directory.exists(test_path1));
@@ -368,7 +390,7 @@ mod tests_mmap_specific {
             drop(_mmap_read);
             // The file should still be in the list of managed file and
             // eventually be deleted once mmap is released.
-            managed_directory.garbage_collect(|| living_files);
+            assert!(managed_directory.garbage_collect(|| living_files).is_ok());
             assert!(!managed_directory.exists(test_path1));
         } else {
             assert!(!managed_directory.exists(test_path1));
@@ -393,6 +415,8 @@ mod tests_mmap_specific {
         write.write_all(&[3u8, 4u8, 5u8]).unwrap();
         write.terminate().unwrap();
 
+        let read_source = managed_directory.open_read(test_path2).unwrap();
+        assert_eq!(read_source.as_slice(), &[3u8, 4u8, 5u8]);
         assert!(managed_directory.list_damaged().unwrap().is_empty());
 
         let mut corrupted_path = tempdir_path.clone();

src/directory/mmap_directory.rs

@@ -174,7 +174,7 @@ impl WatcherWrapper {
                             // We might want to be more accurate than this at one point.
                             if let Some(filename) = changed_path.file_name() {
                                 if filename == *META_FILEPATH {
-                                    watcher_router_clone.broadcast();
+                                    let _ = watcher_router_clone.broadcast();
                                 }
                             }
                         }
@@ -538,16 +538,15 @@ mod tests {
     // The following tests are specific to the MmapDirectory
 
     use super::*;
+    use crate::indexer::LogMergePolicy;
     use crate::schema::{Schema, SchemaBuilder, TEXT};
     use crate::Index;
     use crate::ReloadPolicy;
     use std::fs;
     use std::sync::atomic::{AtomicUsize, Ordering};
-    use std::thread;
-    use std::time::Duration;
 
     #[test]
-    fn test_open_non_existant_path() {
+    fn test_open_non_existent_path() {
         assert!(MmapDirectory::open(PathBuf::from("./nowhere")).is_err());
     }
 
@@ -640,13 +639,18 @@ mod tests {
         let tmp_dir = tempfile::TempDir::new().unwrap();
         let tmp_dirpath = tmp_dir.path().to_owned();
         let mut watch_wrapper = WatcherWrapper::new(&tmp_dirpath).unwrap();
-        let tmp_file = tmp_dirpath.join("coucou");
+        let tmp_file = tmp_dirpath.join(*META_FILEPATH);
         let _handle = watch_wrapper.watch(Box::new(move || {
             counter_clone.fetch_add(1, Ordering::SeqCst);
         }));
+        let (sender, receiver) = crossbeam::channel::unbounded();
+        let _handle2 = watch_wrapper.watch(Box::new(move || {
+            let _ = sender.send(());
+        }));
         assert_eq!(counter.load(Ordering::SeqCst), 0);
         fs::write(&tmp_file, b"whateverwilldo").unwrap();
-        thread::sleep(Duration::new(0, 1_000u32));
+        assert!(receiver.recv().is_ok());
+        assert!(counter.load(Ordering::SeqCst) >= 1);
     }
 
     #[test]
@@ -655,34 +659,42 @@ mod tests {
         let mut schema_builder: SchemaBuilder = Schema::builder();
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
+
         {
             let index = Index::create(mmap_directory.clone(), schema).unwrap();
+
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-            for _num_commits in 0..16 {
+            let mut log_merge_policy = LogMergePolicy::default();
+            log_merge_policy.set_min_merge_size(3);
+            index_writer.set_merge_policy(Box::new(log_merge_policy));
+            for _num_commits in 0..10 {
                 for _ in 0..10 {
                     index_writer.add_document(doc!(text_field=>"abc"));
                 }
                 index_writer.commit().unwrap();
             }
+
             let reader = index
                 .reader_builder()
                 .reload_policy(ReloadPolicy::Manual)
                 .try_into()
                 .unwrap();
-            for _ in 0..30 {
+
+            for _ in 0..4 {
                 index_writer.add_document(doc!(text_field=>"abc"));
                 index_writer.commit().unwrap();
                 reader.reload().unwrap();
             }
             index_writer.wait_merging_threads().unwrap();
+
             reader.reload().unwrap();
             let num_segments = reader.searcher().segment_readers().len();
-            assert_eq!(num_segments, 4);
+            assert!(num_segments <= 4);
             assert_eq!(
                 num_segments * 7,
                 mmap_directory.get_cache_info().mmapped.len()
             );
         }
-        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
+        assert!(mmap_directory.get_cache_info().mmapped.is_empty());
     }
 }

src/directory/mod.rs

@@ -26,6 +26,21 @@ pub use self::read_only_source::ReadOnlySource;
 pub(crate) use self::watch_event_router::WatchCallbackList;
 pub use self::watch_event_router::{WatchCallback, WatchHandle};
 use std::io::{self, BufWriter, Write};
+use std::path::PathBuf;
+
+/// Outcome of the Garbage collection
+pub struct GarbageCollectionResult {
+    /// List of files that were deleted in this cycle
+    pub deleted_files: Vec<PathBuf>,
+    /// List of files that were schedule to be deleted in this cycle,
+    /// but deletion did not work. This typically happens on windows,
+    /// as deleting a memory mapped file is forbidden.
+    ///
+    /// If a searcher is still held, a file cannot be deleted.
+    /// This is not considered a bug, the file will simply be deleted
+    /// in the next GC.
+    pub failed_to_delete_files: Vec<PathBuf>,
+}
 
 #[cfg(feature = "mmap")]
 pub use self::mmap_directory::MmapDirectory;
@@ -33,6 +48,9 @@ pub use self::mmap_directory::MmapDirectory;
 pub use self::managed_directory::ManagedDirectory;
 
 /// Struct used to prevent from calling [`terminate_ref`](trait.TerminatingWrite#method.terminate_ref) directly
+///
+/// The point is that while the type is public, it cannot be built by anyone
+/// outside of this module.
 pub struct AntiCallToken(());
 
 /// Trait used to indicate when no more write need to be done on a writer
@@ -63,6 +81,13 @@ impl<W: TerminatingWrite> TerminatingWrite for BufWriter<W> {
     }
 }
 
+#[cfg(test)]
+impl<'a> TerminatingWrite for &'a mut Vec<u8> {
+    fn terminate_ref(&mut self, _a: AntiCallToken) -> io::Result<()> {
+        self.flush()
+    }
+}
+
 /// Write object for Directory.
 ///
 /// `WritePtr` are required to implement both Write

src/directory/ram_directory.rs

@@ -191,11 +191,11 @@ impl Directory for RAMDirectory {
         // Reserve the path to prevent calls to .write() to succeed.
         self.fs.write().unwrap().write(path_buf.clone(), &[]);
 
-        let mut vec_writer = VecWriter::new(path_buf.clone(), self.clone());
+        let mut vec_writer = VecWriter::new(path_buf, self.clone());
         vec_writer.write_all(data)?;
         vec_writer.flush()?;
         if path == Path::new(&*META_FILEPATH) {
-            self.fs.write().unwrap().watch_router.broadcast();
+            let _ = self.fs.write().unwrap().watch_router.broadcast();
         }
         Ok(())
     }

src/directory/read_only_source.rs

@@ -70,6 +70,12 @@ impl ReadOnlySource {
         (left, right)
     }
 
+    /// Splits into 2 `ReadOnlySource`, at the offset `end - right_len`.
+    pub fn split_from_end(self, right_len: usize) -> (ReadOnlySource, ReadOnlySource) {
+        let left_len = self.len() - right_len;
+        self.split(left_len)
+    }
+
     /// Creates a ReadOnlySource that is just a
     /// view over a slice of the data.
     ///

src/directory/tests.rs

@@ -1,25 +1,117 @@
 use super::*;
+use futures::channel::oneshot;
+use futures::executor::block_on;
 use std::io::Write;
 use std::mem;
 use std::path::{Path, PathBuf};
-use std::sync::atomic::AtomicUsize;
-use std::sync::atomic::Ordering;
+use std::sync::atomic::Ordering::SeqCst;
+use std::sync::atomic::{AtomicBool, AtomicUsize};
 use std::sync::Arc;
-use std::thread;
-use std::time;
 use std::time::Duration;
 
-#[test]
-fn test_ram_directory() {
-    let mut ram_directory = RAMDirectory::create();
-    test_directory(&mut ram_directory);
+#[cfg(feature = "mmap")]
+mod mmap_directory_tests {
+    use crate::directory::MmapDirectory;
+
+    type DirectoryImpl = MmapDirectory;
+
+    fn make_directory() -> DirectoryImpl {
+        MmapDirectory::create_from_tempdir().unwrap()
+    }
+
+    #[test]
+    fn test_simple() {
+        let mut directory = make_directory();
+        super::test_simple(&mut directory);
+    }
+
+    #[test]
+    fn test_write_create_the_file() {
+        let mut directory = make_directory();
+        super::test_write_create_the_file(&mut directory);
+    }
+
+    #[test]
+    fn test_rewrite_forbidden() {
+        let mut directory = make_directory();
+        super::test_rewrite_forbidden(&mut directory);
+    }
+
+    #[test]
+    fn test_directory_delete() {
+        let mut directory = make_directory();
+        super::test_directory_delete(&mut directory);
+    }
+
+    #[test]
+    fn test_lock_non_blocking() {
+        let mut directory = make_directory();
+        super::test_lock_non_blocking(&mut directory);
+    }
+
+    #[test]
+    fn test_lock_blocking() {
+        let mut directory = make_directory();
+        super::test_lock_blocking(&mut directory);
+    }
+
+    #[test]
+    fn test_watch() {
+        let mut directory = make_directory();
+        super::test_watch(&mut directory);
+    }
 }
 
-#[test]
-#[cfg(feature = "mmap")]
-fn test_mmap_directory() {
-    let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
-    test_directory(&mut mmap_directory);
+mod ram_directory_tests {
+    use crate::directory::RAMDirectory;
+
+    type DirectoryImpl = RAMDirectory;
+
+    fn make_directory() -> DirectoryImpl {
+        RAMDirectory::default()
+    }
+
+    #[test]
+    fn test_simple() {
+        let mut directory = make_directory();
+        super::test_simple(&mut directory);
+    }
+
+    #[test]
+    fn test_write_create_the_file() {
+        let mut directory = make_directory();
+        super::test_write_create_the_file(&mut directory);
+    }
+
+    #[test]
+    fn test_rewrite_forbidden() {
+        let mut directory = make_directory();
+        super::test_rewrite_forbidden(&mut directory);
+    }
+
+    #[test]
+    fn test_directory_delete() {
+        let mut directory = make_directory();
+        super::test_directory_delete(&mut directory);
+    }
+
+    #[test]
+    fn test_lock_non_blocking() {
+        let mut directory = make_directory();
+        super::test_lock_non_blocking(&mut directory);
+    }
+
+    #[test]
+    fn test_lock_blocking() {
+        let mut directory = make_directory();
+        super::test_lock_blocking(&mut directory);
+    }
+
+    #[test]
+    fn test_watch() {
+        let mut directory = make_directory();
+        super::test_watch(&mut directory);
+    }
 }
 
 #[test]
@@ -99,48 +191,39 @@ fn test_directory_delete(directory: &mut dyn Directory) {
     assert!(directory.delete(&test_path).is_err());
 }
 
-fn test_directory(directory: &mut dyn Directory) {
-    test_simple(directory);
-    test_rewrite_forbidden(directory);
-    test_write_create_the_file(directory);
-    test_directory_delete(directory);
-    test_lock_non_blocking(directory);
-    test_lock_blocking(directory);
-    test_watch(directory);
-}
-
 fn test_watch(directory: &mut dyn Directory) {
+    let num_progress: Arc<AtomicUsize> = Default::default();
     let counter: Arc<AtomicUsize> = Default::default();
     let counter_clone = counter.clone();
+    let (sender, receiver) = crossbeam::channel::unbounded();
     let watch_callback = Box::new(move || {
-        counter_clone.fetch_add(1, Ordering::SeqCst);
+        counter_clone.fetch_add(1, SeqCst);
     });
-    assert!(directory
-        .atomic_write(Path::new("meta.json"), b"random_test_data")
-        .is_ok());
-    thread::sleep(Duration::new(0, 10_000));
-    assert_eq!(0, counter.load(Ordering::SeqCst));
-
+    // This callback is used to synchronize watching in our unit test.
+    // We bind it to a variable because the callback is removed when that
+    // handle is dropped.
     let watch_handle = directory.watch(watch_callback).unwrap();
+    let _progress_listener = directory
+        .watch(Box::new(move || {
+            let val = num_progress.fetch_add(1, SeqCst);
+            let _ = sender.send(val);
+        }))
+        .unwrap();
+
     for i in 0..10 {
-        assert_eq!(i, counter.load(Ordering::SeqCst));
+        assert_eq!(i, counter.load(SeqCst));
         assert!(directory
             .atomic_write(Path::new("meta.json"), b"random_test_data_2")
             .is_ok());
-        for _ in 0..1_000 {
-            if counter.load(Ordering::SeqCst) > i {
-                break;
-            }
-            thread::sleep(Duration::from_millis(10));
-        }
-        assert_eq!(i + 1, counter.load(Ordering::SeqCst));
+        assert_eq!(receiver.recv_timeout(Duration::from_millis(500)), Ok(i));
+        assert_eq!(i + 1, counter.load(SeqCst));
     }
     mem::drop(watch_handle);
     assert!(directory
         .atomic_write(Path::new("meta.json"), b"random_test_data")
         .is_ok());
-    thread::sleep(Duration::from_millis(200));
-    assert_eq!(10, counter.load(Ordering::SeqCst));
+    assert!(receiver.recv_timeout(Duration::from_millis(500)).is_ok());
+    assert_eq!(10, counter.load(SeqCst));
 }
 
 fn test_lock_non_blocking(directory: &mut dyn Directory) {
@@ -174,9 +257,13 @@ fn test_lock_blocking(directory: &mut dyn Directory) {
         is_blocking: true,
     });
     assert!(lock_a_res.is_ok());
+    let in_thread = Arc::new(AtomicBool::default());
+    let in_thread_clone = in_thread.clone();
+    let (sender, receiver) = oneshot::channel();
     std::thread::spawn(move || {
         //< lock_a_res is sent to the thread.
-        std::thread::sleep(time::Duration::from_millis(10));
+        in_thread_clone.store(true, SeqCst);
+        let _just_sync = block_on(receiver);
         // explicitely droping lock_a_res. It would have been sufficient to just force it
         // to be part of the move, but the intent seems clearer that way.
         drop(lock_a_res);
@@ -189,14 +276,18 @@ fn test_lock_blocking(directory: &mut dyn Directory) {
         });
         assert!(lock_a_res.is_err());
     }
-    {
-        // the blocking call should wait for at least 10ms.
-        let start = time::Instant::now();
-        let lock_a_res = directory.acquire_lock(&Lock {
+    let directory_clone = directory.box_clone();
+    let (sender2, receiver2) = oneshot::channel();
+    let join_handle = std::thread::spawn(move || {
+        assert!(sender2.send(()).is_ok());
+        let lock_a_res = directory_clone.acquire_lock(&Lock {
             filepath: PathBuf::from("a.lock"),
             is_blocking: true,
         });
+        assert!(in_thread.load(SeqCst));
         assert!(lock_a_res.is_ok());
-        assert!(start.elapsed().subsec_millis() >= 10);
-    }
+    });
+    assert!(block_on(receiver2).is_ok());
+    assert!(sender.send(()).is_ok());
+    assert!(join_handle.join().is_ok());
 }

src/directory/watch_event_router.rs

@@ -1,3 +1,5 @@
+use futures::channel::oneshot;
+use futures::{Future, TryFutureExt};
 use std::sync::Arc;
 use std::sync::RwLock;
 use std::sync::Weak;
@@ -47,14 +49,21 @@ impl WatchCallbackList {
     }
 
     /// Triggers all callbacks
-    pub fn broadcast(&self) {
+    pub fn broadcast(&self) -> impl Future<Output = ()> {
         let callbacks = self.list_callback();
+        let (sender, receiver) = oneshot::channel();
+        let result = receiver.unwrap_or_else(|_| ());
+        if callbacks.is_empty() {
+            let _ = sender.send(());
+            return result;
+        }
         let spawn_res = std::thread::Builder::new()
             .name("watch-callbacks".to_string())
             .spawn(move || {
                 for callback in callbacks {
                     callback();
                 }
+                let _ = sender.send(());
             });
         if let Err(err) = spawn_res {
             error!(
@@ -62,19 +71,17 @@ impl WatchCallbackList {
                 err
             );
         }
+        result
     }
 }
 
 #[cfg(test)]
 mod tests {
     use crate::directory::WatchCallbackList;
+    use futures::executor::block_on;
     use std::mem;
     use std::sync::atomic::{AtomicUsize, Ordering};
     use std::sync::Arc;
-    use std::thread;
-    use std::time::Duration;
-
-    const WAIT_TIME: u64 = 20;
 
     #[test]
     fn test_watch_event_router_simple() {
@@ -84,22 +91,22 @@ mod tests {
         let inc_callback = Box::new(move || {
             counter_clone.fetch_add(1, Ordering::SeqCst);
         });
-        watch_event_router.broadcast();
+        block_on(watch_event_router.broadcast());
         assert_eq!(0, counter.load(Ordering::SeqCst));
         let handle_a = watch_event_router.subscribe(inc_callback);
-        thread::sleep(Duration::from_millis(WAIT_TIME));
         assert_eq!(0, counter.load(Ordering::SeqCst));
-        watch_event_router.broadcast();
-        thread::sleep(Duration::from_millis(WAIT_TIME));
+        block_on(watch_event_router.broadcast());
         assert_eq!(1, counter.load(Ordering::SeqCst));
-        watch_event_router.broadcast();
-        watch_event_router.broadcast();
-        watch_event_router.broadcast();
-        thread::sleep(Duration::from_millis(WAIT_TIME));
+        block_on(async {
+            (
+                watch_event_router.broadcast().await,
+                watch_event_router.broadcast().await,
+                watch_event_router.broadcast().await,
+            )
+        });
         assert_eq!(4, counter.load(Ordering::SeqCst));
         mem::drop(handle_a);
-        watch_event_router.broadcast();
-        thread::sleep(Duration::from_millis(WAIT_TIME));
+        block_on(watch_event_router.broadcast());
         assert_eq!(4, counter.load(Ordering::SeqCst));
     }
 
@@ -115,20 +122,20 @@ mod tests {
         };
         let handle_a = watch_event_router.subscribe(inc_callback(1));
         let handle_a2 = watch_event_router.subscribe(inc_callback(10));
-        thread::sleep(Duration::from_millis(WAIT_TIME));
         assert_eq!(0, counter.load(Ordering::SeqCst));
-        watch_event_router.broadcast();
-        watch_event_router.broadcast();
-        thread::sleep(Duration::from_millis(WAIT_TIME));
+        block_on(async {
+            futures::join!(
+                watch_event_router.broadcast(),
+                watch_event_router.broadcast()
+            )
+        });
         assert_eq!(22, counter.load(Ordering::SeqCst));
         mem::drop(handle_a);
-        watch_event_router.broadcast();
-        thread::sleep(Duration::from_millis(WAIT_TIME));
+        block_on(watch_event_router.broadcast());
         assert_eq!(32, counter.load(Ordering::SeqCst));
         mem::drop(handle_a2);
-        watch_event_router.broadcast();
-        watch_event_router.broadcast();
-        thread::sleep(Duration::from_millis(WAIT_TIME));
+        block_on(watch_event_router.broadcast());
+        block_on(watch_event_router.broadcast());
         assert_eq!(32, counter.load(Ordering::SeqCst));
     }
 
@@ -142,14 +149,15 @@ mod tests {
         });
         let handle_a = watch_event_router.subscribe(inc_callback);
         assert_eq!(0, counter.load(Ordering::SeqCst));
-        watch_event_router.broadcast();
-        watch_event_router.broadcast();
-        thread::sleep(Duration::from_millis(WAIT_TIME));
+        block_on(async {
+            let future1 = watch_event_router.broadcast();
+            let future2 = watch_event_router.broadcast();
+            futures::join!(future1, future2)
+        });
         assert_eq!(2, counter.load(Ordering::SeqCst));
-        thread::sleep(Duration::from_millis(WAIT_TIME));
         mem::drop(handle_a);
-        watch_event_router.broadcast();
-        thread::sleep(Duration::from_millis(WAIT_TIME));
+        let _ = watch_event_router.broadcast();
+        block_on(watch_event_router.broadcast());
         assert_eq!(2, counter.load(Ordering::SeqCst));
     }
 }

src/error.rs

@@ -2,8 +2,8 @@
 
 use std::io;
 
-use crate::directory::error::LockError;
 use crate::directory::error::{IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
+use crate::directory::error::{Incompatibility, LockError};
 use crate::fastfield::FastFieldNotAvailableError;
 use crate::query;
 use crate::schema;
@@ -80,6 +80,9 @@ pub enum TantivyError {
     /// System error. (e.g.: We failed spawning a new thread)
     #[fail(display = "System error.'{}'", _0)]
     SystemError(String),
+    /// Index incompatible with current version of tantivy
+    #[fail(display = "{:?}", _0)]
+    IncompatibleIndex(Incompatibility),
 }
 
 impl From<DataCorruption> for TantivyError {
@@ -129,6 +132,9 @@ impl From<OpenReadError> for TantivyError {
         match error {
             OpenReadError::FileDoesNotExist(filepath) => TantivyError::PathDoesNotExist(filepath),
             OpenReadError::IOError(io_error) => TantivyError::IOError(io_error),
+            OpenReadError::IncompatibleIndex(incompatibility) => {
+                TantivyError::IncompatibleIndex(incompatibility)
+            }
         }
     }
 }

src/fastfield/delete.rs

@@ -1,9 +1,8 @@
-use crate::common::HasLen;
+use crate::common::{BitSet, HasLen};
 use crate::directory::ReadOnlySource;
 use crate::directory::WritePtr;
 use crate::space_usage::ByteCount;
 use crate::DocId;
-use bit_set::BitSet;
 use std::io;
 use std::io::Write;
 
@@ -17,7 +16,7 @@ pub fn write_delete_bitset(
 ) -> io::Result<()> {
     let mut byte = 0u8;
     let mut shift = 0u8;
-    for doc in 0..(max_doc as usize) {
+    for doc in 0..max_doc {
         if delete_bitset.contains(doc) {
             byte |= 1 << shift;
         }
@@ -32,7 +31,7 @@ pub fn write_delete_bitset(
     if max_doc % 8 > 0 {
         writer.write_all(&[byte])?;
     }
-    writer.flush()
+    Ok(())
 }
 
 /// Set of deleted `DocId`s.
@@ -86,7 +85,6 @@ impl HasLen for DeleteBitSet {
 mod tests {
     use super::*;
     use crate::directory::*;
-    use bit_set::BitSet;
     use std::path::PathBuf;
 
     fn test_delete_bitset_helper(bitset: &BitSet, max_doc: u32) {
@@ -95,27 +93,26 @@ mod tests {
         {
             let mut writer = directory.open_write(&*test_path).unwrap();
             write_delete_bitset(bitset, max_doc, &mut writer).unwrap();
+            writer.terminate().unwrap();
         }
-        {
-            let source = directory.open_read(&test_path).unwrap();
-            let delete_bitset = DeleteBitSet::open(source);
-            for doc in 0..max_doc as usize {
-                assert_eq!(bitset.contains(doc), delete_bitset.is_deleted(doc as DocId));
-            }
-            assert_eq!(delete_bitset.len(), bitset.len());
+        let source = directory.open_read(&test_path).unwrap();
+        let delete_bitset = DeleteBitSet::open(source);
+        for doc in 0..max_doc {
+            assert_eq!(bitset.contains(doc), delete_bitset.is_deleted(doc as DocId));
         }
+        assert_eq!(delete_bitset.len(), bitset.len());
     }
 
     #[test]
     fn test_delete_bitset() {
         {
-            let mut bitset = BitSet::with_capacity(10);
+            let mut bitset = BitSet::with_max_value(10);
             bitset.insert(1);
             bitset.insert(9);
             test_delete_bitset_helper(&bitset, 10);
         }
         {
-            let mut bitset = BitSet::with_capacity(8);
+            let mut bitset = BitSet::with_max_value(8);
             bitset.insert(1);
             bitset.insert(2);
             bitset.insert(3);

src/fastfield/readers.rs

@@ -156,8 +156,6 @@ impl FastFieldReaders {
     /// If the field is a i64-fast field, return the associated u64 reader. Values are
     /// mapped from i64 to u64 using a (well the, it is unique) monotonic mapping.    ///
     ///
-    ///TODO should it also be lenient with f64?
-    ///
     /// This method is useful when merging segment reader.
     pub(crate) fn u64_lenient(&self, field: Field) -> Option<FastFieldReader<u64>> {
         if let Some(u64_ff_reader) = self.u64(field) {
@@ -166,6 +164,9 @@ impl FastFieldReaders {
         if let Some(i64_ff_reader) = self.i64(field) {
             return Some(i64_ff_reader.into_u64_reader());
         }
+        if let Some(f64_ff_reader) = self.f64(field) {
+            return Some(f64_ff_reader.into_u64_reader());
+        }
         None
     }
 
@@ -202,6 +203,9 @@ impl FastFieldReaders {
         if let Some(i64s_ff_reader) = self.i64s(field) {
             return Some(i64s_ff_reader.into_u64s_reader());
         }
+        if let Some(f64s_ff_reader) = self.f64s(field) {
+            return Some(f64s_ff_reader.into_u64s_reader());
+        }
         None
     }
 

src/indexer/delete_queue.rs

@@ -2,7 +2,7 @@ use super::operation::DeleteOperation;
 use crate::Opstamp;
 use std::mem;
 use std::ops::DerefMut;
-use std::sync::{Arc, RwLock};
+use std::sync::{Arc, RwLock, Weak};
 
 // The DeleteQueue is similar in conceptually to a multiple
 // consumer single producer broadcast channel.
@@ -14,14 +14,15 @@ use std::sync::{Arc, RwLock};
 //
 // New consumer can be created in two ways
 // - calling `delete_queue.cursor()` returns a cursor, that
-//   will include all future delete operation (and no past operations).
+//   will include all future delete operation (and some or none
+//   of the past operations... The client is in charge of checking the opstamps.).
 // - cloning an existing cursor returns a new cursor, that
 //   is at the exact same position, and can now advance independently
 //   from the original cursor.
 #[derive(Default)]
 struct InnerDeleteQueue {
     writer: Vec<DeleteOperation>,
-    last_block: Option<Arc<Block>>,
+    last_block: Weak<Block>,
 }
 
 #[derive(Clone)]
@@ -32,21 +33,31 @@ pub struct DeleteQueue {
 impl DeleteQueue {
     // Creates a new delete queue.
     pub fn new() -> DeleteQueue {
-        let delete_queue = DeleteQueue {
+        DeleteQueue {
             inner: Arc::default(),
-        };
-
-        let next_block = NextBlock::from(delete_queue.clone());
-
-        {
-            let mut delete_queue_wlock = delete_queue.inner.write().unwrap();
-            delete_queue_wlock.last_block = Some(Arc::new(Block {
-                operations: Arc::default(),
-                next: next_block,
-            }));
         }
+    }
 
-        delete_queue
+    fn get_last_block(&self) -> Arc<Block> {
+        {
+            // try get the last block with simply acquiring the read lock.
+            let rlock = self.inner.read().unwrap();
+            if let Some(block) = rlock.last_block.upgrade() {
+                return block;
+            }
+        }
+        // It failed. Let's double check after acquiring the write, as someone could have called
+        // `get_last_block` right after we released the rlock.
+        let mut wlock = self.inner.write().unwrap();
+        if let Some(block) = wlock.last_block.upgrade() {
+            return block;
+        }
+        let block = Arc::new(Block {
+            operations: Arc::default(),
+            next: NextBlock::from(self.clone()),
+        });
+        wlock.last_block = Arc::downgrade(&block);
+        block
     }
 
     // Creates a new cursor that makes it possible to
@@ -54,17 +65,7 @@ impl DeleteQueue {
     //
     // Past delete operations are not accessible.
     pub fn cursor(&self) -> DeleteCursor {
-        let last_block = self
-            .inner
-            .read()
-            .expect("Read lock poisoned when opening delete queue cursor")
-            .last_block
-            .clone()
-            .expect(
-                "Failed to unwrap last_block. This should never happen
-                as the Option<> is only here to make
-                initialization possible",
-            );
+        let last_block = self.get_last_block();
         let operations_len = last_block.operations.len();
         DeleteCursor {
             block: last_block,
@@ -100,23 +101,19 @@ impl DeleteQueue {
             .write()
             .expect("Failed to acquire write lock on delete queue writer");
 
-        let delete_operations;
-        {
-            let writer: &mut Vec<DeleteOperation> = &mut self_wlock.writer;
-            if writer.is_empty() {
-                return None;
-            }
-            delete_operations = mem::replace(writer, vec![]);
+        if self_wlock.writer.is_empty() {
+            return None;
         }
 
-        let next_block = NextBlock::from(self.clone());
-        {
-            self_wlock.last_block = Some(Arc::new(Block {
-                operations: Arc::new(delete_operations),
-                next: next_block,
-            }));
-        }
-        self_wlock.last_block.clone()
+        let delete_operations = mem::replace(&mut self_wlock.writer, vec![]);
+
+        let new_block = Arc::new(Block {
+            operations: Arc::new(delete_operations.into_boxed_slice()),
+            next: NextBlock::from(self.clone()),
+        });
+
+        self_wlock.last_block = Arc::downgrade(&new_block);
+        Some(new_block)
     }
 }
 
@@ -170,7 +167,7 @@ impl NextBlock {
 }
 
 struct Block {
-    operations: Arc<Vec<DeleteOperation>>,
+    operations: Arc<Box<[DeleteOperation]>>,
     next: NextBlock,
 }
 

src/indexer/index_writer.rs

@@ -1,14 +1,15 @@
 use super::operation::{AddOperation, UserOperation};
 use super::segment_updater::SegmentUpdater;
 use super::PreparedCommit;
+use crate::common::BitSet;
 use crate::core::Index;
 use crate::core::Segment;
 use crate::core::SegmentComponent;
 use crate::core::SegmentId;
 use crate::core::SegmentMeta;
 use crate::core::SegmentReader;
-use crate::directory::DirectoryLock;
 use crate::directory::TerminatingWrite;
+use crate::directory::{DirectoryLock, GarbageCollectionResult};
 use crate::docset::DocSet;
 use crate::error::TantivyError;
 use crate::fastfield::write_delete_bitset;
@@ -23,10 +24,9 @@ use crate::schema::Document;
 use crate::schema::IndexRecordOption;
 use crate::schema::Term;
 use crate::Opstamp;
-use crate::Result;
-use bit_set::BitSet;
 use crossbeam::channel;
-use futures::{Canceled, Future};
+use futures::executor::block_on;
+use futures::future::Future;
 use smallvec::smallvec;
 use smallvec::SmallVec;
 use std::mem;
@@ -72,7 +72,7 @@ pub struct IndexWriter {
 
     heap_size_in_bytes_per_thread: usize,
 
-    workers_join_handle: Vec<JoinHandle<Result<()>>>,
+    workers_join_handle: Vec<JoinHandle<crate::Result<()>>>,
 
     operation_receiver: OperationReceiver,
     operation_sender: OperationSender,
@@ -95,7 +95,7 @@ fn compute_deleted_bitset(
     delete_cursor: &mut DeleteCursor,
     doc_opstamps: &DocToOpstampMapping,
     target_opstamp: Opstamp,
-) -> Result<bool> {
+) -> crate::Result<bool> {
     let mut might_have_changed = false;
     while let Some(delete_op) = delete_cursor.get() {
         if delete_op.opstamp > target_opstamp {
@@ -115,7 +115,7 @@ fn compute_deleted_bitset(
             while docset.advance() {
                 let deleted_doc = docset.doc();
                 if deleted_doc < limit_doc {
-                    delete_bitset.insert(deleted_doc as usize);
+                    delete_bitset.insert(deleted_doc);
                     might_have_changed = true;
                 }
             }
@@ -126,51 +126,61 @@ fn compute_deleted_bitset(
     Ok(might_have_changed)
 }
 
-/// Advance delete for the given segment up
-/// to the target opstamp.
+/// Advance delete for the given segment up to the target opstamp.
+///
+/// Note that there are no guarantee that the resulting `segment_entry` delete_opstamp
+/// is `==` target_opstamp.
+/// For instance, there was no delete operation between the state of the `segment_entry` and
+/// the `target_opstamp`, `segment_entry` is not updated.
 pub(crate) fn advance_deletes(
     mut segment: Segment,
     segment_entry: &mut SegmentEntry,
     target_opstamp: Opstamp,
-) -> Result<()> {
-    {
-        if segment_entry.meta().delete_opstamp() == Some(target_opstamp) {
-            // We are already up-to-date here.
-            return Ok(());
-        }
+) -> crate::Result<()> {
+    if segment_entry.meta().delete_opstamp() == Some(target_opstamp) {
+        // We are already up-to-date here.
+        return Ok(());
+    }
 
-        let segment_reader = SegmentReader::open(&segment)?;
+    let mut delete_cursor = segment_entry.delete_cursor().clone();
+    if segment_entry.delete_bitset().is_none() && delete_cursor.get().is_none() {
+        // There has been no `DeleteOperation` between the segment status and `target_opstamp`.
+        return Ok(());
+    }
 
-        let max_doc = segment_reader.max_doc();
-        let mut delete_bitset: BitSet = match segment_entry.delete_bitset() {
-            Some(previous_delete_bitset) => (*previous_delete_bitset).clone(),
-            None => BitSet::with_capacity(max_doc as usize),
-        };
+    let segment_reader = SegmentReader::open(&segment)?;
 
-        let delete_cursor = segment_entry.delete_cursor();
-        compute_deleted_bitset(
-            &mut delete_bitset,
-            &segment_reader,
-            delete_cursor,
-            &DocToOpstampMapping::None,
-            target_opstamp,
-        )?;
+    let max_doc = segment_reader.max_doc();
+    let mut delete_bitset: BitSet = match segment_entry.delete_bitset() {
+        Some(previous_delete_bitset) => (*previous_delete_bitset).clone(),
+        None => BitSet::with_max_value(max_doc),
+    };
 
-        // TODO optimize
+    compute_deleted_bitset(
+        &mut delete_bitset,
+        &segment_reader,
+        &mut delete_cursor,
+        &DocToOpstampMapping::None,
+        target_opstamp,
+    )?;
+
+    // TODO optimize
+    if let Some(seg_delete_bitset) = segment_reader.delete_bitset() {
         for doc in 0u32..max_doc {
-            if segment_reader.is_deleted(doc) {
-                delete_bitset.insert(doc as usize);
+            if seg_delete_bitset.is_deleted(doc) {
+                delete_bitset.insert(doc);
             }
         }
-
-        let num_deleted_docs = delete_bitset.len();
-        if num_deleted_docs > 0 {
-            segment = segment.with_delete_meta(num_deleted_docs as u32, target_opstamp);
-            let mut delete_file = segment.open_write(SegmentComponent::DELETE)?;
-            write_delete_bitset(&delete_bitset, max_doc, &mut delete_file)?;
-            delete_file.terminate()?;
-        }
     }
+
+    let num_deleted_docs = delete_bitset.len();
+    if num_deleted_docs > 0 {
+        segment = segment.with_delete_meta(num_deleted_docs as u32, target_opstamp);
+        let mut delete_file = segment.open_write(SegmentComponent::DELETE)?;
+        write_delete_bitset(&delete_bitset, max_doc, &mut delete_file)?;
+        delete_file.terminate()?;
+    }
+
     segment_entry.set_meta(segment.meta().clone());
     Ok(())
 }
@@ -181,7 +191,7 @@ fn index_documents(
     grouped_document_iterator: &mut dyn Iterator<Item = OperationGroup>,
     segment_updater: &mut SegmentUpdater,
     mut delete_cursor: DeleteCursor,
-) -> Result<bool> {
+) -> crate::Result<bool> {
     let schema = segment.schema();
 
     let mut segment_writer = SegmentWriter::for_segment(memory_budget, segment.clone(), &schema)?;
@@ -227,7 +237,8 @@ fn index_documents(
         delete_cursor,
         delete_bitset_opt,
     );
-    Ok(segment_updater.add_segment(segment_entry))
+    block_on(segment_updater.schedule_add_segment(segment_entry))?;
+    Ok(true)
 }
 
 fn apply_deletes(
@@ -235,7 +246,7 @@ fn apply_deletes(
     mut delete_cursor: &mut DeleteCursor,
     doc_opstamps: &[Opstamp],
     last_docstamp: Opstamp,
-) -> Result<Option<BitSet<u32>>> {
+) -> crate::Result<Option<BitSet>> {
     if delete_cursor.get().is_none() {
         // if there are no delete operation in the queue, no need
         // to even open the segment.
@@ -245,7 +256,7 @@ fn apply_deletes(
     let doc_to_opstamps = DocToOpstampMapping::from(doc_opstamps);
 
     let max_doc = segment.meta().max_doc();
-    let mut deleted_bitset = BitSet::with_capacity(max_doc as usize);
+    let mut deleted_bitset = BitSet::with_max_value(max_doc);
     let may_have_deletes = compute_deleted_bitset(
         &mut deleted_bitset,
         &segment_reader,
@@ -280,7 +291,7 @@ impl IndexWriter {
         num_threads: usize,
         heap_size_in_bytes_per_thread: usize,
         directory_lock: DirectoryLock,
-    ) -> Result<IndexWriter> {
+    ) -> crate::Result<IndexWriter> {
         if heap_size_in_bytes_per_thread < HEAP_SIZE_MIN {
             let err_msg = format!(
                 "The heap size per thread needs to be at least {}.",
@@ -329,12 +340,17 @@ impl IndexWriter {
         Ok(index_writer)
     }
 
+    fn drop_sender(&mut self) {
+        let (sender, _receiver) = channel::bounded(1);
+        mem::replace(&mut self.operation_sender, sender);
+    }
+
     /// If there are some merging threads, blocks until they all finish their work and
     /// then drop the `IndexWriter`.
-    pub fn wait_merging_threads(mut self) -> Result<()> {
+    pub fn wait_merging_threads(mut self) -> crate::Result<()> {
         // this will stop the indexing thread,
         // dropping the last reference to the segment_updater.
-        drop(self.operation_sender);
+        self.drop_sender();
 
         let former_workers_handles = mem::replace(&mut self.workers_join_handle, vec![]);
         for join_handle in former_workers_handles {
@@ -345,7 +361,6 @@ impl IndexWriter {
                     TantivyError::ErrorInThread("Error in indexing worker thread.".into())
                 })?;
         }
-        drop(self.workers_join_handle);
 
         let result = self
             .segment_updater
@@ -360,10 +375,10 @@ impl IndexWriter {
     }
 
     #[doc(hidden)]
-    pub fn add_segment(&mut self, segment_meta: SegmentMeta) {
+    pub fn add_segment(&self, segment_meta: SegmentMeta) -> crate::Result<()> {
         let delete_cursor = self.delete_queue.cursor();
         let segment_entry = SegmentEntry::new(segment_meta, delete_cursor, None);
-        self.segment_updater.add_segment(segment_entry);
+        block_on(self.segment_updater.schedule_add_segment(segment_entry))
     }
 
     /// Creates a new segment.
@@ -380,7 +395,7 @@ impl IndexWriter {
 
     /// Spawns a new worker thread for indexing.
     /// The thread consumes documents from the pipeline.
-    fn add_indexing_worker(&mut self) -> Result<()> {
+    fn add_indexing_worker(&mut self) -> crate::Result<()> {
         let document_receiver_clone = self.operation_receiver.clone();
         let mut segment_updater = self.segment_updater.clone();
 
@@ -388,7 +403,7 @@ impl IndexWriter {
 
         let mem_budget = self.heap_size_in_bytes_per_thread;
         let index = self.index.clone();
-        let join_handle: JoinHandle<Result<()>> = thread::Builder::new()
+        let join_handle: JoinHandle<crate::Result<()>> = thread::Builder::new()
             .name(format!("thrd-tantivy-index{}", self.worker_id))
             .spawn(move || {
                 loop {
@@ -434,22 +449,23 @@ impl IndexWriter {
         self.segment_updater.get_merge_policy()
     }
 
-    /// Set the merge policy.
+    /// Setter for the merge policy.
     pub fn set_merge_policy(&self, merge_policy: Box<dyn MergePolicy>) {
         self.segment_updater.set_merge_policy(merge_policy);
     }
 
-    fn start_workers(&mut self) -> Result<()> {
+    fn start_workers(&mut self) -> crate::Result<()> {
         for _ in 0..self.num_threads {
             self.add_indexing_worker()?;
         }
         Ok(())
     }
 
-    /// Detects and removes the files that
-    /// are not used by the index anymore.
-    pub fn garbage_collect_files(&mut self) -> Result<()> {
-        self.segment_updater.garbage_collect_files().wait()
+    /// Detects and removes the files that are not used by the index anymore.
+    pub fn garbage_collect_files(
+        &self,
+    ) -> impl Future<Output = crate::Result<GarbageCollectionResult>> {
+        self.segment_updater.schedule_garbage_collect()
     }
 
     /// Deletes all documents from the index
@@ -488,7 +504,7 @@ impl IndexWriter {
     ///     Ok(())
     /// }
     /// ```
-    pub fn delete_all_documents(&mut self) -> Result<Opstamp> {
+    pub fn delete_all_documents(&self) -> crate::Result<Opstamp> {
         // Delete segments
         self.segment_updater.remove_all_segments();
         // Return new stamp - reverted stamp
@@ -499,11 +515,9 @@ impl IndexWriter {
     /// Merges a given list of segments
     ///
     /// `segment_ids` is required to be non-empty.
-    pub fn merge(
-        &mut self,
-        segment_ids: &[SegmentId],
-    ) -> Result<impl Future<Item = SegmentMeta, Error = Canceled>> {
-        self.segment_updater.start_merge(segment_ids)
+    pub async fn merge(&mut self, segment_ids: &[SegmentId]) -> crate::Result<SegmentMeta> {
+        let merge_operation = self.segment_updater.make_merge_operation(segment_ids);
+        self.segment_updater.start_merge(merge_operation)?.await
     }
 
     /// Closes the current document channel send.
@@ -529,13 +543,8 @@ impl IndexWriter {
     /// state as it was after the last commit.
     ///
     /// The opstamp at the last commit is returned.
-    pub fn rollback(&mut self) -> Result<Opstamp> {
+    pub fn rollback(&mut self) -> crate::Result<Opstamp> {
         info!("Rolling back to opstamp {}", self.committed_opstamp);
-        self.rollback_impl()
-    }
-
-    /// Private, implementation of rollback
-    fn rollback_impl(&mut self) -> Result<Opstamp> {
         // marks the segment updater as killed. From now on, all
         // segment updates will be ignored.
         self.segment_updater.kill();
@@ -591,7 +600,7 @@ impl IndexWriter {
     /// It is also possible to add a payload to the `commit`
     /// using this API.
     /// See [`PreparedCommit::set_payload()`](PreparedCommit.html)
-    pub fn prepare_commit(&mut self) -> Result<PreparedCommit<'_>> {
+    pub fn prepare_commit(&mut self) -> crate::Result<PreparedCommit> {
         // Here, because we join all of the worker threads,
         // all of the segment update for this commit have been
         // sent.
@@ -638,7 +647,7 @@ impl IndexWriter {
     /// Commit returns the `opstamp` of the last document
     /// that made it in the commit.
     ///
-    pub fn commit(&mut self) -> Result<Opstamp> {
+    pub fn commit(&mut self) -> crate::Result<Opstamp> {
         self.prepare_commit()?.commit()
     }
 
@@ -679,9 +688,6 @@ impl IndexWriter {
     /// The opstamp is an increasing `u64` that can
     /// be used by the client to align commits with its own
     /// document queue.
-    ///
-    /// Currently it represents the number of documents that
-    /// have been added since the creation of the index.
     pub fn add_document(&self, document: Document) -> Opstamp {
         let opstamp = self.stamper.stamp();
         let add_operation = AddOperation { opstamp, document };
@@ -755,6 +761,16 @@ impl IndexWriter {
     }
 }
 
+impl Drop for IndexWriter {
+    fn drop(&mut self) {
+        self.segment_updater.kill();
+        self.drop_sender();
+        for work in self.workers_join_handle.drain(..) {
+            let _ = work.join();
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
 
@@ -764,7 +780,7 @@ mod tests {
     use crate::error::*;
     use crate::indexer::NoMergePolicy;
     use crate::query::TermQuery;
-    use crate::schema::{self, IndexRecordOption};
+    use crate::schema::{self, IndexRecordOption, STRING};
     use crate::Index;
     use crate::ReloadPolicy;
     use crate::Term;
@@ -1189,4 +1205,16 @@ mod tests {
         assert!(clear_again.is_ok());
         assert!(commit_again.is_ok());
     }
+
+    #[test]
+    fn test_index_doc_missing_field() {
+        let mut schema_builder = schema::Schema::builder();
+        let idfield = schema_builder.add_text_field("id", STRING);
+        schema_builder.add_text_field("optfield", STRING);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.add_document(doc!(idfield=>"myid"));
+        let commit = index_writer.commit();
+        assert!(commit.is_ok());
+    }
 }

src/indexer/merge_operation.rs

@@ -2,14 +2,23 @@ use crate::Opstamp;
 use crate::SegmentId;
 use census::{Inventory, TrackedObject};
 use std::collections::HashSet;
+use std::ops::Deref;
 
 #[derive(Default)]
-pub struct MergeOperationInventory(Inventory<InnerMergeOperation>);
+pub(crate) struct MergeOperationInventory(Inventory<InnerMergeOperation>);
+
+impl Deref for MergeOperationInventory {
+    type Target = Inventory<InnerMergeOperation>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
 
 impl MergeOperationInventory {
     pub fn segment_in_merge(&self) -> HashSet<SegmentId> {
         let mut segment_in_merge = HashSet::default();
-        for merge_op in self.0.list() {
+        for merge_op in self.list() {
             for &segment_id in &merge_op.segment_ids {
                 segment_in_merge.insert(segment_id);
             }
@@ -35,13 +44,13 @@ pub struct MergeOperation {
     inner: TrackedObject<InnerMergeOperation>,
 }
 
-struct InnerMergeOperation {
+pub(crate) struct InnerMergeOperation {
     target_opstamp: Opstamp,
     segment_ids: Vec<SegmentId>,
 }
 
 impl MergeOperation {
-    pub fn new(
+    pub(crate) fn new(
         inventory: &MergeOperationInventory,
         target_opstamp: Opstamp,
         segment_ids: Vec<SegmentId>,
@@ -51,7 +60,7 @@ impl MergeOperation {
             segment_ids,
         };
         MergeOperation {
-            inner: inventory.0.track(inner_merge_operation),
+            inner: inventory.track(inner_merge_operation),
         }
     }
 

src/indexer/merger.rs

@@ -709,7 +709,7 @@ mod tests {
     use crate::IndexWriter;
     use crate::Searcher;
     use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
-    use futures::Future;
+    use futures::executor::block_on;
     use std::io::Cursor;
 
     #[test]
@@ -792,11 +792,7 @@ mod tests {
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-            index_writer
-                .merge(&segment_ids)
-                .expect("Failed to initiate merge")
-                .wait()
-                .expect("Merging failed");
+            block_on(index_writer.merge(&segment_ids)).expect("Merging failed");
             index_writer.wait_merging_threads().unwrap();
         }
         {
@@ -1040,11 +1036,7 @@ mod tests {
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            index_writer
-                .merge(&segment_ids)
-                .expect("Failed to initiate merge")
-                .wait()
-                .expect("Merging failed");
+            block_on(index_writer.merge(&segment_ids)).expect("Merging failed");
             reader.reload().unwrap();
             let searcher = reader.searcher();
             assert_eq!(searcher.segment_readers().len(), 1);
@@ -1139,11 +1131,7 @@ mod tests {
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            index_writer
-                .merge(&segment_ids)
-                .expect("Failed to initiate merge")
-                .wait()
-                .expect("Merging failed");
+            block_on(index_writer.merge(&segment_ids)).expect("Merging failed");
             reader.reload().unwrap();
 
             let searcher = reader.searcher();
@@ -1277,11 +1265,7 @@ mod tests {
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-            index_writer
-                .merge(&segment_ids)
-                .expect("Failed to initiate merge")
-                .wait()
-                .expect("Merging failed");
+            block_on(index_writer.merge(&segment_ids)).expect("Merging failed");
             index_writer.wait_merging_threads().unwrap();
             reader.reload().unwrap();
             test_searcher(
@@ -1336,11 +1320,7 @@ mod tests {
         let segment_ids = index
             .searchable_segment_ids()
             .expect("Searchable segments failed.");
-        index_writer
-            .merge(&segment_ids)
-            .expect("Failed to initiate merge")
-            .wait()
-            .expect("Merging failed");
+        block_on(index_writer.merge(&segment_ids)).expect("Merging failed");
         reader.reload().unwrap();
         // commit has not been called yet. The document should still be
         // there.
@@ -1361,22 +1341,18 @@ mod tests {
             let mut doc = Document::default();
             doc.add_u64(int_field, 1);
             index_writer.add_document(doc.clone());
-            index_writer.commit().expect("commit failed");
+            assert!(index_writer.commit().is_ok());
             index_writer.add_document(doc);
-            index_writer.commit().expect("commit failed");
+            assert!(index_writer.commit().is_ok());
             index_writer.delete_term(Term::from_field_u64(int_field, 1));
 
             let segment_ids = index
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
-            index_writer
-                .merge(&segment_ids)
-                .expect("Failed to initiate merge")
-                .wait()
-                .expect("Merging failed");
+            assert!(block_on(index_writer.merge(&segment_ids)).is_ok());
 
             // assert delete has not been committed
-            reader.reload().expect("failed to load searcher 1");
+            assert!(reader.reload().is_ok());
             let searcher = reader.searcher();
             assert_eq!(searcher.num_docs(), 2);
 
@@ -1415,12 +1391,12 @@ mod tests {
             index_doc(&mut index_writer, &[1, 5]);
             index_doc(&mut index_writer, &[3]);
             index_doc(&mut index_writer, &[17]);
-            index_writer.commit().expect("committed");
+            assert!(index_writer.commit().is_ok());
             index_doc(&mut index_writer, &[20]);
-            index_writer.commit().expect("committed");
+            assert!(index_writer.commit().is_ok());
             index_doc(&mut index_writer, &[28, 27]);
             index_doc(&mut index_writer, &[1_000]);
-            index_writer.commit().expect("committed");
+            assert!(index_writer.commit().is_ok());
         }
         let reader = index.reader().unwrap();
         let searcher = reader.searcher();
@@ -1452,15 +1428,6 @@ mod tests {
             assert_eq!(&vals, &[17]);
         }
 
-        println!(
-            "{:?}",
-            searcher
-                .segment_readers()
-                .iter()
-                .map(|reader| reader.max_doc())
-                .collect::<Vec<_>>()
-        );
-
         {
             let segment = searcher.segment_reader(1u32);
             let ff_reader = segment.fast_fields().u64s(int_field).unwrap();
@@ -1484,27 +1451,13 @@ mod tests {
                 .searchable_segment_ids()
                 .expect("Searchable segments failed.");
             let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-            index_writer
-                .merge(&segment_ids)
-                .expect("Failed to initiate merge")
-                .wait()
-                .expect("Merging failed");
-            index_writer
-                .wait_merging_threads()
-                .expect("Wait for merging threads");
+            assert!(block_on(index_writer.merge(&segment_ids)).is_ok());
+            assert!(index_writer.wait_merging_threads().is_ok());
         }
-        reader.reload().expect("Load searcher");
+        assert!(reader.reload().is_ok());
 
         {
             let searcher = reader.searcher();
-            println!(
-                "{:?}",
-                searcher
-                    .segment_readers()
-                    .iter()
-                    .map(|reader| reader.max_doc())
-                    .collect::<Vec<_>>()
-            );
             let segment = searcher.segment_reader(0u32);
             let ff_reader = segment.fast_fields().u64s(int_field).unwrap();
 
@@ -1539,4 +1492,46 @@ mod tests {
             assert_eq!(&vals, &[20]);
         }
     }
+
+    #[test]
+    fn merges_f64_fast_fields_correctly() -> crate::Result<()> {
+        let mut builder = schema::SchemaBuilder::new();
+
+        let fast_multi = IntOptions::default().set_fast(Cardinality::MultiValues);
+
+        let field = builder.add_f64_field("f64", schema::FAST);
+        let multi_field = builder.add_f64_field("f64s", fast_multi);
+
+        let index = Index::create_in_ram(builder.build());
+
+        let mut writer = index.writer_with_num_threads(1, 3_000_000)?;
+
+        // Make sure we'll attempt to merge every created segment
+        let mut policy = crate::indexer::LogMergePolicy::default();
+        policy.set_min_merge_size(2);
+        writer.set_merge_policy(Box::new(policy));
+
+        for i in 0..100 {
+            let mut doc = Document::new();
+            doc.add_f64(field, 42.0);
+
+            doc.add_f64(multi_field, 0.24);
+            doc.add_f64(multi_field, 0.27);
+
+            writer.add_document(doc);
+
+            if i % 5 == 0 {
+                writer.commit()?;
+            }
+        }
+
+        writer.commit()?;
+        writer.wait_merging_threads()?;
+
+        // If a merging thread fails, we should end up with more
+        // than one segment here
+        assert_eq!(1, index.searchable_segments()?.len());
+
+        Ok(())
+    }
 }

src/indexer/mod.rs

@@ -18,7 +18,7 @@ mod stamper;
 
 pub use self::index_writer::IndexWriter;
 pub use self::log_merge_policy::LogMergePolicy;
-pub use self::merge_operation::{MergeOperation, MergeOperationInventory};
+pub use self::merge_operation::MergeOperation;
 pub use self::merge_policy::{MergeCandidate, MergePolicy, NoMergePolicy};
 pub use self::prepared_commit::PreparedCommit;
 pub use self::segment_entry::SegmentEntry;

src/indexer/prepared_commit.rs

@@ -1,6 +1,7 @@
 use super::IndexWriter;
 use crate::Opstamp;
 use crate::Result;
+use futures::executor::block_on;
 
 /// A prepared commit
 pub struct PreparedCommit<'a> {
@@ -32,9 +33,11 @@ impl<'a> PreparedCommit<'a> {
 
     pub fn commit(self) -> Result<Opstamp> {
         info!("committing {}", self.opstamp);
-        self.index_writer
-            .segment_updater()
-            .commit(self.opstamp, self.payload)?;
+        let _ = block_on(
+            self.index_writer
+                .segment_updater()
+                .schedule_commit(self.opstamp, self.payload),
+        );
         Ok(self.opstamp)
     }
 }

src/indexer/segment_entry.rs

@@ -1,7 +1,7 @@
+use crate::common::BitSet;
 use crate::core::SegmentId;
 use crate::core::SegmentMeta;
 use crate::indexer::delete_queue::DeleteCursor;
-use bit_set::BitSet;
 use std::fmt;
 
 /// A segment entry describes the state of

src/indexer/segment_manager.rs

@@ -16,6 +16,28 @@ struct SegmentRegisters {
     committed: SegmentRegister,
 }
 
+#[derive(PartialEq, Eq)]
+pub(crate) enum SegmentsStatus {
+    Committed,
+    Uncommitted,
+}
+
+impl SegmentRegisters {
+    /// Check if all the segments are committed or uncommited.
+    ///
+    /// If some segment is missing or segments are in a different state (this should not happen
+    /// if tantivy is used correctly), returns `None`.
+    fn segments_status(&self, segment_ids: &[SegmentId]) -> Option<SegmentsStatus> {
+        if self.uncommitted.contains_all(segment_ids) {
+            Some(SegmentsStatus::Uncommitted)
+        } else if self.committed.contains_all(segment_ids) {
+            Some(SegmentsStatus::Committed)
+        } else {
+            None
+        }
+    }
+}
+
 /// The segment manager stores the list of segments
 /// as well as their state.
 ///
@@ -153,33 +175,35 @@ impl SegmentManager {
         let mut registers_lock = self.write();
         registers_lock.uncommitted.add_segment_entry(segment_entry);
     }
-
-    pub fn end_merge(
+    // Replace a list of segments for their equivalent merged segment.
+    //
+    // Returns true if these segments are committed, false if the merge segments are uncommited.
+    pub(crate) fn end_merge(
         &self,
         before_merge_segment_ids: &[SegmentId],
         after_merge_segment_entry: SegmentEntry,
-    ) {
+    ) -> crate::Result<SegmentsStatus> {
         let mut registers_lock = self.write();
-        let target_register: &mut SegmentRegister = {
-            if registers_lock
-                .uncommitted
-                .contains_all(before_merge_segment_ids)
-            {
-                &mut registers_lock.uncommitted
-            } else if registers_lock
-                .committed
-                .contains_all(before_merge_segment_ids)
-            {
-                &mut registers_lock.committed
-            } else {
+        let segments_status = registers_lock
+            .segments_status(before_merge_segment_ids)
+            .ok_or_else(|| {
                 warn!("couldn't find segment in SegmentManager");
-                return;
-            }
+                crate::Error::InvalidArgument(
+                    "The segments that were merged could not be found in the SegmentManager. \
+                     This is not necessarily a bug, and can happen after a rollback for instance."
+                        .to_string(),
+                )
+            })?;
+
+        let target_register: &mut SegmentRegister = match segments_status {
+            SegmentsStatus::Uncommitted => &mut registers_lock.uncommitted,
+            SegmentsStatus::Committed => &mut registers_lock.committed,
         };
         for segment_id in before_merge_segment_ids {
             target_register.remove_segment(segment_id);
         }
         target_register.add_segment_entry(after_merge_segment_entry);
+        Ok(segments_status)
     }
 
     pub fn committed_segment_metas(&self) -> Vec<SegmentMeta> {

src/indexer/segment_updater.rs

@@ -6,39 +6,34 @@ use crate::core::SegmentId;
 use crate::core::SegmentMeta;
 use crate::core::SerializableSegment;
 use crate::core::META_FILEPATH;
-use crate::directory::{Directory, DirectoryClone};
-use crate::error::TantivyError;
+use crate::directory::{Directory, DirectoryClone, GarbageCollectionResult};
 use crate::indexer::delete_queue::DeleteCursor;
 use crate::indexer::index_writer::advance_deletes;
 use crate::indexer::merge_operation::MergeOperationInventory;
 use crate::indexer::merger::IndexMerger;
+use crate::indexer::segment_manager::SegmentsStatus;
 use crate::indexer::stamper::Stamper;
-use crate::indexer::MergeOperation;
 use crate::indexer::SegmentEntry;
 use crate::indexer::SegmentSerializer;
 use crate::indexer::{DefaultMergePolicy, MergePolicy};
+use crate::indexer::{MergeCandidate, MergeOperation};
 use crate::schema::Schema;
 use crate::Opstamp;
-use crate::Result;
-use futures::oneshot;
-use futures::sync::oneshot::Receiver;
-use futures::Future;
-use futures_cpupool::Builder as CpuPoolBuilder;
-use futures_cpupool::CpuFuture;
-use futures_cpupool::CpuPool;
+use futures::channel::oneshot;
+use futures::executor::{ThreadPool, ThreadPoolBuilder};
+use futures::future::Future;
+use futures::future::TryFutureExt;
 use serde_json;
 use std::borrow::BorrowMut;
-use std::collections::HashMap;
 use std::collections::HashSet;
 use std::io::Write;
-use std::mem;
-use std::ops::DerefMut;
+use std::ops::Deref;
 use std::path::PathBuf;
-use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::sync::RwLock;
-use std::thread;
-use std::thread::JoinHandle;
+
+const NUM_MERGE_THREADS: usize = 4;
 
 /// Save the index meta file.
 /// This operation is atomic :
@@ -49,7 +44,7 @@ use std::thread::JoinHandle;
 /// and flushed.
 ///
 /// This method is not part of tantivy's public API
-pub fn save_new_metas(schema: Schema, directory: &mut dyn Directory) -> Result<()> {
+pub fn save_new_metas(schema: Schema, directory: &mut dyn Directory) -> crate::Result<()> {
     save_metas(
         &IndexMeta {
             segments: Vec::new(),
@@ -70,7 +65,7 @@ pub fn save_new_metas(schema: Schema, directory: &mut dyn Directory) -> Result<(
 /// and flushed.
 ///
 /// This method is not part of tantivy's public API
-fn save_metas(metas: &IndexMeta, directory: &mut dyn Directory) -> Result<()> {
+fn save_metas(metas: &IndexMeta, directory: &mut dyn Directory) -> crate::Result<()> {
     info!("save metas");
     let mut buffer = serde_json::to_vec_pretty(metas)?;
     // Just adding a new line at the end of the buffer.
@@ -89,21 +84,38 @@ fn save_metas(metas: &IndexMeta, directory: &mut dyn Directory) -> Result<()> {
 // We voluntarily pass a merge_operation ref to guarantee that
 // the merge_operation is alive during the process
 #[derive(Clone)]
-pub struct SegmentUpdater(Arc<InnerSegmentUpdater>);
+pub(crate) struct SegmentUpdater(Arc<InnerSegmentUpdater>);
 
-fn perform_merge(
-    merge_operation: &MergeOperation,
+impl Deref for SegmentUpdater {
+    type Target = InnerSegmentUpdater;
+
+    #[inline]
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+async fn garbage_collect_files(
+    segment_updater: SegmentUpdater,
+) -> crate::Result<GarbageCollectionResult> {
+    info!("Running garbage collection");
+    let mut index = segment_updater.index.clone();
+    index
+        .directory_mut()
+        .garbage_collect(move || segment_updater.list_files())
+}
+
+/// Merges a list of segments the list of segment givens in the `segment_entries`.
+/// This function happens in the calling thread and is computationally expensive.
+fn merge(
     index: &Index,
     mut segment_entries: Vec<SegmentEntry>,
-) -> Result<SegmentEntry> {
-    let target_opstamp = merge_operation.target_opstamp();
-
+    target_opstamp: Opstamp,
+) -> crate::Result<SegmentEntry> {
     // first we need to apply deletes to our segment.
     let mut merged_segment = index.new_segment();
 
-    // TODO add logging
-    let schema = index.schema();
-
+    // First we apply all of the delet to the merged segment, up to the target opstamp.
     for segment_entry in &mut segment_entries {
         let segment = index.segment(segment_entry.meta().clone());
         advance_deletes(segment, segment_entry, target_opstamp)?;
@@ -117,22 +129,19 @@ fn perform_merge(
         .collect();
 
     // An IndexMerger is like a "view" of our merged segments.
-    let merger: IndexMerger = IndexMerger::open(schema, &segments[..])?;
-
-    // ... we just serialize this index merger in our new segment
-    // to merge the two segments.
+    let merger: IndexMerger = IndexMerger::open(index.schema(), &segments[..])?;
 
+    // ... we just serialize this index merger in our new segment to merge the two segments.
     let segment_serializer = SegmentSerializer::for_segment(&mut merged_segment)?;
 
     let num_docs = merger.write(segment_serializer)?;
 
     let segment_meta = index.new_segment_meta(merged_segment.id(), num_docs);
 
-    let after_merge_segment_entry = SegmentEntry::new(segment_meta.clone(), delete_cursor, None);
-    Ok(after_merge_segment_entry)
+    Ok(SegmentEntry::new(segment_meta, delete_cursor, None))
 }
 
-struct InnerSegmentUpdater {
+pub(crate) struct InnerSegmentUpdater {
     // we keep a copy of the current active IndexMeta to
     // avoid loading the file everytime we need it in the
     // `SegmentUpdater`.
@@ -140,12 +149,12 @@ struct InnerSegmentUpdater {
     // This should be up to date as all update happen through
     // the unique active `SegmentUpdater`.
     active_metas: RwLock<Arc<IndexMeta>>,
-    pool: CpuPool,
+    pool: ThreadPool,
+    merge_thread_pool: ThreadPool,
+
     index: Index,
     segment_manager: SegmentManager,
     merge_policy: RwLock<Arc<Box<dyn MergePolicy>>>,
-    merging_thread_id: AtomicUsize,
-    merging_threads: RwLock<HashMap<usize, JoinHandle<Result<()>>>>,
     killed: AtomicBool,
     stamper: Stamper,
     merge_operations: MergeOperationInventory,
@@ -156,22 +165,31 @@ impl SegmentUpdater {
         index: Index,
         stamper: Stamper,
         delete_cursor: &DeleteCursor,
-    ) -> Result<SegmentUpdater> {
+    ) -> crate::Result<SegmentUpdater> {
         let segments = index.searchable_segment_metas()?;
         let segment_manager = SegmentManager::from_segments(segments, delete_cursor);
-        let pool = CpuPoolBuilder::new()
+        let pool = ThreadPoolBuilder::new()
             .name_prefix("segment_updater")
             .pool_size(1)
-            .create();
+            .create()
+            .map_err(|_| {
+                crate::Error::SystemError("Failed to spawn segment updater thread".to_string())
+            })?;
+        let merge_thread_pool = ThreadPoolBuilder::new()
+            .name_prefix("merge_thread")
+            .pool_size(NUM_MERGE_THREADS)
+            .create()
+            .map_err(|_| {
+                crate::Error::SystemError("Failed to spawn segment merging thread".to_string())
+            })?;
         let index_meta = index.load_metas()?;
         Ok(SegmentUpdater(Arc::new(InnerSegmentUpdater {
             active_metas: RwLock::new(Arc::new(index_meta)),
             pool,
+            merge_thread_pool,
             index,
             segment_manager,
             merge_policy: RwLock::new(Arc::new(Box::new(DefaultMergePolicy::default()))),
-            merging_thread_id: AtomicUsize::default(),
-            merging_threads: RwLock::new(HashMap::new()),
             killed: AtomicBool::new(false),
             stamper,
             merge_operations: Default::default(),
@@ -179,67 +197,82 @@ impl SegmentUpdater {
     }
 
     pub fn get_merge_policy(&self) -> Arc<Box<dyn MergePolicy>> {
-        self.0.merge_policy.read().unwrap().clone()
+        self.merge_policy.read().unwrap().clone()
     }
 
     pub fn set_merge_policy(&self, merge_policy: Box<dyn MergePolicy>) {
         let arc_merge_policy = Arc::new(merge_policy);
-        *self.0.merge_policy.write().unwrap() = arc_merge_policy;
+        *self.merge_policy.write().unwrap() = arc_merge_policy;
     }
 
-    fn get_merging_thread_id(&self) -> usize {
-        self.0.merging_thread_id.fetch_add(1, Ordering::SeqCst)
-    }
-
-    fn run_async<T: 'static + Send, F: 'static + Send + FnOnce(SegmentUpdater) -> T>(
+    fn schedule_future<T: 'static + Send, F: Future<Output = crate::Result<T>> + 'static + Send>(
         &self,
         f: F,
-    ) -> CpuFuture<T, TantivyError> {
-        let me_clone = self.clone();
-        self.0.pool.spawn_fn(move || Ok(f(me_clone)))
+    ) -> impl Future<Output = crate::Result<T>> {
+        let (sender, receiver) = oneshot::channel();
+        if self.is_alive() {
+            self.pool.spawn_ok(async move {
+                let _ = sender.send(f.await);
+            });
+        } else {
+            let _ = sender.send(Err(crate::TantivyError::SystemError(
+                "Segment updater killed".to_string(),
+            )));
+        }
+        receiver.unwrap_or_else(|_| {
+            let err_msg =
+                "A segment_updater future did not success. This should never happen.".to_string();
+            Err(crate::Error::SystemError(err_msg))
+        })
     }
 
-    pub fn add_segment(&self, segment_entry: SegmentEntry) -> bool {
-        self.run_async(|segment_updater| {
-            segment_updater.0.segment_manager.add_segment(segment_entry);
-            segment_updater.consider_merge_options();
-            true
+    pub fn schedule_add_segment(
+        &self,
+        segment_entry: SegmentEntry,
+    ) -> impl Future<Output = crate::Result<()>> {
+        let segment_updater = self.clone();
+        self.schedule_future(async move {
+            segment_updater.segment_manager.add_segment(segment_entry);
+            segment_updater.consider_merge_options().await;
+            Ok(())
         })
-        .forget();
-        true
     }
 
     /// Orders `SegmentManager` to remove all segments
     pub(crate) fn remove_all_segments(&self) {
-        self.0.segment_manager.remove_all_segments();
+        self.segment_manager.remove_all_segments();
     }
 
     pub fn kill(&mut self) {
-        self.0.killed.store(true, Ordering::Release);
+        self.killed.store(true, Ordering::Release);
     }
 
     pub fn is_alive(&self) -> bool {
-        !self.0.killed.load(Ordering::Acquire)
+        !self.killed.load(Ordering::Acquire)
     }
 
     /// Apply deletes up to the target opstamp to all segments.
     ///
     /// The method returns copies of the segment entries,
     /// updated with the delete information.
-    fn purge_deletes(&self, target_opstamp: Opstamp) -> Result<Vec<SegmentEntry>> {
-        let mut segment_entries = self.0.segment_manager.segment_entries();
+    fn purge_deletes(&self, target_opstamp: Opstamp) -> crate::Result<Vec<SegmentEntry>> {
+        let mut segment_entries = self.segment_manager.segment_entries();
         for segment_entry in &mut segment_entries {
-            let segment = self.0.index.segment(segment_entry.meta().clone());
+            let segment = self.index.segment(segment_entry.meta().clone());
             advance_deletes(segment, segment_entry, target_opstamp)?;
         }
         Ok(segment_entries)
     }
 
-    pub fn save_metas(&self, opstamp: Opstamp, commit_message: Option<String>) {
+    pub fn save_metas(
+        &self,
+        opstamp: Opstamp,
+        commit_message: Option<String>,
+    ) -> crate::Result<()> {
         if self.is_alive() {
-            let index = &self.0.index;
+            let index = &self.index;
             let directory = index.directory();
-            let mut commited_segment_metas = self.0.segment_manager.committed_segment_metas();
+            let mut commited_segment_metas = self.segment_manager.committed_segment_metas();
 
             // We sort segment_readers by number of documents.
             // This is an heuristic to make multithreading more efficient.
@@ -261,16 +294,18 @@ impl SegmentUpdater {
                 opstamp,
                 payload: commit_message,
             };
-            save_metas(&index_meta, directory.box_clone().borrow_mut())
-                .expect("Could not save metas.");
+            // TODO add context to the error.
+            save_metas(&index_meta, directory.box_clone().borrow_mut())?;
             self.store_meta(&index_meta);
         }
+        Ok(())
     }
 
-    pub fn garbage_collect_files(&self) -> CpuFuture<(), TantivyError> {
-        self.run_async(move |segment_updater| {
-            segment_updater.garbage_collect_files_exec();
-        })
+    pub fn schedule_garbage_collect(
+        &self,
+    ) -> impl Future<Output = crate::Result<GarbageCollectionResult>> {
+        let garbage_collect_future = garbage_collect_files(self.clone());
+        self.schedule_future(garbage_collect_future)
     }
 
     /// List the files that are useful to the index.
@@ -278,148 +313,130 @@ impl SegmentUpdater {
     /// This does not include lock files, or files that are obsolete
     /// but have not yet been deleted by the garbage collector.
     fn list_files(&self) -> HashSet<PathBuf> {
-        let mut files = HashSet::new();
+        let mut files: HashSet<PathBuf> = self
+            .index
+            .list_all_segment_metas()
+            .into_iter()
+            .flat_map(|segment_meta| segment_meta.list_files())
+            .collect();
         files.insert(META_FILEPATH.to_path_buf());
-        for segment_meta in self.0.index.list_all_segment_metas() {
-            files.extend(segment_meta.list_files());
-        }
         files
     }
 
-    fn garbage_collect_files_exec(&self) {
-        info!("Running garbage collection");
-        let mut index = self.0.index.clone();
-        index.directory_mut().garbage_collect(|| self.list_files());
-    }
-
-    pub fn commit(&self, opstamp: Opstamp, payload: Option<String>) -> Result<()> {
-        self.run_async(move |segment_updater| {
-            if segment_updater.is_alive() {
-                let segment_entries = segment_updater
-                    .purge_deletes(opstamp)
-                    .expect("Failed purge deletes");
-                segment_updater.0.segment_manager.commit(segment_entries);
-                segment_updater.save_metas(opstamp, payload);
-                segment_updater.garbage_collect_files_exec();
-                segment_updater.consider_merge_options();
-            }
+    pub fn schedule_commit(
+        &self,
+        opstamp: Opstamp,
+        payload: Option<String>,
+    ) -> impl Future<Output = crate::Result<()>> {
+        let segment_updater: SegmentUpdater = self.clone();
+        self.schedule_future(async move {
+            let segment_entries = segment_updater.purge_deletes(opstamp)?;
+            segment_updater.segment_manager.commit(segment_entries);
+            segment_updater.save_metas(opstamp, payload)?;
+            let _ = garbage_collect_files(segment_updater.clone()).await;
+            segment_updater.consider_merge_options().await;
+            Ok(())
         })
-        .wait()
-    }
-
-    pub fn start_merge(&self, segment_ids: &[SegmentId]) -> Result<Receiver<SegmentMeta>> {
-        let commit_opstamp = self.load_metas().opstamp;
-        let merge_operation = MergeOperation::new(
-            &self.0.merge_operations,
-            commit_opstamp,
-            segment_ids.to_vec(),
-        );
-        self.run_async(move |segment_updater| segment_updater.start_merge_impl(merge_operation))
-            .wait()?
     }
 
     fn store_meta(&self, index_meta: &IndexMeta) {
-        *self.0.active_metas.write().unwrap() = Arc::new(index_meta.clone());
-    }
-    fn load_metas(&self) -> Arc<IndexMeta> {
-        self.0.active_metas.read().unwrap().clone()
+        *self.active_metas.write().unwrap() = Arc::new(index_meta.clone());
     }
 
+    fn load_metas(&self) -> Arc<IndexMeta> {
+        self.active_metas.read().unwrap().clone()
+    }
+
+    pub(crate) fn make_merge_operation(&self, segment_ids: &[SegmentId]) -> MergeOperation {
+        let commit_opstamp = self.load_metas().opstamp;
+        MergeOperation::new(&self.merge_operations, commit_opstamp, segment_ids.to_vec())
+    }
+
+    // Starts a merge operation. This function will block until the merge operation is effectively
+    // started. Note that it does not wait for the merge to terminate.
+    // The calling thread should not be block for a long time, as this only involve waiting for the
+    // `SegmentUpdater` queue which in turns only contains lightweight operations.
+    //
+    // The merge itself happens on a different thread.
+    //
+    // When successful, this function returns a `Future` for a `Result<SegmentMeta>` that represents
+    // the actual outcome of the merge operation.
+    //
+    // It returns an error if for some reason the merge operation could not be started.
+    //
+    // At this point an error is not necessarily the sign of a malfunction.
+    // (e.g. A rollback could have happened, between the instant when the merge operaiton was
+    // suggested and the moment when it ended up being executed.)
+    //
     // `segment_ids` is required to be non-empty.
-    fn start_merge_impl(&self, merge_operation: MergeOperation) -> Result<Receiver<SegmentMeta>> {
+    pub fn start_merge(
+        &self,
+        merge_operation: MergeOperation,
+    ) -> crate::Result<impl Future<Output = crate::Result<SegmentMeta>>> {
         assert!(
             !merge_operation.segment_ids().is_empty(),
             "Segment_ids cannot be empty."
         );
 
-        let segment_updater_clone = self.clone();
+        let segment_updater = self.clone();
         let segment_entries: Vec<SegmentEntry> = self
-            .0
             .segment_manager
             .start_merge(merge_operation.segment_ids())?;
 
-        //        let segment_ids_vec = merge_operation.segment_ids.to_vec();
+        info!("Starting merge  - {:?}", merge_operation.segment_ids());
 
-        let merging_thread_id = self.get_merging_thread_id();
-        info!(
-            "Starting merge thread #{} - {:?}",
-            merging_thread_id,
-            merge_operation.segment_ids()
-        );
-        let (merging_future_send, merging_future_recv) = oneshot();
+        let (merging_future_send, merging_future_recv) =
+            oneshot::channel::<crate::Result<SegmentMeta>>();
 
-        // first we need to apply deletes to our segment.
-        let merging_join_handle = thread::Builder::new()
-            .name(format!("mergingthread-{}", merging_thread_id))
-            .spawn(move || {
-                // first we need to apply deletes to our segment.
-                let merge_result = perform_merge(
-                    &merge_operation,
-                    &segment_updater_clone.0.index,
-                    segment_entries,
-                );
-
-                match merge_result {
-                    Ok(after_merge_segment_entry) => {
-                        let merged_segment_meta = after_merge_segment_entry.meta().clone();
-                        segment_updater_clone
-                            .end_merge(merge_operation, after_merge_segment_entry)
-                            .expect("Segment updater thread is corrupted.");
-
-                        // the future may fail if the listener of the oneshot future
-                        // has been destroyed.
-                        //
-                        // This is not a problem here, so we just ignore any
-                        // possible error.
-                        let _merging_future_res = merging_future_send.send(merged_segment_meta);
-                    }
-                    Err(e) => {
-                        warn!(
-                            "Merge of {:?} was cancelled: {:?}",
-                            merge_operation.segment_ids(),
-                            e
-                        );
-                        // ... cancel merge
-                        if cfg!(test) {
-                            panic!("Merge failed.");
-                        }
-                        // As `merge_operation` will be dropped, the segment in merge state will
-                        // be available for merge again.
-                        // `merging_future_send` will be dropped, sending an error to the future.
+        self.merge_thread_pool.spawn_ok(async move {
+            // The fact that `merge_operation` is moved here is important.
+            // Its lifetime is used to track how many merging thread are currently running,
+            // as well as which segment is currently in merge and therefore should not be
+            // candidate for another merge.
+            match merge(
+                &segment_updater.index,
+                segment_entries,
+                merge_operation.target_opstamp(),
+            ) {
+                Ok(after_merge_segment_entry) => {
+                    let segment_meta = segment_updater
+                        .end_merge(merge_operation, after_merge_segment_entry)
+                        .await;
+                    let _send_result = merging_future_send.send(segment_meta);
+                }
+                Err(e) => {
+                    warn!(
+                        "Merge of {:?} was cancelled: {:?}",
+                        merge_operation.segment_ids().to_vec(),
+                        e
+                    );
+                    // ... cancel merge
+                    if cfg!(test) {
+                        panic!("Merge failed.");
                     }
                 }
-                segment_updater_clone
-                    .0
-                    .merging_threads
-                    .write()
-                    .unwrap()
-                    .remove(&merging_thread_id);
-                Ok(())
-            })
-            .expect("Failed to spawn a thread.");
-        self.0
-            .merging_threads
-            .write()
-            .unwrap()
-            .insert(merging_thread_id, merging_join_handle);
-        Ok(merging_future_recv)
+            }
+        });
+
+        Ok(merging_future_recv
+            .unwrap_or_else(|_| Err(crate::Error::SystemError("Merge failed".to_string()))))
     }
 
-    fn consider_merge_options(&self) {
-        let merge_segment_ids: HashSet<SegmentId> = self.0.merge_operations.segment_in_merge();
+    async fn consider_merge_options(&self) {
+        let merge_segment_ids: HashSet<SegmentId> = self.merge_operations.segment_in_merge();
         let (committed_segments, uncommitted_segments) =
-            get_mergeable_segments(&merge_segment_ids, &self.0.segment_manager);
+            get_mergeable_segments(&merge_segment_ids, &self.segment_manager);
 
         // Committed segments cannot be merged with uncommitted_segments.
         // We therefore consider merges using these two sets of segments independently.
         let merge_policy = self.get_merge_policy();
 
-        let current_opstamp = self.0.stamper.stamp();
+        let current_opstamp = self.stamper.stamp();
         let mut merge_candidates: Vec<MergeOperation> = merge_policy
             .compute_merge_candidates(&uncommitted_segments)
             .into_iter()
             .map(|merge_candidate| {
-                MergeOperation::new(&self.0.merge_operations, current_opstamp, merge_candidate.0)
+                MergeOperation::new(&self.merge_operations, current_opstamp, merge_candidate.0)
             })
             .collect();
 
@@ -427,25 +444,18 @@ impl SegmentUpdater {
         let committed_merge_candidates = merge_policy
             .compute_merge_candidates(&committed_segments)
             .into_iter()
-            .map(|merge_candidate| {
-                MergeOperation::new(&self.0.merge_operations, commit_opstamp, merge_candidate.0)
+            .map(|merge_candidate: MergeCandidate| {
+                MergeOperation::new(&self.merge_operations, commit_opstamp, merge_candidate.0)
             })
             .collect::<Vec<_>>();
         merge_candidates.extend(committed_merge_candidates.into_iter());
 
         for merge_operation in merge_candidates {
-            match self.start_merge_impl(merge_operation) {
-                Ok(merge_future) => {
-                    if let Err(e) = merge_future.fuse().poll() {
-                        error!("The merge task failed quickly after starting: {:?}", e);
-                    }
-                }
-                Err(err) => {
-                    warn!(
-                        "Starting the merge failed for the following reason. This is not fatal. {}",
-                        err
-                    );
-                }
+            if let Err(err) = self.start_merge(merge_operation) {
+                warn!(
+                    "Starting the merge failed for the following reason. This is not fatal. {}",
+                    err
+                );
             }
         }
     }
@@ -454,15 +464,17 @@ impl SegmentUpdater {
         &self,
         merge_operation: MergeOperation,
         mut after_merge_segment_entry: SegmentEntry,
-    ) -> Result<()> {
-        self.run_async(move |segment_updater| {
+    ) -> impl Future<Output = crate::Result<SegmentMeta>> {
+        let segment_updater = self.clone();
+        let after_merge_segment_meta = after_merge_segment_entry.meta().clone();
+        let end_merge_future = self.schedule_future(async move {
             info!("End merge {:?}", after_merge_segment_entry.meta());
             {
                 let mut delete_cursor = after_merge_segment_entry.delete_cursor().clone();
                 if let Some(delete_operation) = delete_cursor.get() {
                     let committed_opstamp = segment_updater.load_metas().opstamp;
                     if delete_operation.opstamp < committed_opstamp {
-                        let index = &segment_updater.0.index;
+                        let index = &segment_updater.index;
                         let segment = index.segment(after_merge_segment_entry.meta().clone());
                         if let Err(e) = advance_deletes(
                             segment,
@@ -480,21 +492,26 @@ impl SegmentUpdater {
                             // ... cancel merge
                             // `merge_operations` are tracked. As it is dropped, the
                             // the segment_ids will be available again for merge.
-                            return;
+                            return Err(e);
                         }
                     }
                 }
                 let previous_metas = segment_updater.load_metas();
-                segment_updater
-                    .0
+                let segments_status = segment_updater
                     .segment_manager
-                    .end_merge(merge_operation.segment_ids(), after_merge_segment_entry);
-                segment_updater.consider_merge_options();
-                segment_updater.save_metas(previous_metas.opstamp, previous_metas.payload.clone());
+                    .end_merge(merge_operation.segment_ids(), after_merge_segment_entry)?;
+
+                if segments_status == SegmentsStatus::Committed {
+                    segment_updater
+                        .save_metas(previous_metas.opstamp, previous_metas.payload.clone())?;
+                }
+
+                segment_updater.consider_merge_options().await;
             } // we drop all possible handle to a now useless `SegmentMeta`.
-            segment_updater.garbage_collect_files_exec();
-        })
-        .wait()
+            let _ = garbage_collect_files(segment_updater).await;
+            Ok(())
+        });
+        end_merge_future.map_ok(|_| after_merge_segment_meta)
     }
 
     /// Wait for current merging threads.
@@ -512,26 +529,9 @@ impl SegmentUpdater {
     ///
     /// Obsolete files will eventually be cleaned up
     /// by the directory garbage collector.
-    pub fn wait_merging_thread(&self) -> Result<()> {
-        loop {
-            let merging_threads: HashMap<usize, JoinHandle<Result<()>>> = {
-                let mut merging_threads = self.0.merging_threads.write().unwrap();
-                mem::replace(merging_threads.deref_mut(), HashMap::new())
-            };
-            if merging_threads.is_empty() {
-                return Ok(());
-            }
-            debug!("wait merging thread {}", merging_threads.len());
-            for (_, merging_thread_handle) in merging_threads {
-                merging_thread_handle
-                    .join()
-                    .map(|_| ())
-                    .map_err(|_| TantivyError::ErrorInThread("Merging thread failed.".into()))?;
-            }
-            // Our merging thread may have queued their completed merged segment.
-            // Let's wait for that too.
-            self.run_async(move |_| {}).wait()?;
-        }
+    pub fn wait_merging_thread(&self) -> crate::Result<()> {
+        self.merge_operations.wait_until_empty();
+        Ok(())
     }
 }
 
@@ -687,7 +687,6 @@ mod tests {
         index_writer.segment_updater().remove_all_segments();
         let seg_vec = index_writer
             .segment_updater()
-            .0
             .segment_manager
             .segment_entries();
         assert!(seg_vec.is_empty());

src/indexer/segment_writer.rs

@@ -249,6 +249,7 @@ impl SegmentWriter {
             }
         }
         doc.filter_fields(|field| schema.get_field_entry(field).is_stored());
+        doc.prepare_for_store();
         let doc_writer = self.segment_serializer.get_store_writer();
         doc_writer.store(&doc)?;
         self.max_doc += 1;

src/lib.rs

@@ -160,7 +160,6 @@ pub use self::snippet::{Snippet, SnippetGenerator};
 
 mod docset;
 pub use self::docset::{DocSet, SkipResult};
-
 pub use crate::common::{f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
 pub use crate::core::SegmentComponent;
 pub use crate::core::{Index, IndexMeta, Searcher, Segment, SegmentId, SegmentMeta};
@@ -170,11 +169,58 @@ pub use crate::indexer::IndexWriter;
 pub use crate::postings::Postings;
 pub use crate::reader::LeasedItem;
 pub use crate::schema::{Document, Term};
+use std::fmt;
 
-/// Expose the current version of tantivy, as well
-/// whether it was compiled with the simd compression.
-pub fn version() -> &'static str {
-    env!("CARGO_PKG_VERSION")
+use once_cell::sync::Lazy;
+
+/// Index format version.
+const INDEX_FORMAT_VERSION: u32 = 1;
+
+/// Structure version for the index.
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Version {
+    major: u32,
+    minor: u32,
+    patch: u32,
+    index_format_version: u32,
+    store_compression: String,
+}
+
+impl fmt::Debug for Version {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.to_string())
+    }
+}
+
+static VERSION: Lazy<Version> = Lazy::new(|| Version {
+    major: env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap(),
+    minor: env!("CARGO_PKG_VERSION_MINOR").parse().unwrap(),
+    patch: env!("CARGO_PKG_VERSION_PATCH").parse().unwrap(),
+    index_format_version: INDEX_FORMAT_VERSION,
+    store_compression: crate::store::COMPRESSION.to_string(),
+});
+
+impl ToString for Version {
+    fn to_string(&self) -> String {
+        format!(
+            "tantivy v{}.{}.{}, index_format v{}, store_compression: {}",
+            self.major, self.minor, self.patch, self.index_format_version, self.store_compression
+        )
+    }
+}
+
+static VERSION_STRING: Lazy<String> = Lazy::new(|| VERSION.to_string());
+
+/// Expose the current version of tantivy as found in Cargo.toml during compilation.
+/// eg. "0.11.0" as well as the compression scheme used in the docstore.
+pub fn version() -> &'static Version {
+    &VERSION
+}
+
+/// Exposes the complete version of tantivy as found in Cargo.toml during compilation as a string.
+/// eg. "tantivy v0.11.0, index_format v1, store_compression: lz4".
+pub fn version_string() -> &'static str {
+    VERSION_STRING.as_str()
 }
 
 /// Defines tantivy's merging strategy
@@ -287,6 +333,18 @@ mod tests {
         sample_with_seed(n, ratio, 4)
     }
 
+    #[test]
+    #[cfg(not(feature = "lz4"))]
+    fn test_version_string() {
+        use regex::Regex;
+        let regex_ptn = Regex::new(
+            "tantivy v[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.{0,10}, index_format v[0-9]{1,5}",
+        )
+        .unwrap();
+        let version = super::version().to_string();
+        assert!(regex_ptn.find(&version).is_some());
+    }
+
     #[test]
     #[cfg(feature = "mmap")]
     fn test_indexing() {

src/macros.rs

@@ -35,9 +35,9 @@
 /// let likes = schema_builder.add_u64_field("num_u64", FAST);
 /// let schema = schema_builder.build();
 /// let doc = doc!(
-/// 	title => "Life Aquatic",
-/// 	author => "Wes Anderson",
-/// 	likes => 4u64
+///     title => "Life Aquatic",
+///     author => "Wes Anderson",
+///     likes => 4u64
 /// );
 /// # }
 /// ```

src/positions/reader.rs

@@ -36,11 +36,10 @@ struct Positions {
 
 impl Positions {
     pub fn new(position_source: ReadOnlySource, skip_source: ReadOnlySource) -> Positions {
-        let skip_len = skip_source.len();
-        let (body, footer) = skip_source.split(skip_len - u32::SIZE_IN_BYTES);
+        let (body, footer) = skip_source.split_from_end(u32::SIZE_IN_BYTES);
         let num_long_skips = u32::deserialize(&mut footer.as_slice()).expect("Index corrupted");
-        let body_split = body.len() - u64::SIZE_IN_BYTES * (num_long_skips as usize);
-        let (skip_source, long_skip_source) = body.split(body_split);
+        let (skip_source, long_skip_source) =
+            body.split_from_end(u64::SIZE_IN_BYTES * (num_long_skips as usize));
         Positions {
             bit_packer: BitPacker4x::new(),
             skip_source,

src/query/exclude.rs

@@ -54,21 +54,21 @@ where
         match self.excluding_state {
             State::ExcludeOne(excluded_doc) => {
                 if doc == excluded_doc {
-                    false
-                } else if excluded_doc > doc {
-                    true
-                } else {
-                    match self.excluding_docset.skip_next(doc) {
-                        SkipResult::OverStep => {
-                            self.excluding_state = State::ExcludeOne(self.excluding_docset.doc());
-                            true
-                        }
-                        SkipResult::End => {
-                            self.excluding_state = State::Finished;
-                            true
-                        }
-                        SkipResult::Reached => false,
+                    return false;
+                }
+                if excluded_doc > doc {
+                    return true;
+                }
+                match self.excluding_docset.skip_next(doc) {
+                    SkipResult::OverStep => {
+                        self.excluding_state = State::ExcludeOne(self.excluding_docset.doc());
+                        true
                     }
+                    SkipResult::End => {
+                        self.excluding_state = State::Finished;
+                        true
+                    }
+                    SkipResult::Reached => false,
                 }
             }
             State::Finished => true,

src/query/fuzzy_query.rs

@@ -33,7 +33,6 @@ static LEV_BUILDER: Lazy<HashMap<(u8, bool), LevenshteinAutomatonBuilder>> = Laz
 /// use tantivy::schema::{Schema, TEXT};
 /// use tantivy::{doc, Index, Result, Term};
 ///
-/// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {
 ///     let mut schema_builder = Schema::builder();
 ///     let title = schema_builder.add_text_field("title", TEXT);
@@ -59,7 +58,6 @@ static LEV_BUILDER: Lazy<HashMap<(u8, bool), LevenshteinAutomatonBuilder>> = Laz
 ///     let searcher = reader.searcher();
 ///
 ///     {
-///
 ///         let term = Term::from_field_text(title, "Diary");
 ///         let query = FuzzyTermQuery::new(term, 1, true);
 ///         let (top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count)).unwrap();
@@ -69,6 +67,7 @@ static LEV_BUILDER: Lazy<HashMap<(u8, bool), LevenshteinAutomatonBuilder>> = Laz
 ///
 ///     Ok(())
 /// }
+/// # assert!(example().is_ok());
 /// ```
 #[derive(Debug, Clone)]
 pub struct FuzzyTermQuery {

src/query/phrase_query/phrase_scorer.rs

@@ -4,6 +4,7 @@ use crate::postings::Postings;
 use crate::query::bm25::BM25Weight;
 use crate::query::{Intersection, Scorer};
 use crate::DocId;
+use std::cmp::Ordering;
 
 struct PostingsWithOffset<TPostings> {
     offset: u32,
@@ -59,12 +60,16 @@ fn intersection_exists(left: &[u32], right: &[u32]) -> bool {
     while left_i < left.len() && right_i < right.len() {
         let left_val = left[left_i];
         let right_val = right[right_i];
-        if left_val < right_val {
-            left_i += 1;
-        } else if right_val < left_val {
-            right_i += 1;
-        } else {
-            return true;
+        match left_val.cmp(&right_val) {
+            Ordering::Less => {
+                left_i += 1;
+            }
+            Ordering::Equal => {
+                return true;
+            }
+            Ordering::Greater => {
+                right_i += 1;
+            }
         }
     }
     false
@@ -77,14 +82,18 @@ fn intersection_count(left: &[u32], right: &[u32]) -> usize {
     while left_i < left.len() && right_i < right.len() {
         let left_val = left[left_i];
         let right_val = right[right_i];
-        if left_val < right_val {
-            left_i += 1;
-        } else if right_val < left_val {
-            right_i += 1;
-        } else {
-            count += 1;
-            left_i += 1;
-            right_i += 1;
+        match left_val.cmp(&right_val) {
+            Ordering::Less => {
+                left_i += 1;
+            }
+            Ordering::Equal => {
+                count += 1;
+                left_i += 1;
+                right_i += 1;
+            }
+            Ordering::Greater => {
+                right_i += 1;
+            }
         }
     }
     count
@@ -103,15 +112,19 @@ fn intersection(left: &mut [u32], right: &[u32]) -> usize {
     while left_i < left_len && right_i < right_len {
         let left_val = left[left_i];
         let right_val = right[right_i];
-        if left_val < right_val {
-            left_i += 1;
-        } else if right_val < left_val {
-            right_i += 1;
-        } else {
-            left[count] = left_val;
-            count += 1;
-            left_i += 1;
-            right_i += 1;
+        match left_val.cmp(&right_val) {
+            Ordering::Less => {
+                left_i += 1;
+            }
+            Ordering::Equal => {
+                left[count] = left_val;
+                count += 1;
+                left_i += 1;
+                right_i += 1;
+            }
+            Ordering::Greater => {
+                right_i += 1;
+            }
         }
     }
     count

src/query/range_query.rs

@@ -38,41 +38,33 @@ fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
 /// # Example
 ///
 /// ```rust
-/// # use tantivy::collector::Count;
-/// # use tantivy::query::RangeQuery;
-/// # use tantivy::schema::{Schema, INDEXED};
-/// # use tantivy::{doc, Index, Result};
-/// #
-/// # fn run() -> Result<()> {
-/// #     let mut schema_builder = Schema::builder();
-/// #     let year_field = schema_builder.add_u64_field("year", INDEXED);
-/// #     let schema = schema_builder.build();
-/// #
-/// #     let index = Index::create_in_ram(schema);
-/// #     {
-/// #         let mut index_writer = index.writer_with_num_threads(1, 6_000_000).unwrap();
-/// #         for year in 1950u64..2017u64 {
-/// #             let num_docs_within_year = 10 + (year - 1950) * (year - 1950);
-/// #             for _ in 0..num_docs_within_year {
-/// #                 index_writer.add_document(doc!(year_field => year));
-/// #             }
-/// #         }
-/// #         index_writer.commit().unwrap();
-/// #     }
-/// #   let reader = index.reader()?;
+/// use tantivy::collector::Count;
+/// use tantivy::query::RangeQuery;
+/// use tantivy::schema::{Schema, INDEXED};
+/// use tantivy::{doc, Index};
+/// # fn test() -> tantivy::Result<()> {
+/// let mut schema_builder = Schema::builder();
+/// let year_field = schema_builder.add_u64_field("year", INDEXED);
+/// let schema = schema_builder.build();
+///
+/// let index = Index::create_in_ram(schema);
+/// let mut index_writer = index.writer_with_num_threads(1, 6_000_000)?;
+/// for year in 1950u64..2017u64 {
+///     let num_docs_within_year = 10 + (year - 1950) * (year - 1950);
+///     for _ in 0..num_docs_within_year {
+///       index_writer.add_document(doc!(year_field => year));
+///     }
+/// }
+/// index_writer.commit()?;
+///
+/// let reader = index.reader()?;
 /// let searcher = reader.searcher();
-///
 /// let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960..1970);
-///
 /// let num_60s_books = searcher.search(&docs_in_the_sixties, &Count)?;
-///
-/// #     assert_eq!(num_60s_books, 2285);
-/// #     Ok(())
-/// # }
-/// #
-/// # fn main() {
-/// #   run().unwrap()
+/// assert_eq!(num_60s_books, 2285);
+/// Ok(())
 /// # }
+/// # assert!(test().is_ok());
 /// ```
 #[derive(Clone, Debug)]
 pub struct RangeQuery {

src/query/regex_query.rs

@@ -15,40 +15,40 @@ use tantivy_fst::Regex;
 /// use tantivy::collector::Count;
 /// use tantivy::query::RegexQuery;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Result, Term};
+/// use tantivy::{doc, Index, Term};
 ///
-/// # fn main() { example().unwrap(); }
-/// fn example() -> Result<()> {
-///     let mut schema_builder = Schema::builder();
-///     let title = schema_builder.add_text_field("title", TEXT);
-///     let schema = schema_builder.build();
-///     let index = Index::create_in_ram(schema);
-///     {
-///         let mut index_writer = index.writer(3_000_000)?;
-///         index_writer.add_document(doc!(
-///             title => "The Name of the Wind",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of Muadib",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "A Dairy Cow",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of a Young Girl",
-///         ));
-///         index_writer.commit().unwrap();
-///     }
-///
-///     let reader = index.reader()?;
-///     let searcher = reader.searcher();
-///
-///     let term = Term::from_field_text(title, "Diary");
-///     let query = RegexQuery::from_pattern("d[ai]{2}ry", title)?;
-///     let count = searcher.search(&query, &Count)?;
-///     assert_eq!(count, 3);
-///     Ok(())
+/// # fn test() -> tantivy::Result<()> {
+/// let mut schema_builder = Schema::builder();
+/// let title = schema_builder.add_text_field("title", TEXT);
+/// let schema = schema_builder.build();
+/// let index = Index::create_in_ram(schema);
+/// {
+///     let mut index_writer = index.writer(3_000_000)?;
+///     index_writer.add_document(doc!(
+///         title => "The Name of the Wind",
+///     ));
+///     index_writer.add_document(doc!(
+///         title => "The Diary of Muadib",
+///     ));
+///     index_writer.add_document(doc!(
+///         title => "A Dairy Cow",
+///     ));
+///     index_writer.add_document(doc!(
+///         title => "The Diary of a Young Girl",
+///     ));
+///     index_writer.commit().unwrap();
 /// }
+///
+/// let reader = index.reader()?;
+/// let searcher = reader.searcher();
+///
+/// let term = Term::from_field_text(title, "Diary");
+/// let query = RegexQuery::from_pattern("d[ai]{2}ry", title)?;
+/// let count = searcher.search(&query, &Count)?;
+/// assert_eq!(count, 3);
+/// Ok(())
+/// # }
+/// # assert!(test().is_ok());
 /// ```
 #[derive(Debug, Clone)]
 pub struct RegexQuery {

src/query/term_query/term_query.rs

@@ -23,42 +23,39 @@ use std::fmt;
 /// use tantivy::collector::{Count, TopDocs};
 /// use tantivy::query::TermQuery;
 /// use tantivy::schema::{Schema, TEXT, IndexRecordOption};
-/// use tantivy::{doc, Index, Result, Term};
-///
-/// # fn main() { example().unwrap(); }
-/// fn example() -> Result<()> {
-///     let mut schema_builder = Schema::builder();
-///     let title = schema_builder.add_text_field("title", TEXT);
-///     let schema = schema_builder.build();
-///     let index = Index::create_in_ram(schema);
-///     {
-///         let mut index_writer = index.writer(3_000_000)?;
-///         index_writer.add_document(doc!(
-///             title => "The Name of the Wind",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of Muadib",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "A Dairy Cow",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of a Young Girl",
-///         ));
-///         index_writer.commit()?;
-///     }
-///     let reader = index.reader()?;
-///     let searcher = reader.searcher();
-///
-///     let query = TermQuery::new(
-///         Term::from_field_text(title, "diary"),
-///         IndexRecordOption::Basic,
-///     );
-///     let (top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count)).unwrap();
-///     assert_eq!(count, 2);
-///
-///     Ok(())
+/// use tantivy::{doc, Index, Term};
+/// # fn test() -> tantivy::Result<()> {
+/// let mut schema_builder = Schema::builder();
+/// let title = schema_builder.add_text_field("title", TEXT);
+/// let schema = schema_builder.build();
+/// let index = Index::create_in_ram(schema);
+/// {
+///     let mut index_writer = index.writer(3_000_000)?;
+///     index_writer.add_document(doc!(
+///         title => "The Name of the Wind",
+///     ));
+///     index_writer.add_document(doc!(
+///         title => "The Diary of Muadib",
+///     ));
+///     index_writer.add_document(doc!(
+///         title => "A Dairy Cow",
+///     ));
+///     index_writer.add_document(doc!(
+///         title => "The Diary of a Young Girl",
+///     ));
+///     index_writer.commit()?;
 /// }
+/// let reader = index.reader()?;
+/// let searcher = reader.searcher();
+/// let query = TermQuery::new(
+///     Term::from_field_text(title, "diary"),
+///     IndexRecordOption::Basic,
+/// );
+/// let (top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count))?;
+/// assert_eq!(count, 2);
+/// Ok(())
+/// # }
+/// # assert!(test().is_ok());
 /// ```
 #[derive(Clone)]
 pub struct TermQuery {

src/reader/mod.rs

@@ -162,6 +162,11 @@ pub struct IndexReader {
 }
 
 impl IndexReader {
+    #[cfg(test)]
+    pub(crate) fn index(&self) -> Index {
+        self.inner.index.clone()
+    }
+
     /// Update searchers so that they reflect the state of the last
     /// `.commit()`.
     ///

src/reader/pool.rs

@@ -167,7 +167,7 @@ mod tests {
 
     use super::Pool;
     use super::Queue;
-    use std::iter;
+    use std::{iter, mem};
 
     #[test]
     fn test_pool() {
@@ -197,33 +197,67 @@ mod tests {
     fn test_pool_dont_panic_on_empty_pop() {
         // When the object pool is exhausted, it shouldn't panic on pop()
         use std::sync::Arc;
-        use std::{thread, time};
+        use std::thread;
 
         // Wrap the pool in an Arc, same way as its used in `core/index.rs`
-        let pool = Arc::new(Pool::new());
+        let pool1 = Arc::new(Pool::new());
         // clone pools outside the move scope of each new thread
-        let pool1 = Arc::clone(&pool);
-        let pool2 = Arc::clone(&pool);
+        let pool2 = Arc::clone(&pool1);
+        let pool3 = Arc::clone(&pool1);
+
         let elements_for_pool = vec![1, 2];
-        pool.publish_new_generation(elements_for_pool);
+        pool1.publish_new_generation(elements_for_pool);
 
         let mut threads = vec![];
-        let sleep_dur = time::Duration::from_millis(10);
         // spawn one more thread than there are elements in the pool
+
+        let (start_1_send, start_1_recv) = crossbeam::bounded(0);
+        let (start_2_send, start_2_recv) = crossbeam::bounded(0);
+        let (start_3_send, start_3_recv) = crossbeam::bounded(0);
+
+        let (event_send1, event_recv) = crossbeam::unbounded();
+        let event_send2 = event_send1.clone();
+        let event_send3 = event_send1.clone();
+
         threads.push(thread::spawn(move || {
-            // leasing to make sure it's not dropped before sleep is called
-            let _leased_searcher = &pool.acquire();
-            thread::sleep(sleep_dur);
-        }));
-        threads.push(thread::spawn(move || {
-            // leasing to make sure it's not dropped before sleep is called
+            assert_eq!(start_1_recv.recv(), Ok("start"));
             let _leased_searcher = &pool1.acquire();
-            thread::sleep(sleep_dur);
+            assert!(event_send1.send("1 acquired").is_ok());
+            assert_eq!(start_1_recv.recv(), Ok("stop"));
+            assert!(event_send1.send("1 stopped").is_ok());
+            mem::drop(_leased_searcher);
         }));
+
         threads.push(thread::spawn(move || {
-            // leasing to make sure it's not dropped before sleep is called
+            assert_eq!(start_2_recv.recv(), Ok("start"));
             let _leased_searcher = &pool2.acquire();
-            thread::sleep(sleep_dur);
+            assert!(event_send2.send("2 acquired").is_ok());
+            assert_eq!(start_2_recv.recv(), Ok("stop"));
+            mem::drop(_leased_searcher);
+            assert!(event_send2.send("2 stopped").is_ok());
         }));
+
+        threads.push(thread::spawn(move || {
+            assert_eq!(start_3_recv.recv(), Ok("start"));
+            let _leased_searcher = &pool3.acquire();
+            assert!(event_send3.send("3 acquired").is_ok());
+            assert_eq!(start_3_recv.recv(), Ok("stop"));
+            mem::drop(_leased_searcher);
+            assert!(event_send3.send("3 stopped").is_ok());
+        }));
+
+        assert!(start_1_send.send("start").is_ok());
+        assert_eq!(event_recv.recv(), Ok("1 acquired"));
+        assert!(start_2_send.send("start").is_ok());
+        assert_eq!(event_recv.recv(), Ok("2 acquired"));
+        assert!(start_3_send.send("start").is_ok());
+        assert!(event_recv.try_recv().is_err());
+        assert!(start_1_send.send("stop").is_ok());
+        assert_eq!(event_recv.recv(), Ok("1 stopped"));
+        assert_eq!(event_recv.recv(), Ok("3 acquired"));
+        assert!(start_3_send.send("stop").is_ok());
+        assert_eq!(event_recv.recv(), Ok("3 stopped"));
+        assert!(start_2_send.send("stop").is_ok());
+        assert_eq!(event_recv.recv(), Ok("2 stopped"));
     }
 }

src/schema/document.rs

@@ -155,6 +155,21 @@ impl Document {
             .find(|field_value| field_value.field() == field)
             .map(FieldValue::value)
     }
+
+    /// Prepares Document for being stored in the document store
+    ///
+    /// Method transforms PreTokenizedString values into String
+    /// values.
+    pub fn prepare_for_store(&mut self) {
+        for field_value in &mut self.field_values {
+            if let Value::PreTokStr(pre_tokenized_text) = field_value.value() {
+                *field_value = FieldValue::new(
+                    field_value.field(),
+                    Value::Str(pre_tokenized_text.text.clone()), //< TODO somehow remove .clone()
+                );
+            }
+        }
+    }
 }
 
 impl BinarySerializable for Document {
@@ -180,6 +195,7 @@ impl BinarySerializable for Document {
 mod tests {
 
     use crate::schema::*;
+    use crate::tokenizer::{PreTokenizedString, Token};
 
     #[test]
     fn test_doc() {
@@ -189,4 +205,38 @@ mod tests {
         doc.add_text(text_field, "My title");
         assert_eq!(doc.field_values().len(), 1);
     }
+
+    #[test]
+    fn test_prepare_for_store() {
+        let mut schema_builder = Schema::builder();
+        let text_field = schema_builder.add_text_field("title", TEXT);
+        let mut doc = Document::default();
+
+        let pre_tokenized_text = PreTokenizedString {
+            text: String::from("A"),
+            tokens: vec![Token {
+                offset_from: 0,
+                offset_to: 1,
+                position: 0,
+                text: String::from("A"),
+                position_length: 1,
+            }],
+        };
+
+        doc.add_pre_tokenized_text(text_field, &pre_tokenized_text);
+        doc.add_text(text_field, "title");
+        doc.prepare_for_store();
+
+        assert_eq!(doc.field_values().len(), 2);
+
+        match doc.field_values()[0].value() {
+            Value::Str(ref text) => assert_eq!(text, "A"),
+            _ => panic!("Incorrect variant of Value"),
+        }
+
+        match doc.field_values()[1].value() {
+            Value::Str(ref text) => assert_eq!(text, "title"),
+            _ => panic!("Incorrect variant of Value"),
+        }
+    }
 }

src/schema/field_type.rs

@@ -6,6 +6,7 @@ use crate::schema::TextFieldIndexing;
 use crate::schema::Value;
 use crate::schema::{IntOptions, TextOptions};
 use crate::tokenizer::PreTokenizedString;
+use chrono::{FixedOffset, Utc};
 use serde_json::Value as JsonValue;
 
 /// Possible error that may occur while parsing a field value
@@ -124,13 +125,20 @@ impl FieldType {
     pub fn value_from_json(&self, json: &JsonValue) -> Result<Value, ValueParsingError> {
         match *json {
             JsonValue::String(ref field_text) => match *self {
-                FieldType::Str(_) => Ok(Value::Str(field_text.clone())),
-                FieldType::U64(_) | FieldType::I64(_) | FieldType::F64(_) | FieldType::Date(_) => {
-                    Err(ValueParsingError::TypeError(format!(
-                        "Expected an integer, got {:?}",
-                        json
-                    )))
+                FieldType::Date(_) => {
+                    let dt_with_fixed_tz: chrono::DateTime<FixedOffset> =
+                        chrono::DateTime::parse_from_rfc3339(field_text).map_err(|err|
+                            ValueParsingError::TypeError(format!(
+                                "Failed to parse date from JSON. Expected rfc3339 format, got {}. {:?}",
+                                field_text, err
+                            ))
+                        )?;
+                    Ok(Value::Date(dt_with_fixed_tz.with_timezone(&Utc)))
                 }
+                FieldType::Str(_) => Ok(Value::Str(field_text.clone())),
+                FieldType::U64(_) | FieldType::I64(_) | FieldType::F64(_) => Err(
+                    ValueParsingError::TypeError(format!("Expected an integer, got {:?}", json)),
+                ),
                 FieldType::HierarchicalFacet => Ok(Value::Facet(Facet::from(field_text))),
                 FieldType::Bytes => decode(field_text).map(Value::Bytes).map_err(|_| {
                     ValueParsingError::InvalidBase64(format!(
@@ -208,7 +216,35 @@ mod tests {
     use crate::schema::field_type::ValueParsingError;
     use crate::schema::TextOptions;
     use crate::schema::Value;
+    use crate::schema::{Schema, INDEXED};
     use crate::tokenizer::{PreTokenizedString, Token};
+    use crate::{DateTime, Document};
+    use chrono::{NaiveDate, NaiveDateTime, NaiveTime, Utc};
+
+    #[test]
+    fn test_deserialize_json_date() {
+        let mut schema_builder = Schema::builder();
+        let date_field = schema_builder.add_date_field("date", INDEXED);
+        let schema = schema_builder.build();
+        let doc_json = r#"{"date": "2019-10-12T07:20:50.52+02:00"}"#;
+        let doc = schema.parse_document(doc_json).unwrap();
+        let date = doc.get_first(date_field).unwrap();
+        assert_eq!(format!("{:?}", date), "Date(2019-10-12T05:20:50.520Z)");
+    }
+
+    #[test]
+    fn test_serialize_json_date() {
+        let mut doc = Document::new();
+        let mut schema_builder = Schema::builder();
+        let date_field = schema_builder.add_date_field("date", INDEXED);
+        let schema = schema_builder.build();
+        let naive_date = NaiveDate::from_ymd(1982, 9, 17);
+        let naive_time = NaiveTime::from_hms(13, 20, 00);
+        let date_time = DateTime::from_utc(NaiveDateTime::new(naive_date, naive_time), Utc);
+        doc.add_date(date_field, &date_time);
+        let doc_json = schema.to_json(&doc);
+        assert_eq!(doc_json, r#"{"date":["1982-09-17T13:20:00+00:00"]}"#);
+    }
 
     #[test]
     fn test_bytes_value_from_json() {

src/schema/flags.rs

@@ -53,7 +53,7 @@ where
     fn bitor(self, head: SchemaFlagList<Head, ()>) -> Self::Output {
         SchemaFlagList {
             head: head.head,
-            tail: self.clone(),
+            tail: self,
         }
     }
 }

src/schema/mod.rs

@@ -44,7 +44,7 @@ We can split the problem of generating a search result page into two phases :
   the search results page. (`doc_ids[] -> Document[]`)
 
 In the first phase, the ability to search for documents by the given field is determined by the
-[`TextIndexingOptions`](enum.TextIndexingOptions.html) of our
+[`IndexRecordOption`](enum.IndexRecordOption.html) of our
 [`TextOptions`](struct.TextOptions.html).
 
 The effect of each possible setting is described more in detail

src/schema/value.rs

@@ -75,7 +75,7 @@ impl Serialize for Value {
             Value::U64(u) => serializer.serialize_u64(u),
             Value::I64(u) => serializer.serialize_i64(u),
             Value::F64(u) => serializer.serialize_f64(u),
-            Value::Date(ref date) => serializer.serialize_i64(date.timestamp()),
+            Value::Date(ref date) => serializer.serialize_str(&date.to_rfc3339()),
             Value::Facet(ref facet) => facet.serialize(serializer),
             Value::Bytes(ref bytes) => serializer.serialize_bytes(bytes),
         }
@@ -96,14 +96,14 @@ impl<'de> Deserialize<'de> for Value {
                 formatter.write_str("a string or u32")
             }
 
-            fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E> {
-                Ok(Value::U64(v))
-            }
-
             fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E> {
                 Ok(Value::I64(v))
             }
 
+            fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E> {
+                Ok(Value::U64(v))
+            }
+
             fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E> {
                 Ok(Value::F64(v))
             }
@@ -233,6 +233,12 @@ impl From<Vec<u8>> for Value {
     }
 }
 
+impl From<PreTokenizedString> for Value {
+    fn from(pretokenized_string: PreTokenizedString) -> Value {
+        Value::PreTokStr(pretokenized_string)
+    }
+}
+
 mod binary_serialize {
     use super::Value;
     use crate::common::{f64_to_u64, u64_to_f64, BinarySerializable};
@@ -356,3 +362,17 @@ mod binary_serialize {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::Value;
+    use crate::DateTime;
+    use std::str::FromStr;
+
+    #[test]
+    fn test_serialize_date() {
+        let value = Value::Date(DateTime::from_str("1996-12-20T00:39:57+00:00").unwrap());
+        let serialized_value_json = serde_json::to_string_pretty(&value).unwrap();
+        assert_eq!(serialized_value_json, r#""1996-12-20T00:39:57+00:00""#);
+    }
+}

src/snippet/mod.rs

@@ -331,9 +331,8 @@ mod tests {
     use std::collections::BTreeMap;
     use std::iter::Iterator;
 
-    const TEST_TEXT: &'static str =
-        r#"Rust is a systems programming language sponsored by Mozilla which
-describes it as a "safe, concurrent, practical language", supporting functional and
+    const TEST_TEXT: &'static str = r#"Rust is a systems programming language sponsored by
+Mozilla which describes it as a "safe, concurrent, practical language", supporting functional and
 imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?],
 but its designers intend it to provide better memory safety while still maintaining
 performance.
@@ -363,13 +362,13 @@ Survey in 2016, 2017, and 2018."#;
         let snippet = select_best_fragment_combination(&fragments[..], &TEST_TEXT);
         assert_eq!(
             snippet.fragments,
-            "Rust is a systems programming language sponsored by \
-             Mozilla which\ndescribes it as a \"safe"
+            "Rust is a systems programming language sponsored by\n\
+             Mozilla which describes it as a \"safe"
         );
         assert_eq!(
             snippet.to_html(),
             "<b>Rust</b> is a systems programming <b>language</b> \
-             sponsored by Mozilla which\ndescribes it as a &quot;safe"
+             sponsored by\nMozilla which describes it as a &quot;safe"
         )
     }
 

src/store/compression_lz4.rs

@@ -1,7 +1,10 @@
-extern crate lz4;
-
 use std::io::{self, Read, Write};
 
+/// Name of the compression scheme used in the doc store.
+///
+/// This name is appended to the version string of tantivy.
+pub const COMPRESSION: &'static str = "lz4";
+
 pub fn compress(uncompressed: &[u8], compressed: &mut Vec<u8>) -> io::Result<()> {
     compressed.clear();
     let mut encoder = lz4::EncoderBuilder::new().build(compressed)?;

src/store/compression_snap.rs

@@ -2,6 +2,11 @@ use snap;
 
 use std::io::{self, Read, Write};
 
+/// Name of the compression scheme used in the doc store.
+///
+/// This name is appended to the version string of tantivy.
+pub const COMPRESSION: &str = "snappy";
+
 pub fn compress(uncompressed: &[u8], compressed: &mut Vec<u8>) -> io::Result<()> {
     compressed.clear();
     let mut encoder = snap::Writer::new(compressed);

src/store/mod.rs

@@ -42,12 +42,16 @@ pub use self::writer::StoreWriter;
 #[cfg(feature = "lz4")]
 mod compression_lz4;
 #[cfg(feature = "lz4")]
-use self::compression_lz4::*;
+pub use self::compression_lz4::COMPRESSION;
+#[cfg(feature = "lz4")]
+use self::compression_lz4::{compress, decompress};
 
 #[cfg(not(feature = "lz4"))]
 mod compression_snap;
 #[cfg(not(feature = "lz4"))]
-use self::compression_snap::*;
+pub use self::compression_snap::COMPRESSION;
+#[cfg(not(feature = "lz4"))]
+use self::compression_snap::{compress, decompress};
 
 #[cfg(test)]
 pub mod tests {

src/tokenizer/alphanum_only.rs

@@ -2,8 +2,6 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! # fn main() {
-//!
 //! let tokenizer = RawTokenizer
 //!   .filter(AlphaNumOnlyFilter);
 //!
@@ -20,7 +18,6 @@
 //! assert!(stream.next().is_some());
 //! // the "emoji" is dropped because its not an alphanum
 //! assert!(stream.next().is_none());
-//! # }
 //! ```
 use super::{Token, TokenFilter, TokenStream};
 

src/tokenizer/mod.rs

@@ -7,7 +7,6 @@
 //! ```rust
 //! use tantivy::schema::*;
 //!
-//! # fn main() {
 //! let mut schema_builder = Schema::builder();
 //!
 //! let text_options = TextOptions::default()
@@ -31,7 +30,6 @@
 //! schema_builder.add_text_field("uuid", id_options);
 //!
 //! let schema = schema_builder.build();
-//! # }
 //! ```
 //!
 //! By default, `tantivy` offers the following tokenizers:
@@ -66,12 +64,10 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! # fn main() {
 //! let en_stem = SimpleTokenizer
 //!     .filter(RemoveLongFilter::limit(40))
 //!     .filter(LowerCaser)
 //!     .filter(Stemmer::new(Language::English));
-//! # }
 //! ```
 //!
 //! Once your tokenizer is defined, you need to
@@ -81,13 +77,12 @@
 //! # use tantivy::schema::Schema;
 //! # use tantivy::tokenizer::*;
 //! # use tantivy::Index;
-//! # fn main() {
-//! # let custom_en_tokenizer = SimpleTokenizer;
+//! #
+//! let custom_en_tokenizer = SimpleTokenizer;
 //! # let schema = Schema::builder().build();
 //! let index = Index::create_in_ram(schema);
 //! index.tokenizers()
 //!      .register("custom_en", custom_en_tokenizer);
-//! # }
 //! ```
 //!
 //! If you built your schema programmatically, a complete example
@@ -102,7 +97,6 @@
 //! use tantivy::tokenizer::*;
 //! use tantivy::Index;
 //!
-//! # fn main() {
 //! let mut schema_builder = Schema::builder();
 //! let text_field_indexing = TextFieldIndexing::default()
 //!     .set_tokenizer("custom_en")
@@ -121,8 +115,6 @@
 //! index
 //!     .tokenizers()
 //!     .register("custom_en", custom_en_tokenizer);
-//! // ...
-//! # }
 //! ```
 //!
 mod alphanum_only;

src/tokenizer/ngram_tokenizer.rs

@@ -31,7 +31,7 @@ use super::{Token, TokenStream, Tokenizer};
 ///
 /// ```rust
 /// use tantivy::tokenizer::*;
-/// # fn main() {
+///
 /// let tokenizer = NgramTokenizer::new(2, 3, false);
 /// let mut stream = tokenizer.token_stream("hello");
 /// {
@@ -77,7 +77,6 @@ use super::{Token, TokenStream, Tokenizer};
 ///   assert_eq!(token.offset_to, 5);
 /// }
 /// assert!(stream.next().is_none());
-/// # }
 /// ```
 #[derive(Clone)]
 pub struct NgramTokenizer {

src/tokenizer/remove_long.rs

@@ -2,8 +2,6 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! # fn main() {
-//!
 //! let tokenizer = SimpleTokenizer
 //!   .filter(RemoveLongFilter::limit(5));
 //!
@@ -12,7 +10,6 @@
 //! // out of the token stream.
 //! assert_eq!(stream.next().unwrap().text, "nice");
 //! assert!(stream.next().is_none());
-//! # }
 //! ```
 //!
 use super::{Token, TokenFilter, TokenStream};

src/tokenizer/stemmer.rs

@@ -15,6 +15,7 @@ pub enum Language {
     Greek,
     Hungarian,
     Italian,
+    Norwegian,
     Portuguese,
     Romanian,
     Russian,
@@ -38,6 +39,7 @@ impl Language {
             Greek => Algorithm::Greek,
             Hungarian => Algorithm::Hungarian,
             Italian => Algorithm::Italian,
+            Norwegian => Algorithm::Norwegian,
             Portuguese => Algorithm::Portuguese,
             Romanian => Algorithm::Romanian,
             Russian => Algorithm::Russian,

src/tokenizer/stop_word_filter.rs

@@ -2,7 +2,6 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! # fn main() {
 //! let tokenizer = SimpleTokenizer
 //!   .filter(StopWordFilter::remove(vec!["the".to_string(), "is".to_string()]));
 //!
@@ -10,7 +9,6 @@
 //! assert_eq!(stream.next().unwrap().text, "fox");
 //! assert_eq!(stream.next().unwrap().text, "crafty");
 //! assert!(stream.next().is_none());
-//! # }
 //! ```
 use super::{Token, TokenFilter, TokenStream};
 use fnv::FnvHasher;
@@ -46,7 +44,7 @@ impl StopWordFilter {
             "there", "these", "they", "this", "to", "was", "will", "with",
         ];
 
-        StopWordFilter::remove(words.iter().map(|s| s.to_string()).collect())
+        StopWordFilter::remove(words.iter().map(|&s| s.to_string()).collect())
     }
 }
 

src/tokenizer/tokenizer.rs

@@ -58,12 +58,10 @@ pub trait Tokenizer<'a>: Sized + Clone {
     /// ```rust
     /// use tantivy::tokenizer::*;
     ///
-    /// # fn main() {
     /// let en_stem = SimpleTokenizer
     ///     .filter(RemoveLongFilter::limit(40))
     ///     .filter(LowerCaser)
     ///     .filter(Stemmer::default());
-    /// # }
     /// ```
     ///
     fn filter<NewFilter>(self, new_filter: NewFilter) -> ChainTokenizer<NewFilter, Self>
@@ -188,7 +186,6 @@ impl<'b> TokenStream for Box<dyn TokenStream + 'b> {
 /// ```
 /// use tantivy::tokenizer::*;
 ///
-/// # fn main() {
 /// let tokenizer = SimpleTokenizer
 ///        .filter(RemoveLongFilter::limit(40))
 ///        .filter(LowerCaser);
@@ -207,7 +204,6 @@ impl<'b> TokenStream for Box<dyn TokenStream + 'b> {
 ///     assert_eq!(token.offset_to, 12);
 ///     assert_eq!(token.position, 1);
 /// }
-/// # }
 /// ```
 ///
 pub trait TokenStream {
@@ -227,17 +223,15 @@ pub trait TokenStream {
     /// and `.token()`.
     ///
     /// ```
-    /// # use tantivy::tokenizer::*;
-    /// #
-    /// # fn main() {
-    /// # let tokenizer = SimpleTokenizer
-    /// #       .filter(RemoveLongFilter::limit(40))
-    /// #       .filter(LowerCaser);
+    /// use tantivy::tokenizer::*;
+    ///
+    /// let tokenizer = SimpleTokenizer
+    ///       .filter(RemoveLongFilter::limit(40))
+    ///       .filter(LowerCaser);
     /// let mut token_stream = tokenizer.token_stream("Hello, happy tax payer");
     /// while let Some(token) = token_stream.next() {
     ///     println!("Token {:?}", token.text);
     /// }
-    /// # }
     /// ```
     fn next(&mut self) -> Option<&Token> {
         if self.advance() {

tests/failpoints/mod.rs

@@ -28,11 +28,11 @@ fn test_failpoints_managed_directory_gc_if_delete_fails() {
     // The initial 1*off is there to allow for the removal of the
     // lock file.
     fail::cfg("RAMDirectory::delete", "1*off->1*return").unwrap();
-    managed_directory.garbage_collect(Default::default);
+    assert!(managed_directory.garbage_collect(Default::default).is_ok());
     assert!(managed_directory.exists(test_path));
 
     // running the gc a second time should remove the file.
-    managed_directory.garbage_collect(Default::default);
+    assert!(managed_directory.garbage_collect(Default::default).is_ok());
     assert!(
         !managed_directory.exists(test_path),
         "The file should have been deleted"