first stab

2025-12-28 13:02:55 +00:00 · 2019-11-01 09:05:35 +09:00
106 changed files with 1937 additions and 3627 deletions
--- a/.github/FUNDING.yml
+++ b/.github/FUNDING.yml
@@ -1,12 +0,0 @@
-# These are supported funding model platforms
-
-github: fulmicoton
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,30 +1,3 @@
-Tantivy 0.12.0
-======================
- Removing static dispatch in tokenizers for simplicity. (#762)
- Added backward iteration for `TermDictionary` stream. (@halvorboe)
- Fixed a performance issue when searching for the posting lists of a missing term (@audunhalland)
- Added a configurable maximum number of docs (10M by default) for a segment to be considered for merge (@hntd187, landed by @halvorboe #713) 
-
-## How to update?
-
-Crates relying on custom tokenizer, or registering tokenizer in the manager will require some 
-minor changes. Check https://github.com/tantivy-search/tantivy/blob/master/examples/custom_tokenizer.rs
-to check for some code sample.
-
-Tantivy 0.11.3
-=======================
- Fixed DateTime as a fast field (#735)
-
-Tantivy 0.11.2
-=======================
- The future returned by `IndexWriter::merge` does not borrow `self` mutably anymore (#732)
- Exposing a constructor for `WatchHandle` (#731)
-
-Tantivy 0.11.1
-=====================
- Bug fix #729
-
-
 Tantivy 0.11.0
 =====================

@@ -36,19 +9,14 @@ Tantivy 0.11.0
 - API change around `Box<BoxableTokenizer>`. See detail in #629
 - Avoid rebuilding Regex automaton whenever a regex query is reused. #639 (@brainlock)
 - Add footer with some metadata to index files. #605 (@fdb-hiroshima)
- Add a method to check the compatibility of the footer in the index with the running version of tantivy (@petr-tik)
 - TopDocs collector: ensure stable sorting on equal score. #671 (@brainlock)
- Added handling of pre-tokenized text fields (#642), which will enable users to
-  load tokens created outside tantivy. See usage in examples/pre_tokenized_text. (@kkoziara)
- Fix crash when committing multiple times with deleted documents. #681 (@brainlock)
-
+ 
 ## How to update?

- The index format is changed. You are required to reindex your data to use tantivy 0.11. 
 - `Box<dyn BoxableTokenizer>` has been replaced by a `BoxedTokenizer` struct.
 - Regex are now compiled when the `RegexQuery` instance is built. As a result, it can now return
 an error and handling the `Result` is required.
- `tantivy::version()` now returns a `Version` object. This object implements `ToString()`
+

 Tantivy 0.10.2
 =====================
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.12.0"
+version = "0.11.0"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -18,7 +18,7 @@ byteorder = "1.0"
 crc32fast = "1.2.0"
 once_cell = "1.0"
 regex ={version = "1.3.0", default-features = false, features = ["std"]}
-tantivy-fst = "0.2.1"
+tantivy-fst = "0.1"
 memmap = {version = "0.7", optional=true}
 lz4 = {version="1.20", optional=true}
 snap = {version="0.2"}
@@ -33,25 +33,27 @@ fs2={version="0.4", optional=true}
 itertools = "0.8"
 levenshtein_automata = {version="0.1", features=["fst_automaton"]}
 notify = {version="4", optional=true}
+bit-set = "0.5"
 uuid = { version = "0.8", features = ["v4", "serde"] }
 crossbeam = "0.7"
-futures = {version = "0.3",  features=["thread-pool"] }
+futures = "0.1"
+futures-cpupool = "0.1"
 owning_ref = "0.4"
 stable_deref_trait = "1.0.0"
-rust-stemmers = "1.2"
+rust-stemmers = "1.1"
 downcast-rs = { version="1.0" }
-tantivy-query-grammar = { version="0.11", path="./query-grammar" }
+tantivy-query-grammar = { path="./query-grammar" }
 bitpacking = {version="0.8", default-features = false, features=["bitpacker4x"]}
-census = "0.4"
+census = "0.2"
 fnv = "1.0.6"
 owned-read = "0.4"
 failure = "0.1"
 htmlescape = "0.3.1"
 fail = "0.3"
+scoped-pool = "1.0"
 murmurhash32 = "0.2"
 chrono = "0.4"
-smallvec = "1.0"
-rayon = "1"
+smallvec = "0.6"

 [target.'cfg(windows)'.dependencies]
 winapi = "0.3"
@@ -60,10 +62,7 @@ winapi = "0.3"
 rand = "0.7"
 maplit = "1"
 matches = "0.1.8"
-
-[dev-dependencies.fail]
-version = "0.3"
-features = ["failpoints"]
+time = "0.1.42"

 [profile.release]
 opt-level = 3
@@ -88,6 +87,10 @@ members = ["query-grammar"]
 [badges]
 travis-ci = { repository = "tantivy-search/tantivy" }

+[dev-dependencies.fail]
+version = "0.3"
+features = ["failpoints"]
+
 # Following the "fail" crate best practises, we isolate
 # tests that define specific behavior in fail check points
 # in a different binary.
--- a/examples/faceted_search.rs
+++ b/examples/faceted_search.rs
@@ -13,100 +13,63 @@
 // ---
 // Importing tantivy...
 use tantivy::collector::FacetCollector;
-use tantivy::query::{AllQuery, TermQuery};
+use tantivy::query::AllQuery;
 use tantivy::schema::*;
 use tantivy::{doc, Index};
+use tempfile::TempDir;

 fn main() -> tantivy::Result<()> {
-    // Let's create a temporary directory for the sake of this example
+    // Let's create a temporary directory for the
+    // sake of this example
+    let index_path = TempDir::new()?;
    let mut schema_builder = Schema::builder();

-    let name = schema_builder.add_text_field("felin_name", TEXT | STORED);
-    // this is our faceted field: its scientific classification
-    let classification = schema_builder.add_facet_field("classification");
+    schema_builder.add_text_field("name", TEXT | STORED);
+
+    // this is our faceted field
+    schema_builder.add_facet_field("tags");

    let schema = schema_builder.build();
-    let index = Index::create_in_ram(schema);

-    let mut index_writer = index.writer(30_000_000)?;
+    let index = Index::create_in_dir(&index_path, schema.clone())?;
+
+    let mut index_writer = index.writer(50_000_000)?;
+
+    let name = schema.get_field("name").unwrap();
+    let tags = schema.get_field("tags").unwrap();

    // For convenience, tantivy also comes with a macro to
    // reduce the boilerplate above.
    index_writer.add_document(doc!(
-        name => "Cat",
-        classification => Facet::from("/Felidae/Felinae/Felis")
+        name => "the ditch",
+        tags => Facet::from("/pools/north")
    ));
+
    index_writer.add_document(doc!(
-        name => "Canada lynx",
-        classification => Facet::from("/Felidae/Felinae/Lynx")
-    ));
-    index_writer.add_document(doc!(
-        name => "Cheetah",
-        classification => Facet::from("/Felidae/Felinae/Acinonyx")
-    ));
-    index_writer.add_document(doc!(
-        name => "Tiger",
-        classification => Facet::from("/Felidae/Pantherinae/Panthera")
-    ));
-    index_writer.add_document(doc!(
-        name => "Lion",
-        classification => Facet::from("/Felidae/Pantherinae/Panthera")
-    ));
-    index_writer.add_document(doc!(
-        name => "Jaguar",
-        classification => Facet::from("/Felidae/Pantherinae/Panthera")
-    ));
-    index_writer.add_document(doc!(
-        name => "Sunda clouded leopard",
-        classification => Facet::from("/Felidae/Pantherinae/Neofelis")
-    ));
-    index_writer.add_document(doc!(
-        name => "Fossa",
-        classification => Facet::from("/Eupleridae/Cryptoprocta")
+        name => "little stacey",
+        tags => Facet::from("/pools/south")
    ));
+
    index_writer.commit()?;

    let reader = index.reader()?;
+
    let searcher = reader.searcher();
-    {
-        let mut facet_collector = FacetCollector::for_field(classification);
-        facet_collector.add_facet("/Felidae");
-        let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
-        // This lists all of the facet counts, right below "/Felidae".
-        let facets: Vec<(&Facet, u64)> = facet_counts.get("/Felidae").collect();
-        assert_eq!(
-            facets,
-            vec![
-                (&Facet::from("/Felidae/Felinae"), 3),
-                (&Facet::from("/Felidae/Pantherinae"), 4),
-            ]
-        );
-    }

-    // Facets are also searchable.
-    //
-    // For instance a common UI pattern is to allow the user someone to click on a facet link
-    // (e.g: `Pantherinae`) to drill down and filter the current result set with this subfacet.
-    //
-    // The search would then look as follows.
+    let mut facet_collector = FacetCollector::for_field(tags);
+    facet_collector.add_facet("/pools");

-    // Check the reference doc for different ways to create a `Facet` object.
-    {
-        let facet = Facet::from_text("/Felidae/Pantherinae");
-        let facet_term = Term::from_facet(classification, &facet);
-        let facet_term_query = TermQuery::new(facet_term, IndexRecordOption::Basic);
-        let mut facet_collector = FacetCollector::for_field(classification);
-        facet_collector.add_facet("/Felidae/Pantherinae");
-        let facet_counts = searcher.search(&facet_term_query, &facet_collector)?;
-        let facets: Vec<(&Facet, u64)> = facet_counts.get("/Felidae/Pantherinae").collect();
-        assert_eq!(
-            facets,
-            vec![
-                (&Facet::from("/Felidae/Pantherinae/Neofelis"), 1),
-                (&Facet::from("/Felidae/Pantherinae/Panthera"), 3),
-            ]
-        );
-    }
+    let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
+
+    // This lists all of the facet counts
+    let facets: Vec<(&Facet, u64)> = facet_counts.get("/pools").collect();
+    assert_eq!(
+        facets,
+        vec![
+            (&Facet::from("/pools/north"), 1),
+            (&Facet::from("/pools/south"), 1),
+        ]
+    );

    Ok(())
 }
--- a/examples/pre_tokenized_text.rs
+++ b/examples/pre_tokenized_text.rs
@@ -1,140 +0,0 @@
-// # Pre-tokenized text example
-//
-// This example shows how to use pre-tokenized text. Sometimes yout might
-// want to index and search through text which is already split into
-// tokens by some external tool.
-//
-// In this example we will:
-// - use tantivy tokenizer to create tokens and load them directly into tantivy,
-// - import tokenized text straight from json,
-// - perform a search on documents with pre-tokenized text
-
-use tantivy::tokenizer::{PreTokenizedString, SimpleTokenizer, Token, Tokenizer};
-
-use tantivy::collector::{Count, TopDocs};
-use tantivy::query::TermQuery;
-use tantivy::schema::*;
-use tantivy::{doc, Index, ReloadPolicy};
-use tempfile::TempDir;
-
-fn pre_tokenize_text(text: &str) -> Vec<Token> {
-    let mut token_stream = SimpleTokenizer.token_stream(text);
-    let mut tokens = vec![];
-    while token_stream.advance() {
-        tokens.push(token_stream.token().clone());
-    }
-    tokens
-}
-
-fn main() -> tantivy::Result<()> {
-    let index_path = TempDir::new()?;
-
-    let mut schema_builder = Schema::builder();
-
-    schema_builder.add_text_field("title", TEXT | STORED);
-    schema_builder.add_text_field("body", TEXT);
-
-    let schema = schema_builder.build();
-
-    let index = Index::create_in_dir(&index_path, schema.clone())?;
-
-    let mut index_writer = index.writer(50_000_000)?;
-
-    // We can create a document manually, by setting the fields
-    // one by one in a Document object.
-    let title = schema.get_field("title").unwrap();
-    let body = schema.get_field("body").unwrap();
-
-    let title_text = "The Old Man and the Sea";
-    let body_text = "He was an old man who fished alone in a skiff in the Gulf Stream";
-
-    // Content of our first document
-    // We create `PreTokenizedString` which contains original text and vector of tokens
-    let title_tok = PreTokenizedString {
-        text: String::from(title_text),
-        tokens: pre_tokenize_text(title_text),
-    };
-
-    println!(
-        "Original text: \"{}\" and tokens: {:?}",
-        title_tok.text, title_tok.tokens
-    );
-
-    let body_tok = PreTokenizedString {
-        text: String::from(body_text),
-        tokens: pre_tokenize_text(body_text),
-    };
-
-    // Now lets create a document and add our `PreTokenizedString`
-    let old_man_doc = doc!(title => title_tok, body => body_tok);
-
-    // ... now let's just add it to the IndexWriter
-    index_writer.add_document(old_man_doc);
-
-    // Pretokenized text can also be fed as JSON
-    let short_man_json = r#"{
-        "title":[{
-            "text":"The Old Man",
-            "tokens":[
-                {"offset_from":0,"offset_to":3,"position":0,"text":"The","position_length":1},
-                {"offset_from":4,"offset_to":7,"position":1,"text":"Old","position_length":1},
-                {"offset_from":8,"offset_to":11,"position":2,"text":"Man","position_length":1}
-            ]
-        }]
-    }"#;
-
-    let short_man_doc = schema.parse_document(&short_man_json)?;
-
-    index_writer.add_document(short_man_doc);
-
-    // Let's commit changes
-    index_writer.commit()?;
-
-    // ... and now is the time to query our index
-
-    let reader = index
-        .reader_builder()
-        .reload_policy(ReloadPolicy::OnCommit)
-        .try_into()?;
-
-    let searcher = reader.searcher();
-
-    // We want to get documents with token "Man", we will use TermQuery to do it
-    // Using PreTokenizedString means the tokens are stored as is avoiding stemming
-    // and lowercasing, which preserves full words in their original form
-    let query = TermQuery::new(
-        Term::from_field_text(title, "Man"),
-        IndexRecordOption::Basic,
-    );
-
-    let (top_docs, count) = searcher
-        .search(&query, &(TopDocs::with_limit(2), Count))
-        .unwrap();
-
-    assert_eq!(count, 2);
-
-    // Now let's print out the results.
-    // Note that the tokens are not stored along with the original text
-    // in the document store
-    for (_score, doc_address) in top_docs {
-        let retrieved_doc = searcher.doc(doc_address)?;
-        println!("Document: {}", schema.to_json(&retrieved_doc));
-    }
-
-    // In contrary to the previous query, when we search for the "man" term we
-    // should get no results, as it's not one of the indexed tokens. SimpleTokenizer
-    // only splits text on whitespace / punctuation.
-
-    let query = TermQuery::new(
-        Term::from_field_text(title, "man"),
-        IndexRecordOption::Basic,
-    );
-
-    let (_top_docs, count) = searcher
-        .search(&query, &(TopDocs::with_limit(2), Count))
-        .unwrap();
-
-    assert_eq!(count, 0);
-
-    Ok(())
-}
--- a/examples/stop_words.rs
+++ b/examples/stop_words.rs
@@ -50,7 +50,7 @@ fn main() -> tantivy::Result<()> {

    // This tokenizer lowers all of the text (to help with stop word matching)
    // then removes all instances of `the` and `and` from the corpus
-    let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+    let tokenizer = SimpleTokenizer
        .filter(LowerCaser)
        .filter(StopWordFilter::remove(vec![
            "the".to_string(),
--- a/query-grammar/README.md
+++ b/query-grammar/README.md
@@ -1,3 +0,0 @@
-# Tantivy Query Grammar
-
-This crate is used by tantivy to parse queries.
--- a/src/collector/count_collector.rs
+++ b/src/collector/count_collector.rs
@@ -1,6 +1,7 @@
 use super::Collector;
 use crate::collector::SegmentCollector;
 use crate::DocId;
+use crate::Result;
 use crate::Score;
 use crate::SegmentLocalId;
 use crate::SegmentReader;
@@ -12,29 +13,44 @@ use crate::SegmentReader;
 /// use tantivy::collector::Count;
 /// use tantivy::query::QueryParser;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index};
+/// use tantivy::{doc, Index, Result};
 ///
-/// let mut schema_builder = Schema::builder();
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
+/// # fn main() { example().unwrap(); }
+/// fn example() -> Result<()> {
+///     let mut schema_builder = Schema::builder();
+///     let title = schema_builder.add_text_field("title", TEXT);
+///     let schema = schema_builder.build();
+///     let index = Index::create_in_ram(schema);
+///     {
+///         let mut index_writer = index.writer(3_000_000)?;
+///         index_writer.add_document(doc!(
+///             title => "The Name of the Wind",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of Muadib",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "A Dairy Cow",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of a Young Girl",
+///         ));
+///         index_writer.commit().unwrap();
+///     }
 ///
-/// let mut index_writer = index.writer(3_000_000).unwrap();
-/// index_writer.add_document(doc!(title => "The Name of the Wind"));
-/// index_writer.add_document(doc!(title => "The Diary of Muadib"));
-/// index_writer.add_document(doc!(title => "A Dairy Cow"));
-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"));
-/// assert!(index_writer.commit().is_ok());
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
-/// let reader = index.reader().unwrap();
-/// let searcher = reader.searcher();
+///     {
+///         let query_parser = QueryParser::for_index(&index, vec![title]);
+///         let query = query_parser.parse_query("diary")?;
+///         let count = searcher.search(&query, &Count).unwrap();
 ///
-/// // Here comes the important part
-/// let query_parser = QueryParser::for_index(&index, vec![title]);
-/// let query = query_parser.parse_query("diary").unwrap();
-/// let count = searcher.search(&query, &Count).unwrap();
+///         assert_eq!(count, 2);
+///     }
 ///
-/// assert_eq!(count, 2);
+///     Ok(())
+/// }
 /// ```
 pub struct Count;

@@ -43,11 +59,7 @@ impl Collector for Count {

    type Child = SegmentCountCollector;

-    fn for_segment(
-        &self,
-        _: SegmentLocalId,
-        _: &SegmentReader,
-    ) -> crate::Result<SegmentCountCollector> {
+    fn for_segment(&self, _: SegmentLocalId, _: &SegmentReader) -> Result<SegmentCountCollector> {
        Ok(SegmentCountCollector::default())
    }

@@ -55,7 +67,7 @@ impl Collector for Count {
        false
    }

-    fn merge_fruits(&self, segment_counts: Vec<usize>) -> crate::Result<usize> {
+    fn merge_fruits(&self, segment_counts: Vec<usize>) -> Result<usize> {
        Ok(segment_counts.into_iter().sum())
    }
 }
--- a/src/collector/custom_score_top_collector.rs
+++ b/src/collector/custom_score_top_collector.rs
@@ -1,5 +1,6 @@
 use crate::collector::top_collector::{TopCollector, TopSegmentCollector};
 use crate::collector::{Collector, SegmentCollector};
+use crate::Result;
 use crate::{DocAddress, DocId, Score, SegmentReader};

 pub(crate) struct CustomScoreTopCollector<TCustomScorer, TScore = Score> {
@@ -41,7 +42,7 @@ pub trait CustomScorer<TScore>: Sync {
    type Child: CustomSegmentScorer<TScore>;
    /// Builds a child scorer for a specific segment. The child scorer is associated to
    /// a specific segment.
-    fn segment_scorer(&self, segment_reader: &SegmentReader) -> crate::Result<Self::Child>;
+    fn segment_scorer(&self, segment_reader: &SegmentReader) -> Result<Self::Child>;
 }

 impl<TCustomScorer, TScore> Collector for CustomScoreTopCollector<TCustomScorer, TScore>
@@ -57,7 +58,7 @@ where
        &self,
        segment_local_id: u32,
        segment_reader: &SegmentReader,
-    ) -> crate::Result<Self::Child> {
+    ) -> Result<Self::Child> {
        let segment_scorer = self.custom_scorer.segment_scorer(segment_reader)?;
        let segment_collector = self
            .collector
@@ -72,7 +73,7 @@ where
        false
    }

-    fn merge_fruits(&self, segment_fruits: Vec<Self::Fruit>) -> crate::Result<Self::Fruit> {
+    fn merge_fruits(&self, segment_fruits: Vec<Self::Fruit>) -> Result<Self::Fruit> {
        self.collector.merge_fruits(segment_fruits)
    }
 }
@@ -110,7 +111,7 @@ where
 {
    type Child = T;

-    fn segment_scorer(&self, segment_reader: &SegmentReader) -> crate::Result<Self::Child> {
+    fn segment_scorer(&self, segment_reader: &SegmentReader) -> Result<Self::Child> {
        Ok((self)(segment_reader))
    }
 }
--- a/src/collector/facet_collector.rs
+++ b/src/collector/facet_collector.rs
@@ -5,6 +5,7 @@ use crate::fastfield::FacetReader;
 use crate::schema::Facet;
 use crate::schema::Field;
 use crate::DocId;
+use crate::Result;
 use crate::Score;
 use crate::SegmentLocalId;
 use crate::SegmentReader;
@@ -83,9 +84,10 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 /// use tantivy::collector::FacetCollector;
 /// use tantivy::query::AllQuery;
 /// use tantivy::schema::{Facet, Schema, TEXT};
-/// use tantivy::{doc, Index};
+/// use tantivy::{doc, Index, Result};
 ///
-/// fn example() -> tantivy::Result<()> {
+/// # fn main() { example().unwrap(); }
+/// fn example() -> Result<()> {
 ///     let mut schema_builder = Schema::builder();
 ///
 ///     // Facet have their own specific type.
@@ -125,7 +127,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///     let searcher = reader.searcher();
 ///
 ///     {
-///         let mut facet_collector = FacetCollector::for_field(facet);
+///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/lang");
 ///         facet_collector.add_facet("/category");
 ///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
@@ -141,7 +143,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///     }
 ///
 ///     {
-///         let mut facet_collector = FacetCollector::for_field(facet);
+///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/category/fiction");
 ///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
@@ -156,8 +158,8 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///         ]);
 ///     }
 ///
-///     {
-///         let mut facet_collector = FacetCollector::for_field(facet);
+///    {
+///			let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/category/fiction");
 ///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
@@ -170,7 +172,6 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///
 ///     Ok(())
 /// }
-/// # assert!(example().is_ok());
 /// ```
 pub struct FacetCollector {
    field: Field,
@@ -261,7 +262,7 @@ impl Collector for FacetCollector {
        &self,
        _: SegmentLocalId,
        reader: &SegmentReader,
-    ) -> crate::Result<FacetSegmentCollector> {
+    ) -> Result<FacetSegmentCollector> {
        let field_name = reader.schema().get_field_name(self.field);
        let facet_reader = reader.facet_reader(self.field).ok_or_else(|| {
            TantivyError::SchemaError(format!("Field {:?} is not a facet field.", field_name))
@@ -327,7 +328,7 @@ impl Collector for FacetCollector {
        false
    }

-    fn merge_fruits(&self, segments_facet_counts: Vec<FacetCounts>) -> crate::Result<FacetCounts> {
+    fn merge_fruits(&self, segments_facet_counts: Vec<FacetCounts>) -> Result<FacetCounts> {
        let mut facet_counts: BTreeMap<Facet, u64> = BTreeMap::new();
        for segment_facet_counts in segments_facet_counts {
            for (facet, count) in segment_facet_counts.facet_counts {
@@ -451,11 +452,9 @@ impl FacetCounts {
 #[cfg(test)]
 mod tests {
    use super::{FacetCollector, FacetCounts};
-    use crate::collector::Count;
    use crate::core::Index;
-    use crate::query::{AllQuery, QueryParser, TermQuery};
-    use crate::schema::{Document, Facet, Field, IndexRecordOption, Schema};
-    use crate::Term;
+    use crate::query::AllQuery;
+    use crate::schema::{Document, Facet, Field, Schema};
    use rand::distributions::Uniform;
    use rand::prelude::SliceRandom;
    use rand::{thread_rng, Rng};
@@ -545,56 +544,6 @@ mod tests {
        assert_eq!(facets[0].1, 1);
    }

-    #[test]
-    fn test_doc_search_by_facet() {
-        let mut schema_builder = Schema::builder();
-        let facet_field = schema_builder.add_facet_field("facet");
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-        index_writer.add_document(doc!(
-            facet_field => Facet::from_text(&"/A/A"),
-        ));
-        index_writer.add_document(doc!(
-            facet_field => Facet::from_text(&"/A/B"),
-        ));
-        index_writer.add_document(doc!(
-            facet_field => Facet::from_text(&"/A/C/A"),
-        ));
-        index_writer.add_document(doc!(
-            facet_field => Facet::from_text(&"/D/C/A"),
-        ));
-        index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-        let searcher = reader.searcher();
-        assert_eq!(searcher.num_docs(), 4);
-
-        let count_facet = |facet_str: &str| {
-            let term = Term::from_facet(facet_field, &Facet::from_text(facet_str));
-            searcher
-                .search(&TermQuery::new(term, IndexRecordOption::Basic), &Count)
-                .unwrap()
-        };
-
-        assert_eq!(count_facet("/"), 4);
-        assert_eq!(count_facet("/A"), 3);
-        assert_eq!(count_facet("/A/B"), 1);
-        assert_eq!(count_facet("/A/C"), 1);
-        assert_eq!(count_facet("/A/C/A"), 1);
-        assert_eq!(count_facet("/C/A"), 0);
-        {
-            let query_parser = QueryParser::for_index(&index, vec![]);
-            {
-                let query = query_parser.parse_query("facet:/A/B").unwrap();
-                assert_eq!(1, searcher.search(&query, &Count).unwrap());
-            }
-            {
-                let query = query_parser.parse_query("facet:/A").unwrap();
-                assert_eq!(3, searcher.search(&query, &Count).unwrap());
-            }
-        }
-    }
-
    #[test]
    fn test_non_used_facet_collector() {
        let mut facet_collector = FacetCollector::for_field(Field::from_field_id(0));
--- a/src/collector/mod.rs
+++ b/src/collector/mod.rs
@@ -85,6 +85,7 @@ See the `custom_collector` example.
 */

 use crate::DocId;
+use crate::Result;
 use crate::Score;
 use crate::SegmentLocalId;
 use crate::SegmentReader;
@@ -146,14 +147,14 @@ pub trait Collector: Sync {
        &self,
        segment_local_id: SegmentLocalId,
        segment: &SegmentReader,
-    ) -> crate::Result<Self::Child>;
+    ) -> Result<Self::Child>;

    /// Returns true iff the collector requires to compute scores for documents.
    fn requires_scoring(&self) -> bool;

    /// Combines the fruit associated to the collection of each segments
    /// into one fruit.
-    fn merge_fruits(&self, segment_fruits: Vec<Self::Fruit>) -> crate::Result<Self::Fruit>;
+    fn merge_fruits(&self, segment_fruits: Vec<Self::Fruit>) -> Result<Self::Fruit>;
 }

 /// The `SegmentCollector` is the trait in charge of defining the
@@ -184,11 +185,7 @@ where
    type Fruit = (Left::Fruit, Right::Fruit);
    type Child = (Left::Child, Right::Child);

-    fn for_segment(
-        &self,
-        segment_local_id: u32,
-        segment: &SegmentReader,
-    ) -> crate::Result<Self::Child> {
+    fn for_segment(&self, segment_local_id: u32, segment: &SegmentReader) -> Result<Self::Child> {
        let left = self.0.for_segment(segment_local_id, segment)?;
        let right = self.1.for_segment(segment_local_id, segment)?;
        Ok((left, right))
@@ -201,7 +198,7 @@ where
    fn merge_fruits(
        &self,
        children: Vec<(Left::Fruit, Right::Fruit)>,
-    ) -> crate::Result<(Left::Fruit, Right::Fruit)> {
+    ) -> Result<(Left::Fruit, Right::Fruit)> {
        let mut left_fruits = vec![];
        let mut right_fruits = vec![];
        for (left_fruit, right_fruit) in children {
@@ -243,11 +240,7 @@ where
    type Fruit = (One::Fruit, Two::Fruit, Three::Fruit);
    type Child = (One::Child, Two::Child, Three::Child);

-    fn for_segment(
-        &self,
-        segment_local_id: u32,
-        segment: &SegmentReader,
-    ) -> crate::Result<Self::Child> {
+    fn for_segment(&self, segment_local_id: u32, segment: &SegmentReader) -> Result<Self::Child> {
        let one = self.0.for_segment(segment_local_id, segment)?;
        let two = self.1.for_segment(segment_local_id, segment)?;
        let three = self.2.for_segment(segment_local_id, segment)?;
@@ -258,7 +251,7 @@ where
        self.0.requires_scoring() || self.1.requires_scoring() || self.2.requires_scoring()
    }

-    fn merge_fruits(&self, children: Vec<Self::Fruit>) -> crate::Result<Self::Fruit> {
+    fn merge_fruits(&self, children: Vec<Self::Fruit>) -> Result<Self::Fruit> {
        let mut one_fruits = vec![];
        let mut two_fruits = vec![];
        let mut three_fruits = vec![];
@@ -306,11 +299,7 @@ where
    type Fruit = (One::Fruit, Two::Fruit, Three::Fruit, Four::Fruit);
    type Child = (One::Child, Two::Child, Three::Child, Four::Child);

-    fn for_segment(
-        &self,
-        segment_local_id: u32,
-        segment: &SegmentReader,
-    ) -> crate::Result<Self::Child> {
+    fn for_segment(&self, segment_local_id: u32, segment: &SegmentReader) -> Result<Self::Child> {
        let one = self.0.for_segment(segment_local_id, segment)?;
        let two = self.1.for_segment(segment_local_id, segment)?;
        let three = self.2.for_segment(segment_local_id, segment)?;
@@ -325,7 +314,7 @@ where
            || self.3.requires_scoring()
    }

-    fn merge_fruits(&self, children: Vec<Self::Fruit>) -> crate::Result<Self::Fruit> {
+    fn merge_fruits(&self, children: Vec<Self::Fruit>) -> Result<Self::Fruit> {
        let mut one_fruits = vec![];
        let mut two_fruits = vec![];
        let mut three_fruits = vec![];
--- a/src/collector/multi_collector.rs
+++ b/src/collector/multi_collector.rs
@@ -2,6 +2,7 @@ use super::Collector;
 use super::SegmentCollector;
 use crate::collector::Fruit;
 use crate::DocId;
+use crate::Result;
 use crate::Score;
 use crate::SegmentLocalId;
 use crate::SegmentReader;
@@ -23,7 +24,7 @@ impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> {
        &self,
        segment_local_id: u32,
        reader: &SegmentReader,
-    ) -> crate::Result<Box<dyn BoxableSegmentCollector>> {
+    ) -> Result<Box<dyn BoxableSegmentCollector>> {
        let child = self.0.for_segment(segment_local_id, reader)?;
        Ok(Box::new(SegmentCollectorWrapper(child)))
    }
@@ -32,10 +33,7 @@ impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> {
        self.0.requires_scoring()
    }

-    fn merge_fruits(
-        &self,
-        children: Vec<<Self as Collector>::Fruit>,
-    ) -> crate::Result<Box<dyn Fruit>> {
+    fn merge_fruits(&self, children: Vec<<Self as Collector>::Fruit>) -> Result<Box<dyn Fruit>> {
        let typed_fruit: Vec<TCollector::Fruit> = children
            .into_iter()
            .map(|untyped_fruit| {
@@ -46,7 +44,7 @@ impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> {
                        TantivyError::InvalidArgument("Failed to cast child fruit.".to_string())
                    })
            })
-            .collect::<crate::Result<_>>()?;
+            .collect::<Result<_>>()?;
        let merged_fruit = self.0.merge_fruits(typed_fruit)?;
        Ok(Box::new(merged_fruit))
    }
@@ -110,35 +108,49 @@ impl<TFruit: Fruit> FruitHandle<TFruit> {
 /// use tantivy::collector::{Count, TopDocs, MultiCollector};
 /// use tantivy::query::QueryParser;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index};
+/// use tantivy::{doc, Index, Result};
 ///
-/// let mut schema_builder = Schema::builder();
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
+/// # fn main() { example().unwrap(); }
+/// fn example() -> Result<()> {
+///     let mut schema_builder = Schema::builder();
+///     let title = schema_builder.add_text_field("title", TEXT);
+///     let schema = schema_builder.build();
+///     let index = Index::create_in_ram(schema);
+///     {
+///         let mut index_writer = index.writer(3_000_000)?;
+///         index_writer.add_document(doc!(
+///             title => "The Name of the Wind",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of Muadib",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "A Dairy Cow",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of a Young Girl",
+///         ));
+///         index_writer.commit().unwrap();
+///     }
 ///
-/// let mut index_writer = index.writer(3_000_000).unwrap();
-/// index_writer.add_document(doc!(title => "The Name of the Wind"));
-/// index_writer.add_document(doc!(title => "The Diary of Muadib"));
-/// index_writer.add_document(doc!(title => "A Dairy Cow"));
-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"));
-/// assert!(index_writer.commit().is_ok());
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
-/// let reader = index.reader().unwrap();
-/// let searcher = reader.searcher();
+///     let mut collectors = MultiCollector::new();
+///     let top_docs_handle = collectors.add_collector(TopDocs::with_limit(2));
+///     let count_handle = collectors.add_collector(Count);
+///     let query_parser = QueryParser::for_index(&index, vec![title]);
+///     let query = query_parser.parse_query("diary")?;
+///     let mut multi_fruit = searcher.search(&query, &collectors)?;
 ///
-/// let mut collectors = MultiCollector::new();
-/// let top_docs_handle = collectors.add_collector(TopDocs::with_limit(2));
-/// let count_handle = collectors.add_collector(Count);
-/// let query_parser = QueryParser::for_index(&index, vec![title]);
-/// let query = query_parser.parse_query("diary").unwrap();
-/// let mut multi_fruit = searcher.search(&query, &collectors).unwrap();
+///     let count = count_handle.extract(&mut multi_fruit);
+///     let top_docs = top_docs_handle.extract(&mut multi_fruit);
 ///
-/// let count = count_handle.extract(&mut multi_fruit);
-/// let top_docs = top_docs_handle.extract(&mut multi_fruit);
+///     # assert_eq!(count, 2);
+///     # assert_eq!(top_docs.len(), 2);
 ///
-/// assert_eq!(count, 2);
-/// assert_eq!(top_docs.len(), 2);
+///     Ok(())
+/// }
 /// ```
 #[allow(clippy::type_complexity)]
 #[derive(Default)]
@@ -177,12 +189,12 @@ impl<'a> Collector for MultiCollector<'a> {
        &self,
        segment_local_id: SegmentLocalId,
        segment: &SegmentReader,
-    ) -> crate::Result<MultiCollectorChild> {
+    ) -> Result<MultiCollectorChild> {
        let children = self
            .collector_wrappers
            .iter()
            .map(|collector_wrapper| collector_wrapper.for_segment(segment_local_id, segment))
-            .collect::<crate::Result<Vec<_>>>()?;
+            .collect::<Result<Vec<_>>>()?;
        Ok(MultiCollectorChild { children })
    }

@@ -193,7 +205,7 @@ impl<'a> Collector for MultiCollector<'a> {
            .any(Collector::requires_scoring)
    }

-    fn merge_fruits(&self, segments_multifruits: Vec<MultiFruit>) -> crate::Result<MultiFruit> {
+    fn merge_fruits(&self, segments_multifruits: Vec<MultiFruit>) -> Result<MultiFruit> {
        let mut segment_fruits_list: Vec<Vec<Box<dyn Fruit>>> = (0..self.collector_wrappers.len())
            .map(|_| Vec::with_capacity(segments_multifruits.len()))
            .collect::<Vec<_>>();
@@ -211,7 +223,7 @@ impl<'a> Collector for MultiCollector<'a> {
            .map(|(child_collector, segment_fruits)| {
                Ok(Some(child_collector.merge_fruits(segment_fruits)?))
            })
-            .collect::<crate::Result<_>>()?;
+            .collect::<Result<_>>()?;
        Ok(MultiFruit { sub_fruits })
    }
 }
--- a/src/collector/tests.rs
+++ b/src/collector/tests.rs
@@ -55,7 +55,7 @@ impl Collector for TestCollector {
        &self,
        segment_id: SegmentLocalId,
        _reader: &SegmentReader,
-    ) -> crate::Result<TestSegmentCollector> {
+    ) -> Result<TestSegmentCollector> {
        Ok(TestSegmentCollector {
            segment_id,
            fruit: TestFruit::default(),
@@ -66,7 +66,7 @@ impl Collector for TestCollector {
        self.compute_score
    }

-    fn merge_fruits(&self, mut children: Vec<TestFruit>) -> crate::Result<TestFruit> {
+    fn merge_fruits(&self, mut children: Vec<TestFruit>) -> Result<TestFruit> {
        children.sort_by_key(|fruit| {
            if fruit.docs().is_empty() {
                0
@@ -124,7 +124,7 @@ impl Collector for FastFieldTestCollector {
        &self,
        _: SegmentLocalId,
        segment_reader: &SegmentReader,
-    ) -> crate::Result<FastFieldSegmentCollector> {
+    ) -> Result<FastFieldSegmentCollector> {
        let reader = segment_reader
            .fast_fields()
            .u64(self.field)
@@ -139,7 +139,7 @@ impl Collector for FastFieldTestCollector {
        false
    }

-    fn merge_fruits(&self, children: Vec<Vec<u64>>) -> crate::Result<Vec<u64>> {
+    fn merge_fruits(&self, children: Vec<Vec<u64>>) -> Result<Vec<u64>> {
        Ok(children.into_iter().flat_map(|v| v.into_iter()).collect())
    }
 }
@@ -184,7 +184,7 @@ impl Collector for BytesFastFieldTestCollector {
        &self,
        _segment_local_id: u32,
        segment_reader: &SegmentReader,
-    ) -> crate::Result<BytesFastFieldSegmentCollector> {
+    ) -> Result<BytesFastFieldSegmentCollector> {
        Ok(BytesFastFieldSegmentCollector {
            vals: Vec::new(),
            reader: segment_reader
@@ -198,7 +198,7 @@ impl Collector for BytesFastFieldTestCollector {
        false
    }

-    fn merge_fruits(&self, children: Vec<Vec<u8>>) -> crate::Result<Vec<u8>> {
+    fn merge_fruits(&self, children: Vec<Vec<u8>>) -> Result<Vec<u8>> {
        Ok(children.into_iter().flat_map(|c| c.into_iter()).collect())
    }
 }
--- a/src/collector/top_collector.rs
+++ b/src/collector/top_collector.rs
@@ -1,5 +1,6 @@
 use crate::DocAddress;
 use crate::DocId;
+use crate::Result;
 use crate::SegmentLocalId;
 use crate::SegmentReader;
 use serde::export::PhantomData;
@@ -85,7 +86,7 @@ where
    pub fn merge_fruits(
        &self,
        children: Vec<Vec<(T, DocAddress)>>,
-    ) -> crate::Result<Vec<(T, DocAddress)>> {
+    ) -> Result<Vec<(T, DocAddress)>> {
        if self.limit == 0 {
            return Ok(Vec::new());
        }
@@ -112,7 +113,7 @@ where
        &self,
        segment_id: SegmentLocalId,
        _: &SegmentReader,
-    ) -> crate::Result<TopSegmentCollector<F>> {
+    ) -> Result<TopSegmentCollector<F>> {
        Ok(TopSegmentCollector::new(segment_id, self.limit))
    }
 }
--- a/src/collector/top_score_collector.rs
+++ b/src/collector/top_score_collector.rs
@@ -6,10 +6,10 @@ use crate::collector::tweak_score_top_collector::TweakedScoreTopCollector;
 use crate::collector::{
    CustomScorer, CustomSegmentScorer, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector,
 };
-use crate::fastfield::FastFieldReader;
 use crate::schema::Field;
 use crate::DocAddress;
 use crate::DocId;
+use crate::Result;
 use crate::Score;
 use crate::SegmentLocalId;
 use crate::SegmentReader;
@@ -29,29 +29,43 @@ use std::fmt;
 /// use tantivy::collector::TopDocs;
 /// use tantivy::query::QueryParser;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, DocAddress, Index};
+/// use tantivy::{doc, DocAddress, Index, Result};
 ///
-/// let mut schema_builder = Schema::builder();
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
+/// # fn main() { example().unwrap(); }
+/// fn example() -> Result<()> {
+///     let mut schema_builder = Schema::builder();
+///     let title = schema_builder.add_text_field("title", TEXT);
+///     let schema = schema_builder.build();
+///     let index = Index::create_in_ram(schema);
+///     {
+///         let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
+///         index_writer.add_document(doc!(
+///             title => "The Name of the Wind",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of Muadib",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "A Dairy Cow",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of a Young Girl",
+///         ));
+///         index_writer.commit().unwrap();
+///     }
 ///
-/// let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-/// index_writer.add_document(doc!(title => "The Name of the Wind"));
-/// index_writer.add_document(doc!(title => "The Diary of Muadib"));
-/// index_writer.add_document(doc!(title => "A Dairy Cow"));
-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"));
-/// assert!(index_writer.commit().is_ok());
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
 ///
-/// let reader = index.reader().unwrap();
-/// let searcher = reader.searcher();
+///     let query_parser = QueryParser::for_index(&index, vec![title]);
+///     let query = query_parser.parse_query("diary")?;
+///     let top_docs = searcher.search(&query, &TopDocs::with_limit(2))?;
 ///
-/// let query_parser = QueryParser::for_index(&index, vec![title]);
-/// let query = query_parser.parse_query("diary").unwrap();
-/// let top_docs = searcher.search(&query, &TopDocs::with_limit(2)).unwrap();
+///     assert_eq!(&top_docs[0], &(0.7261542, DocAddress(0, 1)));
+///     assert_eq!(&top_docs[1], &(0.6099695, DocAddress(0, 3)));
 ///
-/// assert_eq!(&top_docs[0], &(0.7261542, DocAddress(0, 1)));
-/// assert_eq!(&top_docs[1], &(0.6099695, DocAddress(0, 3)));
+///     Ok(())
+/// }
 /// ```
 pub struct TopDocs(TopCollector<Score>);

@@ -61,36 +75,6 @@ impl fmt::Debug for TopDocs {
    }
 }

-struct ScorerByFastFieldReader {
-    ff_reader: FastFieldReader<u64>,
-}
-
-impl CustomSegmentScorer<u64> for ScorerByFastFieldReader {
-    fn score(&self, doc: DocId) -> u64 {
-        self.ff_reader.get_u64(u64::from(doc))
-    }
-}
-
-struct ScorerByField {
-    field: Field,
-}
-
-impl CustomScorer<u64> for ScorerByField {
-    type Child = ScorerByFastFieldReader;
-
-    fn segment_scorer(&self, segment_reader: &SegmentReader) -> crate::Result<Self::Child> {
-        let ff_reader = segment_reader
-            .fast_fields()
-            .u64(self.field)
-            .ok_or_else(|| {
-                crate::TantivyError::SchemaError(format!(
-                    "Field requested is not a i64/u64 fast field."
-                ))
-            })?;
-        Ok(ScorerByFastFieldReader { ff_reader })
-    }
-}
-
 impl TopDocs {
    /// Creates a top score collector, with a number of documents equal to "limit".
    ///
@@ -104,7 +88,7 @@ impl TopDocs {
    ///
    /// ```rust
    /// # use tantivy::schema::{Schema, FAST, TEXT};
-    /// # use tantivy::{doc, Index, DocAddress};
+    /// # use tantivy::{doc, Index, Result, DocAddress};
    /// # use tantivy::query::{Query, QueryParser};
    /// use tantivy::Searcher;
    /// use tantivy::collector::TopDocs;
@@ -118,12 +102,15 @@ impl TopDocs {
    /// #  
    /// #   let index = Index::create_in_ram(schema);
    /// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
-    /// #   index_writer.add_document(doc!(title => "The Name of the Wind", rating => 92u64));
+    /// #   index_writer.add_document(doc!(
+    /// #       title => "The Name of the Wind",
+    /// #       rating => 92u64,
+    /// #   ));
    /// #   index_writer.add_document(doc!(title => "The Diary of Muadib", rating => 97u64));
    /// #   index_writer.add_document(doc!(title => "A Dairy Cow", rating => 63u64));
    /// #   index_writer.add_document(doc!(title => "The Diary of a Young Girl", rating => 80u64));
-    /// #   assert!(index_writer.commit().is_ok());
-    /// #   let reader = index.reader().unwrap();
+    /// #   index_writer.commit()?;
+    /// #   let reader = index.reader()?;
    /// #   let query = QueryParser::for_index(&index, vec![title]).parse_query("diary")?;
    /// #   let top_docs = docs_sorted_by_rating(&reader.searcher(), &query, rating)?;
    /// #   assert_eq!(top_docs,
@@ -141,7 +128,7 @@ impl TopDocs {
    /// fn docs_sorted_by_rating(searcher: &Searcher,
    ///                          query: &dyn Query,
    ///                          sort_by_field: Field)
-    ///     -> tantivy::Result<Vec<(u64, DocAddress)>> {
+    ///     -> Result<Vec<(u64, DocAddress)>> {
    ///
    ///     // This is where we build our topdocs collector
    ///     //
@@ -173,7 +160,14 @@ impl TopDocs {
        self,
        field: Field,
    ) -> impl Collector<Fruit = Vec<(u64, DocAddress)>> {
-        self.custom_score(ScorerByField { field })
+        self.custom_score(move |segment_reader: &SegmentReader| {
+            let ff_reader = segment_reader
+                .fast_fields()
+                .u64(field)
+                .expect("Field requested is not a i64/u64 fast field.");
+            //TODO error message missmatch actual behavior for i64
+            move |doc: DocId| ff_reader.get(doc)
+        })
    }

    /// Ranks the documents using a custom score.
@@ -208,33 +202,27 @@ impl TopDocs {
    /// use tantivy::collector::TopDocs;
    /// use tantivy::schema::Field;
    ///
-    /// fn create_schema() -> Schema {
-    ///    let mut schema_builder = Schema::builder();
-    ///    schema_builder.add_text_field("product_name", TEXT);
-    ///    schema_builder.add_u64_field("popularity", FAST);
-    ///    schema_builder.build()
-    /// }
-    ///
-    /// fn create_index() -> tantivy::Result<Index> {
-    ///   let schema = create_schema();
-    ///   let index = Index::create_in_ram(schema);
-    ///   let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
-    ///   let product_name = index.schema().get_field("product_name").unwrap();
-    ///   let popularity: Field = index.schema().get_field("popularity").unwrap();
-    ///   index_writer.add_document(doc!(product_name => "The Diary of Muadib", popularity => 1u64));
-    ///   index_writer.add_document(doc!(product_name => "A Dairy Cow", popularity => 10u64));
-    ///   index_writer.add_document(doc!(product_name => "The Diary of a Young Girl", popularity => 15u64));
-    ///   index_writer.commit()?;
-    ///   Ok(index)
-    /// }
-    ///
-    /// let index = create_index().unwrap();
-    /// let product_name = index.schema().get_field("product_name").unwrap();
+    /// # fn create_schema() -> Schema {
+    /// #    let mut schema_builder = Schema::builder();
+    /// #    schema_builder.add_text_field("product_name", TEXT);
+    /// #    schema_builder.add_u64_field("popularity", FAST);
+    /// #    schema_builder.build()
+    /// # }
+    /// #
+    /// # fn main() -> tantivy::Result<()> {
+    /// #   let schema = create_schema();
+    /// #   let index = Index::create_in_ram(schema);
+    /// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
+    /// #   let product_name = index.schema().get_field("product_name").unwrap();
+    /// #   
    /// let popularity: Field = index.schema().get_field("popularity").unwrap();
-    ///
-    /// let user_query_str = "diary";
-    /// let query_parser = QueryParser::for_index(&index, vec![product_name]);
-    /// let query = query_parser.parse_query(user_query_str).unwrap();
+    /// #   index_writer.add_document(doc!(product_name => "The Diary of Muadib", popularity => 1u64));
+    /// #   index_writer.add_document(doc!(product_name => "A Dairy Cow", popularity => 10u64));
+    /// #   index_writer.add_document(doc!(product_name => "The Diary of a Young Girl", popularity => 15u64));
+    /// #   index_writer.commit()?;
+    /// // ...
+    /// # let user_query = "diary";
+    /// # let query = QueryParser::for_index(&index, vec![product_name]).parse_query(user_query)?;
    ///
    /// // This is where we build our collector with our custom score.
    /// let top_docs_by_custom_score = TopDocs
@@ -261,12 +249,15 @@ impl TopDocs {
    ///                 popularity_boost_score * original_score
    ///             }
    ///           });
-    /// let reader = index.reader().unwrap();
-    /// let searcher = reader.searcher();
+    /// # let reader = index.reader()?;
+    /// # let searcher = reader.searcher();
    /// // ... and here are our documents. Note this is a simple vec.
    /// // The `Score` in the pair is our tweaked score.
    /// let resulting_docs: Vec<(Score, DocAddress)> =
-    ///      searcher.search(&query, &top_docs_by_custom_score).unwrap();
+    ///      searcher.search(&*query, &top_docs_by_custom_score)?;
+    ///
+    /// # Ok(())
+    /// # }
    /// ```
    ///
    /// # See also
@@ -407,7 +398,7 @@ impl Collector for TopDocs {
        &self,
        segment_local_id: SegmentLocalId,
        reader: &SegmentReader,
-    ) -> crate::Result<Self::Child> {
+    ) -> Result<Self::Child> {
        let collector = self.0.for_segment(segment_local_id, reader)?;
        Ok(TopScoreSegmentCollector(collector))
    }
@@ -416,10 +407,7 @@ impl Collector for TopDocs {
        true
    }

-    fn merge_fruits(
-        &self,
-        child_fruits: Vec<Vec<(Score, DocAddress)>>,
-    ) -> crate::Result<Self::Fruit> {
+    fn merge_fruits(&self, child_fruits: Vec<Vec<(Score, DocAddress)>>) -> Result<Self::Fruit> {
        self.0.merge_fruits(child_fruits)
    }
 }
@@ -598,6 +586,7 @@ mod tests {
    }

    #[test]
+    #[should_panic(expected = "Field requested is not a i64/u64 fast field")]
    fn test_field_not_fast_field() {
        let mut schema_builder = Schema::builder();
        let title = schema_builder.add_text_field(TITLE, TEXT);
@@ -612,12 +601,7 @@ mod tests {
        let searcher = index.reader().unwrap().searcher();
        let segment = searcher.segment_reader(0);
        let top_collector = TopDocs::with_limit(4).order_by_u64_field(size);
-        let err = top_collector.for_segment(0, segment);
-        if let Err(crate::TantivyError::SchemaError(msg)) = err {
-            assert_eq!(msg, "Field requested is not a i64/u64 fast field.");
-        } else {
-            assert!(false);
-        }
+        assert!(top_collector.for_segment(0, segment).is_ok());
    }

    fn index(
--- a/src/core/executor.rs
+++ b/src/core/executor.rs
@@ -1,5 +1,6 @@
+use crate::Result;
 use crossbeam::channel;
-use rayon::{ThreadPool, ThreadPoolBuilder};
+use scoped_pool::{Pool, ThreadConfig};

 /// Search executor whether search request are single thread or multithread.
 ///
@@ -9,10 +10,8 @@ use rayon::{ThreadPool, ThreadPoolBuilder};
 /// API of a dependency, knowing it might conflict with a different version
 /// used by the client. Second, we may stop using rayon in the future.
 pub enum Executor {
-    /// Single thread variant of an Executor
    SingleThread,
-    /// Thread pool variant of an Executor
-    ThreadPool(ThreadPool),
+    ThreadPool(Pool),
 }

 impl Executor {
@@ -21,39 +20,37 @@ impl Executor {
        Executor::SingleThread
    }

-    /// Creates an Executor that dispatches the tasks in a thread pool.
-    pub fn multi_thread(num_threads: usize, prefix: &'static str) -> crate::Result<Executor> {
-        let pool = ThreadPoolBuilder::new()
-            .num_threads(num_threads)
-            .thread_name(move |num| format!("{}{}", prefix, num))
-            .build()?;
-        Ok(Executor::ThreadPool(pool))
+    // Creates an Executor that dispatches the tasks in a thread pool.
+    pub fn multi_thread(num_threads: usize, prefix: &'static str) -> Executor {
+        let thread_config = ThreadConfig::new().prefix(prefix);
+        let pool = Pool::with_thread_config(num_threads, thread_config);
+        Executor::ThreadPool(pool)
    }

-    /// Perform a map in the thread pool.
-    ///
-    /// Regardless of the executor (`SingleThread` or `ThreadPool`), panics in the task
-    /// will propagate to the caller.
+    // Perform a map in the thread pool.
+    //
+    // Regardless of the executor (`SingleThread` or `ThreadPool`), panics in the task
+    // will propagate to the caller.
    pub fn map<
        A: Send,
        R: Send,
        AIterator: Iterator<Item = A>,
-        F: Sized + Sync + Fn(A) -> crate::Result<R>,
+        F: Sized + Sync + Fn(A) -> Result<R>,
    >(
        &self,
        f: F,
        args: AIterator,
-    ) -> crate::Result<Vec<R>> {
+    ) -> Result<Vec<R>> {
        match self {
-            Executor::SingleThread => args.map(f).collect::<crate::Result<_>>(),
+            Executor::SingleThread => args.map(f).collect::<Result<_>>(),
            Executor::ThreadPool(pool) => {
                let args_with_indices: Vec<(usize, A)> = args.enumerate().collect();
                let num_fruits = args_with_indices.len();
                let fruit_receiver = {
                    let (fruit_sender, fruit_receiver) = channel::unbounded();
-                    pool.scope(|scope| {
+                    pool.scoped(|scope| {
                        for arg_with_idx in args_with_indices {
-                            scope.spawn(|_| {
+                            scope.execute(|| {
                                let (idx, arg) = arg_with_idx;
                                let fruit = f(arg);
                                if let Err(err) = fruit_sender.send((idx, fruit)) {
@@ -106,7 +103,6 @@ mod tests {
    #[should_panic] //< unfortunately the panic message is not propagated
    fn test_panic_propagates_multi_thread() {
        let _result: Vec<usize> = Executor::multi_thread(1, "search-test")
-            .unwrap()
            .map(
                |_| {
                    panic!("panic should propagate");
@@ -130,7 +126,6 @@ mod tests {
    #[test]
    fn test_map_multithread() {
        let result: Vec<usize> = Executor::multi_thread(3, "search-test")
-            .unwrap()
            .map(|i| Ok(i * 2), 0..10)
            .unwrap();
        assert_eq!(result.len(), 10);
--- a/src/core/index.rs
+++ b/src/core/index.rs
@@ -20,8 +20,10 @@ use crate::reader::IndexReaderBuilder;
 use crate::schema::Field;
 use crate::schema::FieldType;
 use crate::schema::Schema;
-use crate::tokenizer::{TextAnalyzer, TokenizerManager};
+use crate::tokenizer::BoxedTokenizer;
+use crate::tokenizer::TokenizerManager;
 use crate::IndexWriter;
+use crate::Result;
 use num_cpus;
 use std::borrow::BorrowMut;
 use std::collections::HashSet;
@@ -30,10 +32,7 @@ use std::fmt;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;

-fn load_metas(
-    directory: &dyn Directory,
-    inventory: &SegmentMetaInventory,
-) -> crate::Result<IndexMeta> {
+fn load_metas(directory: &dyn Directory, inventory: &SegmentMetaInventory) -> Result<IndexMeta> {
    let meta_data = directory.atomic_read(&META_FILEPATH)?;
    let meta_string = String::from_utf8_lossy(&meta_data);
    IndexMeta::deserialize(&meta_string, &inventory)
@@ -74,16 +73,15 @@ impl Index {

    /// Replace the default single thread search executor pool
    /// by a thread pool with a given number of threads.
-    pub fn set_multithread_executor(&mut self, num_threads: usize) -> crate::Result<()> {
-        self.executor = Arc::new(Executor::multi_thread(num_threads, "thrd-tantivy-search-")?);
-        Ok(())
+    pub fn set_multithread_executor(&mut self, num_threads: usize) {
+        self.executor = Arc::new(Executor::multi_thread(num_threads, "thrd-tantivy-search-"));
    }

    /// Replace the default single thread search executor pool
    /// by a thread pool with a given number of threads.
-    pub fn set_default_multithread_executor(&mut self) -> crate::Result<()> {
+    pub fn set_default_multithread_executor(&mut self) {
        let default_num_threads = num_cpus::get();
-        self.set_multithread_executor(default_num_threads)
+        self.set_multithread_executor(default_num_threads);
    }

    /// Creates a new index using the `RAMDirectory`.
@@ -100,29 +98,28 @@ impl Index {
    ///
    /// If a previous index was in this directory, then its meta file will be destroyed.
    #[cfg(feature = "mmap")]
-    pub fn create_in_dir<P: AsRef<Path>>(
-        directory_path: P,
-        schema: Schema,
-    ) -> crate::Result<Index> {
+    pub fn create_in_dir<P: AsRef<Path>>(directory_path: P, schema: Schema) -> Result<Index> {
        let mmap_directory = MmapDirectory::open(directory_path)?;
        if Index::exists(&mmap_directory) {
            return Err(TantivyError::IndexAlreadyExists);
        }
+
        Index::create(mmap_directory, schema)
    }

    /// Opens or creates a new index in the provided directory
-    pub fn open_or_create<Dir: Directory>(dir: Dir, schema: Schema) -> crate::Result<Index> {
-        if !Index::exists(&dir) {
-            return Index::create(dir, schema);
-        }
-        let index = Index::open(dir)?;
-        if index.schema() == schema {
-            Ok(index)
+    pub fn open_or_create<Dir: Directory>(dir: Dir, schema: Schema) -> Result<Index> {
+        if Index::exists(&dir) {
+            let index = Index::open(dir)?;
+            if index.schema() == schema {
+                Ok(index)
+            } else {
+                Err(TantivyError::SchemaError(
+                    "An index exists but the schema does not match.".to_string(),
+                ))
+            }
        } else {
-            Err(TantivyError::SchemaError(
-                "An index exists but the schema does not match.".to_string(),
-            ))
+            Index::create(dir, schema)
        }
    }

@@ -135,13 +132,13 @@ impl Index {
    /// The temp directory is only used for testing the `MmapDirectory`.
    /// For other unit tests, prefer the `RAMDirectory`, see: `create_in_ram`.
    #[cfg(feature = "mmap")]
-    pub fn create_from_tempdir(schema: Schema) -> crate::Result<Index> {
+    pub fn create_from_tempdir(schema: Schema) -> Result<Index> {
        let mmap_directory = MmapDirectory::create_from_tempdir()?;
        Index::create(mmap_directory, schema)
    }

    /// Creates a new index given an implementation of the trait `Directory`
-    pub fn create<Dir: Directory>(dir: Dir, schema: Schema) -> crate::Result<Index> {
+    pub fn create<Dir: Directory>(dir: Dir, schema: Schema) -> Result<Index> {
        let directory = ManagedDirectory::wrap(dir)?;
        Index::from_directory(directory, schema)
    }
@@ -149,7 +146,7 @@ impl Index {
    /// Create a new index from a directory.
    ///
    /// This will overwrite existing meta.json
-    fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> crate::Result<Index> {
+    fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> Result<Index> {
        save_new_metas(schema.clone(), directory.borrow_mut())?;
        let metas = IndexMeta::with_schema(schema);
        Index::create_from_metas(directory, &metas, SegmentMetaInventory::default())
@@ -160,7 +157,7 @@ impl Index {
        directory: ManagedDirectory,
        metas: &IndexMeta,
        inventory: SegmentMetaInventory,
-    ) -> crate::Result<Index> {
+    ) -> Result<Index> {
        let schema = metas.schema.clone();
        Ok(Index {
            directory,
@@ -177,11 +174,11 @@ impl Index {
    }

    /// Helper to access the tokenizer associated to a specific field.
-    pub fn tokenizer_for_field(&self, field: Field) -> crate::Result<TextAnalyzer> {
+    pub fn tokenizer_for_field(&self, field: Field) -> Result<BoxedTokenizer> {
        let field_entry = self.schema.get_field_entry(field);
        let field_type = field_entry.field_type();
        let tokenizer_manager: &TokenizerManager = self.tokenizers();
-        let tokenizer_name_opt: Option<TextAnalyzer> = match field_type {
+        let tokenizer_name_opt: Option<BoxedTokenizer> = match field_type {
            FieldType::Str(text_options) => text_options
                .get_indexing_options()
                .map(|text_indexing_options| text_indexing_options.tokenizer().to_string())
@@ -200,7 +197,7 @@ impl Index {
    /// Create a default `IndexReader` for the given index.
    ///
    /// See [`Index.reader_builder()`](#method.reader_builder).
-    pub fn reader(&self) -> crate::Result<IndexReader> {
+    pub fn reader(&self) -> Result<IndexReader> {
        self.reader_builder().try_into()
    }

@@ -215,7 +212,7 @@ impl Index {

    /// Opens a new directory from an index path.
    #[cfg(feature = "mmap")]
-    pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> crate::Result<Index> {
+    pub fn open_in_dir<P: AsRef<Path>>(directory_path: P) -> Result<Index> {
        let mmap_directory = MmapDirectory::open(directory_path)?;
        Index::open(mmap_directory)
    }
@@ -239,7 +236,7 @@ impl Index {
    }

    /// Open the index using the provided directory
-    pub fn open<D: Directory>(directory: D) -> crate::Result<Index> {
+    pub fn open<D: Directory>(directory: D) -> Result<Index> {
        let directory = ManagedDirectory::wrap(directory)?;
        let inventory = SegmentMetaInventory::default();
        let metas = load_metas(&directory, &inventory)?;
@@ -247,7 +244,7 @@ impl Index {
    }

    /// Reads the index meta file from the directory.
-    pub fn load_metas(&self) -> crate::Result<IndexMeta> {
+    pub fn load_metas(&self) -> Result<IndexMeta> {
        load_metas(self.directory(), &self.inventory)
    }

@@ -275,7 +272,7 @@ impl Index {
        &self,
        num_threads: usize,
        overall_heap_size_in_bytes: usize,
-    ) -> crate::Result<IndexWriter> {
+    ) -> Result<IndexWriter> {
        let directory_lock = self
            .directory
            .acquire_lock(&INDEX_WRITER_LOCK)
@@ -310,7 +307,7 @@ impl Index {
    /// If the lockfile already exists, returns `Error::FileAlreadyExists`.
    /// # Panics
    /// If the heap size per thread is too small, panics.
-    pub fn writer(&self, overall_heap_size_in_bytes: usize) -> crate::Result<IndexWriter> {
+    pub fn writer(&self, overall_heap_size_in_bytes: usize) -> Result<IndexWriter> {
        let mut num_threads = num_cpus::get();
        let heap_size_in_bytes_per_thread = overall_heap_size_in_bytes / num_threads;
        if heap_size_in_bytes_per_thread < HEAP_SIZE_MIN {
@@ -327,7 +324,7 @@ impl Index {
    }

    /// Returns the list of segments that are searchable
-    pub fn searchable_segments(&self) -> crate::Result<Vec<Segment>> {
+    pub fn searchable_segments(&self) -> Result<Vec<Segment>> {
        Ok(self
            .searchable_segment_metas()?
            .into_iter()
@@ -360,12 +357,12 @@ impl Index {

    /// Reads the meta.json and returns the list of
    /// `SegmentMeta` from the last commit.
-    pub fn searchable_segment_metas(&self) -> crate::Result<Vec<SegmentMeta>> {
+    pub fn searchable_segment_metas(&self) -> Result<Vec<SegmentMeta>> {
        Ok(self.load_metas()?.segments)
    }

    /// Returns the list of segment ids that are searchable.
-    pub fn searchable_segment_ids(&self) -> crate::Result<Vec<SegmentId>> {
+    pub fn searchable_segment_ids(&self) -> Result<Vec<SegmentId>> {
        Ok(self
            .searchable_segment_metas()?
            .iter()
@@ -374,7 +371,7 @@ impl Index {
    }

    /// Returns the set of corrupted files
-    pub fn validate_checksum(&self) -> crate::Result<HashSet<PathBuf>> {
+    pub fn validate_checksum(&self) -> Result<HashSet<PathBuf>> {
        self.directory.list_damaged().map_err(Into::into)
    }
 }
@@ -390,9 +387,12 @@ mod tests {
    use crate::directory::RAMDirectory;
    use crate::schema::Field;
    use crate::schema::{Schema, INDEXED, TEXT};
+    use crate::Index;
    use crate::IndexReader;
+    use crate::IndexWriter;
    use crate::ReloadPolicy;
-    use crate::{Directory, Index};
+    use std::thread;
+    use std::time::Duration;

    #[test]
    fn test_indexer_for_field() {
@@ -470,14 +470,14 @@ mod tests {
            .try_into()
            .unwrap();
        assert_eq!(reader.searcher().num_docs(), 0);
-        test_index_on_commit_reload_policy_aux(field, &index, &reader);
+        let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
    }

    #[cfg(feature = "mmap")]
    mod mmap_specific {

        use super::*;
-        use crate::Directory;
        use std::path::PathBuf;
        use tempfile::TempDir;

@@ -488,20 +488,22 @@ mod tests {
            let tempdir = TempDir::new().unwrap();
            let tempdir_path = PathBuf::from(tempdir.path());
            let index = Index::create_in_dir(&tempdir_path, schema).unwrap();
+            let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+            writer.commit().unwrap();
            let reader = index
                .reader_builder()
                .reload_policy(ReloadPolicy::OnCommit)
                .try_into()
                .unwrap();
            assert_eq!(reader.searcher().num_docs(), 0);
-            test_index_on_commit_reload_policy_aux(field, &index, &reader);
+            test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
        }

        #[test]
        fn test_index_manual_policy_mmap() {
            let schema = throw_away_schema();
            let field = schema.get_field("num_likes").unwrap();
-            let mut index = Index::create_from_tempdir(schema).unwrap();
+            let index = Index::create_from_tempdir(schema).unwrap();
            let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
            writer.commit().unwrap();
            let reader = index
@@ -511,12 +513,8 @@ mod tests {
                .unwrap();
            assert_eq!(reader.searcher().num_docs(), 0);
            writer.add_document(doc!(field=>1u64));
-            let (sender, receiver) = crossbeam::channel::unbounded();
-            let _handle = index.directory_mut().watch(Box::new(move || {
-                let _ = sender.send(());
-            }));
            writer.commit().unwrap();
-            assert!(receiver.recv().is_ok());
+            thread::sleep(Duration::from_millis(500));
            assert_eq!(reader.searcher().num_docs(), 0);
            reader.reload().unwrap();
            assert_eq!(reader.searcher().num_docs(), 1);
@@ -536,26 +534,39 @@ mod tests {
                .try_into()
                .unwrap();
            assert_eq!(reader.searcher().num_docs(), 0);
-            test_index_on_commit_reload_policy_aux(field, &write_index, &reader);
+            let mut writer = write_index.writer_with_num_threads(1, 3_000_000).unwrap();
+            test_index_on_commit_reload_policy_aux(field, &mut writer, &reader);
        }
    }

-    fn test_index_on_commit_reload_policy_aux(field: Field, index: &Index, reader: &IndexReader) {
-        let mut reader_index = reader.index();
-        let (sender, receiver) = crossbeam::channel::unbounded();
-        let _watch_handle = reader_index.directory_mut().watch(Box::new(move || {
-            let _ = sender.send(());
-        }));
-        let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+    fn test_index_on_commit_reload_policy_aux(
+        field: Field,
+        writer: &mut IndexWriter,
+        reader: &IndexReader,
+    ) {
        assert_eq!(reader.searcher().num_docs(), 0);
        writer.add_document(doc!(field=>1u64));
        writer.commit().unwrap();
-        assert!(receiver.recv().is_ok());
-        assert_eq!(reader.searcher().num_docs(), 1);
+        let mut count = 0;
+        for _ in 0..100 {
+            count = reader.searcher().num_docs();
+            if count > 0 {
+                break;
+            }
+            thread::sleep(Duration::from_millis(100));
+        }
+        assert_eq!(count, 1);
        writer.add_document(doc!(field=>2u64));
        writer.commit().unwrap();
-        assert!(receiver.recv().is_ok());
-        assert_eq!(reader.searcher().num_docs(), 2);
+        let mut count = 0;
+        for _ in 0..10 {
+            count = reader.searcher().num_docs();
+            if count > 1 {
+                break;
+            }
+            thread::sleep(Duration::from_millis(100));
+        }
+        assert_eq!(count, 2);
    }

    // This test will not pass on windows, because windows
@@ -572,13 +583,9 @@ mod tests {
        for i in 0u64..8_000u64 {
            writer.add_document(doc!(field => i));
        }
-        let (sender, receiver) = crossbeam::channel::unbounded();
-        let _handle = directory.watch(Box::new(move || {
-            let _ = sender.send(());
-        }));
        writer.commit().unwrap();
        let mem_right_after_commit = directory.total_mem_usage();
-        assert!(receiver.recv().is_ok());
+        thread::sleep(Duration::from_millis(1_000));
        let reader = index
            .reader_builder()
            .reload_policy(ReloadPolicy::Manual)
@@ -592,11 +599,6 @@ mod tests {
        reader.reload().unwrap();
        let searcher = reader.searcher();
        assert_eq!(searcher.num_docs(), 8_000);
-        assert!(
-            mem_right_after_merge_finished < mem_right_after_commit,
-            "(mem after merge){} is expected < (mem before merge){}",
-            mem_right_after_merge_finished,
-            mem_right_after_commit
-        );
+        assert!(mem_right_after_merge_finished < mem_right_after_commit);
    }
 }
--- a/src/core/index_meta.rs
+++ b/src/core/index_meta.rs
@@ -150,21 +150,6 @@ impl SegmentMeta {
        self.num_deleted_docs() > 0
    }

-    /// Updates the max_doc value from the `SegmentMeta`.
-    ///
-    /// This method is only used when updating `max_doc` from 0
-    /// as we finalize a fresh new segment.
-    pub(crate) fn with_max_doc(self, max_doc: u32) -> SegmentMeta {
-        assert_eq!(self.tracked.max_doc, 0);
-        assert!(self.tracked.deletes.is_none());
-        let tracked = self.tracked.map(move |inner_meta| InnerSegmentMeta {
-            segment_id: inner_meta.segment_id,
-            max_doc,
-            deletes: None,
-        });
-        SegmentMeta { tracked }
-    }
-
    #[doc(hidden)]
    pub fn with_delete_meta(self, num_deleted_docs: u32, opstamp: Opstamp) -> SegmentMeta {
        let delete_meta = DeleteMeta {
@@ -300,9 +285,6 @@ mod tests {
            payload: None,
        };
        let json = serde_json::ser::to_string(&index_metas).expect("serialization failed");
-        assert_eq!(
-            json,
-            r#"{"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","tokenizer":"default"},"stored":false}}],"opstamp":0}"#
-        );
+        assert_eq!(json, r#"{"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","tokenizer":"default"},"stored":false}}],"opstamp":0}"#);
    }
 }
--- a/src/core/inverted_index_reader.rs
+++ b/src/core/inverted_index_reader.rs
@@ -60,7 +60,7 @@ impl InvertedIndexReader {
            .get_index_record_option()
            .unwrap_or(IndexRecordOption::Basic);
        InvertedIndexReader {
-            termdict: TermDictionary::empty(),
+            termdict: TermDictionary::empty(&field_type),
            postings_source: ReadOnlySource::empty(),
            positions_source: ReadOnlySource::empty(),
            positions_idx_source: ReadOnlySource::empty(),
--- a/src/core/searcher.rs
+++ b/src/core/searcher.rs
@@ -14,6 +14,7 @@ use crate::store::StoreReader;
 use crate::termdict::TermMerger;
 use crate::DocAddress;
 use crate::Index;
+use crate::Result;
 use std::fmt;
 use std::sync::Arc;

@@ -22,7 +23,7 @@ fn collect_segment<C: Collector>(
    weight: &dyn Weight,
    segment_ord: u32,
    segment_reader: &SegmentReader,
-) -> crate::Result<C::Fruit> {
+) -> Result<C::Fruit> {
    let mut scorer = weight.scorer(segment_reader)?;
    let mut segment_collector = collector.for_segment(segment_ord as u32, segment_reader)?;
    if let Some(delete_bitset) = segment_reader.delete_bitset() {
@@ -77,7 +78,7 @@ impl Searcher {
    ///
    /// The searcher uses the segment ordinal to route the
    /// the request to the right `Segment`.
-    pub fn doc(&self, doc_address: DocAddress) -> crate::Result<Document> {
+    pub fn doc(&self, doc_address: DocAddress) -> Result<Document> {
        let DocAddress(segment_local_id, doc_id) = doc_address;
        let store_reader = &self.store_readers[segment_local_id as usize];
        store_reader.get(doc_id)
@@ -131,11 +132,7 @@ impl Searcher {
    ///
    ///  Finally, the Collector merges each of the child collectors into itself for result usability
    ///  by the caller.
-    pub fn search<C: Collector>(
-        &self,
-        query: &dyn Query,
-        collector: &C,
-    ) -> crate::Result<C::Fruit> {
+    pub fn search<C: Collector>(&self, query: &dyn Query, collector: &C) -> Result<C::Fruit> {
        let executor = self.index.search_executor();
        self.search_with_executor(query, collector, executor)
    }
@@ -157,7 +154,7 @@ impl Searcher {
        query: &dyn Query,
        collector: &C,
        executor: &Executor,
-    ) -> crate::Result<C::Fruit> {
+    ) -> Result<C::Fruit> {
        let scoring_enabled = collector.requires_scoring();
        let weight = query.weight(self, scoring_enabled)?;
        let segment_readers = self.segment_readers();
--- a/src/core/segment.rs
+++ b/src/core/segment.rs
@@ -8,8 +8,10 @@ use crate::directory::{ReadOnlySource, WritePtr};
 use crate::indexer::segment_serializer::SegmentSerializer;
 use crate::schema::Schema;
 use crate::Opstamp;
+use crate::Result;
 use std::fmt;
 use std::path::PathBuf;
+use std::result;

 /// A segment is a piece of the index.
 #[derive(Clone)]
@@ -48,17 +50,6 @@ impl Segment {
        &self.meta
    }

-    /// Updates the max_doc value from the `SegmentMeta`.
-    ///
-    /// This method is only used when updating `max_doc` from 0
-    /// as we finalize a fresh new segment.
-    pub(crate) fn with_max_doc(self, max_doc: u32) -> Segment {
-        Segment {
-            index: self.index,
-            meta: self.meta.with_max_doc(max_doc),
-        }
-    }
-
    #[doc(hidden)]
    pub fn with_delete_meta(self, num_deleted_docs: u32, opstamp: Opstamp) -> Segment {
        Segment {
@@ -81,14 +72,20 @@ impl Segment {
    }

    /// Open one of the component file for a *regular* read.
-    pub fn open_read(&self, component: SegmentComponent) -> Result<ReadOnlySource, OpenReadError> {
+    pub fn open_read(
+        &self,
+        component: SegmentComponent,
+    ) -> result::Result<ReadOnlySource, OpenReadError> {
        let path = self.relative_path(component);
        let source = self.index.directory().open_read(&path)?;
        Ok(source)
    }

    /// Open one of the component file for *regular* write.
-    pub fn open_write(&mut self, component: SegmentComponent) -> Result<WritePtr, OpenWriteError> {
+    pub fn open_write(
+        &mut self,
+        component: SegmentComponent,
+    ) -> result::Result<WritePtr, OpenWriteError> {
        let path = self.relative_path(component);
        let write = self.index.directory_mut().open_write(&path)?;
        Ok(write)
@@ -101,5 +98,5 @@ pub trait SerializableSegment {
    ///
    /// # Returns
    /// The number of documents in the segment.
-    fn write(&self, serializer: SegmentSerializer) -> crate::Result<u32>;
+    fn write(&self, serializer: SegmentSerializer) -> Result<u32>;
 }
--- a/src/core/segment_reader.rs
+++ b/src/core/segment_reader.rs
@@ -16,6 +16,7 @@ use crate::space_usage::SegmentSpaceUsage;
 use crate::store::StoreReader;
 use crate::termdict::TermDictionary;
 use crate::DocId;
+use crate::Result;
 use fail::fail_point;
 use std::collections::HashMap;
 use std::fmt;
@@ -144,7 +145,7 @@ impl SegmentReader {
    }

    /// Open a new segment for reading.
-    pub fn open(segment: &Segment) -> crate::Result<SegmentReader> {
+    pub fn open(segment: &Segment) -> Result<SegmentReader> {
        let termdict_source = segment.open_read(SegmentComponent::TERMS)?;
        let termdict_composite = CompositeFile::open(&termdict_source)?;

--- a/src/directory/directory.rs
+++ b/src/directory/directory.rs
@@ -119,7 +119,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
    /// Specifically, subsequent writes or flushes should
    /// have no effect on the returned `ReadOnlySource` object.
    ///
-    /// You should only use this to read files create with [Directory::open_write].
+    /// You should only use this to read files create with [`open_write`]
    fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError>;

    /// Removes a file
@@ -160,7 +160,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
    ///
    /// This should only be used for small files.
    ///
-    /// You should only use this to read files create with [Directory::atomic_write].
+    /// You should only use this to read files create with [`atomic_write`]
    fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError>;

    /// Atomically replace the content of a file with data.
@@ -197,7 +197,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
    /// Registers a callback that will be called whenever a change on the `meta.json`
    /// using the `atomic_write` API is detected.
    ///
-    /// The behavior when using `.watch()` on a file using [Directory::open_write] is, on the other
+    /// The behavior when using `.watch()` on a file using `.open_write(...)` is, on the other
    /// hand, undefined.
    ///
    /// The file will be watched for the lifetime of the returned `WatchHandle`. The caller is
--- a/src/directory/error.rs
+++ b/src/directory/error.rs
@@ -1,4 +1,3 @@
-use crate::Version;
 use std::error::Error as StdError;
 use std::fmt;
 use std::io;
@@ -157,65 +156,6 @@ impl StdError for OpenWriteError {
    }
 }

-/// Type of index incompatibility between the library and the index found on disk
-/// Used to catch and provide a hint to solve this incompatibility issue
-pub enum Incompatibility {
-    /// This library cannot decompress the index found on disk
-    CompressionMismatch {
-        /// Compression algorithm used by the current version of tantivy
-        library_compression_format: String,
-        /// Compression algorithm that was used to serialise the index
-        index_compression_format: String,
-    },
-    /// The index format found on disk isn't supported by this version of the library
-    IndexMismatch {
-        /// Version used by the library
-        library_version: Version,
-        /// Version the index was built with
-        index_version: Version,
-    },
-}
-
-impl fmt::Debug for Incompatibility {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
-        match self {
-            Incompatibility::CompressionMismatch {
-                library_compression_format,
-                index_compression_format,
-            } => {
-                let err = format!(
-                    "Library was compiled with {:?} compression, index was compressed with {:?}",
-                    library_compression_format, index_compression_format
-                );
-                let advice = format!(
-                    "Change the feature flag to {:?} and rebuild the library",
-                    index_compression_format
-                );
-                write!(f, "{}. {}", err, advice)?;
-            }
-            Incompatibility::IndexMismatch {
-                library_version,
-                index_version,
-            } => {
-                let err = format!(
-                    "Library version: {}, index version: {}",
-                    library_version.index_format_version, index_version.index_format_version
-                );
-                // TODO make a more useful error message
-                // include the version range that supports this index_format_version
-                let advice = format!(
-                    "Change tantivy to a version compatible with index format {} (e.g. {}.{}.x) \
-                     and rebuild your project.",
-                    index_version.index_format_version, index_version.major, index_version.minor
-                );
-                write!(f, "{}. {}", err, advice)?;
-            }
-        }
-
-        Ok(())
-    }
-}
-
 /// Error that may occur when accessing a file read
 #[derive(Debug)]
 pub enum OpenReadError {
@@ -224,8 +164,6 @@ pub enum OpenReadError {
    /// Any kind of IO error that happens when
    /// interacting with the underlying IO device.
    IOError(IOError),
-    /// This library doesn't support the index version found on disk
-    IncompatibleIndex(Incompatibility),
 }

 impl From<IOError> for OpenReadError {
@@ -245,9 +183,19 @@ impl fmt::Display for OpenReadError {
                "an io error occurred while opening a file for reading: '{}'",
                err
            ),
-            OpenReadError::IncompatibleIndex(ref footer) => {
-                write!(f, "Incompatible index format: {:?}", footer)
-            }
+        }
+    }
+}
+
+impl StdError for OpenReadError {
+    fn description(&self) -> &str {
+        "error occurred while opening a file for reading"
+    }
+
+    fn cause(&self) -> Option<&dyn StdError> {
+        match *self {
+            OpenReadError::FileDoesNotExist(_) => None,
+            OpenReadError::IOError(ref err) => Some(err),
        }
    }
 }
@@ -268,12 +216,6 @@ impl From<IOError> for DeleteError {
    }
 }

-impl From<Incompatibility> for OpenReadError {
-    fn from(incompatibility: Incompatibility) -> Self {
-        OpenReadError::IncompatibleIndex(incompatibility)
-    }
-}
-
 impl fmt::Display for DeleteError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match *self {
--- a/src/directory/footer.rs
+++ b/src/directory/footer.rs
@@ -1,175 +1,159 @@
-use crate::common::{BinarySerializable, CountingWriter, FixedSize, VInt};
-use crate::directory::error::Incompatibility;
 use crate::directory::read_only_source::ReadOnlySource;
 use crate::directory::{AntiCallToken, TerminatingWrite};
-use crate::Version;
-use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
+use byteorder::{ByteOrder, LittleEndian};
 use crc32fast::Hasher;
 use std::io;
 use std::io::Write;

-type CrcHashU32 = u32;
+const COMMON_FOOTER_SIZE: usize = 4 * 5;

 #[derive(Debug, Clone, PartialEq)]
 pub struct Footer {
-    pub version: Version,
+    pub tantivy_version: (u32, u32, u32),
    pub meta: String,
    pub versioned_footer: VersionedFooter,
 }

-/// Serialises the footer to a byte-array
-/// - versioned_footer_len : 4 bytes
-///-  versioned_footer: variable bytes
-/// - meta_len: 4 bytes
-/// - meta: variable bytes
-/// - version_len: 4 bytes
-/// - version json: variable bytes
-impl BinarySerializable for Footer {
-    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
-        BinarySerializable::serialize(&self.versioned_footer, writer)?;
-        BinarySerializable::serialize(&self.meta, writer)?;
-        let version_string =
-            serde_json::to_string(&self.version).map_err(|_err| io::ErrorKind::InvalidInput)?;
-        BinarySerializable::serialize(&version_string, writer)?;
-        Ok(())
-    }
-
-    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-        let versioned_footer = VersionedFooter::deserialize(reader)?;
-        let meta = String::deserialize(reader)?;
-        let version_json = String::deserialize(reader)?;
-        let version = serde_json::from_str(&version_json)?;
-        Ok(Footer {
-            version,
-            meta,
-            versioned_footer,
-        })
-    }
-}
-
 impl Footer {
    pub fn new(versioned_footer: VersionedFooter) -> Self {
-        let version = crate::VERSION.clone();
-        let meta = version.to_string();
+        let tantivy_version = (
+            env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap(),
+            env!("CARGO_PKG_VERSION_MINOR").parse().unwrap(),
+            env!("CARGO_PKG_VERSION_PATCH").parse().unwrap(),
+        );
        Footer {
-            version,
-            meta,
+            tantivy_version,
+            meta: format!(
+                "tantivy {}.{}.{}, index v{}",
+                tantivy_version.0,
+                tantivy_version.1,
+                tantivy_version.2,
+                versioned_footer.version()
+            ),
            versioned_footer,
        }
    }

-    pub fn append_footer<W: io::Write>(&self, mut write: &mut W) -> io::Result<()> {
-        let mut counting_write = CountingWriter::wrap(&mut write);
-        self.serialize(&mut counting_write)?;
-        let written_len = counting_write.written_bytes();
-        write.write_u32::<LittleEndian>(written_len as u32)?;
-        Ok(())
+    pub fn to_bytes(&self) -> Vec<u8> {
+        let mut res = self.versioned_footer.to_bytes();
+        res.extend_from_slice(self.meta.as_bytes());
+        let len = res.len();
+        res.resize(len + COMMON_FOOTER_SIZE, 0);
+        let mut common_footer = &mut res[len..];
+        LittleEndian::write_u32(&mut common_footer, self.meta.len() as u32);
+        LittleEndian::write_u32(&mut common_footer[4..], self.tantivy_version.0);
+        LittleEndian::write_u32(&mut common_footer[8..], self.tantivy_version.1);
+        LittleEndian::write_u32(&mut common_footer[12..], self.tantivy_version.2);
+        LittleEndian::write_u32(&mut common_footer[16..], (len + COMMON_FOOTER_SIZE) as u32);
+        res
    }

-    pub fn extract_footer(source: ReadOnlySource) -> Result<(Footer, ReadOnlySource), io::Error> {
-        if source.len() < 4 {
+    pub fn from_bytes(data: &[u8]) -> Result<Self, io::Error> {
+        let len = data.len();
+        if len < COMMON_FOOTER_SIZE + 4 {
+            // 4 bytes for index version, stored in versioned footer
+            return Err(io::Error::new(
+                    io::ErrorKind::UnexpectedEof,
+                    format!("File corrupted. The footer len must be over 24, while the entire file len is {}", len)
+                    )
+            );
+        }
+
+        let size = LittleEndian::read_u32(&data[len - 4..]) as usize;
+        if len < size as usize {
            return Err(io::Error::new(
                io::ErrorKind::UnexpectedEof,
                format!(
-                    "File corrupted. The file is smaller than 4 bytes (len={}).",
-                    source.len()
+                    "File corrupted. The footer len is {}, while the entire file len is {}",
+                    size, len
                ),
            ));
        }
-        let (body_footer, footer_len_bytes) = source.split_from_end(u32::SIZE_IN_BYTES);
-        let footer_len = LittleEndian::read_u32(footer_len_bytes.as_slice()) as usize;
-        let body_len = body_footer.len() - footer_len;
-        let (body, footer_data) = body_footer.split(body_len);
-        let mut cursor = footer_data.as_slice();
-        let footer = Footer::deserialize(&mut cursor)?;
-        Ok((footer, body))
+        let footer = &data[len - size as usize..];
+        let meta_len = LittleEndian::read_u32(&footer[size - 20..]) as usize;
+        let tantivy_major = LittleEndian::read_u32(&footer[size - 16..]);
+        let tantivy_minor = LittleEndian::read_u32(&footer[size - 12..]);
+        let tantivy_patch = LittleEndian::read_u32(&footer[size - 8..]);
+        Ok(Footer {
+            tantivy_version: (tantivy_major, tantivy_minor, tantivy_patch),
+            meta: String::from_utf8_lossy(&footer[size - meta_len - 20..size - 20]).into_owned(),
+            versioned_footer: VersionedFooter::from_bytes(&footer[..size - meta_len - 20])?,
+        })
    }

-    /// Confirms that the index will be read correctly by this version of tantivy
-    /// Has to be called after `extract_footer` to make sure it's not accessing uninitialised memory
-    pub fn is_compatible(&self) -> Result<(), Incompatibility> {
-        let library_version = crate::version();
-        match &self.versioned_footer {
-            VersionedFooter::V1 {
-                crc32: _crc,
-                store_compression: compression,
-            } => {
-                if &library_version.store_compression != compression {
-                    return Err(Incompatibility::CompressionMismatch {
-                        library_compression_format: library_version.store_compression.to_string(),
-                        index_compression_format: compression.to_string(),
-                    });
-                }
-                Ok(())
-            }
-            VersionedFooter::UnknownVersion => Err(Incompatibility::IndexMismatch {
-                library_version: library_version.clone(),
-                index_version: self.version.clone(),
-            }),
-        }
+    pub fn extract_footer(source: ReadOnlySource) -> Result<(Footer, ReadOnlySource), io::Error> {
+        let footer = Footer::from_bytes(source.as_slice())?;
+        let reader = source.slice_to(source.as_slice().len() - footer.size());
+        Ok((footer, reader))
+    }
+
+    pub fn size(&self) -> usize {
+        self.versioned_footer.size() as usize + self.meta.len() + 20
    }
 }

-/// Footer that includes a crc32 hash that enables us to checksum files in the index
 #[derive(Debug, Clone, PartialEq)]
 pub enum VersionedFooter {
-    UnknownVersion,
-    V1 {
-        crc32: CrcHashU32,
-        store_compression: String,
-    },
-}
-
-impl BinarySerializable for VersionedFooter {
-    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
-        let mut buf = Vec::new();
-        match self {
-            VersionedFooter::V1 {
-                crc32,
-                store_compression: compression,
-            } => {
-                // Serializes a valid `VersionedFooter` or panics if the version is unknown
-                // [   version    |   crc_hash  | compression_mode ]
-                // [    0..4      |     4..8    |     variable     ]
-                BinarySerializable::serialize(&1u32, &mut buf)?;
-                BinarySerializable::serialize(crc32, &mut buf)?;
-                BinarySerializable::serialize(compression, &mut buf)?;
-            }
-            VersionedFooter::UnknownVersion => {
-                return Err(io::Error::new(
-                    io::ErrorKind::InvalidInput,
-                    "Cannot serialize an unknown versioned footer ",
-                ));
-            }
-        }
-        BinarySerializable::serialize(&VInt(buf.len() as u64), writer)?;
-        writer.write_all(&buf[..])?;
-        Ok(())
-    }
-
-    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-        let len = VInt::deserialize(reader)?.0 as usize;
-        let mut buf = vec![0u8; len];
-        reader.read_exact(&mut buf[..])?;
-        let mut cursor = &buf[..];
-        let version = u32::deserialize(&mut cursor)?;
-        if version == 1 {
-            let crc32 = u32::deserialize(&mut cursor)?;
-            let compression = String::deserialize(&mut cursor)?;
-            Ok(VersionedFooter::V1 {
-                crc32,
-                store_compression: compression,
-            })
-        } else {
-            Ok(VersionedFooter::UnknownVersion)
-        }
-    }
+    UnknownVersion { version: u32, size: u32 },
+    V0(u32), // crc
 }

 impl VersionedFooter {
-    pub fn crc(&self) -> Option<CrcHashU32> {
+    pub fn to_bytes(&self) -> Vec<u8> {
        match self {
-            VersionedFooter::V1 { crc32, .. } => Some(*crc32),
+            VersionedFooter::V0(crc) => {
+                let mut res = vec![0; 8];
+                LittleEndian::write_u32(&mut res, 0);
+                LittleEndian::write_u32(&mut res[4..], *crc);
+                res
+            }
+            VersionedFooter::UnknownVersion { .. } => {
+                panic!("Unsupported index should never get serialized");
+            }
+        }
+    }
+
+    pub fn from_bytes(footer: &[u8]) -> Result<Self, io::Error> {
+        assert!(footer.len() >= 4);
+        let version = LittleEndian::read_u32(footer);
+        match version {
+            0 => {
+                if footer.len() == 8 {
+                    Ok(VersionedFooter::V0(LittleEndian::read_u32(&footer[4..])))
+                } else {
+                    Err(io::Error::new(
+                        io::ErrorKind::UnexpectedEof,
+                        format!(
+                            "File corrupted. The versioned footer len is {}, while it should be 8",
+                            footer.len()
+                        ),
+                    ))
+                }
+            }
+            version => Ok(VersionedFooter::UnknownVersion {
+                version,
+                size: footer.len() as u32,
+            }),
+        }
+    }
+
+    pub fn size(&self) -> u32 {
+        match self {
+            VersionedFooter::V0(_) => 8,
+            VersionedFooter::UnknownVersion { size, .. } => *size,
+        }
+    }
+
+    pub fn version(&self) -> u32 {
+        match self {
+            VersionedFooter::V0(_) => 0,
+            VersionedFooter::UnknownVersion { version, .. } => *version,
+        }
+    }
+
+    pub fn crc(&self) -> Option<u32> {
+        match self {
+            VersionedFooter::V0(crc) => Some(*crc),
            VersionedFooter::UnknownVersion { .. } => None,
        }
    }
@@ -205,135 +189,25 @@ impl<W: TerminatingWrite> Write for FooterProxy<W> {

 impl<W: TerminatingWrite> TerminatingWrite for FooterProxy<W> {
    fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()> {
-        let crc32 = self.hasher.take().unwrap().finalize();
-        let footer = Footer::new(VersionedFooter::V1 {
-            crc32,
-            store_compression: crate::store::COMPRESSION.to_string(),
-        });
+        let crc = self.hasher.take().unwrap().finalize();
+
+        let footer = Footer::new(VersionedFooter::V0(crc)).to_bytes();
        let mut writer = self.writer.take().unwrap();
-        footer.append_footer(&mut writer)?;
+        writer.write_all(&footer)?;
        writer.terminate()
    }
 }

 #[cfg(test)]
 mod tests {
-
-    use super::CrcHashU32;
-    use super::FooterProxy;
-    use crate::common::BinarySerializable;
    use crate::directory::footer::{Footer, VersionedFooter};
-    use crate::directory::TerminatingWrite;
-    use byteorder::{ByteOrder, LittleEndian};
-    use regex::Regex;
-
-    #[test]
-    fn test_versioned_footer() {
-        let mut vec = Vec::new();
-        let footer_proxy = FooterProxy::new(&mut vec);
-        assert!(footer_proxy.terminate().is_ok());
-        assert_eq!(vec.len(), 167);
-        let footer = Footer::deserialize(&mut &vec[..]).unwrap();
-        if let VersionedFooter::V1 {
-            crc32: _,
-            store_compression,
-        } = footer.versioned_footer
-        {
-            assert_eq!(store_compression, crate::store::COMPRESSION);
-        } else {
-            panic!("Versioned footer should be V1.");
-        }
-        assert_eq!(&footer.version, crate::version());
-    }

    #[test]
    fn test_serialize_deserialize_footer() {
-        let mut buffer = Vec::new();
-        let crc32 = 123456u32;
-        let footer: Footer = Footer::new(VersionedFooter::V1 {
-            crc32,
-            store_compression: "lz4".to_string(),
-        });
-        footer.serialize(&mut buffer).unwrap();
-        let footer_deser = Footer::deserialize(&mut &buffer[..]).unwrap();
-        assert_eq!(footer_deser, footer);
-    }
+        let crc = 123456;
+        let footer = Footer::new(VersionedFooter::V0(crc));
+        let footer_bytes = footer.to_bytes();

-    #[test]
-    fn footer_length() {
-        let crc32 = 1111111u32;
-        let versioned_footer = VersionedFooter::V1 {
-            crc32,
-            store_compression: "lz4".to_string(),
-        };
-        let mut buf = Vec::new();
-        versioned_footer.serialize(&mut buf).unwrap();
-        assert_eq!(buf.len(), 13);
-        let footer = Footer::new(versioned_footer);
-        let regex_ptn = Regex::new(
-            "tantivy v[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.{0,10}, index_format v[0-9]{1,5}",
-        )
-        .unwrap();
-        assert!(regex_ptn.is_match(&footer.meta));
-    }
-
-    #[test]
-    fn versioned_footer_from_bytes() {
-        let v_footer_bytes = vec![
-            // versionned footer length
-            12 | 128,
-            // index format version
-            1,
-            0,
-            0,
-            0,
-            // crc 32
-            12,
-            35,
-            89,
-            18,
-            // compression format
-            3 | 128,
-            b'l',
-            b'z',
-            b'4',
-        ];
-        let mut cursor = &v_footer_bytes[..];
-        let versioned_footer = VersionedFooter::deserialize(&mut cursor).unwrap();
-        assert!(cursor.is_empty());
-        let expected_crc: u32 = LittleEndian::read_u32(&v_footer_bytes[5..9]) as CrcHashU32;
-        let expected_versioned_footer: VersionedFooter = VersionedFooter::V1 {
-            crc32: expected_crc,
-            store_compression: "lz4".to_string(),
-        };
-        assert_eq!(versioned_footer, expected_versioned_footer);
-        let mut buffer = Vec::new();
-        assert!(versioned_footer.serialize(&mut buffer).is_ok());
-        assert_eq!(&v_footer_bytes[..], &buffer[..]);
-    }
-
-    #[test]
-    fn versioned_footer_panic() {
-        let v_footer_bytes = vec![6u8 | 128u8, 3u8, 0u8, 0u8, 1u8, 0u8, 0u8];
-        let mut b = &v_footer_bytes[..];
-        let versioned_footer = VersionedFooter::deserialize(&mut b).unwrap();
-        assert!(b.is_empty());
-        let expected_versioned_footer = VersionedFooter::UnknownVersion;
-        assert_eq!(versioned_footer, expected_versioned_footer);
-        let mut buf = Vec::new();
-        assert!(versioned_footer.serialize(&mut buf).is_err());
-    }
-
-    #[test]
-    #[cfg(not(feature = "lz4"))]
-    fn compression_mismatch() {
-        let crc32 = 1111111u32;
-        let versioned_footer = VersionedFooter::V1 {
-            crc32,
-            store_compression: "lz4".to_string(),
-        };
-        let footer = Footer::new(versioned_footer);
-        let res = footer.is_compatible();
-        assert!(res.is_err());
+        assert_eq!(Footer::from_bytes(&footer_bytes).unwrap(), footer);
    }
 }
--- a/src/directory/managed_directory.rs
+++ b/src/directory/managed_directory.rs
@@ -2,14 +2,13 @@ use crate::core::MANAGED_FILEPATH;
 use crate::directory::error::{DeleteError, IOError, LockError, OpenReadError, OpenWriteError};
 use crate::directory::footer::{Footer, FooterProxy};
 use crate::directory::DirectoryLock;
-use crate::directory::GarbageCollectionResult;
 use crate::directory::Lock;
 use crate::directory::META_LOCK;
 use crate::directory::{ReadOnlySource, WritePtr};
 use crate::directory::{WatchCallback, WatchHandle};
 use crate::error::DataCorruption;
 use crate::Directory;
-
+use crate::Result;
 use crc32fast::Hasher;
 use serde_json;
 use std::collections::HashSet;
@@ -65,7 +64,7 @@ fn save_managed_paths(

 impl ManagedDirectory {
    /// Wraps a directory as managed directory.
-    pub fn wrap<Dir: Directory>(directory: Dir) -> crate::Result<ManagedDirectory> {
+    pub fn wrap<Dir: Directory>(directory: Dir) -> Result<ManagedDirectory> {
        match directory.atomic_read(&MANAGED_FILEPATH) {
            Ok(data) => {
                let managed_files_json = String::from_utf8_lossy(&data);
@@ -88,11 +87,6 @@ impl ManagedDirectory {
                meta_informations: Arc::default(),
            }),
            Err(OpenReadError::IOError(e)) => Err(From::from(e)),
-            Err(OpenReadError::IncompatibleIndex(incompatibility)) => {
-                // For the moment, this should never happen  `meta.json`
-                // do not have any footer and cannot detect incompatibility.
-                Err(crate::TantivyError::IncompatibleIndex(incompatibility))
-            }
        }
    }

@@ -110,10 +104,7 @@ impl ManagedDirectory {
    /// If a file cannot be deleted (for permission reasons for instance)
    /// an error is simply logged, and the file remains in the list of managed
    /// files.
-    pub fn garbage_collect<L: FnOnce() -> HashSet<PathBuf>>(
-        &mut self,
-        get_living_files: L,
-    ) -> crate::Result<GarbageCollectionResult> {
+    pub fn garbage_collect<L: FnOnce() -> HashSet<PathBuf>>(&mut self, get_living_files: L) {
        info!("Garbage collect");
        let mut files_to_delete = vec![];

@@ -139,25 +130,19 @@ impl ManagedDirectory {
            // 2) writer change meta.json (for instance after a merge or a commit)
            // 3) gc kicks in.
            // 4) gc removes a file that was useful for process B, before process B opened it.
-            match self.acquire_lock(&META_LOCK) {
-                Ok(_meta_lock) => {
-                    let living_files = get_living_files();
-                    for managed_path in &meta_informations_rlock.managed_paths {
-                        if !living_files.contains(managed_path) {
-                            files_to_delete.push(managed_path.clone());
-                        }
+            if let Ok(_meta_lock) = self.acquire_lock(&META_LOCK) {
+                let living_files = get_living_files();
+                for managed_path in &meta_informations_rlock.managed_paths {
+                    if !living_files.contains(managed_path) {
+                        files_to_delete.push(managed_path.clone());
                    }
                }
-                Err(err) => {
-                    error!("Failed to acquire lock for GC");
-                    return Err(crate::TantivyError::from(err));
-                }
+            } else {
+                error!("Failed to acquire lock for GC");
            }
        }

-        let mut failed_to_delete_files = vec![];
        let mut deleted_files = vec![];
-
        for file_to_delete in files_to_delete {
            match self.delete(&file_to_delete) {
                Ok(_) => {
@@ -167,10 +152,9 @@ impl ManagedDirectory {
                Err(file_error) => {
                    match file_error {
                        DeleteError::FileDoesNotExist(_) => {
-                            deleted_files.push(file_to_delete.clone());
+                            deleted_files.push(file_to_delete);
                        }
                        DeleteError::IOError(_) => {
-                            failed_to_delete_files.push(file_to_delete.clone());
                            if !cfg!(target_os = "windows") {
                                // On windows, delete is expected to fail if the file
                                // is mmapped.
@@ -193,13 +177,10 @@ impl ManagedDirectory {
            for delete_file in &deleted_files {
                managed_paths_write.remove(delete_file);
            }
-            save_managed_paths(self.directory.as_mut(), &meta_informations_wlock)?;
+            if save_managed_paths(self.directory.as_mut(), &meta_informations_wlock).is_err() {
+                error!("Failed to save the list of managed files.");
+            }
        }
-
-        Ok(GarbageCollectionResult {
-            deleted_files,
-            failed_to_delete_files,
-        })
    }

    /// Registers a file as managed
@@ -266,9 +247,8 @@ impl ManagedDirectory {
 impl Directory for ManagedDirectory {
    fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
        let read_only_source = self.directory.open_read(path)?;
-        let (footer, reader) = Footer::extract_footer(read_only_source)
+        let (_footer, reader) = Footer::extract_footer(read_only_source)
            .map_err(|err| IOError::with_path(path.to_path_buf(), err))?;
-        footer.is_compatible()?;
        Ok(reader)
    }

@@ -347,8 +327,9 @@ mod tests_mmap_specific {
                .unwrap();
            assert!(managed_directory.exists(test_path1));
            assert!(managed_directory.exists(test_path2));
-            let living_files: HashSet<PathBuf> = [test_path1.to_owned()].iter().cloned().collect();
-            assert!(managed_directory.garbage_collect(|| living_files).is_ok());
+            let living_files: HashSet<PathBuf> =
+                [test_path1.to_owned()].into_iter().cloned().collect();
+            managed_directory.garbage_collect(|| living_files);
            assert!(managed_directory.exists(test_path1));
            assert!(!managed_directory.exists(test_path2));
        }
@@ -358,7 +339,7 @@ mod tests_mmap_specific {
            assert!(managed_directory.exists(test_path1));
            assert!(!managed_directory.exists(test_path2));
            let living_files: HashSet<PathBuf> = HashSet::new();
-            assert!(managed_directory.garbage_collect(|| living_files).is_ok());
+            managed_directory.garbage_collect(|| living_files);
            assert!(!managed_directory.exists(test_path1));
            assert!(!managed_directory.exists(test_path2));
        }
@@ -380,9 +361,7 @@ mod tests_mmap_specific {
        assert!(managed_directory.exists(test_path1));

        let _mmap_read = managed_directory.open_read(test_path1).unwrap();
-        assert!(managed_directory
-            .garbage_collect(|| living_files.clone())
-            .is_ok());
+        managed_directory.garbage_collect(|| living_files.clone());
        if cfg!(target_os = "windows") {
            // On Windows, gc should try and fail the file as it is mmapped.
            assert!(managed_directory.exists(test_path1));
@@ -390,7 +369,7 @@ mod tests_mmap_specific {
            drop(_mmap_read);
            // The file should still be in the list of managed file and
            // eventually be deleted once mmap is released.
-            assert!(managed_directory.garbage_collect(|| living_files).is_ok());
+            managed_directory.garbage_collect(|| living_files);
            assert!(!managed_directory.exists(test_path1));
        } else {
            assert!(!managed_directory.exists(test_path1));
@@ -415,8 +394,6 @@ mod tests_mmap_specific {
        write.write_all(&[3u8, 4u8, 5u8]).unwrap();
        write.terminate().unwrap();

-        let read_source = managed_directory.open_read(test_path2).unwrap();
-        assert_eq!(read_source.as_slice(), &[3u8, 4u8, 5u8]);
        assert!(managed_directory.list_damaged().unwrap().is_empty());

        let mut corrupted_path = tempdir_path.clone();
--- a/src/directory/mmap_directory.rs
+++ b/src/directory/mmap_directory.rs
@@ -131,18 +131,19 @@ impl MmapCache {
        }
        self.cache.remove(full_path);
        self.counters.miss += 1;
-        let mmap_opt = open_mmap(full_path)?;
-        Ok(mmap_opt.map(|mmap| {
+        Ok(if let Some(mmap) = open_mmap(full_path)? {
            let mmap_arc: Arc<BoxedData> = Arc::new(Box::new(mmap));
            let mmap_weak = Arc::downgrade(&mmap_arc);
            self.cache.insert(full_path.to_owned(), mmap_weak);
-            mmap_arc
-        }))
+            Some(mmap_arc)
+        } else {
+            None
+        })
    }
 }

 struct WatcherWrapper {
-    _watcher: Mutex<notify::PollWatcher>,
+    _watcher: Mutex<notify::RecommendedWatcher>,
    watcher_router: Arc<WatchCallbackList>,
 }

@@ -150,7 +151,7 @@ impl WatcherWrapper {
    pub fn new(path: &Path) -> Result<Self, OpenDirectoryError> {
        let (tx, watcher_recv): (Sender<RawEvent>, Receiver<RawEvent>) = channel();
        // We need to initialize the
-        let watcher = notify::poll::PollWatcher::with_delay_ms(tx, 1)
+        let watcher = notify::raw_watcher(tx)
            .and_then(|mut watcher| {
                watcher.watch(path, RecursiveMode::Recursive)?;
                Ok(watcher)
@@ -173,7 +174,7 @@ impl WatcherWrapper {
                            // We might want to be more accurate than this at one point.
                            if let Some(filename) = changed_path.file_name() {
                                if filename == *META_FILEPATH {
-                                    let _ = watcher_router_clone.broadcast();
+                                    watcher_router_clone.broadcast();
                                }
                            }
                        }
@@ -537,15 +538,16 @@ mod tests {
    // The following tests are specific to the MmapDirectory

    use super::*;
-    use crate::indexer::LogMergePolicy;
    use crate::schema::{Schema, SchemaBuilder, TEXT};
    use crate::Index;
    use crate::ReloadPolicy;
    use std::fs;
    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::thread;
+    use std::time::Duration;

    #[test]
-    fn test_open_non_existent_path() {
+    fn test_open_non_existant_path() {
        assert!(MmapDirectory::open(PathBuf::from("./nowhere")).is_err());
    }

@@ -638,18 +640,13 @@ mod tests {
        let tmp_dir = tempfile::TempDir::new().unwrap();
        let tmp_dirpath = tmp_dir.path().to_owned();
        let mut watch_wrapper = WatcherWrapper::new(&tmp_dirpath).unwrap();
-        let tmp_file = tmp_dirpath.join(*META_FILEPATH);
+        let tmp_file = tmp_dirpath.join("coucou");
        let _handle = watch_wrapper.watch(Box::new(move || {
            counter_clone.fetch_add(1, Ordering::SeqCst);
        }));
-        let (sender, receiver) = crossbeam::channel::unbounded();
-        let _handle2 = watch_wrapper.watch(Box::new(move || {
-            let _ = sender.send(());
-        }));
        assert_eq!(counter.load(Ordering::SeqCst), 0);
        fs::write(&tmp_file, b"whateverwilldo").unwrap();
-        assert!(receiver.recv().is_ok());
-        assert!(counter.load(Ordering::SeqCst) >= 1);
+        thread::sleep(Duration::new(0, 1_000u32));
    }

    #[test]
@@ -658,42 +655,34 @@ mod tests {
        let mut schema_builder: SchemaBuilder = Schema::builder();
        let text_field = schema_builder.add_text_field("text", TEXT);
        let schema = schema_builder.build();
-
        {
            let index = Index::create(mmap_directory.clone(), schema).unwrap();
-
            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-            let mut log_merge_policy = LogMergePolicy::default();
-            log_merge_policy.set_min_merge_size(3);
-            index_writer.set_merge_policy(Box::new(log_merge_policy));
-            for _num_commits in 0..10 {
+            for _num_commits in 0..16 {
                for _ in 0..10 {
                    index_writer.add_document(doc!(text_field=>"abc"));
                }
                index_writer.commit().unwrap();
            }
-
            let reader = index
                .reader_builder()
                .reload_policy(ReloadPolicy::Manual)
                .try_into()
                .unwrap();
-
-            for _ in 0..4 {
+            for _ in 0..30 {
                index_writer.add_document(doc!(text_field=>"abc"));
                index_writer.commit().unwrap();
                reader.reload().unwrap();
            }
            index_writer.wait_merging_threads().unwrap();
-
            reader.reload().unwrap();
            let num_segments = reader.searcher().segment_readers().len();
-            assert!(num_segments <= 4);
+            assert_eq!(num_segments, 4);
            assert_eq!(
                num_segments * 7,
                mmap_directory.get_cache_info().mmapped.len()
            );
        }
-        assert!(mmap_directory.get_cache_info().mmapped.is_empty());
+        assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
    }
 }
--- a/src/directory/mod.rs
+++ b/src/directory/mod.rs
@@ -23,22 +23,9 @@ pub use self::directory::{Directory, DirectoryClone};
 pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};
 pub use self::ram_directory::RAMDirectory;
 pub use self::read_only_source::ReadOnlySource;
-pub use self::watch_event_router::{WatchCallback, WatchCallbackList, WatchHandle};
+pub(crate) use self::watch_event_router::WatchCallbackList;
+pub use self::watch_event_router::{WatchCallback, WatchHandle};
 use std::io::{self, BufWriter, Write};
-use std::path::PathBuf;
-/// Outcome of the Garbage collection
-pub struct GarbageCollectionResult {
-    /// List of files that were deleted in this cycle
-    pub deleted_files: Vec<PathBuf>,
-    /// List of files that were schedule to be deleted in this cycle,
-    /// but deletion did not work. This typically happens on windows,
-    /// as deleting a memory mapped file is forbidden.
-    ///
-    /// If a searcher is still held, a file cannot be deleted.
-    /// This is not considered a bug, the file will simply be deleted
-    /// in the next GC.
-    pub failed_to_delete_files: Vec<PathBuf>,
-}

 #[cfg(feature = "mmap")]
 pub use self::mmap_directory::MmapDirectory;
@@ -46,9 +33,6 @@ pub use self::mmap_directory::MmapDirectory;
 pub use self::managed_directory::ManagedDirectory;

 /// Struct used to prevent from calling [`terminate_ref`](trait.TerminatingWrite#method.terminate_ref) directly
-///
-/// The point is that while the type is public, it cannot be built by anyone
-/// outside of this module.
 pub struct AntiCallToken(());

 /// Trait used to indicate when no more write need to be done on a writer
@@ -79,13 +63,6 @@ impl<W: TerminatingWrite> TerminatingWrite for BufWriter<W> {
    }
 }

-#[cfg(test)]
-impl<'a> TerminatingWrite for &'a mut Vec<u8> {
-    fn terminate_ref(&mut self, _a: AntiCallToken) -> io::Result<()> {
-        self.flush()
-    }
-}
-
 /// Write object for Directory.
 ///
 /// `WritePtr` are required to implement both Write
--- a/src/directory/ram_directory.rs
+++ b/src/directory/ram_directory.rs
@@ -191,11 +191,11 @@ impl Directory for RAMDirectory {
        // Reserve the path to prevent calls to .write() to succeed.
        self.fs.write().unwrap().write(path_buf.clone(), &[]);

-        let mut vec_writer = VecWriter::new(path_buf, self.clone());
+        let mut vec_writer = VecWriter::new(path_buf.clone(), self.clone());
        vec_writer.write_all(data)?;
        vec_writer.flush()?;
        if path == Path::new(&*META_FILEPATH) {
-            let _ = self.fs.write().unwrap().watch_router.broadcast();
+            self.fs.write().unwrap().watch_router.broadcast();
        }
        Ok(())
    }
--- a/src/directory/read_only_source.rs
+++ b/src/directory/read_only_source.rs
@@ -70,12 +70,6 @@ impl ReadOnlySource {
        (left, right)
    }

-    /// Splits into 2 `ReadOnlySource`, at the offset `end - right_len`.
-    pub fn split_from_end(self, right_len: usize) -> (ReadOnlySource, ReadOnlySource) {
-        let left_len = self.len() - right_len;
-        self.split(left_len)
-    }
-
    /// Creates a ReadOnlySource that is just a
    /// view over a slice of the data.
    ///
--- a/src/directory/tests.rs
+++ b/src/directory/tests.rs
@@ -1,117 +1,25 @@
 use super::*;
-use futures::channel::oneshot;
-use futures::executor::block_on;
 use std::io::Write;
 use std::mem;
 use std::path::{Path, PathBuf};
-use std::sync::atomic::Ordering::SeqCst;
-use std::sync::atomic::{AtomicBool, AtomicUsize};
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
 use std::sync::Arc;
+use std::thread;
+use std::time;
 use std::time::Duration;

-#[cfg(feature = "mmap")]
-mod mmap_directory_tests {
-    use crate::directory::MmapDirectory;
-
-    type DirectoryImpl = MmapDirectory;
-
-    fn make_directory() -> DirectoryImpl {
-        MmapDirectory::create_from_tempdir().unwrap()
-    }
-
-    #[test]
-    fn test_simple() {
-        let mut directory = make_directory();
-        super::test_simple(&mut directory);
-    }
-
-    #[test]
-    fn test_write_create_the_file() {
-        let mut directory = make_directory();
-        super::test_write_create_the_file(&mut directory);
-    }
-
-    #[test]
-    fn test_rewrite_forbidden() {
-        let mut directory = make_directory();
-        super::test_rewrite_forbidden(&mut directory);
-    }
-
-    #[test]
-    fn test_directory_delete() {
-        let mut directory = make_directory();
-        super::test_directory_delete(&mut directory);
-    }
-
-    #[test]
-    fn test_lock_non_blocking() {
-        let mut directory = make_directory();
-        super::test_lock_non_blocking(&mut directory);
-    }
-
-    #[test]
-    fn test_lock_blocking() {
-        let mut directory = make_directory();
-        super::test_lock_blocking(&mut directory);
-    }
-
-    #[test]
-    fn test_watch() {
-        let mut directory = make_directory();
-        super::test_watch(&mut directory);
-    }
+#[test]
+fn test_ram_directory() {
+    let mut ram_directory = RAMDirectory::create();
+    test_directory(&mut ram_directory);
 }

-mod ram_directory_tests {
-    use crate::directory::RAMDirectory;
-
-    type DirectoryImpl = RAMDirectory;
-
-    fn make_directory() -> DirectoryImpl {
-        RAMDirectory::default()
-    }
-
-    #[test]
-    fn test_simple() {
-        let mut directory = make_directory();
-        super::test_simple(&mut directory);
-    }
-
-    #[test]
-    fn test_write_create_the_file() {
-        let mut directory = make_directory();
-        super::test_write_create_the_file(&mut directory);
-    }
-
-    #[test]
-    fn test_rewrite_forbidden() {
-        let mut directory = make_directory();
-        super::test_rewrite_forbidden(&mut directory);
-    }
-
-    #[test]
-    fn test_directory_delete() {
-        let mut directory = make_directory();
-        super::test_directory_delete(&mut directory);
-    }
-
-    #[test]
-    fn test_lock_non_blocking() {
-        let mut directory = make_directory();
-        super::test_lock_non_blocking(&mut directory);
-    }
-
-    #[test]
-    fn test_lock_blocking() {
-        let mut directory = make_directory();
-        super::test_lock_blocking(&mut directory);
-    }
-
-    #[test]
-    fn test_watch() {
-        let mut directory = make_directory();
-        super::test_watch(&mut directory);
-    }
+#[test]
+#[cfg(feature = "mmap")]
+fn test_mmap_directory() {
+    let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
+    test_directory(&mut mmap_directory);
 }

 #[test]
@@ -191,39 +99,48 @@ fn test_directory_delete(directory: &mut dyn Directory) {
    assert!(directory.delete(&test_path).is_err());
 }

+fn test_directory(directory: &mut dyn Directory) {
+    test_simple(directory);
+    test_rewrite_forbidden(directory);
+    test_write_create_the_file(directory);
+    test_directory_delete(directory);
+    test_lock_non_blocking(directory);
+    test_lock_blocking(directory);
+    test_watch(directory);
+}
+
 fn test_watch(directory: &mut dyn Directory) {
-    let num_progress: Arc<AtomicUsize> = Default::default();
    let counter: Arc<AtomicUsize> = Default::default();
    let counter_clone = counter.clone();
-    let (sender, receiver) = crossbeam::channel::unbounded();
    let watch_callback = Box::new(move || {
-        counter_clone.fetch_add(1, SeqCst);
+        counter_clone.fetch_add(1, Ordering::SeqCst);
    });
-    // This callback is used to synchronize watching in our unit test.
-    // We bind it to a variable because the callback is removed when that
-    // handle is dropped.
-    let watch_handle = directory.watch(watch_callback).unwrap();
-    let _progress_listener = directory
-        .watch(Box::new(move || {
-            let val = num_progress.fetch_add(1, SeqCst);
-            let _ = sender.send(val);
-        }))
-        .unwrap();
+    assert!(directory
+        .atomic_write(Path::new("meta.json"), b"random_test_data")
+        .is_ok());
+    thread::sleep(Duration::new(0, 10_000));
+    assert_eq!(0, counter.load(Ordering::SeqCst));

+    let watch_handle = directory.watch(watch_callback).unwrap();
    for i in 0..10 {
-        assert_eq!(i, counter.load(SeqCst));
+        assert_eq!(i, counter.load(Ordering::SeqCst));
        assert!(directory
            .atomic_write(Path::new("meta.json"), b"random_test_data_2")
            .is_ok());
-        assert_eq!(receiver.recv_timeout(Duration::from_millis(500)), Ok(i));
-        assert_eq!(i + 1, counter.load(SeqCst));
+        for _ in 0..1_000 {
+            if counter.load(Ordering::SeqCst) > i {
+                break;
+            }
+            thread::sleep(Duration::from_millis(10));
+        }
+        assert_eq!(i + 1, counter.load(Ordering::SeqCst));
    }
    mem::drop(watch_handle);
    assert!(directory
        .atomic_write(Path::new("meta.json"), b"random_test_data")
        .is_ok());
-    assert!(receiver.recv_timeout(Duration::from_millis(500)).is_ok());
-    assert_eq!(10, counter.load(SeqCst));
+    thread::sleep(Duration::from_millis(200));
+    assert_eq!(10, counter.load(Ordering::SeqCst));
 }

 fn test_lock_non_blocking(directory: &mut dyn Directory) {
@@ -257,13 +174,9 @@ fn test_lock_blocking(directory: &mut dyn Directory) {
        is_blocking: true,
    });
    assert!(lock_a_res.is_ok());
-    let in_thread = Arc::new(AtomicBool::default());
-    let in_thread_clone = in_thread.clone();
-    let (sender, receiver) = oneshot::channel();
    std::thread::spawn(move || {
        //< lock_a_res is sent to the thread.
-        in_thread_clone.store(true, SeqCst);
-        let _just_sync = block_on(receiver);
+        std::thread::sleep(time::Duration::from_millis(10));
        // explicitely droping lock_a_res. It would have been sufficient to just force it
        // to be part of the move, but the intent seems clearer that way.
        drop(lock_a_res);
@@ -276,18 +189,14 @@ fn test_lock_blocking(directory: &mut dyn Directory) {
        });
        assert!(lock_a_res.is_err());
    }
-    let directory_clone = directory.box_clone();
-    let (sender2, receiver2) = oneshot::channel();
-    let join_handle = std::thread::spawn(move || {
-        assert!(sender2.send(()).is_ok());
-        let lock_a_res = directory_clone.acquire_lock(&Lock {
+    {
+        // the blocking call should wait for at least 10ms.
+        let start = time::Instant::now();
+        let lock_a_res = directory.acquire_lock(&Lock {
            filepath: PathBuf::from("a.lock"),
            is_blocking: true,
        });
-        assert!(in_thread.load(SeqCst));
        assert!(lock_a_res.is_ok());
-    });
-    assert!(block_on(receiver2).is_ok());
-    assert!(sender.send(()).is_ok());
-    assert!(join_handle.join().is_ok());
+        assert!(start.elapsed().subsec_millis() >= 10);
+    }
 }
--- a/src/directory/watch_event_router.rs
+++ b/src/directory/watch_event_router.rs
@@ -1,5 +1,3 @@
-use futures::channel::oneshot;
-use futures::{Future, TryFutureExt};
 use std::sync::Arc;
 use std::sync::RwLock;
 use std::sync::Weak;
@@ -24,20 +22,13 @@ pub struct WatchCallbackList {
 #[derive(Clone)]
 pub struct WatchHandle(Arc<WatchCallback>);

-impl WatchHandle {
-    /// Create a WatchHandle handle.
-    pub fn new(watch_callback: Arc<WatchCallback>) -> WatchHandle {
-        WatchHandle(watch_callback)
-    }
-}
-
 impl WatchCallbackList {
    /// Suscribes a new callback and returns a handle that controls the lifetime of the callback.
    pub fn subscribe(&self, watch_callback: WatchCallback) -> WatchHandle {
        let watch_callback_arc = Arc::new(watch_callback);
        let watch_callback_weak = Arc::downgrade(&watch_callback_arc);
        self.router.write().unwrap().push(watch_callback_weak);
-        WatchHandle::new(watch_callback_arc)
+        WatchHandle(watch_callback_arc)
    }

    fn list_callback(&self) -> Vec<Arc<WatchCallback>> {
@@ -56,21 +47,14 @@ impl WatchCallbackList {
    }

    /// Triggers all callbacks
-    pub fn broadcast(&self) -> impl Future<Output = ()> {
+    pub fn broadcast(&self) {
        let callbacks = self.list_callback();
-        let (sender, receiver) = oneshot::channel();
-        let result = receiver.unwrap_or_else(|_| ());
-        if callbacks.is_empty() {
-            let _ = sender.send(());
-            return result;
-        }
        let spawn_res = std::thread::Builder::new()
            .name("watch-callbacks".to_string())
            .spawn(move || {
                for callback in callbacks {
                    callback();
                }
-                let _ = sender.send(());
            });
        if let Err(err) = spawn_res {
            error!(
@@ -78,17 +62,19 @@ impl WatchCallbackList {
                err
            );
        }
-        result
    }
 }

 #[cfg(test)]
 mod tests {
    use crate::directory::WatchCallbackList;
-    use futures::executor::block_on;
    use std::mem;
    use std::sync::atomic::{AtomicUsize, Ordering};
    use std::sync::Arc;
+    use std::thread;
+    use std::time::Duration;
+
+    const WAIT_TIME: u64 = 20;

    #[test]
    fn test_watch_event_router_simple() {
@@ -98,22 +84,22 @@ mod tests {
        let inc_callback = Box::new(move || {
            counter_clone.fetch_add(1, Ordering::SeqCst);
        });
-        block_on(watch_event_router.broadcast());
+        watch_event_router.broadcast();
        assert_eq!(0, counter.load(Ordering::SeqCst));
        let handle_a = watch_event_router.subscribe(inc_callback);
+        thread::sleep(Duration::from_millis(WAIT_TIME));
        assert_eq!(0, counter.load(Ordering::SeqCst));
-        block_on(watch_event_router.broadcast());
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
        assert_eq!(1, counter.load(Ordering::SeqCst));
-        block_on(async {
-            (
-                watch_event_router.broadcast().await,
-                watch_event_router.broadcast().await,
-                watch_event_router.broadcast().await,
-            )
-        });
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
        assert_eq!(4, counter.load(Ordering::SeqCst));
        mem::drop(handle_a);
-        block_on(watch_event_router.broadcast());
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
        assert_eq!(4, counter.load(Ordering::SeqCst));
    }

@@ -129,20 +115,20 @@ mod tests {
        };
        let handle_a = watch_event_router.subscribe(inc_callback(1));
        let handle_a2 = watch_event_router.subscribe(inc_callback(10));
+        thread::sleep(Duration::from_millis(WAIT_TIME));
        assert_eq!(0, counter.load(Ordering::SeqCst));
-        block_on(async {
-            futures::join!(
-                watch_event_router.broadcast(),
-                watch_event_router.broadcast()
-            )
-        });
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
        assert_eq!(22, counter.load(Ordering::SeqCst));
        mem::drop(handle_a);
-        block_on(watch_event_router.broadcast());
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
        assert_eq!(32, counter.load(Ordering::SeqCst));
        mem::drop(handle_a2);
-        block_on(watch_event_router.broadcast());
-        block_on(watch_event_router.broadcast());
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
        assert_eq!(32, counter.load(Ordering::SeqCst));
    }

@@ -156,15 +142,14 @@ mod tests {
        });
        let handle_a = watch_event_router.subscribe(inc_callback);
        assert_eq!(0, counter.load(Ordering::SeqCst));
-        block_on(async {
-            let future1 = watch_event_router.broadcast();
-            let future2 = watch_event_router.broadcast();
-            futures::join!(future1, future2)
-        });
+        watch_event_router.broadcast();
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
        assert_eq!(2, counter.load(Ordering::SeqCst));
+        thread::sleep(Duration::from_millis(WAIT_TIME));
        mem::drop(handle_a);
-        let _ = watch_event_router.broadcast();
-        block_on(watch_event_router.broadcast());
+        watch_event_router.broadcast();
+        thread::sleep(Duration::from_millis(WAIT_TIME));
        assert_eq!(2, counter.load(Ordering::SeqCst));
    }
 }
--- a/src/error.rs
+++ b/src/error.rs
@@ -2,8 +2,8 @@

 use std::io;

+use crate::directory::error::LockError;
 use crate::directory::error::{IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
-use crate::directory::error::{Incompatibility, LockError};
 use crate::fastfield::FastFieldNotAvailableError;
 use crate::query;
 use crate::schema;
@@ -80,9 +80,6 @@ pub enum TantivyError {
    /// System error. (e.g.: We failed spawning a new thread)
    #[fail(display = "System error.'{}'", _0)]
    SystemError(String),
-    /// Index incompatible with current version of tantivy
-    #[fail(display = "{:?}", _0)]
-    IncompatibleIndex(Incompatibility),
 }

 impl From<DataCorruption> for TantivyError {
@@ -132,9 +129,6 @@ impl From<OpenReadError> for TantivyError {
        match error {
            OpenReadError::FileDoesNotExist(filepath) => TantivyError::PathDoesNotExist(filepath),
            OpenReadError::IOError(io_error) => TantivyError::IOError(io_error),
-            OpenReadError::IncompatibleIndex(incompatibility) => {
-                TantivyError::IncompatibleIndex(incompatibility)
-            }
        }
    }
 }
@@ -176,9 +170,3 @@ impl From<serde_json::Error> for TantivyError {
        TantivyError::IOError(io_err.into())
    }
 }
-
-impl From<rayon::ThreadPoolBuildError> for TantivyError {
-    fn from(error: rayon::ThreadPoolBuildError) -> TantivyError {
-        TantivyError::SystemError(error.to_string())
-    }
-}
--- a/src/fastfield/delete.rs
+++ b/src/fastfield/delete.rs
@@ -1,19 +1,17 @@
-use crate::common::{BitSet, HasLen};
+use crate::common::HasLen;
 use crate::directory::ReadOnlySource;
 use crate::directory::WritePtr;
 use crate::space_usage::ByteCount;
 use crate::DocId;
+use bit_set::BitSet;
 use std::io;
 use std::io::Write;

 /// Write a delete `BitSet`
 ///
 /// where `delete_bitset` is the set of deleted `DocId`.
-pub fn write_delete_bitset(
-    delete_bitset: &BitSet,
-    max_doc: u32,
-    writer: &mut WritePtr,
-) -> io::Result<()> {
+pub fn write_delete_bitset(delete_bitset: &BitSet, writer: &mut WritePtr) -> io::Result<()> {
+    let max_doc = delete_bitset.capacity();
    let mut byte = 0u8;
    let mut shift = 0u8;
    for doc in 0..max_doc {
@@ -31,7 +29,7 @@ pub fn write_delete_bitset(
    if max_doc % 8 > 0 {
        writer.write_all(&[byte])?;
    }
-    Ok(())
+    writer.flush()
 }

 /// Set of deleted `DocId`s.
@@ -85,40 +83,43 @@ impl HasLen for DeleteBitSet {
 mod tests {
    use super::*;
    use crate::directory::*;
+    use bit_set::BitSet;
    use std::path::PathBuf;

-    fn test_delete_bitset_helper(bitset: &BitSet, max_doc: u32) {
+    fn test_delete_bitset_helper(bitset: &BitSet) {
        let test_path = PathBuf::from("test");
        let mut directory = RAMDirectory::create();
        {
            let mut writer = directory.open_write(&*test_path).unwrap();
-            write_delete_bitset(bitset, max_doc, &mut writer).unwrap();
-            writer.terminate().unwrap();
+            write_delete_bitset(bitset, &mut writer).unwrap();
        }
-        let source = directory.open_read(&test_path).unwrap();
-        let delete_bitset = DeleteBitSet::open(source);
-        for doc in 0..max_doc {
-            assert_eq!(bitset.contains(doc), delete_bitset.is_deleted(doc as DocId));
+        {
+            let source = directory.open_read(&test_path).unwrap();
+            let delete_bitset = DeleteBitSet::open(source);
+            let n = bitset.capacity();
+            for doc in 0..n {
+                assert_eq!(bitset.contains(doc), delete_bitset.is_deleted(doc as DocId));
+            }
+            assert_eq!(delete_bitset.len(), bitset.len());
        }
-        assert_eq!(delete_bitset.len(), bitset.len());
    }

    #[test]
    fn test_delete_bitset() {
        {
-            let mut bitset = BitSet::with_max_value(10);
+            let mut bitset = BitSet::with_capacity(10);
            bitset.insert(1);
            bitset.insert(9);
-            test_delete_bitset_helper(&bitset, 10);
+            test_delete_bitset_helper(&bitset);
        }
        {
-            let mut bitset = BitSet::with_max_value(8);
+            let mut bitset = BitSet::with_capacity(8);
            bitset.insert(1);
            bitset.insert(2);
            bitset.insert(3);
            bitset.insert(5);
            bitset.insert(7);
-            test_delete_bitset_helper(&bitset, 8);
+            test_delete_bitset_helper(&bitset);
        }
    }
 }
--- a/src/fastfield/mod.rs
+++ b/src/fastfield/mod.rs
@@ -33,7 +33,6 @@ pub use self::reader::FastFieldReader;
 pub use self::readers::FastFieldReaders;
 pub use self::serializer::FastFieldSerializer;
 pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
-use crate::chrono::{NaiveDateTime, Utc};
 use crate::common;
 use crate::schema::Cardinality;
 use crate::schema::FieldType;
@@ -50,7 +49,7 @@ mod serializer;
 mod writer;

 /// Trait for types that are allowed for fast fields: (u64, i64 and f64).
-pub trait FastValue: Clone + Copy + Send + Sync + PartialOrd {
+pub trait FastValue: Default + Clone + Copy + Send + Sync + PartialOrd {
    /// Converts a value from u64
    ///
    /// Internally all fast field values are encoded as u64.
@@ -70,12 +69,6 @@ pub trait FastValue: Clone + Copy + Send + Sync + PartialOrd {
    /// Cast value to `u64`.
    /// The value is just reinterpreted in memory.
    fn as_u64(&self) -> u64;
-
-    /// Build a default value. This default value is never used, so the value does not
-    /// really matter.
-    fn make_zero() -> Self {
-        Self::from_u64(0i64.to_u64())
-    }
 }

 impl FastValue for u64 {
@@ -142,34 +135,11 @@ impl FastValue for f64 {
    }
 }

-impl FastValue for crate::DateTime {
-    fn from_u64(timestamp_u64: u64) -> Self {
-        let timestamp_i64 = i64::from_u64(timestamp_u64);
-        crate::DateTime::from_utc(NaiveDateTime::from_timestamp(timestamp_i64, 0), Utc)
-    }
-
-    fn to_u64(&self) -> u64 {
-        self.timestamp().to_u64()
-    }
-
-    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
-        match *field_type {
-            FieldType::Date(ref integer_options) => integer_options.get_fastfield_cardinality(),
-            _ => None,
-        }
-    }
-
-    fn as_u64(&self) -> u64 {
-        self.timestamp().as_u64()
-    }
-}
-
 fn value_to_u64(value: &Value) -> u64 {
    match *value {
        Value::U64(ref val) => *val,
        Value::I64(ref val) => common::i64_to_u64(*val),
        Value::F64(ref val) => common::f64_to_u64(*val),
-        Value::Date(ref datetime) => common::i64_to_u64(datetime.timestamp()),
        _ => panic!("Expected a u64/i64/f64 field, got {:?} ", value),
    }
 }
@@ -181,12 +151,10 @@ mod tests {
    use crate::common::CompositeFile;
    use crate::directory::{Directory, RAMDirectory, WritePtr};
    use crate::fastfield::FastFieldReader;
-    use crate::merge_policy::NoMergePolicy;
+    use crate::schema::Document;
    use crate::schema::Field;
    use crate::schema::Schema;
    use crate::schema::FAST;
-    use crate::schema::{Document, IntOptions};
-    use crate::{Index, SegmentId, SegmentReader};
    use once_cell::sync::Lazy;
    use rand::prelude::SliceRandom;
    use rand::rngs::StdRng;
@@ -210,12 +178,6 @@ mod tests {
        assert_eq!(test_fastfield.get(2), 300);
    }

-    #[test]
-    pub fn test_fastfield_i64_u64() {
-        let datetime = crate::DateTime::from_utc(NaiveDateTime::from_timestamp(0i64, 0), Utc);
-        assert_eq!(i64::from_u64(datetime.to_u64()), 0i64);
-    }
-
    #[test]
    fn test_intfastfield_small() {
        let path = Path::new("test");
@@ -467,93 +429,6 @@ mod tests {
            }
        }
    }
-
-    #[test]
-    fn test_merge_missing_date_fast_field() {
-        let mut schema_builder = Schema::builder();
-        let date_field = schema_builder.add_date_field("date", FAST);
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-        index_writer.set_merge_policy(Box::new(NoMergePolicy));
-        index_writer.add_document(doc!(date_field =>crate::chrono::prelude::Utc::now()));
-        index_writer.commit().unwrap();
-        index_writer.add_document(doc!());
-        index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-        let segment_ids: Vec<SegmentId> = reader
-            .searcher()
-            .segment_readers()
-            .iter()
-            .map(SegmentReader::segment_id)
-            .collect();
-        assert_eq!(segment_ids.len(), 2);
-        let merge_future = index_writer.merge(&segment_ids[..]);
-        let merge_res = futures::executor::block_on(merge_future);
-        assert!(merge_res.is_ok());
-        assert!(reader.reload().is_ok());
-        assert_eq!(reader.searcher().segment_readers().len(), 1);
-    }
-
-    #[test]
-    fn test_default_datetime() {
-        assert_eq!(crate::DateTime::make_zero().timestamp(), 0i64);
-    }
-
-    #[test]
-    fn test_datefastfield() {
-        use crate::fastfield::FastValue;
-        let mut schema_builder = Schema::builder();
-        let date_field = schema_builder.add_date_field("date", FAST);
-        let multi_date_field = schema_builder.add_date_field(
-            "multi_date",
-            IntOptions::default().set_fast(Cardinality::MultiValues),
-        );
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-        index_writer.set_merge_policy(Box::new(NoMergePolicy));
-        index_writer.add_document(doc!(
-            date_field => crate::DateTime::from_u64(1i64.to_u64()),
-            multi_date_field => crate::DateTime::from_u64(2i64.to_u64()),
-            multi_date_field => crate::DateTime::from_u64(3i64.to_u64())
-        ));
-        index_writer.add_document(doc!(
-            date_field => crate::DateTime::from_u64(4i64.to_u64())
-        ));
-        index_writer.add_document(doc!(
-            multi_date_field => crate::DateTime::from_u64(5i64.to_u64()),
-            multi_date_field => crate::DateTime::from_u64(6i64.to_u64())
-        ));
-        index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-        let searcher = reader.searcher();
-        assert_eq!(searcher.segment_readers().len(), 1);
-        let segment_reader = searcher.segment_reader(0);
-        let fast_fields = segment_reader.fast_fields();
-        let date_fast_field = fast_fields.date(date_field).unwrap();
-        let dates_fast_field = fast_fields.dates(multi_date_field).unwrap();
-        let mut dates = vec![];
-        {
-            assert_eq!(date_fast_field.get(0u32).timestamp(), 1i64);
-            dates_fast_field.get_vals(0u32, &mut dates);
-            assert_eq!(dates.len(), 2);
-            assert_eq!(dates[0].timestamp(), 2i64);
-            assert_eq!(dates[1].timestamp(), 3i64);
-        }
-        {
-            assert_eq!(date_fast_field.get(1u32).timestamp(), 4i64);
-            dates_fast_field.get_vals(1u32, &mut dates);
-            assert!(dates.is_empty());
-        }
-        {
-            assert_eq!(date_fast_field.get(2u32).timestamp(), 0i64);
-            dates_fast_field.get_vals(2u32, &mut dates);
-            assert_eq!(dates.len(), 2);
-            assert_eq!(dates[0].timestamp(), 5i64);
-            assert_eq!(dates[1].timestamp(), 6i64);
-        }
-    }
 }

 #[cfg(all(test, feature = "unstable"))]
--- a/src/fastfield/multivalued/mod.rs
+++ b/src/fastfield/multivalued/mod.rs
@@ -7,6 +7,9 @@ pub use self::writer::MultiValueIntFastFieldWriter;
 #[cfg(test)]
 mod tests {

+    use time;
+
+    use self::time::Duration;
    use crate::collector::TopDocs;
    use crate::query::QueryParser;
    use crate::schema::Cardinality;
@@ -14,7 +17,6 @@ mod tests {
    use crate::schema::IntOptions;
    use crate::schema::Schema;
    use crate::Index;
-    use chrono::Duration;

    #[test]
    fn test_multivalued_u64() {
--- a/src/fastfield/multivalued/reader.rs
+++ b/src/fastfield/multivalued/reader.rs
@@ -45,7 +45,7 @@ impl<Item: FastValue> MultiValueIntFastFieldReader<Item> {
    pub fn get_vals(&self, doc: DocId, vals: &mut Vec<Item>) {
        let (start, stop) = self.range(doc);
        let len = (stop - start) as usize;
-        vals.resize(len, Item::make_zero());
+        vals.resize(len, Item::default());
        self.vals_reader.get_range_u64(start, &mut vals[..]);
    }

--- a/src/fastfield/readers.rs
+++ b/src/fastfield/readers.rs
@@ -4,6 +4,7 @@ use crate::fastfield::MultiValueIntFastFieldReader;
 use crate::fastfield::{FastFieldNotAvailableError, FastFieldReader};
 use crate::schema::{Cardinality, Field, FieldType, Schema};
 use crate::space_usage::PerFieldSpaceUsage;
+use crate::Result;
 use std::collections::HashMap;

 /// Provides access to all of the FastFieldReader.
@@ -14,11 +15,9 @@ pub struct FastFieldReaders {
    fast_field_i64: HashMap<Field, FastFieldReader<i64>>,
    fast_field_u64: HashMap<Field, FastFieldReader<u64>>,
    fast_field_f64: HashMap<Field, FastFieldReader<f64>>,
-    fast_field_date: HashMap<Field, FastFieldReader<crate::DateTime>>,
    fast_field_i64s: HashMap<Field, MultiValueIntFastFieldReader<i64>>,
    fast_field_u64s: HashMap<Field, MultiValueIntFastFieldReader<u64>>,
    fast_field_f64s: HashMap<Field, MultiValueIntFastFieldReader<f64>>,
-    fast_field_dates: HashMap<Field, MultiValueIntFastFieldReader<crate::DateTime>>,
    fast_bytes: HashMap<Field, BytesFastFieldReader>,
    fast_fields_composite: CompositeFile,
 }
@@ -27,7 +26,6 @@ enum FastType {
    I64,
    U64,
    F64,
-    Date,
 }

 fn type_and_cardinality(field_type: &FieldType) -> Option<(FastType, Cardinality)> {
@@ -41,9 +39,6 @@ fn type_and_cardinality(field_type: &FieldType) -> Option<(FastType, Cardinality
        FieldType::F64(options) => options
            .get_fastfield_cardinality()
            .map(|cardinality| (FastType::F64, cardinality)),
-        FieldType::Date(options) => options
-            .get_fastfield_cardinality()
-            .map(|cardinality| (FastType::Date, cardinality)),
        FieldType::HierarchicalFacet => Some((FastType::U64, Cardinality::MultiValues)),
        _ => None,
    }
@@ -53,16 +48,14 @@ impl FastFieldReaders {
    pub(crate) fn load_all(
        schema: &Schema,
        fast_fields_composite: &CompositeFile,
-    ) -> crate::Result<FastFieldReaders> {
+    ) -> Result<FastFieldReaders> {
        let mut fast_field_readers = FastFieldReaders {
            fast_field_i64: Default::default(),
            fast_field_u64: Default::default(),
            fast_field_f64: Default::default(),
-            fast_field_date: Default::default(),
            fast_field_i64s: Default::default(),
            fast_field_u64s: Default::default(),
            fast_field_f64s: Default::default(),
-            fast_field_dates: Default::default(),
            fast_bytes: Default::default(),
            fast_fields_composite: fast_fields_composite.clone(),
        };
@@ -102,12 +95,6 @@ impl FastFieldReaders {
                                        FastFieldReader::open(fast_field_data.clone()),
                                    );
                                }
-                                FastType::Date => {
-                                    fast_field_readers.fast_field_date.insert(
-                                        field,
-                                        FastFieldReader::open(fast_field_data.clone()),
-                                    );
-                                }
                            }
                        } else {
                            return Err(From::from(FastFieldNotAvailableError::new(field_entry)));
@@ -143,14 +130,6 @@ impl FastFieldReaders {
                                        .fast_field_f64s
                                        .insert(field, multivalued_int_fast_field);
                                }
-                                FastType::Date => {
-                                    let vals_reader = FastFieldReader::open(fast_field_data);
-                                    let multivalued_int_fast_field =
-                                        MultiValueIntFastFieldReader::open(idx_reader, vals_reader);
-                                    fast_field_readers
-                                        .fast_field_dates
-                                        .insert(field, multivalued_int_fast_field);
-                                }
                            }
                        } else {
                            return Err(From::from(FastFieldNotAvailableError::new(field_entry)));
@@ -177,6 +156,8 @@ impl FastFieldReaders {
    /// If the field is a i64-fast field, return the associated u64 reader. Values are
    /// mapped from i64 to u64 using a (well the, it is unique) monotonic mapping.    ///
    ///
+    ///TODO should it also be lenient with f64?
+    ///
    /// This method is useful when merging segment reader.
    pub(crate) fn u64_lenient(&self, field: Field) -> Option<FastFieldReader<u64>> {
        if let Some(u64_ff_reader) = self.u64(field) {
@@ -185,12 +166,6 @@ impl FastFieldReaders {
        if let Some(i64_ff_reader) = self.i64(field) {
            return Some(i64_ff_reader.into_u64_reader());
        }
-        if let Some(f64_ff_reader) = self.f64(field) {
-            return Some(f64_ff_reader.into_u64_reader());
-        }
-        if let Some(date_ff_reader) = self.date(field) {
-            return Some(date_ff_reader.into_u64_reader());
-        }
        None
    }

@@ -201,13 +176,6 @@ impl FastFieldReaders {
        self.fast_field_i64.get(&field).cloned()
    }

-    /// Returns the `i64` fast field reader reader associated to `field`.
-    ///
-    /// If `field` is not a i64 fast field, this method returns `None`.
-    pub fn date(&self, field: Field) -> Option<FastFieldReader<crate::DateTime>> {
-        self.fast_field_date.get(&field).cloned()
-    }
-
    /// Returns the `f64` fast field reader reader associated to `field`.
    ///
    /// If `field` is not a f64 fast field, this method returns `None`.
@@ -234,9 +202,6 @@ impl FastFieldReaders {
        if let Some(i64s_ff_reader) = self.i64s(field) {
            return Some(i64s_ff_reader.into_u64s_reader());
        }
-        if let Some(f64s_ff_reader) = self.f64s(field) {
-            return Some(f64s_ff_reader.into_u64s_reader());
-        }
        None
    }

@@ -254,13 +219,6 @@ impl FastFieldReaders {
        self.fast_field_f64s.get(&field).cloned()
    }

-    /// Returns a `crate::DateTime` multi-valued fast field reader reader associated to `field`.
-    ///
-    /// If `field` is not a `crate::DateTime` multi-valued fast field, this method returns `None`.
-    pub fn dates(&self, field: Field) -> Option<MultiValueIntFastFieldReader<crate::DateTime>> {
-        self.fast_field_dates.get(&field).cloned()
-    }
-
    /// Returns the `bytes` fast field reader associated to `field`.
    ///
    /// If `field` is not a bytes fast field, returns `None`.
--- a/src/fastfield/writer.rs
+++ b/src/fastfield/writer.rs
@@ -4,7 +4,7 @@ use crate::common::BinarySerializable;
 use crate::common::VInt;
 use crate::fastfield::{BytesFastFieldWriter, FastFieldSerializer};
 use crate::postings::UnorderedTermId;
-use crate::schema::{Cardinality, Document, Field, FieldEntry, FieldType, Schema};
+use crate::schema::{Cardinality, Document, Field, FieldType, Schema};
 use crate::termdict::TermOrdinal;
 use fnv::FnvHashMap;
 use std::collections::HashMap;
@@ -17,14 +17,6 @@ pub struct FastFieldsWriter {
    bytes_value_writers: Vec<BytesFastFieldWriter>,
 }

-fn fast_field_default_value(field_entry: &FieldEntry) -> u64 {
-    match *field_entry.field_type() {
-        FieldType::I64(_) | FieldType::Date(_) => common::i64_to_u64(0i64),
-        FieldType::F64(_) => common::f64_to_u64(0.0f64),
-        _ => 0u64,
-    }
-}
-
 impl FastFieldsWriter {
    /// Create all `FastFieldWriter` required by the schema.
    pub fn from_schema(schema: &Schema) -> FastFieldsWriter {
@@ -33,15 +25,18 @@ impl FastFieldsWriter {
        let mut bytes_value_writers = Vec::new();

        for (field, field_entry) in schema.fields() {
+            let default_value = match *field_entry.field_type() {
+                FieldType::I64(_) => common::i64_to_u64(0i64),
+                FieldType::F64(_) => common::f64_to_u64(0.0f64),
+                _ => 0u64,
+            };
            match *field_entry.field_type() {
                FieldType::I64(ref int_options)
                | FieldType::U64(ref int_options)
-                | FieldType::F64(ref int_options)
-                | FieldType::Date(ref int_options) => {
+                | FieldType::F64(ref int_options) => {
                    match int_options.get_fastfield_cardinality() {
                        Some(Cardinality::SingleValue) => {
                            let mut fast_field_writer = IntFastFieldWriter::new(field);
-                            let default_value = fast_field_default_value(field_entry);
                            fast_field_writer.set_val_if_missing(default_value);
                            single_value_writers.push(fast_field_writer);
                        }
--- a/src/indexer/delete_queue.rs
+++ b/src/indexer/delete_queue.rs
@@ -2,7 +2,7 @@ use super::operation::DeleteOperation;
 use crate::Opstamp;
 use std::mem;
 use std::ops::DerefMut;
-use std::sync::{Arc, RwLock, Weak};
+use std::sync::{Arc, RwLock};

 // The DeleteQueue is similar in conceptually to a multiple
 // consumer single producer broadcast channel.
@@ -14,15 +14,14 @@ use std::sync::{Arc, RwLock, Weak};
 //
 // New consumer can be created in two ways
 // - calling `delete_queue.cursor()` returns a cursor, that
-//   will include all future delete operation (and some or none
-//   of the past operations... The client is in charge of checking the opstamps.).
+//   will include all future delete operation (and no past operations).
 // - cloning an existing cursor returns a new cursor, that
 //   is at the exact same position, and can now advance independently
 //   from the original cursor.
 #[derive(Default)]
 struct InnerDeleteQueue {
    writer: Vec<DeleteOperation>,
-    last_block: Weak<Block>,
+    last_block: Option<Arc<Block>>,
 }

 #[derive(Clone)]
@@ -33,31 +32,21 @@ pub struct DeleteQueue {
 impl DeleteQueue {
    // Creates a new delete queue.
    pub fn new() -> DeleteQueue {
-        DeleteQueue {
+        let delete_queue = DeleteQueue {
            inner: Arc::default(),
-        }
-    }
+        };
+
+        let next_block = NextBlock::from(delete_queue.clone());

-    fn get_last_block(&self) -> Arc<Block> {
        {
-            // try get the last block with simply acquiring the read lock.
-            let rlock = self.inner.read().unwrap();
-            if let Some(block) = rlock.last_block.upgrade() {
-                return block;
-            }
+            let mut delete_queue_wlock = delete_queue.inner.write().unwrap();
+            delete_queue_wlock.last_block = Some(Arc::new(Block {
+                operations: Arc::default(),
+                next: next_block,
+            }));
        }
-        // It failed. Let's double check after acquiring the write, as someone could have called
-        // `get_last_block` right after we released the rlock.
-        let mut wlock = self.inner.write().unwrap();
-        if let Some(block) = wlock.last_block.upgrade() {
-            return block;
-        }
-        let block = Arc::new(Block {
-            operations: Arc::default(),
-            next: NextBlock::from(self.clone()),
-        });
-        wlock.last_block = Arc::downgrade(&block);
-        block
+
+        delete_queue
    }

    // Creates a new cursor that makes it possible to
@@ -65,7 +54,17 @@ impl DeleteQueue {
    //
    // Past delete operations are not accessible.
    pub fn cursor(&self) -> DeleteCursor {
-        let last_block = self.get_last_block();
+        let last_block = self
+            .inner
+            .read()
+            .expect("Read lock poisoned when opening delete queue cursor")
+            .last_block
+            .clone()
+            .expect(
+                "Failed to unwrap last_block. This should never happen
+                as the Option<> is only here to make
+                initialization possible",
+            );
        let operations_len = last_block.operations.len();
        DeleteCursor {
            block: last_block,
@@ -101,19 +100,23 @@ impl DeleteQueue {
            .write()
            .expect("Failed to acquire write lock on delete queue writer");

-        if self_wlock.writer.is_empty() {
-            return None;
+        let delete_operations;
+        {
+            let writer: &mut Vec<DeleteOperation> = &mut self_wlock.writer;
+            if writer.is_empty() {
+                return None;
+            }
+            delete_operations = mem::replace(writer, vec![]);
        }

-        let delete_operations = mem::replace(&mut self_wlock.writer, vec![]);
-
-        let new_block = Arc::new(Block {
-            operations: Arc::new(delete_operations.into_boxed_slice()),
-            next: NextBlock::from(self.clone()),
-        });
-
-        self_wlock.last_block = Arc::downgrade(&new_block);
-        Some(new_block)
+        let next_block = NextBlock::from(self.clone());
+        {
+            self_wlock.last_block = Some(Arc::new(Block {
+                operations: Arc::new(delete_operations),
+                next: next_block,
+            }));
+        }
+        self_wlock.last_block.clone()
    }
 }

@@ -167,7 +170,7 @@ impl NextBlock {
 }

 struct Block {
-    operations: Arc<Box<[DeleteOperation]>>,
+    operations: Arc<Vec<DeleteOperation>>,
    next: NextBlock,
 }

--- a/src/indexer/index_writer.rs
+++ b/src/indexer/index_writer.rs
@@ -1,15 +1,14 @@
 use super::operation::{AddOperation, UserOperation};
 use super::segment_updater::SegmentUpdater;
 use super::PreparedCommit;
-use crate::common::BitSet;
 use crate::core::Index;
 use crate::core::Segment;
 use crate::core::SegmentComponent;
 use crate::core::SegmentId;
 use crate::core::SegmentMeta;
 use crate::core::SegmentReader;
+use crate::directory::DirectoryLock;
 use crate::directory::TerminatingWrite;
-use crate::directory::{DirectoryLock, GarbageCollectionResult};
 use crate::docset::DocSet;
 use crate::error::TantivyError;
 use crate::fastfield::write_delete_bitset;
@@ -24,9 +23,10 @@ use crate::schema::Document;
 use crate::schema::IndexRecordOption;
 use crate::schema::Term;
 use crate::Opstamp;
+use crate::Result;
+use bit_set::BitSet;
 use crossbeam::channel;
-use futures::executor::block_on;
-use futures::future::Future;
+use futures::{Canceled, Future};
 use smallvec::smallvec;
 use smallvec::SmallVec;
 use std::mem;
@@ -72,7 +72,7 @@ pub struct IndexWriter {

    heap_size_in_bytes_per_thread: usize,

-    workers_join_handle: Vec<JoinHandle<crate::Result<()>>>,
+    workers_join_handle: Vec<JoinHandle<Result<()>>>,

    operation_receiver: OperationReceiver,
    operation_sender: OperationSender,
@@ -95,7 +95,7 @@ fn compute_deleted_bitset(
    delete_cursor: &mut DeleteCursor,
    doc_opstamps: &DocToOpstampMapping,
    target_opstamp: Opstamp,
-) -> crate::Result<bool> {
+) -> Result<bool> {
    let mut might_have_changed = false;
    while let Some(delete_op) = delete_cursor.get() {
        if delete_op.opstamp > target_opstamp {
@@ -115,7 +115,7 @@ fn compute_deleted_bitset(
            while docset.advance() {
                let deleted_doc = docset.doc();
                if deleted_doc < limit_doc {
-                    delete_bitset.insert(deleted_doc);
+                    delete_bitset.insert(deleted_doc as usize);
                    might_have_changed = true;
                }
            }
@@ -126,73 +126,65 @@ fn compute_deleted_bitset(
    Ok(might_have_changed)
 }

-/// Advance delete for the given segment up to the target opstamp.
-///
-/// Note that there are no guarantee that the resulting `segment_entry` delete_opstamp
-/// is `==` target_opstamp.
-/// For instance, there was no delete operation between the state of the `segment_entry` and
-/// the `target_opstamp`, `segment_entry` is not updated.
+/// Advance delete for the given segment up
+/// to the target opstamp.
 pub(crate) fn advance_deletes(
    mut segment: Segment,
    segment_entry: &mut SegmentEntry,
    target_opstamp: Opstamp,
-) -> crate::Result<()> {
-    if segment_entry.meta().delete_opstamp() == Some(target_opstamp) {
-        // We are already up-to-date here.
-        return Ok(());
-    }
+) -> Result<()> {
+    {
+        if segment_entry.meta().delete_opstamp() == Some(target_opstamp) {
+            // We are already up-to-date here.
+            return Ok(());
+        }

-    if segment_entry.delete_bitset().is_none() && segment_entry.delete_cursor().get().is_none() {
-        // There has been no `DeleteOperation` between the segment status and `target_opstamp`.
-        return Ok(());
-    }
+        let segment_reader = SegmentReader::open(&segment)?;

-    let segment_reader = SegmentReader::open(&segment)?;
+        let max_doc = segment_reader.max_doc();
+        let mut delete_bitset: BitSet = match segment_entry.delete_bitset() {
+            Some(previous_delete_bitset) => (*previous_delete_bitset).clone(),
+            None => BitSet::with_capacity(max_doc as usize),
+        };

-    let max_doc = segment_reader.max_doc();
-    let mut delete_bitset: BitSet = match segment_entry.delete_bitset() {
-        Some(previous_delete_bitset) => (*previous_delete_bitset).clone(),
-        None => BitSet::with_max_value(max_doc),
-    };
+        let delete_cursor = segment_entry.delete_cursor();

-    compute_deleted_bitset(
-        &mut delete_bitset,
-        &segment_reader,
-        segment_entry.delete_cursor(),
-        &DocToOpstampMapping::None,
-        target_opstamp,
-    )?;
+        compute_deleted_bitset(
+            &mut delete_bitset,
+            &segment_reader,
+            delete_cursor,
+            &DocToOpstampMapping::None,
+            target_opstamp,
+        )?;

-    // TODO optimize
-    if let Some(seg_delete_bitset) = segment_reader.delete_bitset() {
+        // TODO optimize
        for doc in 0u32..max_doc {
-            if seg_delete_bitset.is_deleted(doc) {
-                delete_bitset.insert(doc);
+            if segment_reader.is_deleted(doc) {
+                delete_bitset.insert(doc as usize);
            }
        }
-    }

-    let num_deleted_docs = delete_bitset.len();
-    if num_deleted_docs > 0 {
-        segment = segment.with_delete_meta(num_deleted_docs as u32, target_opstamp);
-        let mut delete_file = segment.open_write(SegmentComponent::DELETE)?;
-        write_delete_bitset(&delete_bitset, max_doc, &mut delete_file)?;
-        delete_file.terminate()?;
+        let num_deleted_docs = delete_bitset.len();
+        if num_deleted_docs > 0 {
+            segment = segment.with_delete_meta(num_deleted_docs as u32, target_opstamp);
+            let mut delete_file = segment.open_write(SegmentComponent::DELETE)?;
+            write_delete_bitset(&delete_bitset, &mut delete_file)?;
+            delete_file.terminate()?;
+        }
    }
-
    segment_entry.set_meta(segment.meta().clone());
    Ok(())
 }

 fn index_documents(
    memory_budget: usize,
-    segment: Segment,
+    segment: &Segment,
    grouped_document_iterator: &mut dyn Iterator<Item = OperationGroup>,
    segment_updater: &mut SegmentUpdater,
    mut delete_cursor: DeleteCursor,
-) -> crate::Result<bool> {
+) -> Result<bool> {
    let schema = segment.schema();
-
+    let segment_id = segment.id();
    let mut segment_writer = SegmentWriter::for_segment(memory_budget, segment.clone(), &schema)?;
    for document_group in grouped_document_iterator {
        for doc in document_group {
@@ -212,32 +204,22 @@ fn index_documents(
        return Ok(false);
    }

-    let max_doc = segment_writer.max_doc();
+    let num_docs = segment_writer.max_doc();

    // this is ensured by the call to peek before starting
    // the worker thread.
-    assert!(max_doc > 0);
+    assert!(num_docs > 0);

    let doc_opstamps: Vec<Opstamp> = segment_writer.finalize()?;
-
-    let segment_with_max_doc = segment.with_max_doc(max_doc);
+    let segment_meta = segment.index().new_segment_meta(segment_id, num_docs);

    let last_docstamp: Opstamp = *(doc_opstamps.last().unwrap());

-    let delete_bitset_opt = apply_deletes(
-        &segment_with_max_doc,
-        &mut delete_cursor,
-        &doc_opstamps,
-        last_docstamp,
-    )?;
+    let delete_bitset_opt =
+        apply_deletes(&segment, &mut delete_cursor, &doc_opstamps, last_docstamp)?;

-    let segment_entry = SegmentEntry::new(
-        segment_with_max_doc.meta().clone(),
-        delete_cursor,
-        delete_bitset_opt,
-    );
-    block_on(segment_updater.schedule_add_segment(segment_entry))?;
-    Ok(true)
+    let segment_entry = SegmentEntry::new(segment_meta, delete_cursor, delete_bitset_opt);
+    Ok(segment_updater.add_segment(segment_entry))
 }

 fn apply_deletes(
@@ -245,7 +227,7 @@ fn apply_deletes(
    mut delete_cursor: &mut DeleteCursor,
    doc_opstamps: &[Opstamp],
    last_docstamp: Opstamp,
-) -> crate::Result<Option<BitSet>> {
+) -> Result<Option<BitSet<u32>>> {
    if delete_cursor.get().is_none() {
        // if there are no delete operation in the queue, no need
        // to even open the segment.
@@ -253,9 +235,7 @@ fn apply_deletes(
    }
    let segment_reader = SegmentReader::open(segment)?;
    let doc_to_opstamps = DocToOpstampMapping::from(doc_opstamps);
-
-    let max_doc = segment.meta().max_doc();
-    let mut deleted_bitset = BitSet::with_max_value(max_doc);
+    let mut deleted_bitset = BitSet::with_capacity(segment_reader.max_doc() as usize);
    let may_have_deletes = compute_deleted_bitset(
        &mut deleted_bitset,
        &segment_reader,
@@ -290,7 +270,7 @@ impl IndexWriter {
        num_threads: usize,
        heap_size_in_bytes_per_thread: usize,
        directory_lock: DirectoryLock,
-    ) -> crate::Result<IndexWriter> {
+    ) -> Result<IndexWriter> {
        if heap_size_in_bytes_per_thread < HEAP_SIZE_MIN {
            let err_msg = format!(
                "The heap size per thread needs to be at least {}.",
@@ -339,17 +319,12 @@ impl IndexWriter {
        Ok(index_writer)
    }

-    fn drop_sender(&mut self) {
-        let (sender, _receiver) = channel::bounded(1);
-        mem::replace(&mut self.operation_sender, sender);
-    }
-
    /// If there are some merging threads, blocks until they all finish their work and
    /// then drop the `IndexWriter`.
-    pub fn wait_merging_threads(mut self) -> crate::Result<()> {
+    pub fn wait_merging_threads(mut self) -> Result<()> {
        // this will stop the indexing thread,
        // dropping the last reference to the segment_updater.
-        self.drop_sender();
+        drop(self.operation_sender);

        let former_workers_handles = mem::replace(&mut self.workers_join_handle, vec![]);
        for join_handle in former_workers_handles {
@@ -360,6 +335,7 @@ impl IndexWriter {
                    TantivyError::ErrorInThread("Error in indexing worker thread.".into())
                })?;
        }
+        drop(self.workers_join_handle);

        let result = self
            .segment_updater
@@ -374,10 +350,10 @@ impl IndexWriter {
    }

    #[doc(hidden)]
-    pub fn add_segment(&self, segment_meta: SegmentMeta) -> crate::Result<()> {
+    pub fn add_segment(&mut self, segment_meta: SegmentMeta) {
        let delete_cursor = self.delete_queue.cursor();
        let segment_entry = SegmentEntry::new(segment_meta, delete_cursor, None);
-        block_on(self.segment_updater.schedule_add_segment(segment_entry))
+        self.segment_updater.add_segment(segment_entry);
    }

    /// Creates a new segment.
@@ -394,7 +370,7 @@ impl IndexWriter {

    /// Spawns a new worker thread for indexing.
    /// The thread consumes documents from the pipeline.
-    fn add_indexing_worker(&mut self) -> crate::Result<()> {
+    fn add_indexing_worker(&mut self) -> Result<()> {
        let document_receiver_clone = self.operation_receiver.clone();
        let mut segment_updater = self.segment_updater.clone();

@@ -402,7 +378,7 @@ impl IndexWriter {

        let mem_budget = self.heap_size_in_bytes_per_thread;
        let index = self.index.clone();
-        let join_handle: JoinHandle<crate::Result<()>> = thread::Builder::new()
+        let join_handle: JoinHandle<Result<()>> = thread::Builder::new()
            .name(format!("thrd-tantivy-index{}", self.worker_id))
            .spawn(move || {
                loop {
@@ -431,7 +407,7 @@ impl IndexWriter {
                    let segment = index.new_segment();
                    index_documents(
                        mem_budget,
-                        segment,
+                        &segment,
                        &mut document_iterator,
                        &mut segment_updater,
                        delete_cursor.clone(),
@@ -448,23 +424,22 @@ impl IndexWriter {
        self.segment_updater.get_merge_policy()
    }

-    /// Setter for the merge policy.
+    /// Set the merge policy.
    pub fn set_merge_policy(&self, merge_policy: Box<dyn MergePolicy>) {
        self.segment_updater.set_merge_policy(merge_policy);
    }

-    fn start_workers(&mut self) -> crate::Result<()> {
+    fn start_workers(&mut self) -> Result<()> {
        for _ in 0..self.num_threads {
            self.add_indexing_worker()?;
        }
        Ok(())
    }

-    /// Detects and removes the files that are not used by the index anymore.
-    pub fn garbage_collect_files(
-        &self,
-    ) -> impl Future<Output = crate::Result<GarbageCollectionResult>> {
-        self.segment_updater.schedule_garbage_collect()
+    /// Detects and removes the files that
+    /// are not used by the index anymore.
+    pub fn garbage_collect_files(&mut self) -> Result<()> {
+        self.segment_updater.garbage_collect_files().wait()
    }

    /// Deletes all documents from the index
@@ -503,7 +478,7 @@ impl IndexWriter {
    ///     Ok(())
    /// }
    /// ```
-    pub fn delete_all_documents(&self) -> crate::Result<Opstamp> {
+    pub fn delete_all_documents(&mut self) -> Result<Opstamp> {
        // Delete segments
        self.segment_updater.remove_all_segments();
        // Return new stamp - reverted stamp
@@ -517,10 +492,8 @@ impl IndexWriter {
    pub fn merge(
        &mut self,
        segment_ids: &[SegmentId],
-    ) -> impl Future<Output = crate::Result<SegmentMeta>> {
-        let merge_operation = self.segment_updater.make_merge_operation(segment_ids);
-        let segment_updater = self.segment_updater.clone();
-        async move { segment_updater.start_merge(merge_operation)?.await }
+    ) -> Result<impl Future<Item = SegmentMeta, Error = Canceled>> {
+        self.segment_updater.start_merge(segment_ids)
    }

    /// Closes the current document channel send.
@@ -546,8 +519,13 @@ impl IndexWriter {
    /// state as it was after the last commit.
    ///
    /// The opstamp at the last commit is returned.
-    pub fn rollback(&mut self) -> crate::Result<Opstamp> {
+    pub fn rollback(&mut self) -> Result<Opstamp> {
        info!("Rolling back to opstamp {}", self.committed_opstamp);
+        self.rollback_impl()
+    }
+
+    /// Private, implementation of rollback
+    fn rollback_impl(&mut self) -> Result<Opstamp> {
        // marks the segment updater as killed. From now on, all
        // segment updates will be ignored.
        self.segment_updater.kill();
@@ -603,7 +581,7 @@ impl IndexWriter {
    /// It is also possible to add a payload to the `commit`
    /// using this API.
    /// See [`PreparedCommit::set_payload()`](PreparedCommit.html)
-    pub fn prepare_commit(&mut self) -> crate::Result<PreparedCommit> {
+    pub fn prepare_commit(&mut self) -> Result<PreparedCommit<'_>> {
        // Here, because we join all of the worker threads,
        // all of the segment update for this commit have been
        // sent.
@@ -650,7 +628,7 @@ impl IndexWriter {
    /// Commit returns the `opstamp` of the last document
    /// that made it in the commit.
    ///
-    pub fn commit(&mut self) -> crate::Result<Opstamp> {
+    pub fn commit(&mut self) -> Result<Opstamp> {
        self.prepare_commit()?.commit()
    }

@@ -691,6 +669,9 @@ impl IndexWriter {
    /// The opstamp is an increasing `u64` that can
    /// be used by the client to align commits with its own
    /// document queue.
+    ///
+    /// Currently it represents the number of documents that
+    /// have been added since the creation of the index.
    pub fn add_document(&self, document: Document) -> Opstamp {
        let opstamp = self.stamper.stamp();
        let add_operation = AddOperation { opstamp, document };
@@ -751,7 +732,7 @@ impl IndexWriter {
                }
                UserOperation::Add(document) => {
                    let add_operation = AddOperation { opstamp, document };
-                    adds.push(add_operation);
+                    adds.push(add_operpation);
                }
            }
        }
@@ -764,16 +745,6 @@ impl IndexWriter {
    }
 }

-impl Drop for IndexWriter {
-    fn drop(&mut self) {
-        self.segment_updater.kill();
-        self.drop_sender();
-        for work in self.workers_join_handle.drain(..) {
-            let _ = work.join();
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {

@@ -783,7 +754,7 @@ mod tests {
    use crate::error::*;
    use crate::indexer::NoMergePolicy;
    use crate::query::TermQuery;
-    use crate::schema::{self, IndexRecordOption, STRING};
+    use crate::schema::{self, IndexRecordOption};
    use crate::Index;
    use crate::ReloadPolicy;
    use crate::Term;
@@ -897,7 +868,7 @@ mod tests {
        let index_writer = index.writer(3_000_000).unwrap();
        assert_eq!(
            format!("{:?}", index_writer.get_merge_policy()),
-            "LogMergePolicy { min_merge_size: 8, max_merge_size: 10000000, min_layer_size: 10000, \
+            "LogMergePolicy { min_merge_size: 8, min_layer_size: 10000, \
             level_log_size: 0.75 }"
        );
        let merge_policy = Box::new(NoMergePolicy::default());
@@ -1208,16 +1179,4 @@ mod tests {
        assert!(clear_again.is_ok());
        assert!(commit_again.is_ok());
    }
-
-    #[test]
-    fn test_index_doc_missing_field() {
-        let mut schema_builder = schema::Schema::builder();
-        let idfield = schema_builder.add_text_field("id", STRING);
-        schema_builder.add_text_field("optfield", STRING);
-        let index = Index::create_in_ram(schema_builder.build());
-        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-        index_writer.add_document(doc!(idfield=>"myid"));
-        let commit = index_writer.commit();
-        assert!(commit.is_ok());
-    }
 }
--- a/src/indexer/log_merge_policy.rs
+++ b/src/indexer/log_merge_policy.rs
@@ -6,14 +6,12 @@ use std::f64;
 const DEFAULT_LEVEL_LOG_SIZE: f64 = 0.75;
 const DEFAULT_MIN_LAYER_SIZE: u32 = 10_000;
 const DEFAULT_MIN_MERGE_SIZE: usize = 8;
-const DEFAULT_MAX_MERGE_SIZE: usize = 10_000_000;

 /// `LogMergePolicy` tries tries to merge segments that have a similar number of
 /// documents.
 #[derive(Debug, Clone)]
 pub struct LogMergePolicy {
    min_merge_size: usize,
-    max_merge_size: usize,
    min_layer_size: u32,
    level_log_size: f64,
 }
@@ -28,12 +26,6 @@ impl LogMergePolicy {
        self.min_merge_size = min_merge_size;
    }

-    /// Set the maximum number docs in a segment for it to be considered for
-    /// merging.
-    pub fn set_max_merge_size(&mut self, max_merge_size: usize) {
-        self.max_merge_size = max_merge_size;
-    }
-
    /// Set the minimum segment size under which all segment belong
    /// to the same level.
    pub fn set_min_layer_size(&mut self, min_layer_size: u32) {
@@ -61,7 +53,6 @@ impl MergePolicy for LogMergePolicy {
        let mut size_sorted_tuples = segments
            .iter()
            .map(SegmentMeta::num_docs)
-            .filter(|s| s <= &(self.max_merge_size as u32))
            .enumerate()
            .collect::<Vec<(usize, u32)>>();

@@ -95,7 +86,6 @@ impl Default for LogMergePolicy {
    fn default() -> LogMergePolicy {
        LogMergePolicy {
            min_merge_size: DEFAULT_MIN_MERGE_SIZE,
-            max_merge_size: DEFAULT_MAX_MERGE_SIZE,
            min_layer_size: DEFAULT_MIN_LAYER_SIZE,
            level_log_size: DEFAULT_LEVEL_LOG_SIZE,
        }
@@ -114,7 +104,6 @@ mod tests {
    fn test_merge_policy() -> LogMergePolicy {
        let mut log_merge_policy = LogMergePolicy::default();
        log_merge_policy.set_min_merge_size(3);
-        log_merge_policy.set_max_merge_size(100_000);
        log_merge_policy.set_min_layer_size(2);
        log_merge_policy
    }
@@ -152,11 +141,11 @@ mod tests {
            create_random_segment_meta(10),
            create_random_segment_meta(10),
            create_random_segment_meta(10),
-            create_random_segment_meta(1_000),
-            create_random_segment_meta(1_000),
-            create_random_segment_meta(1_000),
-            create_random_segment_meta(10_000),
-            create_random_segment_meta(10_000),
+            create_random_segment_meta(1000),
+            create_random_segment_meta(1000),
+            create_random_segment_meta(1000),
+            create_random_segment_meta(10000),
+            create_random_segment_meta(10000),
            create_random_segment_meta(10),
            create_random_segment_meta(10),
            create_random_segment_meta(10),
@@ -193,19 +182,4 @@ mod tests {
        let result_list = test_merge_policy().compute_merge_candidates(&test_input);
        assert_eq!(result_list.len(), 1);
    }
-
-    #[test]
-    fn test_large_merge_segments() {
-        let test_input = vec![
-            create_random_segment_meta(1_000_000),
-            create_random_segment_meta(100_001),
-            create_random_segment_meta(100_000),
-            create_random_segment_meta(100_000),
-            create_random_segment_meta(100_000),
-        ];
-        let result_list = test_merge_policy().compute_merge_candidates(&test_input);
-        // Do not include large segments
-        assert_eq!(result_list.len(), 1);
-        assert_eq!(result_list[0].0.len(), 3)
-    }
 }
--- a/src/indexer/merge_operation.rs
+++ b/src/indexer/merge_operation.rs
@@ -2,23 +2,18 @@ use crate::Opstamp;
 use crate::SegmentId;
 use census::{Inventory, TrackedObject};
 use std::collections::HashSet;
-use std::ops::Deref;

 #[derive(Default)]
-pub(crate) struct MergeOperationInventory(Inventory<InnerMergeOperation>);
-
-impl Deref for MergeOperationInventory {
-    type Target = Inventory<InnerMergeOperation>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
+pub struct MergeOperationInventory(Inventory<InnerMergeOperation>);

 impl MergeOperationInventory {
+    pub fn num_merge_operations(&self) -> usize {
+        self.0.list().len()
+    }
+
    pub fn segment_in_merge(&self) -> HashSet<SegmentId> {
        let mut segment_in_merge = HashSet::default();
-        for merge_op in self.list() {
+        for merge_op in self.0.list() {
            for &segment_id in &merge_op.segment_ids {
                segment_in_merge.insert(segment_id);
            }
@@ -44,13 +39,13 @@ pub struct MergeOperation {
    inner: TrackedObject<InnerMergeOperation>,
 }

-pub(crate) struct InnerMergeOperation {
+struct InnerMergeOperation {
    target_opstamp: Opstamp,
    segment_ids: Vec<SegmentId>,
 }

 impl MergeOperation {
-    pub(crate) fn new(
+    pub fn new(
        inventory: &MergeOperationInventory,
        target_opstamp: Opstamp,
        segment_ids: Vec<SegmentId>,
@@ -60,7 +55,7 @@ impl MergeOperation {
            segment_ids,
        };
        MergeOperation {
-            inner: inventory.track(inner_merge_operation),
+            inner: inventory.0.track(inner_merge_operation),
        }
    }

--- a/src/indexer/merge_policy.rs
+++ b/src/indexer/merge_policy.rs
@@ -12,6 +12,10 @@ pub struct MergeCandidate(pub Vec<SegmentId>);
 /// Every time a the list of segments changes, the segment updater
 /// asks the merge policy if some segments should be merged.
 pub trait MergePolicy: marker::Send + marker::Sync + Debug {
+    fn maximum_num_threads(&self) -> Option<usize> {
+        None
+    }
+
    /// Given the list of segment metas, returns the list of merge candidates.
    ///
    /// This call happens on the segment updater thread, and will block
--- a/src/indexer/merger.rs
+++ b/src/indexer/merger.rs
@@ -21,6 +21,8 @@ use crate::store::StoreWriter;
 use crate::termdict::TermMerger;
 use crate::termdict::TermOrdinal;
 use crate::DocId;
+use crate::Result;
+use crate::TantivyError;
 use itertools::Itertools;
 use std::cmp;
 use std::collections::HashMap;
@@ -141,7 +143,7 @@ impl DeltaComputer {
 }

 impl IndexMerger {
-    pub fn open(schema: Schema, segments: &[Segment]) -> crate::Result<IndexMerger> {
+    pub fn open(schema: Schema, segments: &[Segment]) -> Result<IndexMerger> {
        let mut readers = vec![];
        let mut max_doc: u32 = 0u32;
        for segment in segments {
@@ -157,7 +159,7 @@ impl IndexMerger {
                 which exceeds the limit {}.",
                max_doc, MAX_DOC_LIMIT
            );
-            return Err(crate::TantivyError::InvalidArgument(err_msg));
+            return Err(TantivyError::InvalidArgument(err_msg));
        }
        Ok(IndexMerger {
            schema,
@@ -166,10 +168,7 @@ impl IndexMerger {
        })
    }

-    fn write_fieldnorms(
-        &self,
-        fieldnorms_serializer: &mut FieldNormsSerializer,
-    ) -> crate::Result<()> {
+    fn write_fieldnorms(&self, fieldnorms_serializer: &mut FieldNormsSerializer) -> Result<()> {
        let fields = FieldNormsWriter::fields_with_fieldnorm(&self.schema);
        let mut fieldnorms_data = Vec::with_capacity(self.max_doc as usize);
        for field in fields {
@@ -190,7 +189,7 @@ impl IndexMerger {
        &self,
        fast_field_serializer: &mut FastFieldSerializer,
        mut term_ord_mappings: HashMap<Field, TermOrdinalMapping>,
-    ) -> crate::Result<()> {
+    ) -> Result<()> {
        for (field, field_entry) in self.schema.fields() {
            let field_type = field_entry.field_type();
            match *field_type {
@@ -235,7 +234,7 @@ impl IndexMerger {
        &self,
        field: Field,
        fast_field_serializer: &mut FastFieldSerializer,
-    ) -> crate::Result<()> {
+    ) -> Result<()> {
        let mut u64_readers = vec![];
        let mut min_value = u64::max_value();
        let mut max_value = u64::min_value();
@@ -285,7 +284,7 @@ impl IndexMerger {
        &self,
        field: Field,
        fast_field_serializer: &mut FastFieldSerializer,
-    ) -> crate::Result<()> {
+    ) -> Result<()> {
        let mut total_num_vals = 0u64;
        let mut u64s_readers: Vec<MultiValueIntFastFieldReader<u64>> = Vec::new();

@@ -332,7 +331,7 @@ impl IndexMerger {
        field: Field,
        term_ordinal_mappings: &TermOrdinalMapping,
        fast_field_serializer: &mut FastFieldSerializer,
-    ) -> crate::Result<()> {
+    ) -> Result<()> {
        // Multifastfield consists in 2 fastfields.
        // The first serves as an index into the second one and is stricly increasing.
        // The second contains the actual values.
@@ -372,7 +371,7 @@ impl IndexMerger {
        &self,
        field: Field,
        fast_field_serializer: &mut FastFieldSerializer,
-    ) -> crate::Result<()> {
+    ) -> Result<()> {
        // Multifastfield consists in 2 fastfields.
        // The first serves as an index into the second one and is stricly increasing.
        // The second contains the actual values.
@@ -437,7 +436,7 @@ impl IndexMerger {
        &self,
        field: Field,
        fast_field_serializer: &mut FastFieldSerializer,
-    ) -> crate::Result<()> {
+    ) -> Result<()> {
        let mut total_num_vals = 0u64;
        let mut bytes_readers: Vec<BytesFastFieldReader> = Vec::new();

@@ -493,7 +492,7 @@ impl IndexMerger {
        indexed_field: Field,
        field_type: &FieldType,
        serializer: &mut InvertedIndexSerializer,
-    ) -> crate::Result<Option<TermOrdinalMapping>> {
+    ) -> Result<Option<TermOrdinalMapping>> {
        let mut positions_buffer: Vec<u32> = Vec::with_capacity(1_000);
        let mut delta_computer = DeltaComputer::new();
        let field_readers = self
@@ -647,7 +646,7 @@ impl IndexMerger {
    fn write_postings(
        &self,
        serializer: &mut InvertedIndexSerializer,
-    ) -> crate::Result<HashMap<Field, TermOrdinalMapping>> {
+    ) -> Result<HashMap<Field, TermOrdinalMapping>> {
        let mut term_ordinal_mappings = HashMap::new();
        for (field, field_entry) in self.schema.fields() {
            if field_entry.is_indexed() {
@@ -661,7 +660,7 @@ impl IndexMerger {
        Ok(term_ordinal_mappings)
    }

-    fn write_storable_fields(&self, store_writer: &mut StoreWriter) -> crate::Result<()> {
+    fn write_storable_fields(&self, store_writer: &mut StoreWriter) -> Result<()> {
        for reader in &self.readers {
            let store_reader = reader.get_store_reader();
            if reader.num_deleted_docs() > 0 {
@@ -678,7 +677,7 @@ impl IndexMerger {
 }

 impl SerializableSegment for IndexMerger {
-    fn write(&self, mut serializer: SegmentSerializer) -> crate::Result<u32> {
+    fn write(&self, mut serializer: SegmentSerializer) -> Result<u32> {
        let term_ord_mappings = self.write_postings(serializer.get_postings_serializer())?;
        self.write_fieldnorms(serializer.get_fieldnorms_serializer())?;
        self.write_fast_fields(serializer.get_fast_field_serializer(), term_ord_mappings)?;
@@ -710,7 +709,7 @@ mod tests {
    use crate::IndexWriter;
    use crate::Searcher;
    use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
-    use futures::executor::block_on;
+    use futures::Future;
    use std::io::Cursor;

    #[test]
@@ -793,7 +792,11 @@ mod tests {
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-            block_on(index_writer.merge(&segment_ids)).expect("Merging failed");
+            index_writer
+                .merge(&segment_ids)
+                .expect("Failed to initiate merge")
+                .wait()
+                .expect("Merging failed");
            index_writer.wait_merging_threads().unwrap();
        }
        {
@@ -1037,7 +1040,11 @@ mod tests {
            let segment_ids = index
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
-            block_on(index_writer.merge(&segment_ids)).expect("Merging failed");
+            index_writer
+                .merge(&segment_ids)
+                .expect("Failed to initiate merge")
+                .wait()
+                .expect("Merging failed");
            reader.reload().unwrap();
            let searcher = reader.searcher();
            assert_eq!(searcher.segment_readers().len(), 1);
@@ -1132,7 +1139,11 @@ mod tests {
            let segment_ids = index
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
-            block_on(index_writer.merge(&segment_ids)).expect("Merging failed");
+            index_writer
+                .merge(&segment_ids)
+                .expect("Failed to initiate merge")
+                .wait()
+                .expect("Merging failed");
            reader.reload().unwrap();

            let searcher = reader.searcher();
@@ -1266,7 +1277,11 @@ mod tests {
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-            block_on(index_writer.merge(&segment_ids)).expect("Merging failed");
+            index_writer
+                .merge(&segment_ids)
+                .expect("Failed to initiate merge")
+                .wait()
+                .expect("Merging failed");
            index_writer.wait_merging_threads().unwrap();
            reader.reload().unwrap();
            test_searcher(
@@ -1321,7 +1336,11 @@ mod tests {
        let segment_ids = index
            .searchable_segment_ids()
            .expect("Searchable segments failed.");
-        block_on(index_writer.merge(&segment_ids)).expect("Merging failed");
+        index_writer
+            .merge(&segment_ids)
+            .expect("Failed to initiate merge")
+            .wait()
+            .expect("Merging failed");
        reader.reload().unwrap();
        // commit has not been called yet. The document should still be
        // there.
@@ -1342,18 +1361,22 @@ mod tests {
            let mut doc = Document::default();
            doc.add_u64(int_field, 1);
            index_writer.add_document(doc.clone());
-            assert!(index_writer.commit().is_ok());
+            index_writer.commit().expect("commit failed");
            index_writer.add_document(doc);
-            assert!(index_writer.commit().is_ok());
+            index_writer.commit().expect("commit failed");
            index_writer.delete_term(Term::from_field_u64(int_field, 1));

            let segment_ids = index
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
-            assert!(block_on(index_writer.merge(&segment_ids)).is_ok());
+            index_writer
+                .merge(&segment_ids)
+                .expect("Failed to initiate merge")
+                .wait()
+                .expect("Merging failed");

            // assert delete has not been committed
-            assert!(reader.reload().is_ok());
+            reader.reload().expect("failed to load searcher 1");
            let searcher = reader.searcher();
            assert_eq!(searcher.num_docs(), 2);

@@ -1392,12 +1415,12 @@ mod tests {
            index_doc(&mut index_writer, &[1, 5]);
            index_doc(&mut index_writer, &[3]);
            index_doc(&mut index_writer, &[17]);
-            assert!(index_writer.commit().is_ok());
+            index_writer.commit().expect("committed");
            index_doc(&mut index_writer, &[20]);
-            assert!(index_writer.commit().is_ok());
+            index_writer.commit().expect("committed");
            index_doc(&mut index_writer, &[28, 27]);
            index_doc(&mut index_writer, &[1_000]);
-            assert!(index_writer.commit().is_ok());
+            index_writer.commit().expect("committed");
        }
        let reader = index.reader().unwrap();
        let searcher = reader.searcher();
@@ -1429,6 +1452,15 @@ mod tests {
            assert_eq!(&vals, &[17]);
        }

+        println!(
+            "{:?}",
+            searcher
+                .segment_readers()
+                .iter()
+                .map(|reader| reader.max_doc())
+                .collect::<Vec<_>>()
+        );
+
        {
            let segment = searcher.segment_reader(1u32);
            let ff_reader = segment.fast_fields().u64s(int_field).unwrap();
@@ -1452,13 +1484,27 @@ mod tests {
                .searchable_segment_ids()
                .expect("Searchable segments failed.");
            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-            assert!(block_on(index_writer.merge(&segment_ids)).is_ok());
-            assert!(index_writer.wait_merging_threads().is_ok());
+            index_writer
+                .merge(&segment_ids)
+                .expect("Failed to initiate merge")
+                .wait()
+                .expect("Merging failed");
+            index_writer
+                .wait_merging_threads()
+                .expect("Wait for merging threads");
        }
-        assert!(reader.reload().is_ok());
+        reader.reload().expect("Load searcher");

        {
            let searcher = reader.searcher();
+            println!(
+                "{:?}",
+                searcher
+                    .segment_readers()
+                    .iter()
+                    .map(|reader| reader.max_doc())
+                    .collect::<Vec<_>>()
+            );
            let segment = searcher.segment_reader(0u32);
            let ff_reader = segment.fast_fields().u64s(int_field).unwrap();

@@ -1493,46 +1539,4 @@ mod tests {
            assert_eq!(&vals, &[20]);
        }
    }
-
-    #[test]
-    fn merges_f64_fast_fields_correctly() -> crate::Result<()> {
-        let mut builder = schema::SchemaBuilder::new();
-
-        let fast_multi = IntOptions::default().set_fast(Cardinality::MultiValues);
-
-        let field = builder.add_f64_field("f64", schema::FAST);
-        let multi_field = builder.add_f64_field("f64s", fast_multi);
-
-        let index = Index::create_in_ram(builder.build());
-
-        let mut writer = index.writer_with_num_threads(1, 3_000_000)?;
-
-        // Make sure we'll attempt to merge every created segment
-        let mut policy = crate::indexer::LogMergePolicy::default();
-        policy.set_min_merge_size(2);
-        writer.set_merge_policy(Box::new(policy));
-
-        for i in 0..100 {
-            let mut doc = Document::new();
-            doc.add_f64(field, 42.0);
-
-            doc.add_f64(multi_field, 0.24);
-            doc.add_f64(multi_field, 0.27);
-
-            writer.add_document(doc);
-
-            if i % 5 == 0 {
-                writer.commit()?;
-            }
-        }
-
-        writer.commit()?;
-        writer.wait_merging_threads()?;
-
-        // If a merging thread fails, we should end up with more
-        // than one segment here
-        assert_eq!(1, index.searchable_segments()?.len());
-
-        Ok(())
-    }
 }
--- a/src/indexer/mod.rs
+++ b/src/indexer/mod.rs
@@ -18,7 +18,7 @@ mod stamper;

 pub use self::index_writer::IndexWriter;
 pub use self::log_merge_policy::LogMergePolicy;
-pub use self::merge_operation::MergeOperation;
+pub use self::merge_operation::{MergeOperation, MergeOperationInventory};
 pub use self::merge_policy::{MergeCandidate, MergePolicy, NoMergePolicy};
 pub use self::prepared_commit::PreparedCommit;
 pub use self::segment_entry::SegmentEntry;
@@ -28,26 +28,3 @@ pub use self::segment_writer::SegmentWriter;

 /// Alias for the default merge policy, which is the `LogMergePolicy`.
 pub type DefaultMergePolicy = LogMergePolicy;
-
-#[cfg(test)]
-mod tests {
-    use crate::schema::{self, Schema};
-    use crate::{Index, Term};
-
-    #[test]
-    fn test_advance_delete_bug() {
-        let mut schema_builder = Schema::builder();
-        let text_field = schema_builder.add_text_field("text", schema::TEXT);
-        let index = Index::create_from_tempdir(schema_builder.build()).unwrap();
-        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-        // there must be one deleted document in the segment
-        index_writer.add_document(doc!(text_field=>"b"));
-        index_writer.delete_term(Term::from_field_text(text_field, "b"));
-        // we need enough data to trigger the bug (at least 32 documents)
-        for _ in 0..32 {
-            index_writer.add_document(doc!(text_field=>"c"));
-        }
-        index_writer.commit().unwrap();
-        index_writer.commit().unwrap();
-    }
-}
--- a/src/indexer/operation.rs
+++ b/src/indexer/operation.rs
@@ -19,8 +19,6 @@ pub struct AddOperation {
 /// UserOperation is an enum type that encapsulates other operation types.
 #[derive(Eq, PartialEq, Debug)]
 pub enum UserOperation {
-    /// Add operation
    Add(Document),
-    /// Delete operation
    Delete(Term),
 }
--- a/src/indexer/prepared_commit.rs
+++ b/src/indexer/prepared_commit.rs
@@ -1,6 +1,6 @@
 use super::IndexWriter;
 use crate::Opstamp;
-use futures::executor::block_on;
+use crate::Result;

 /// A prepared commit
 pub struct PreparedCommit<'a> {
@@ -26,17 +26,15 @@ impl<'a> PreparedCommit<'a> {
        self.payload = Some(payload.to_string())
    }

-    pub fn abort(self) -> crate::Result<Opstamp> {
+    pub fn abort(self) -> Result<Opstamp> {
        self.index_writer.rollback()
    }

-    pub fn commit(self) -> crate::Result<Opstamp> {
+    pub fn commit(self) -> Result<Opstamp> {
        info!("committing {}", self.opstamp);
-        let _ = block_on(
-            self.index_writer
-                .segment_updater()
-                .schedule_commit(self.opstamp, self.payload),
-        );
+        self.index_writer
+            .segment_updater()
+            .commit(self.opstamp, self.payload)?;
        Ok(self.opstamp)
    }
 }
--- a/src/indexer/segment_entry.rs
+++ b/src/indexer/segment_entry.rs
@@ -1,7 +1,7 @@
-use crate::common::BitSet;
 use crate::core::SegmentId;
 use crate::core::SegmentMeta;
 use crate::indexer::delete_queue::DeleteCursor;
+use bit_set::BitSet;
 use std::fmt;

 /// A segment entry describes the state of
--- a/src/indexer/segment_manager.rs
+++ b/src/indexer/segment_manager.rs
@@ -4,6 +4,7 @@ use crate::core::SegmentMeta;
 use crate::error::TantivyError;
 use crate::indexer::delete_queue::DeleteCursor;
 use crate::indexer::SegmentEntry;
+use crate::Result as TantivyResult;
 use std::collections::hash_set::HashSet;
 use std::fmt::{self, Debug, Formatter};
 use std::sync::RwLock;
@@ -15,28 +16,6 @@ struct SegmentRegisters {
    committed: SegmentRegister,
 }

-#[derive(PartialEq, Eq)]
-pub(crate) enum SegmentsStatus {
-    Committed,
-    Uncommitted,
-}
-
-impl SegmentRegisters {
-    /// Check if all the segments are committed or uncommited.
-    ///
-    /// If some segment is missing or segments are in a different state (this should not happen
-    /// if tantivy is used correctly), returns `None`.
-    fn segments_status(&self, segment_ids: &[SegmentId]) -> Option<SegmentsStatus> {
-        if self.uncommitted.contains_all(segment_ids) {
-            Some(SegmentsStatus::Uncommitted)
-        } else if self.committed.contains_all(segment_ids) {
-            Some(SegmentsStatus::Committed)
-        } else {
-            None
-        }
-    }
-}
-
 /// The segment manager stores the list of segments
 /// as well as their state.
 ///
@@ -48,7 +27,7 @@ pub struct SegmentManager {
 }

 impl Debug for SegmentManager {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> {
        let lock = self.read();
        write!(
            f,
@@ -144,7 +123,7 @@ impl SegmentManager {
    /// Returns an error if some segments are missing, or if
    /// the `segment_ids` are not either all committed or all
    /// uncommitted.
-    pub fn start_merge(&self, segment_ids: &[SegmentId]) -> crate::Result<Vec<SegmentEntry>> {
+    pub fn start_merge(&self, segment_ids: &[SegmentId]) -> TantivyResult<Vec<SegmentEntry>> {
        let registers_lock = self.read();
        let mut segment_entries = vec![];
        if registers_lock.uncommitted.contains_all(segment_ids) {
@@ -174,35 +153,33 @@ impl SegmentManager {
        let mut registers_lock = self.write();
        registers_lock.uncommitted.add_segment_entry(segment_entry);
    }
-    // Replace a list of segments for their equivalent merged segment.
-    //
-    // Returns true if these segments are committed, false if the merge segments are uncommited.
-    pub(crate) fn end_merge(
+
+    pub fn end_merge(
        &self,
        before_merge_segment_ids: &[SegmentId],
        after_merge_segment_entry: SegmentEntry,
-    ) -> crate::Result<SegmentsStatus> {
+    ) {
        let mut registers_lock = self.write();
-        let segments_status = registers_lock
-            .segments_status(before_merge_segment_ids)
-            .ok_or_else(|| {
+        let target_register: &mut SegmentRegister = {
+            if registers_lock
+                .uncommitted
+                .contains_all(before_merge_segment_ids)
+            {
+                &mut registers_lock.uncommitted
+            } else if registers_lock
+                .committed
+                .contains_all(before_merge_segment_ids)
+            {
+                &mut registers_lock.committed
+            } else {
                warn!("couldn't find segment in SegmentManager");
-                crate::TantivyError::InvalidArgument(
-                    "The segments that were merged could not be found in the SegmentManager. \
-                     This is not necessarily a bug, and can happen after a rollback for instance."
-                        .to_string(),
-                )
-            })?;
-
-        let target_register: &mut SegmentRegister = match segments_status {
-            SegmentsStatus::Uncommitted => &mut registers_lock.uncommitted,
-            SegmentsStatus::Committed => &mut registers_lock.committed,
+                return;
+            }
        };
        for segment_id in before_merge_segment_ids {
            target_register.remove_segment(segment_id);
        }
        target_register.add_segment_entry(after_merge_segment_entry);
-        Ok(segments_status)
    }

    pub fn committed_segment_metas(&self) -> Vec<SegmentMeta> {
--- a/src/indexer/segment_serializer.rs
+++ b/src/indexer/segment_serializer.rs
@@ -1,3 +1,5 @@
+use crate::Result;
+
 use crate::core::Segment;
 use crate::core::SegmentComponent;
 use crate::fastfield::FastFieldSerializer;
@@ -16,7 +18,7 @@ pub struct SegmentSerializer {

 impl SegmentSerializer {
    /// Creates a new `SegmentSerializer`.
-    pub fn for_segment(segment: &mut Segment) -> crate::Result<SegmentSerializer> {
+    pub fn for_segment(segment: &mut Segment) -> Result<SegmentSerializer> {
        let store_write = segment.open_write(SegmentComponent::STORE)?;

        let fast_field_write = segment.open_write(SegmentComponent::FASTFIELDS)?;
@@ -55,7 +57,7 @@ impl SegmentSerializer {
    }

    /// Finalize the segment serialization.
-    pub fn close(self) -> crate::Result<()> {
+    pub fn close(self) -> Result<()> {
        self.fast_field_serializer.close()?;
        self.postings_serializer.close()?;
        self.store_writer.close()?;
--- a/src/indexer/segment_updater.rs
+++ b/src/indexer/segment_updater.rs
@@ -6,34 +6,40 @@ use crate::core::SegmentId;
 use crate::core::SegmentMeta;
 use crate::core::SerializableSegment;
 use crate::core::META_FILEPATH;
-use crate::directory::{Directory, DirectoryClone, GarbageCollectionResult};
+use crate::directory::{Directory, DirectoryClone};
+use crate::error::TantivyError;
 use crate::indexer::delete_queue::DeleteCursor;
 use crate::indexer::index_writer::advance_deletes;
 use crate::indexer::merge_operation::MergeOperationInventory;
 use crate::indexer::merger::IndexMerger;
-use crate::indexer::segment_manager::SegmentsStatus;
 use crate::indexer::stamper::Stamper;
+use crate::indexer::MergeOperation;
 use crate::indexer::SegmentEntry;
 use crate::indexer::SegmentSerializer;
 use crate::indexer::{DefaultMergePolicy, MergePolicy};
-use crate::indexer::{MergeCandidate, MergeOperation};
 use crate::schema::Schema;
 use crate::Opstamp;
-use futures::channel::oneshot;
-use futures::executor::{ThreadPool, ThreadPoolBuilder};
-use futures::future::Future;
-use futures::future::TryFutureExt;
+use crate::Result;
+use futures::oneshot;
+use futures::sync::oneshot::Receiver;
+use futures::Future;
+use futures_cpupool::Builder as CpuPoolBuilder;
+use futures_cpupool::CpuFuture;
+use futures_cpupool::CpuPool;
 use serde_json;
 use std::borrow::BorrowMut;
+use std::collections::HashMap;
 use std::collections::HashSet;
 use std::io::Write;
-use std::ops::Deref;
+use std::mem;
+use std::ops::DerefMut;
 use std::path::PathBuf;
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::sync::RwLock;
-
-const NUM_MERGE_THREADS: usize = 4;
+use std::thread;
+use std::thread::JoinHandle;
+use std::time::Duration;

 /// Save the index meta file.
 /// This operation is atomic :
@@ -44,7 +50,7 @@ const NUM_MERGE_THREADS: usize = 4;
 /// and flushed.
 ///
 /// This method is not part of tantivy's public API
-pub fn save_new_metas(schema: Schema, directory: &mut dyn Directory) -> crate::Result<()> {
+pub fn save_new_metas(schema: Schema, directory: &mut dyn Directory) -> Result<()> {
    save_metas(
        &IndexMeta {
            segments: Vec::new(),
@@ -65,7 +71,7 @@ pub fn save_new_metas(schema: Schema, directory: &mut dyn Directory) -> crate::R
 /// and flushed.
 ///
 /// This method is not part of tantivy's public API
-fn save_metas(metas: &IndexMeta, directory: &mut dyn Directory) -> crate::Result<()> {
+fn save_metas(metas: &IndexMeta, directory: &mut dyn Directory) -> Result<()> {
    info!("save metas");
    let mut buffer = serde_json::to_vec_pretty(metas)?;
    // Just adding a new line at the end of the buffer.
@@ -84,38 +90,21 @@ fn save_metas(metas: &IndexMeta, directory: &mut dyn Directory) -> crate::Result
 // We voluntarily pass a merge_operation ref to guarantee that
 // the merge_operation is alive during the process
 #[derive(Clone)]
-pub(crate) struct SegmentUpdater(Arc<InnerSegmentUpdater>);
+pub struct SegmentUpdater(Arc<InnerSegmentUpdater>);

-impl Deref for SegmentUpdater {
-    type Target = InnerSegmentUpdater;
-
-    #[inline]
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-async fn garbage_collect_files(
-    segment_updater: SegmentUpdater,
-) -> crate::Result<GarbageCollectionResult> {
-    info!("Running garbage collection");
-    let mut index = segment_updater.index.clone();
-    index
-        .directory_mut()
-        .garbage_collect(move || segment_updater.list_files())
-}
-
-/// Merges a list of segments the list of segment givens in the `segment_entries`.
-/// This function happens in the calling thread and is computationally expensive.
-fn merge(
+fn perform_merge(
+    merge_operation: &MergeOperation,
    index: &Index,
    mut segment_entries: Vec<SegmentEntry>,
-    target_opstamp: Opstamp,
-) -> crate::Result<SegmentEntry> {
+) -> Result<SegmentEntry> {
+    let target_opstamp = merge_operation.target_opstamp();
+
    // first we need to apply deletes to our segment.
    let mut merged_segment = index.new_segment();

-    // First we apply all of the delet to the merged segment, up to the target opstamp.
+    // TODO add logging
+    let schema = index.schema();
+
    for segment_entry in &mut segment_entries {
        let segment = index.segment(segment_entry.meta().clone());
        advance_deletes(segment, segment_entry, target_opstamp)?;
@@ -129,19 +118,22 @@ fn merge(
        .collect();

    // An IndexMerger is like a "view" of our merged segments.
-    let merger: IndexMerger = IndexMerger::open(index.schema(), &segments[..])?;
+    let merger: IndexMerger = IndexMerger::open(schema, &segments[..])?;
+
+    // ... we just serialize this index merger in our new segment
+    // to merge the two segments.

-    // ... we just serialize this index merger in our new segment to merge the two segments.
    let segment_serializer = SegmentSerializer::for_segment(&mut merged_segment)?;

    let num_docs = merger.write(segment_serializer)?;

    let segment_meta = index.new_segment_meta(merged_segment.id(), num_docs);

-    Ok(SegmentEntry::new(segment_meta, delete_cursor, None))
+    let after_merge_segment_entry = SegmentEntry::new(segment_meta.clone(), delete_cursor, None);
+    Ok(after_merge_segment_entry)
 }

-pub(crate) struct InnerSegmentUpdater {
+struct InnerSegmentUpdater {
    // we keep a copy of the current active IndexMeta to
    // avoid loading the file everytime we need it in the
    // `SegmentUpdater`.
@@ -149,12 +141,12 @@ pub(crate) struct InnerSegmentUpdater {
    // This should be up to date as all update happen through
    // the unique active `SegmentUpdater`.
    active_metas: RwLock<Arc<IndexMeta>>,
-    pool: ThreadPool,
-    merge_thread_pool: ThreadPool,
-
+    pool: CpuPool,
    index: Index,
    segment_manager: SegmentManager,
    merge_policy: RwLock<Arc<Box<dyn MergePolicy>>>,
+    merging_thread_id: AtomicUsize,
+    merging_threads: RwLock<HashMap<usize, JoinHandle<Result<()>>>>,
    killed: AtomicBool,
    stamper: Stamper,
    merge_operations: MergeOperationInventory,
@@ -165,35 +157,22 @@ impl SegmentUpdater {
        index: Index,
        stamper: Stamper,
        delete_cursor: &DeleteCursor,
-    ) -> crate::Result<SegmentUpdater> {
+    ) -> Result<SegmentUpdater> {
        let segments = index.searchable_segment_metas()?;
        let segment_manager = SegmentManager::from_segments(segments, delete_cursor);
-        let pool = ThreadPoolBuilder::new()
+        let pool = CpuPoolBuilder::new()
            .name_prefix("segment_updater")
            .pool_size(1)
-            .create()
-            .map_err(|_| {
-                crate::TantivyError::SystemError(
-                    "Failed to spawn segment updater thread".to_string(),
-                )
-            })?;
-        let merge_thread_pool = ThreadPoolBuilder::new()
-            .name_prefix("merge_thread")
-            .pool_size(NUM_MERGE_THREADS)
-            .create()
-            .map_err(|_| {
-                crate::TantivyError::SystemError(
-                    "Failed to spawn segment merging thread".to_string(),
-                )
-            })?;
+            .create();
        let index_meta = index.load_metas()?;
        Ok(SegmentUpdater(Arc::new(InnerSegmentUpdater {
            active_metas: RwLock::new(Arc::new(index_meta)),
            pool,
-            merge_thread_pool,
            index,
            segment_manager,
            merge_policy: RwLock::new(Arc::new(Box::new(DefaultMergePolicy::default()))),
+            merging_thread_id: AtomicUsize::default(),
+            merging_threads: RwLock::new(HashMap::new()),
            killed: AtomicBool::new(false),
            stamper,
            merge_operations: Default::default(),
@@ -201,82 +180,73 @@ impl SegmentUpdater {
    }

    pub fn get_merge_policy(&self) -> Arc<Box<dyn MergePolicy>> {
-        self.merge_policy.read().unwrap().clone()
+        self.0.merge_policy.read().unwrap().clone()
    }

    pub fn set_merge_policy(&self, merge_policy: Box<dyn MergePolicy>) {
        let arc_merge_policy = Arc::new(merge_policy);
-        *self.merge_policy.write().unwrap() = arc_merge_policy;
+        *self.0.merge_policy.write().unwrap() = arc_merge_policy;
    }

-    fn schedule_future<T: 'static + Send, F: Future<Output = crate::Result<T>> + 'static + Send>(
+    fn get_merging_thread_id(&self) -> usize {
+        self.0.merging_thread_id.fetch_add(1, Ordering::SeqCst)
+    }
+
+    fn run_async<T: 'static + Send, F: 'static + Send + FnOnce(SegmentUpdater) -> T>(
        &self,
        f: F,
-    ) -> impl Future<Output = crate::Result<T>> {
-        let (sender, receiver) = oneshot::channel();
-        if self.is_alive() {
-            self.pool.spawn_ok(async move {
-                let _ = sender.send(f.await);
-            });
-        } else {
-            let _ = sender.send(Err(crate::TantivyError::SystemError(
-                "Segment updater killed".to_string(),
-            )));
-        }
-        receiver.unwrap_or_else(|_| {
-            let err_msg =
-                "A segment_updater future did not success. This should never happen.".to_string();
-            Err(crate::TantivyError::SystemError(err_msg))
-        })
+    ) -> CpuFuture<T, TantivyError> {
+        let me_clone = self.clone();
+        self.0.pool.spawn_fn(move || Ok(f(me_clone)))
    }

-    pub fn schedule_add_segment(
-        &self,
-        segment_entry: SegmentEntry,
-    ) -> impl Future<Output = crate::Result<()>> {
-        let segment_updater = self.clone();
-        self.schedule_future(async move {
-            segment_updater.segment_manager.add_segment(segment_entry);
-            segment_updater.consider_merge_options().await;
-            Ok(())
+    pub fn add_segment(&self, segment_entry: SegmentEntry) -> bool {
+        let max_num_threads_opt = self.0.merge_policy.read().unwrap().maximum_num_threads();
+        if let Some(max_num_threads) = max_num_threads_opt {
+            while self.0.merge_operations.num_merge_operations() >= max_num_threads_opt {
+                std::thread::sleep(Duration::from_secs(1u64));
+            }
+        }
+        self.run_async(|segment_updater| {
+            segment_updater.0.segment_manager.add_segment(segment_entry);
+            segment_updater.consider_merge_options();
+            true
        })
+        .forget();
+        true
    }

    /// Orders `SegmentManager` to remove all segments
    pub(crate) fn remove_all_segments(&self) {
-        self.segment_manager.remove_all_segments();
+        self.0.segment_manager.remove_all_segments();
    }

    pub fn kill(&mut self) {
-        self.killed.store(true, Ordering::Release);
+        self.0.killed.store(true, Ordering::Release);
    }

    pub fn is_alive(&self) -> bool {
-        !self.killed.load(Ordering::Acquire)
+        !self.0.killed.load(Ordering::Acquire)
    }

    /// Apply deletes up to the target opstamp to all segments.
    ///
    /// The method returns copies of the segment entries,
    /// updated with the delete information.
-    fn purge_deletes(&self, target_opstamp: Opstamp) -> crate::Result<Vec<SegmentEntry>> {
-        let mut segment_entries = self.segment_manager.segment_entries();
+    fn purge_deletes(&self, target_opstamp: Opstamp) -> Result<Vec<SegmentEntry>> {
+        let mut segment_entries = self.0.segment_manager.segment_entries();
        for segment_entry in &mut segment_entries {
-            let segment = self.index.segment(segment_entry.meta().clone());
+            let segment = self.0.index.segment(segment_entry.meta().clone());
            advance_deletes(segment, segment_entry, target_opstamp)?;
        }
        Ok(segment_entries)
    }

-    pub fn save_metas(
-        &self,
-        opstamp: Opstamp,
-        commit_message: Option<String>,
-    ) -> crate::Result<()> {
+    pub fn save_metas(&self, opstamp: Opstamp, commit_message: Option<String>) {
        if self.is_alive() {
-            let index = &self.index;
+            let index = &self.0.index;
            let directory = index.directory();
-            let mut commited_segment_metas = self.segment_manager.committed_segment_metas();
+            let mut commited_segment_metas = self.0.segment_manager.committed_segment_metas();

            // We sort segment_readers by number of documents.
            // This is an heuristic to make multithreading more efficient.
@@ -298,18 +268,16 @@ impl SegmentUpdater {
                opstamp,
                payload: commit_message,
            };
-            // TODO add context to the error.
-            save_metas(&index_meta, directory.box_clone().borrow_mut())?;
+            save_metas(&index_meta, directory.box_clone().borrow_mut())
+                .expect("Could not save metas.");
            self.store_meta(&index_meta);
        }
-        Ok(())
    }

-    pub fn schedule_garbage_collect(
-        &self,
-    ) -> impl Future<Output = crate::Result<GarbageCollectionResult>> {
-        let garbage_collect_future = garbage_collect_files(self.clone());
-        self.schedule_future(garbage_collect_future)
+    pub fn garbage_collect_files(&self) -> CpuFuture<(), TantivyError> {
+        self.run_async(move |segment_updater| {
+            segment_updater.garbage_collect_files_exec();
+        })
    }

    /// List the files that are useful to the index.
@@ -317,130 +285,148 @@ impl SegmentUpdater {
    /// This does not include lock files, or files that are obsolete
    /// but have not yet been deleted by the garbage collector.
    fn list_files(&self) -> HashSet<PathBuf> {
-        let mut files: HashSet<PathBuf> = self
-            .index
-            .list_all_segment_metas()
-            .into_iter()
-            .flat_map(|segment_meta| segment_meta.list_files())
-            .collect();
+        let mut files = HashSet::new();
        files.insert(META_FILEPATH.to_path_buf());
+        for segment_meta in self.0.index.list_all_segment_metas() {
+            files.extend(segment_meta.list_files());
+        }
        files
    }

-    pub fn schedule_commit(
-        &self,
-        opstamp: Opstamp,
-        payload: Option<String>,
-    ) -> impl Future<Output = crate::Result<()>> {
-        let segment_updater: SegmentUpdater = self.clone();
-        self.schedule_future(async move {
-            let segment_entries = segment_updater.purge_deletes(opstamp)?;
-            segment_updater.segment_manager.commit(segment_entries);
-            segment_updater.save_metas(opstamp, payload)?;
-            let _ = garbage_collect_files(segment_updater.clone()).await;
-            segment_updater.consider_merge_options().await;
-            Ok(())
+    fn garbage_collect_files_exec(&self) {
+        info!("Running garbage collection");
+        let mut index = self.0.index.clone();
+        index.directory_mut().garbage_collect(|| self.list_files());
+    }
+
+    pub fn commit(&self, opstamp: Opstamp, payload: Option<String>) -> Result<()> {
+        self.run_async(move |segment_updater| {
+            if segment_updater.is_alive() {
+                let segment_entries = segment_updater
+                    .purge_deletes(opstamp)
+                    .expect("Failed purge deletes");
+                segment_updater.0.segment_manager.commit(segment_entries);
+                segment_updater.save_metas(opstamp, payload);
+                segment_updater.garbage_collect_files_exec();
+                segment_updater.consider_merge_options();
+            }
        })
+        .wait()
+    }
+
+    pub fn start_merge(&self, segment_ids: &[SegmentId]) -> Result<Receiver<SegmentMeta>> {
+        let commit_opstamp = self.load_metas().opstamp;
+        let merge_operation = MergeOperation::new(
+            &self.0.merge_operations,
+            commit_opstamp,
+            segment_ids.to_vec(),
+        );
+        self.run_async(move |segment_updater| segment_updater.start_merge_impl(merge_operation))
+            .wait()?
    }

    fn store_meta(&self, index_meta: &IndexMeta) {
-        *self.active_metas.write().unwrap() = Arc::new(index_meta.clone());
+        *self.0.active_metas.write().unwrap() = Arc::new(index_meta.clone());
    }
-
    fn load_metas(&self) -> Arc<IndexMeta> {
-        self.active_metas.read().unwrap().clone()
+        self.0.active_metas.read().unwrap().clone()
    }

-    pub(crate) fn make_merge_operation(&self, segment_ids: &[SegmentId]) -> MergeOperation {
-        let commit_opstamp = self.load_metas().opstamp;
-        MergeOperation::new(&self.merge_operations, commit_opstamp, segment_ids.to_vec())
-    }
-
-    // Starts a merge operation. This function will block until the merge operation is effectively
-    // started. Note that it does not wait for the merge to terminate.
-    // The calling thread should not be block for a long time, as this only involve waiting for the
-    // `SegmentUpdater` queue which in turns only contains lightweight operations.
-    //
-    // The merge itself happens on a different thread.
-    //
-    // When successful, this function returns a `Future` for a `Result<SegmentMeta>` that represents
-    // the actual outcome of the merge operation.
-    //
-    // It returns an error if for some reason the merge operation could not be started.
-    //
-    // At this point an error is not necessarily the sign of a malfunction.
-    // (e.g. A rollback could have happened, between the instant when the merge operaiton was
-    // suggested and the moment when it ended up being executed.)
-    //
    // `segment_ids` is required to be non-empty.
-    pub fn start_merge(
-        &self,
-        merge_operation: MergeOperation,
-    ) -> crate::Result<impl Future<Output = crate::Result<SegmentMeta>>> {
+    fn start_merge_impl(&self, merge_operation: MergeOperation) -> Result<Receiver<SegmentMeta>> {
        assert!(
            !merge_operation.segment_ids().is_empty(),
            "Segment_ids cannot be empty."
        );

-        let segment_updater = self.clone();
+        let segment_updater_clone = self.clone();
        let segment_entries: Vec<SegmentEntry> = self
+            .0
            .segment_manager
            .start_merge(merge_operation.segment_ids())?;

-        info!("Starting merge  - {:?}", merge_operation.segment_ids());
+        //        let segment_ids_vec = merge_operation.segment_ids.to_vec();

-        let (merging_future_send, merging_future_recv) =
-            oneshot::channel::<crate::Result<SegmentMeta>>();
+        let merging_thread_id = self.get_merging_thread_id();
+        info!(
+            "Starting merge thread #{} - {:?}",
+            merging_thread_id,
+            merge_operation.segment_ids()
+        );
+        let (merging_future_send, merging_future_recv) = oneshot();

-        self.merge_thread_pool.spawn_ok(async move {
-            // The fact that `merge_operation` is moved here is important.
-            // Its lifetime is used to track how many merging thread are currently running,
-            // as well as which segment is currently in merge and therefore should not be
-            // candidate for another merge.
-            match merge(
-                &segment_updater.index,
-                segment_entries,
-                merge_operation.target_opstamp(),
-            ) {
-                Ok(after_merge_segment_entry) => {
-                    let segment_meta = segment_updater
-                        .end_merge(merge_operation, after_merge_segment_entry)
-                        .await;
-                    let _send_result = merging_future_send.send(segment_meta);
-                }
-                Err(e) => {
-                    warn!(
-                        "Merge of {:?} was cancelled: {:?}",
-                        merge_operation.segment_ids().to_vec(),
-                        e
-                    );
-                    // ... cancel merge
-                    if cfg!(test) {
-                        panic!("Merge failed.");
+        // first we need to apply deletes to our segment.
+        let merging_join_handle = thread::Builder::new()
+            .name(format!("mergingthread-{}", merging_thread_id))
+            .spawn(move || {
+                // first we need to apply deletes to our segment.
+                let merge_result = perform_merge(
+                    &merge_operation,
+                    &segment_updater_clone.0.index,
+                    segment_entries,
+                );
+
+                match merge_result {
+                    Ok(after_merge_segment_entry) => {
+                        let merged_segment_meta = after_merge_segment_entry.meta().clone();
+                        segment_updater_clone
+                            .end_merge(merge_operation, after_merge_segment_entry)
+                            .expect("Segment updater thread is corrupted.");
+
+                        // the future may fail if the listener of the oneshot future
+                        // has been destroyed.
+                        //
+                        // This is not a problem here, so we just ignore any
+                        // possible error.
+                        let _merging_future_res = merging_future_send.send(merged_segment_meta);
+                    }
+                    Err(e) => {
+                        warn!(
+                            "Merge of {:?} was cancelled: {:?}",
+                            merge_operation.segment_ids(),
+                            e
+                        );
+                        // ... cancel merge
+                        if cfg!(test) {
+                            panic!("Merge failed.");
+                        }
+                        // As `merge_operation` will be dropped, the segment in merge state will
+                        // be available for merge again.
+                        // `merging_future_send` will be dropped, sending an error to the future.
                    }
                }
-            }
-        });
-
-        Ok(merging_future_recv
-            .unwrap_or_else(|_| Err(crate::TantivyError::SystemError("Merge failed".to_string()))))
+                segment_updater_clone
+                    .0
+                    .merging_threads
+                    .write()
+                    .unwrap()
+                    .remove(&merging_thread_id);
+                Ok(())
+            })
+            .expect("Failed to spawn a thread.");
+        self.0
+            .merging_threads
+            .write()
+            .unwrap()
+            .insert(merging_thread_id, merging_join_handle);
+        Ok(merging_future_recv)
    }

-    async fn consider_merge_options(&self) {
-        let merge_segment_ids: HashSet<SegmentId> = self.merge_operations.segment_in_merge();
+    fn consider_merge_options(&self) {
+        let merge_segment_ids: HashSet<SegmentId> = self.0.merge_operations.segment_in_merge();
        let (committed_segments, uncommitted_segments) =
-            get_mergeable_segments(&merge_segment_ids, &self.segment_manager);
+            get_mergeable_segments(&merge_segment_ids, &self.0.segment_manager);

        // Committed segments cannot be merged with uncommitted_segments.
        // We therefore consider merges using these two sets of segments independently.
        let merge_policy = self.get_merge_policy();

-        let current_opstamp = self.stamper.stamp();
+        let current_opstamp = self.0.stamper.stamp();
        let mut merge_candidates: Vec<MergeOperation> = merge_policy
            .compute_merge_candidates(&uncommitted_segments)
            .into_iter()
            .map(|merge_candidate| {
-                MergeOperation::new(&self.merge_operations, current_opstamp, merge_candidate.0)
+                MergeOperation::new(&self.0.merge_operations, current_opstamp, merge_candidate.0)
            })
            .collect();

@@ -448,18 +434,25 @@ impl SegmentUpdater {
        let committed_merge_candidates = merge_policy
            .compute_merge_candidates(&committed_segments)
            .into_iter()
-            .map(|merge_candidate: MergeCandidate| {
-                MergeOperation::new(&self.merge_operations, commit_opstamp, merge_candidate.0)
+            .map(|merge_candidate| {
+                MergeOperation::new(&self.0.merge_operations, commit_opstamp, merge_candidate.0)
            })
            .collect::<Vec<_>>();
        merge_candidates.extend(committed_merge_candidates.into_iter());

        for merge_operation in merge_candidates {
-            if let Err(err) = self.start_merge(merge_operation) {
-                warn!(
-                    "Starting the merge failed for the following reason. This is not fatal. {}",
-                    err
-                );
+            match self.start_merge_impl(merge_operation) {
+                Ok(merge_future) => {
+                    if let Err(e) = merge_future.fuse().poll() {
+                        error!("The merge task failed quickly after starting: {:?}", e);
+                    }
+                }
+                Err(err) => {
+                    warn!(
+                        "Starting the merge failed for the following reason. This is not fatal. {}",
+                        err
+                    );
+                }
            }
        }
    }
@@ -468,17 +461,15 @@ impl SegmentUpdater {
        &self,
        merge_operation: MergeOperation,
        mut after_merge_segment_entry: SegmentEntry,
-    ) -> impl Future<Output = crate::Result<SegmentMeta>> {
-        let segment_updater = self.clone();
-        let after_merge_segment_meta = after_merge_segment_entry.meta().clone();
-        let end_merge_future = self.schedule_future(async move {
+    ) -> Result<()> {
+        self.run_async(move |segment_updater| {
            info!("End merge {:?}", after_merge_segment_entry.meta());
            {
                let mut delete_cursor = after_merge_segment_entry.delete_cursor().clone();
                if let Some(delete_operation) = delete_cursor.get() {
                    let committed_opstamp = segment_updater.load_metas().opstamp;
                    if delete_operation.opstamp < committed_opstamp {
-                        let index = &segment_updater.index;
+                        let index = &segment_updater.0.index;
                        let segment = index.segment(after_merge_segment_entry.meta().clone());
                        if let Err(e) = advance_deletes(
                            segment,
@@ -496,26 +487,21 @@ impl SegmentUpdater {
                            // ... cancel merge
                            // `merge_operations` are tracked. As it is dropped, the
                            // the segment_ids will be available again for merge.
-                            return Err(e);
+                            return;
                        }
                    }
                }
                let previous_metas = segment_updater.load_metas();
-                let segments_status = segment_updater
+                segment_updater
+                    .0
                    .segment_manager
-                    .end_merge(merge_operation.segment_ids(), after_merge_segment_entry)?;
-
-                if segments_status == SegmentsStatus::Committed {
-                    segment_updater
-                        .save_metas(previous_metas.opstamp, previous_metas.payload.clone())?;
-                }
-
-                segment_updater.consider_merge_options().await;
+                    .end_merge(merge_operation.segment_ids(), after_merge_segment_entry);
+                segment_updater.consider_merge_options();
+                segment_updater.save_metas(previous_metas.opstamp, previous_metas.payload.clone());
            } // we drop all possible handle to a now useless `SegmentMeta`.
-            let _ = garbage_collect_files(segment_updater).await;
-            Ok(())
-        });
-        end_merge_future.map_ok(|_| after_merge_segment_meta)
+            segment_updater.garbage_collect_files_exec();
+        })
+        .wait()
    }

    /// Wait for current merging threads.
@@ -533,9 +519,26 @@ impl SegmentUpdater {
    ///
    /// Obsolete files will eventually be cleaned up
    /// by the directory garbage collector.
-    pub fn wait_merging_thread(&self) -> crate::Result<()> {
-        self.merge_operations.wait_until_empty();
-        Ok(())
+    pub fn wait_merging_thread(&self) -> Result<()> {
+        loop {
+            let merging_threads: HashMap<usize, JoinHandle<Result<()>>> = {
+                let mut merging_threads = self.0.merging_threads.write().unwrap();
+                mem::replace(merging_threads.deref_mut(), HashMap::new())
+            };
+            if merging_threads.is_empty() {
+                return Ok(());
+            }
+            debug!("wait merging thread {}", merging_threads.len());
+            for (_, merging_thread_handle) in merging_threads {
+                merging_thread_handle
+                    .join()
+                    .map(|_| ())
+                    .map_err(|_| TantivyError::ErrorInThread("Merging thread failed.".into()))?;
+            }
+            // Our merging thread may have queued their completed merged segment.
+            // Let's wait for that too.
+            self.run_async(move |_| {}).wait()?;
+        }
    }
 }

@@ -691,6 +694,7 @@ mod tests {
        index_writer.segment_updater().remove_all_segments();
        let seg_vec = index_writer
            .segment_updater()
+            .0
            .segment_manager
            .segment_entries();
        assert!(seg_vec.is_empty());
--- a/src/indexer/segment_writer.rs
+++ b/src/indexer/segment_writer.rs
@@ -11,18 +11,20 @@ use crate::schema::Schema;
 use crate::schema::Term;
 use crate::schema::Value;
 use crate::schema::{Field, FieldEntry};
-use crate::tokenizer::{BoxTokenStream, PreTokenizedStream};
-use crate::tokenizer::{FacetTokenizer, TextAnalyzer};
-use crate::tokenizer::{TokenStreamChain, Tokenizer};
+use crate::tokenizer::BoxedTokenizer;
+use crate::tokenizer::FacetTokenizer;
+use crate::tokenizer::{TokenStream, Tokenizer};
 use crate::DocId;
 use crate::Opstamp;
+use crate::Result;
+use crate::TantivyError;
 use std::io;
 use std::str;

 /// Computes the initial size of the hash table.
 ///
 /// Returns a number of bit `b`, such that the recommended initial table size is 2^b.
-fn initial_table_size(per_thread_memory_budget: usize) -> crate::Result<usize> {
+fn initial_table_size(per_thread_memory_budget: usize) -> Result<usize> {
    let table_memory_upper_bound = per_thread_memory_budget / 3;
    if let Some(limit) = (10..)
        .take_while(|num_bits: &usize| compute_table_size(*num_bits) < table_memory_upper_bound)
@@ -30,7 +32,7 @@ fn initial_table_size(per_thread_memory_budget: usize) -> crate::Result<usize> {
    {
        Ok(limit.min(19)) // we cap it at 2^19 = 512K.
    } else {
-        Err(crate::TantivyError::InvalidArgument(
+        Err(TantivyError::InvalidArgument(
            format!("per thread memory budget (={}) is too small. Raise the memory budget or lower the number of threads.", per_thread_memory_budget)))
    }
 }
@@ -47,7 +49,7 @@ pub struct SegmentWriter {
    fast_field_writers: FastFieldsWriter,
    fieldnorms_writer: FieldNormsWriter,
    doc_opstamps: Vec<Opstamp>,
-    tokenizers: Vec<Option<TextAnalyzer>>,
+    tokenizers: Vec<Option<BoxedTokenizer>>,
 }

 impl SegmentWriter {
@@ -64,7 +66,7 @@ impl SegmentWriter {
        memory_budget: usize,
        mut segment: Segment,
        schema: &Schema,
-    ) -> crate::Result<SegmentWriter> {
+    ) -> Result<SegmentWriter> {
        let table_num_bits = initial_table_size(memory_budget)?;
        let segment_serializer = SegmentSerializer::for_segment(&mut segment)?;
        let multifield_postings = MultiFieldPostingsWriter::new(schema, table_num_bits);
@@ -97,7 +99,7 @@ impl SegmentWriter {
    ///
    /// Finalize consumes the `SegmentWriter`, so that it cannot
    /// be used afterwards.
-    pub fn finalize(mut self) -> crate::Result<Vec<u64>> {
+    pub fn finalize(mut self) -> Result<Vec<u64>> {
        self.fieldnorms_writer.fill_up_to_max_doc(self.max_doc);
        write(
            &self.multifield_postings,
@@ -156,43 +158,26 @@ impl SegmentWriter {
                    }
                }
                FieldType::Str(_) => {
-                    let mut token_streams: Vec<BoxTokenStream> = vec![];
-                    let mut offsets = vec![];
-                    let mut total_offset = 0;
-
-                    for field_value in field_values {
-                        match field_value.value() {
-                            Value::PreTokStr(tok_str) => {
-                                offsets.push(total_offset);
-                                if let Some(last_token) = tok_str.tokens.last() {
-                                    total_offset += last_token.offset_to;
-                                }
-
-                                token_streams
-                                    .push(PreTokenizedStream::from(tok_str.clone()).into());
-                            }
-                            Value::Str(ref text) => {
-                                if let Some(ref mut tokenizer) =
-                                    self.tokenizers[field.field_id() as usize]
-                                {
-                                    offsets.push(total_offset);
-                                    total_offset += text.len();
-
-                                    token_streams.push(tokenizer.token_stream(text));
-                                }
-                            }
-                            _ => (),
+                    let num_tokens = if let Some(ref mut tokenizer) =
+                        self.tokenizers[field.field_id() as usize]
+                    {
+                        let texts: Vec<&str> = field_values
+                            .iter()
+                            .flat_map(|field_value| match *field_value.value() {
+                                Value::Str(ref text) => Some(text.as_str()),
+                                _ => None,
+                            })
+                            .collect();
+                        if texts.is_empty() {
+                            0
+                        } else {
+                            let mut token_stream = tokenizer.token_stream_texts(&texts[..]);
+                            self.multifield_postings
+                                .index_text(doc_id, field, &mut token_stream)
                        }
-                    }
-
-                    let num_tokens = if token_streams.is_empty() {
-                        0
                    } else {
-                        let mut token_stream = TokenStreamChain::new(offsets, token_streams);
-                        self.multifield_postings
-                            .index_text(doc_id, field, &mut token_stream)
+                        0
                    };
-
                    self.fieldnorms_writer.record(doc_id, field, num_tokens);
                }
                FieldType::U64(ref int_option) => {
@@ -245,7 +230,6 @@ impl SegmentWriter {
            }
        }
        doc.filter_fields(|field| schema.get_field_entry(field).is_stored());
-        doc.prepare_for_store();
        let doc_writer = self.segment_serializer.get_store_writer();
        doc_writer.store(&doc)?;
        self.max_doc += 1;
@@ -279,7 +263,7 @@ fn write(
    fast_field_writers: &FastFieldsWriter,
    fieldnorms_writer: &FieldNormsWriter,
    mut serializer: SegmentSerializer,
-) -> crate::Result<()> {
+) -> Result<()> {
    let term_ord_map = multifield_postings.serialize(serializer.get_postings_serializer())?;
    fast_field_writers.serialize(serializer.get_fast_field_serializer(), &term_ord_map)?;
    fieldnorms_writer.serialize(serializer.get_fieldnorms_serializer())?;
@@ -288,7 +272,7 @@ fn write(
 }

 impl SerializableSegment for SegmentWriter {
-    fn write(&self, serializer: SegmentSerializer) -> crate::Result<u32> {
+    fn write(&self, serializer: SegmentSerializer) -> Result<u32> {
        let max_doc = self.max_doc;
        write(
            &self.multifield_postings,
--- a/src/indexer/stamper.rs
+++ b/src/indexer/stamper.rs
@@ -1,76 +1,18 @@
 use crate::Opstamp;
 use std::ops::Range;
-use std::sync::atomic::Ordering;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;

-#[cfg(not(target_arch = "arm"))]
-mod atomic_impl {
-
-    use crate::Opstamp;
-    use std::sync::atomic::{AtomicU64, Ordering};
-
-    #[derive(Default)]
-    pub struct AtomicU64Wrapper(AtomicU64);
-
-    impl AtomicU64Wrapper {
-        pub fn new(first_opstamp: Opstamp) -> AtomicU64Wrapper {
-            AtomicU64Wrapper(AtomicU64::new(first_opstamp as u64))
-        }
-
-        pub fn fetch_add(&self, val: u64, order: Ordering) -> u64 {
-            self.0.fetch_add(val as u64, order) as u64
-        }
-
-        pub fn revert(&self, val: u64, order: Ordering) -> u64 {
-            self.0.store(val, order);
-            val
-        }
-    }
-}
-
-#[cfg(target_arch = "arm")]
-mod atomic_impl {
-
-    use crate::Opstamp;
-    /// Under other architecture, we rely on a mutex.
-    use std::sync::atomic::Ordering;
-    use std::sync::RwLock;
-
-    #[derive(Default)]
-    pub struct AtomicU64Wrapper(RwLock<u64>);
-
-    impl AtomicU64Wrapper {
-        pub fn new(first_opstamp: Opstamp) -> AtomicU64Wrapper {
-            AtomicU64Wrapper(RwLock::new(first_opstamp))
-        }
-
-        pub fn fetch_add(&self, incr: u64, _order: Ordering) -> u64 {
-            let mut lock = self.0.write().unwrap();
-            let previous_val = *lock;
-            *lock = previous_val + incr;
-            previous_val
-        }
-
-        pub fn revert(&self, val: u64, _order: Ordering) -> u64 {
-            let mut lock = self.0.write().unwrap();
-            *lock = val;
-            val
-        }
-    }
-}
-
-use self::atomic_impl::AtomicU64Wrapper;
-
 /// Stamper provides Opstamps, which is just an auto-increment id to label
 /// an operation.
 ///
 /// Cloning does not "fork" the stamp generation. The stamper actually wraps an `Arc`.
 #[derive(Clone, Default)]
-pub struct Stamper(Arc<AtomicU64Wrapper>);
+pub struct Stamper(Arc<AtomicU64>);

 impl Stamper {
    pub fn new(first_opstamp: Opstamp) -> Stamper {
-        Stamper(Arc::new(AtomicU64Wrapper::new(first_opstamp)))
+        Stamper(Arc::new(AtomicU64::new(first_opstamp)))
    }

    pub fn stamp(&self) -> Opstamp {
@@ -89,7 +31,8 @@ impl Stamper {

    /// Reverts the stamper to a given `Opstamp` value and returns it
    pub fn revert(&self, to_opstamp: Opstamp) -> Opstamp {
-        self.0.revert(to_opstamp, Ordering::SeqCst)
+        self.0.store(to_opstamp, Ordering::SeqCst);
+        to_opstamp
    }
 }

--- a/src/lib.rs
+++ b/src/lib.rs
@@ -121,13 +121,13 @@ mod functional_test;
 mod macros;

 pub use crate::error::TantivyError;
+
+#[deprecated(since = "0.7.0", note = "please use `tantivy::TantivyError` instead")]
+pub use crate::error::TantivyError as Error;
 pub use chrono;

 /// Tantivy result.
-///
-/// Within tantivy, please avoid importing `Result` using `use crate::Result`
-/// and instead, refer to this as `crate::Result<T>`.
-pub type Result<T> = std::result::Result<T, TantivyError>;
+pub type Result<T> = std::result::Result<T, error::TantivyError>;

 /// Tantivy DateTime
 pub type DateTime = chrono::DateTime<chrono::Utc>;
@@ -160,68 +160,21 @@ pub use self::snippet::{Snippet, SnippetGenerator};

 mod docset;
 pub use self::docset::{DocSet, SkipResult};
+
 pub use crate::common::{f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
-pub use crate::core::{Executor, SegmentComponent};
+pub use crate::core::SegmentComponent;
 pub use crate::core::{Index, IndexMeta, Searcher, Segment, SegmentId, SegmentMeta};
 pub use crate::core::{InvertedIndexReader, SegmentReader};
 pub use crate::directory::Directory;
-pub use crate::indexer::operation::UserOperation;
 pub use crate::indexer::IndexWriter;
 pub use crate::postings::Postings;
 pub use crate::reader::LeasedItem;
 pub use crate::schema::{Document, Term};
-use std::fmt;

-use once_cell::sync::Lazy;
-
-/// Index format version.
-const INDEX_FORMAT_VERSION: u32 = 1;
-
-/// Structure version for the index.
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Version {
-    major: u32,
-    minor: u32,
-    patch: u32,
-    index_format_version: u32,
-    store_compression: String,
-}
-
-impl fmt::Debug for Version {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", self.to_string())
-    }
-}
-
-static VERSION: Lazy<Version> = Lazy::new(|| Version {
-    major: env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap(),
-    minor: env!("CARGO_PKG_VERSION_MINOR").parse().unwrap(),
-    patch: env!("CARGO_PKG_VERSION_PATCH").parse().unwrap(),
-    index_format_version: INDEX_FORMAT_VERSION,
-    store_compression: crate::store::COMPRESSION.to_string(),
-});
-
-impl ToString for Version {
-    fn to_string(&self) -> String {
-        format!(
-            "tantivy v{}.{}.{}, index_format v{}, store_compression: {}",
-            self.major, self.minor, self.patch, self.index_format_version, self.store_compression
-        )
-    }
-}
-
-static VERSION_STRING: Lazy<String> = Lazy::new(|| VERSION.to_string());
-
-/// Expose the current version of tantivy as found in Cargo.toml during compilation.
-/// eg. "0.11.0" as well as the compression scheme used in the docstore.
-pub fn version() -> &'static Version {
-    &VERSION
-}
-
-/// Exposes the complete version of tantivy as found in Cargo.toml during compilation as a string.
-/// eg. "tantivy v0.11.0, index_format v1, store_compression: lz4".
-pub fn version_string() -> &'static str {
-    VERSION_STRING.as_str()
+/// Expose the current version of tantivy, as well
+/// whether it was compiled with the simd compression.
+pub fn version() -> &'static str {
+    env!("CARGO_PKG_VERSION")
 }

 /// Defines tantivy's merging strategy
@@ -334,18 +287,6 @@ mod tests {
        sample_with_seed(n, ratio, 4)
    }

-    #[test]
-    #[cfg(not(feature = "lz4"))]
-    fn test_version_string() {
-        use regex::Regex;
-        let regex_ptn = Regex::new(
-            "tantivy v[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.{0,10}, index_format v[0-9]{1,5}",
-        )
-        .unwrap();
-        let version = super::version().to_string();
-        assert!(regex_ptn.find(&version).is_some());
-    }
-
    #[test]
    #[cfg(feature = "mmap")]
    fn test_indexing() {
@@ -941,73 +882,4 @@ mod tests {
            assert_eq!(fast_field_reader.get(0), 4f64)
        }
    }
-
-    // motivated by #729
-    #[test]
-    fn test_update_via_delete_insert() {
-        use crate::collector::Count;
-        use crate::indexer::NoMergePolicy;
-        use crate::query::AllQuery;
-        use crate::SegmentId;
-        use futures::executor::block_on;
-
-        const DOC_COUNT: u64 = 2u64;
-
-        let mut schema_builder = SchemaBuilder::default();
-        let id = schema_builder.add_u64_field("id", INDEXED);
-        let schema = schema_builder.build();
-
-        let index = Index::create_in_ram(schema.clone());
-        let index_reader = index.reader().unwrap();
-
-        let mut index_writer = index.writer(3_000_000).unwrap();
-        index_writer.set_merge_policy(Box::new(NoMergePolicy));
-
-        for doc_id in 0u64..DOC_COUNT {
-            index_writer.add_document(doc!(id => doc_id));
-        }
-        index_writer.commit().unwrap();
-
-        index_reader.reload().unwrap();
-        let searcher = index_reader.searcher();
-
-        assert_eq!(
-            searcher.search(&AllQuery, &Count).unwrap(),
-            DOC_COUNT as usize
-        );
-
-        // update the 10 elements by deleting and re-adding
-        for doc_id in 0u64..DOC_COUNT {
-            index_writer.delete_term(Term::from_field_u64(id, doc_id));
-            index_writer.commit().unwrap();
-            index_reader.reload().unwrap();
-            let doc = doc!(id =>  doc_id);
-            index_writer.add_document(doc);
-            index_writer.commit().unwrap();
-            index_reader.reload().unwrap();
-            let searcher = index_reader.searcher();
-            // The number of document should be stable.
-            assert_eq!(
-                searcher.search(&AllQuery, &Count).unwrap(),
-                DOC_COUNT as usize
-            );
-        }
-
-        index_reader.reload().unwrap();
-        let searcher = index_reader.searcher();
-        let segment_ids: Vec<SegmentId> = searcher
-            .segment_readers()
-            .into_iter()
-            .map(|reader| reader.segment_id())
-            .collect();
-        block_on(index_writer.merge(&segment_ids)).unwrap();
-
-        index_reader.reload().unwrap();
-        let searcher = index_reader.searcher();
-
-        assert_eq!(
-            searcher.search(&AllQuery, &Count).unwrap(),
-            DOC_COUNT as usize
-        );
-    }
 }
--- a/src/macros.rs
+++ b/src/macros.rs
@@ -35,9 +35,9 @@
 /// let likes = schema_builder.add_u64_field("num_u64", FAST);
 /// let schema = schema_builder.build();
 /// let doc = doc!(
-///     title => "Life Aquatic",
-///     author => "Wes Anderson",
-///     likes => 4u64
+/// 	title => "Life Aquatic",
+/// 	author => "Wes Anderson",
+/// 	likes => 4u64
 /// );
 /// # }
 /// ```
--- a/src/positions/reader.rs
+++ b/src/positions/reader.rs
@@ -36,10 +36,11 @@ struct Positions {

 impl Positions {
    pub fn new(position_source: ReadOnlySource, skip_source: ReadOnlySource) -> Positions {
-        let (body, footer) = skip_source.split_from_end(u32::SIZE_IN_BYTES);
+        let skip_len = skip_source.len();
+        let (body, footer) = skip_source.split(skip_len - u32::SIZE_IN_BYTES);
        let num_long_skips = u32::deserialize(&mut footer.as_slice()).expect("Index corrupted");
-        let (skip_source, long_skip_source) =
-            body.split_from_end(u64::SIZE_IN_BYTES * (num_long_skips as usize));
+        let body_split = body.len() - u64::SIZE_IN_BYTES * (num_long_skips as usize);
+        let (skip_source, long_skip_source) = body.split(body_split);
        Positions {
            bit_packer: BitPacker4x::new(),
            skip_source,
--- a/src/postings/postings_writer.rs
+++ b/src/postings/postings_writer.rs
@@ -11,6 +11,7 @@ use crate::termdict::TermOrdinal;
 use crate::tokenizer::TokenStream;
 use crate::tokenizer::{Token, MAX_TOKEN_LEN};
 use crate::DocId;
+use crate::Result;
 use fnv::FnvHashMap;
 use std::collections::HashMap;
 use std::io;
@@ -128,7 +129,7 @@ impl MultiFieldPostingsWriter {
    pub fn serialize(
        &self,
        serializer: &mut InvertedIndexSerializer,
-    ) -> crate::Result<HashMap<Field, FnvHashMap<UnorderedTermId, TermOrdinal>>> {
+    ) -> Result<HashMap<Field, FnvHashMap<UnorderedTermId, TermOrdinal>>> {
        let mut term_offsets: Vec<(&[u8], Addr, UnorderedTermId)> =
            self.term_index.iter().collect();
        term_offsets.sort_unstable_by_key(|&(k, _, _)| k);
--- a/src/postings/serializer.rs
+++ b/src/postings/serializer.rs
@@ -11,6 +11,7 @@ use crate::schema::Schema;
 use crate::schema::{Field, FieldEntry, FieldType};
 use crate::termdict::{TermDictionaryBuilder, TermOrdinal};
 use crate::DocId;
+use crate::Result;
 use std::io::{self, Write};

 /// `InvertedIndexSerializer` is in charge of serializing
@@ -60,7 +61,7 @@ impl InvertedIndexSerializer {
        positions_write: CompositeWrite<WritePtr>,
        positionsidx_write: CompositeWrite<WritePtr>,
        schema: Schema,
-    ) -> crate::Result<InvertedIndexSerializer> {
+    ) -> Result<InvertedIndexSerializer> {
        Ok(InvertedIndexSerializer {
            terms_write,
            postings_write,
@@ -71,7 +72,7 @@ impl InvertedIndexSerializer {
    }

    /// Open a new `PostingsSerializer` for the given segment
-    pub fn open(segment: &mut Segment) -> crate::Result<InvertedIndexSerializer> {
+    pub fn open(segment: &mut Segment) -> Result<InvertedIndexSerializer> {
        use crate::SegmentComponent::{POSITIONS, POSITIONSSKIP, POSTINGS, TERMS};
        InvertedIndexSerializer::create(
            CompositeWrite::wrap(segment.open_write(TERMS)?),
@@ -147,7 +148,8 @@ impl<'a> FieldSerializer<'a> {
            }
            _ => (false, false),
        };
-        let term_dictionary_builder = TermDictionaryBuilder::create(term_dictionary_write)?;
+        let term_dictionary_builder =
+            TermDictionaryBuilder::create(term_dictionary_write, &field_type)?;
        let postings_serializer =
            PostingsSerializer::new(postings_write, term_freq_enabled, position_enabled);
        let positions_serializer_opt = if position_enabled {
--- a/src/query/all_query.rs
+++ b/src/query/all_query.rs
@@ -4,6 +4,7 @@ use crate::docset::DocSet;
 use crate::query::explanation::does_not_match;
 use crate::query::{Explanation, Query, Scorer, Weight};
 use crate::DocId;
+use crate::Result;
 use crate::Score;

 /// Query that matches all of the documents.
@@ -13,7 +14,7 @@ use crate::Score;
 pub struct AllQuery;

 impl Query for AllQuery {
-    fn weight(&self, _: &Searcher, _: bool) -> crate::Result<Box<dyn Weight>> {
+    fn weight(&self, _: &Searcher, _: bool) -> Result<Box<dyn Weight>> {
        Ok(Box::new(AllWeight))
    }
 }
@@ -22,7 +23,7 @@ impl Query for AllQuery {
 pub struct AllWeight;

 impl Weight for AllWeight {
-    fn scorer(&self, reader: &SegmentReader) -> crate::Result<Box<dyn Scorer>> {
+    fn scorer(&self, reader: &SegmentReader) -> Result<Box<dyn Scorer>> {
        Ok(Box::new(AllScorer {
            state: State::NotStarted,
            doc: 0u32,
@@ -30,7 +31,7 @@ impl Weight for AllWeight {
        }))
    }

-    fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
+    fn explain(&self, reader: &SegmentReader, doc: DocId) -> Result<Explanation> {
        if doc >= reader.max_doc() {
            return Err(does_not_match(doc));
        }
--- a/src/query/boolean_query/boolean_query.rs
+++ b/src/query/boolean_query/boolean_query.rs
@@ -5,6 +5,7 @@ use crate::query::TermQuery;
 use crate::query::Weight;
 use crate::schema::IndexRecordOption;
 use crate::schema::Term;
+use crate::Result;
 use crate::Searcher;
 use std::collections::BTreeSet;

@@ -29,9 +30,9 @@ use std::collections::BTreeSet;
 ///use tantivy::query::{BooleanQuery, Occur, PhraseQuery, Query, TermQuery};
 ///use tantivy::schema::{IndexRecordOption, Schema, TEXT};
 ///use tantivy::Term;
-///use tantivy::Index;
+///use tantivy::{Index, Result};
 ///
-///fn main() -> tantivy::Result<()> {
+///fn main() -> Result<()> {
 ///    let mut schema_builder = Schema::builder();
 ///    let title = schema_builder.add_text_field("title", TEXT);
 ///    let body = schema_builder.add_text_field("body", TEXT);
@@ -148,14 +149,14 @@ impl From<Vec<(Occur, Box<dyn Query>)>> for BooleanQuery {
 }

 impl Query for BooleanQuery {
-    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> Result<Box<dyn Weight>> {
        let sub_weights = self
            .subqueries
            .iter()
            .map(|&(ref occur, ref subquery)| {
                Ok((*occur, subquery.weight(searcher, scoring_enabled)?))
            })
-            .collect::<crate::Result<_>>()?;
+            .collect::<Result<_>>()?;
        Ok(Box::new(BooleanWeight::new(sub_weights, scoring_enabled)))
    }

--- a/src/query/boolean_query/boolean_weight.rs
+++ b/src/query/boolean_query/boolean_weight.rs
@@ -10,6 +10,7 @@ use crate::query::Scorer;
 use crate::query::Union;
 use crate::query::Weight;
 use crate::query::{intersect_scorers, Explanation};
+use crate::Result;
 use crate::{DocId, SkipResult};
 use std::collections::HashMap;

@@ -55,7 +56,7 @@ impl BooleanWeight {
    fn per_occur_scorers(
        &self,
        reader: &SegmentReader,
-    ) -> crate::Result<HashMap<Occur, Vec<Box<dyn Scorer>>>> {
+    ) -> Result<HashMap<Occur, Vec<Box<dyn Scorer>>>> {
        let mut per_occur_scorers: HashMap<Occur, Vec<Box<dyn Scorer>>> = HashMap::new();
        for &(ref occur, ref subweight) in &self.weights {
            let sub_scorer: Box<dyn Scorer> = subweight.scorer(reader)?;
@@ -70,7 +71,7 @@ impl BooleanWeight {
    fn complex_scorer<TScoreCombiner: ScoreCombiner>(
        &self,
        reader: &SegmentReader,
-    ) -> crate::Result<Box<dyn Scorer>> {
+    ) -> Result<Box<dyn Scorer>> {
        let mut per_occur_scorers = self.per_occur_scorers(reader)?;

        let should_scorer_opt: Option<Box<dyn Scorer>> = per_occur_scorers
@@ -112,7 +113,7 @@ impl BooleanWeight {
 }

 impl Weight for BooleanWeight {
-    fn scorer(&self, reader: &SegmentReader) -> crate::Result<Box<dyn Scorer>> {
+    fn scorer(&self, reader: &SegmentReader) -> Result<Box<dyn Scorer>> {
        if self.weights.is_empty() {
            Ok(Box::new(EmptyScorer))
        } else if self.weights.len() == 1 {
@@ -129,7 +130,7 @@ impl Weight for BooleanWeight {
        }
    }

-    fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
+    fn explain(&self, reader: &SegmentReader, doc: DocId) -> Result<Explanation> {
        let mut scorer = self.scorer(reader)?;
        if scorer.skip_next(doc) != SkipResult::Reached {
            return Err(does_not_match(doc));
--- a/src/query/empty_query.rs
+++ b/src/query/empty_query.rs
@@ -4,6 +4,7 @@ use crate::query::Weight;
 use crate::query::{Explanation, Query};
 use crate::DocId;
 use crate::DocSet;
+use crate::Result;
 use crate::Score;
 use crate::Searcher;
 use crate::SegmentReader;
@@ -15,15 +16,11 @@ use crate::SegmentReader;
 pub struct EmptyQuery;

 impl Query for EmptyQuery {
-    fn weight(
-        &self,
-        _searcher: &Searcher,
-        _scoring_enabled: bool,
-    ) -> crate::Result<Box<dyn Weight>> {
+    fn weight(&self, _searcher: &Searcher, _scoring_enabled: bool) -> Result<Box<dyn Weight>> {
        Ok(Box::new(EmptyWeight))
    }

-    fn count(&self, _searcher: &Searcher) -> crate::Result<usize> {
+    fn count(&self, _searcher: &Searcher) -> Result<usize> {
        Ok(0)
    }
 }
@@ -33,11 +30,11 @@ impl Query for EmptyQuery {
 /// It is useful for tests and handling edge cases.
 pub struct EmptyWeight;
 impl Weight for EmptyWeight {
-    fn scorer(&self, _reader: &SegmentReader) -> crate::Result<Box<dyn Scorer>> {
+    fn scorer(&self, _reader: &SegmentReader) -> Result<Box<dyn Scorer>> {
        Ok(Box::new(EmptyScorer))
    }

-    fn explain(&self, _reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
+    fn explain(&self, _reader: &SegmentReader, doc: DocId) -> Result<Explanation> {
        Err(does_not_match(doc))
    }
 }
--- a/src/query/exclude.rs
+++ b/src/query/exclude.rs
@@ -54,21 +54,21 @@ where
        match self.excluding_state {
            State::ExcludeOne(excluded_doc) => {
                if doc == excluded_doc {
-                    return false;
-                }
-                if excluded_doc > doc {
-                    return true;
-                }
-                match self.excluding_docset.skip_next(doc) {
-                    SkipResult::OverStep => {
-                        self.excluding_state = State::ExcludeOne(self.excluding_docset.doc());
-                        true
+                    false
+                } else if excluded_doc > doc {
+                    true
+                } else {
+                    match self.excluding_docset.skip_next(doc) {
+                        SkipResult::OverStep => {
+                            self.excluding_state = State::ExcludeOne(self.excluding_docset.doc());
+                            true
+                        }
+                        SkipResult::End => {
+                            self.excluding_state = State::Finished;
+                            true
+                        }
+                        SkipResult::Reached => false,
                    }
-                    SkipResult::End => {
-                        self.excluding_state = State::Finished;
-                        true
-                    }
-                    SkipResult::Reached => false,
                }
            }
            State::Finished => true,
--- a/src/query/fuzzy_query.rs
+++ b/src/query/fuzzy_query.rs
@@ -1,7 +1,8 @@
+use crate::error::TantivyError::InvalidArgument;
 use crate::query::{AutomatonWeight, Query, Weight};
 use crate::schema::Term;
+use crate::Result;
 use crate::Searcher;
-use crate::TantivyError::InvalidArgument;
 use levenshtein_automata::{LevenshteinAutomatonBuilder, DFA};
 use once_cell::sync::Lazy;
 use std::collections::HashMap;
@@ -30,9 +31,10 @@ static LEV_BUILDER: Lazy<HashMap<(u8, bool), LevenshteinAutomatonBuilder>> = Laz
 /// use tantivy::collector::{Count, TopDocs};
 /// use tantivy::query::FuzzyTermQuery;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Term};
+/// use tantivy::{doc, Index, Result, Term};
 ///
-/// fn example() -> tantivy::Result<()> {
+/// # fn main() { example().unwrap(); }
+/// fn example() -> Result<()> {
 ///     let mut schema_builder = Schema::builder();
 ///     let title = schema_builder.add_text_field("title", TEXT);
 ///     let schema = schema_builder.build();
@@ -57,6 +59,7 @@ static LEV_BUILDER: Lazy<HashMap<(u8, bool), LevenshteinAutomatonBuilder>> = Laz
 ///     let searcher = reader.searcher();
 ///
 ///     {
+///
 ///         let term = Term::from_field_text(title, "Diary");
 ///         let query = FuzzyTermQuery::new(term, 1, true);
 ///         let (top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count)).unwrap();
@@ -66,7 +69,6 @@ static LEV_BUILDER: Lazy<HashMap<(u8, bool), LevenshteinAutomatonBuilder>> = Laz
 ///
 ///     Ok(())
 /// }
-/// # assert!(example().is_ok());
 /// ```
 #[derive(Debug, Clone)]
 pub struct FuzzyTermQuery {
@@ -101,7 +103,7 @@ impl FuzzyTermQuery {
        }
    }

-    fn specialized_weight(&self) -> crate::Result<AutomatonWeight<DFA>> {
+    fn specialized_weight(&self) -> Result<AutomatonWeight<DFA>> {
        // LEV_BUILDER is a HashMap, whose `get` method returns an Option
        match LEV_BUILDER.get(&(self.distance, false)) {
            // Unwrap the option and build the Ok(AutomatonWeight)
@@ -118,11 +120,7 @@ impl FuzzyTermQuery {
 }

 impl Query for FuzzyTermQuery {
-    fn weight(
-        &self,
-        _searcher: &Searcher,
-        _scoring_enabled: bool,
-    ) -> crate::Result<Box<dyn Weight>> {
+    fn weight(&self, _searcher: &Searcher, _scoring_enabled: bool) -> Result<Box<dyn Weight>> {
        Ok(Box::new(self.specialized_weight()?))
    }
 }
--- a/src/query/phrase_query/phrase_query.rs
+++ b/src/query/phrase_query/phrase_query.rs
@@ -1,10 +1,12 @@
 use super::PhraseWeight;
 use crate::core::searcher::Searcher;
+use crate::error::TantivyError;
 use crate::query::bm25::BM25Weight;
 use crate::query::Query;
 use crate::query::Weight;
 use crate::schema::IndexRecordOption;
 use crate::schema::{Field, Term};
+use crate::Result;
 use std::collections::BTreeSet;

 /// `PhraseQuery` matches a specific sequence of words.
@@ -79,7 +81,7 @@ impl PhraseQuery {
        &self,
        searcher: &Searcher,
        scoring_enabled: bool,
-    ) -> crate::Result<PhraseWeight> {
+    ) -> Result<PhraseWeight> {
        let schema = searcher.schema();
        let field_entry = schema.get_field_entry(self.field);
        let has_positions = field_entry
@@ -89,7 +91,7 @@ impl PhraseQuery {
            .unwrap_or(false);
        if !has_positions {
            let field_name = field_entry.name();
-            return Err(crate::TantivyError::SchemaError(format!(
+            return Err(TantivyError::SchemaError(format!(
                "Applied phrase query on field {:?}, which does not have positions indexed",
                field_name
            )));
@@ -108,7 +110,7 @@ impl Query for PhraseQuery {
    /// Create the weight associated to a query.
    ///
    /// See [`Weight`](./trait.Weight.html).
-    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> Result<Box<dyn Weight>> {
        let phrase_weight = self.phrase_weight(searcher, scoring_enabled)?;
        Ok(Box::new(phrase_weight))
    }
--- a/src/query/phrase_query/phrase_scorer.rs
+++ b/src/query/phrase_query/phrase_scorer.rs
@@ -4,7 +4,6 @@ use crate::postings::Postings;
 use crate::query::bm25::BM25Weight;
 use crate::query::{Intersection, Scorer};
 use crate::DocId;
-use std::cmp::Ordering;

 struct PostingsWithOffset<TPostings> {
    offset: u32,
@@ -60,16 +59,12 @@ fn intersection_exists(left: &[u32], right: &[u32]) -> bool {
    while left_i < left.len() && right_i < right.len() {
        let left_val = left[left_i];
        let right_val = right[right_i];
-        match left_val.cmp(&right_val) {
-            Ordering::Less => {
-                left_i += 1;
-            }
-            Ordering::Equal => {
-                return true;
-            }
-            Ordering::Greater => {
-                right_i += 1;
-            }
+        if left_val < right_val {
+            left_i += 1;
+        } else if right_val < left_val {
+            right_i += 1;
+        } else {
+            return true;
        }
    }
    false
@@ -82,18 +77,14 @@ fn intersection_count(left: &[u32], right: &[u32]) -> usize {
    while left_i < left.len() && right_i < right.len() {
        let left_val = left[left_i];
        let right_val = right[right_i];
-        match left_val.cmp(&right_val) {
-            Ordering::Less => {
-                left_i += 1;
-            }
-            Ordering::Equal => {
-                count += 1;
-                left_i += 1;
-                right_i += 1;
-            }
-            Ordering::Greater => {
-                right_i += 1;
-            }
+        if left_val < right_val {
+            left_i += 1;
+        } else if right_val < left_val {
+            right_i += 1;
+        } else {
+            count += 1;
+            left_i += 1;
+            right_i += 1;
        }
    }
    count
@@ -112,19 +103,15 @@ fn intersection(left: &mut [u32], right: &[u32]) -> usize {
    while left_i < left_len && right_i < right_len {
        let left_val = left[left_i];
        let right_val = right[right_i];
-        match left_val.cmp(&right_val) {
-            Ordering::Less => {
-                left_i += 1;
-            }
-            Ordering::Equal => {
-                left[count] = left_val;
-                count += 1;
-                left_i += 1;
-                right_i += 1;
-            }
-            Ordering::Greater => {
-                right_i += 1;
-            }
+        if left_val < right_val {
+            left_i += 1;
+        } else if right_val < left_val {
+            right_i += 1;
+        } else {
+            left[count] = left_val;
+            count += 1;
+            left_i += 1;
+            right_i += 1;
        }
    }
    count
--- a/src/query/query.rs
+++ b/src/query/query.rs
@@ -2,6 +2,7 @@ use super::Weight;
 use crate::core::searcher::Searcher;
 use crate::query::Explanation;
 use crate::DocAddress;
+use crate::Result;
 use crate::Term;
 use downcast_rs::impl_downcast;
 use std::collections::BTreeSet;
@@ -47,17 +48,17 @@ pub trait Query: QueryClone + downcast_rs::Downcast + fmt::Debug {
    /// can increase performances.
    ///
    /// See [`Weight`](./trait.Weight.html).
-    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>>;
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> Result<Box<dyn Weight>>;

    /// Returns an `Explanation` for the score of the document.
-    fn explain(&self, searcher: &Searcher, doc_address: DocAddress) -> crate::Result<Explanation> {
+    fn explain(&self, searcher: &Searcher, doc_address: DocAddress) -> Result<Explanation> {
        let reader = searcher.segment_reader(doc_address.segment_ord());
        let weight = self.weight(searcher, true)?;
        weight.explain(reader, doc_address.doc())
    }

    /// Returns the number of documents matching the query.
-    fn count(&self, searcher: &Searcher) -> crate::Result<usize> {
+    fn count(&self, searcher: &Searcher) -> Result<usize> {
        let weight = self.weight(searcher, false)?;
        let mut result = 0;
        for reader in searcher.segment_readers() {
@@ -85,11 +86,11 @@ where
 }

 impl Query for Box<dyn Query> {
-    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> Result<Box<dyn Weight>> {
        self.as_ref().weight(searcher, scoring_enabled)
    }

-    fn count(&self, searcher: &Searcher) -> crate::Result<usize> {
+    fn count(&self, searcher: &Searcher) -> Result<usize> {
        self.as_ref().count(searcher)
    }

--- a/src/query/query_parser/query_parser.rs
+++ b/src/query/query_parser/query_parser.rs
@@ -8,7 +8,7 @@ use crate::query::PhraseQuery;
 use crate::query::Query;
 use crate::query::RangeQuery;
 use crate::query::TermQuery;
-use crate::schema::{Facet, IndexRecordOption};
+use crate::schema::IndexRecordOption;
 use crate::schema::{Field, Schema};
 use crate::schema::{FieldType, Term};
 use crate::tokenizer::TokenizerManager;
@@ -319,10 +319,7 @@ impl QueryParser {
                    ))
                }
            }
-            FieldType::HierarchicalFacet => {
-                let facet = Facet::from_text(phrase);
-                Ok(vec![(0, Term::from_field_text(field, facet.encoded_str()))])
-            }
+            FieldType::HierarchicalFacet => Ok(vec![(0, Term::from_field_text(field, phrase))]),
            FieldType::Bytes => {
                let field_name = self.schema.get_field_name(field).to_string();
                Err(QueryParserError::FieldNotIndexed(field_name))
@@ -533,7 +530,7 @@ mod test {
    use crate::schema::{IndexRecordOption, TextFieldIndexing, TextOptions};
    use crate::schema::{Schema, Term, INDEXED, STORED, STRING, TEXT};
    use crate::tokenizer::{
-        LowerCaser, SimpleTokenizer, StopWordFilter, TextAnalyzer, TokenizerManager,
+        LowerCaser, SimpleTokenizer, StopWordFilter, Tokenizer, TokenizerManager,
    };
    use crate::Index;
    use matches::assert_matches;
@@ -557,13 +554,12 @@ mod test {
        schema_builder.add_text_field("with_stop_words", text_options);
        schema_builder.add_date_field("date", INDEXED);
        schema_builder.add_f64_field("float", INDEXED);
-        schema_builder.add_facet_field("facet");
        let schema = schema_builder.build();
        let default_fields = vec![title, text];
        let tokenizer_manager = TokenizerManager::default();
        tokenizer_manager.register(
            "en_with_stop_words",
-            TextAnalyzer::from(SimpleTokenizer)
+            SimpleTokenizer
                .filter(LowerCaser)
                .filter(StopWordFilter::remove(vec!["the".to_string()])),
        );
@@ -592,13 +588,9 @@ mod test {
    }

    #[test]
-    pub fn test_parse_query_facet() {
+    pub fn test_parse_query_simple() {
        let query_parser = make_query_parser();
-        let query = query_parser.parse_query("facet:/root/branch/leaf").unwrap();
-        assert_eq!(
-            format!("{:?}", query),
-            "TermQuery(Term(field=11,bytes=[114, 111, 111, 116, 0, 98, 114, 97, 110, 99, 104, 0, 108, 101, 97, 102]))"
-        );
+        assert!(query_parser.parse_query("toto").is_ok());
    }

    #[test]
--- a/src/query/range_query.rs
+++ b/src/query/range_query.rs
@@ -38,33 +38,41 @@ fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
 /// # Example
 ///
 /// ```rust
-/// use tantivy::collector::Count;
-/// use tantivy::query::RangeQuery;
-/// use tantivy::schema::{Schema, INDEXED};
-/// use tantivy::{doc, Index};
-/// # fn test() -> tantivy::Result<()> {
-/// let mut schema_builder = Schema::builder();
-/// let year_field = schema_builder.add_u64_field("year", INDEXED);
-/// let schema = schema_builder.build();
-///
-/// let index = Index::create_in_ram(schema);
-/// let mut index_writer = index.writer_with_num_threads(1, 6_000_000)?;
-/// for year in 1950u64..2017u64 {
-///     let num_docs_within_year = 10 + (year - 1950) * (year - 1950);
-///     for _ in 0..num_docs_within_year {
-///       index_writer.add_document(doc!(year_field => year));
-///     }
-/// }
-/// index_writer.commit()?;
-///
-/// let reader = index.reader()?;
+/// # use tantivy::collector::Count;
+/// # use tantivy::query::RangeQuery;
+/// # use tantivy::schema::{Schema, INDEXED};
+/// # use tantivy::{doc, Index, Result};
+/// #
+/// # fn run() -> Result<()> {
+/// #     let mut schema_builder = Schema::builder();
+/// #     let year_field = schema_builder.add_u64_field("year", INDEXED);
+/// #     let schema = schema_builder.build();
+/// #
+/// #     let index = Index::create_in_ram(schema);
+/// #     {
+/// #         let mut index_writer = index.writer_with_num_threads(1, 6_000_000).unwrap();
+/// #         for year in 1950u64..2017u64 {
+/// #             let num_docs_within_year = 10 + (year - 1950) * (year - 1950);
+/// #             for _ in 0..num_docs_within_year {
+/// #                 index_writer.add_document(doc!(year_field => year));
+/// #             }
+/// #         }
+/// #         index_writer.commit().unwrap();
+/// #     }
+/// #   let reader = index.reader()?;
 /// let searcher = reader.searcher();
+///
 /// let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960..1970);
+///
 /// let num_60s_books = searcher.search(&docs_in_the_sixties, &Count)?;
-/// assert_eq!(num_60s_books, 2285);
-/// Ok(())
+///
+/// #     assert_eq!(num_60s_books, 2285);
+/// #     Ok(())
+/// # }
+/// #
+/// # fn main() {
+/// #   run().unwrap()
 /// # }
-/// # assert!(test().is_ok());
 /// ```
 #[derive(Clone, Debug)]
 pub struct RangeQuery {
--- a/src/query/regex_query.rs
+++ b/src/query/regex_query.rs
@@ -1,6 +1,7 @@
 use crate::error::TantivyError;
 use crate::query::{AutomatonWeight, Query, Weight};
 use crate::schema::Field;
+use crate::Result;
 use crate::Searcher;
 use std::clone::Clone;
 use std::sync::Arc;
@@ -14,40 +15,40 @@ use tantivy_fst::Regex;
 /// use tantivy::collector::Count;
 /// use tantivy::query::RegexQuery;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Term};
+/// use tantivy::{doc, Index, Result, Term};
 ///
-/// # fn test() -> tantivy::Result<()> {
-/// let mut schema_builder = Schema::builder();
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
-/// {
-///     let mut index_writer = index.writer(3_000_000)?;
-///     index_writer.add_document(doc!(
-///         title => "The Name of the Wind",
-///     ));
-///     index_writer.add_document(doc!(
-///         title => "The Diary of Muadib",
-///     ));
-///     index_writer.add_document(doc!(
-///         title => "A Dairy Cow",
-///     ));
-///     index_writer.add_document(doc!(
-///         title => "The Diary of a Young Girl",
-///     ));
-///     index_writer.commit().unwrap();
+/// # fn main() { example().unwrap(); }
+/// fn example() -> Result<()> {
+///     let mut schema_builder = Schema::builder();
+///     let title = schema_builder.add_text_field("title", TEXT);
+///     let schema = schema_builder.build();
+///     let index = Index::create_in_ram(schema);
+///     {
+///         let mut index_writer = index.writer(3_000_000)?;
+///         index_writer.add_document(doc!(
+///             title => "The Name of the Wind",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of Muadib",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "A Dairy Cow",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of a Young Girl",
+///         ));
+///         index_writer.commit().unwrap();
+///     }
+///
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
+///
+///     let term = Term::from_field_text(title, "Diary");
+///     let query = RegexQuery::from_pattern("d[ai]{2}ry", title)?;
+///     let count = searcher.search(&query, &Count)?;
+///     assert_eq!(count, 3);
+///     Ok(())
 /// }
-///
-/// let reader = index.reader()?;
-/// let searcher = reader.searcher();
-///
-/// let term = Term::from_field_text(title, "Diary");
-/// let query = RegexQuery::from_pattern("d[ai]{2}ry", title)?;
-/// let count = searcher.search(&query, &Count)?;
-/// assert_eq!(count, 3);
-/// Ok(())
-/// # }
-/// # assert!(test().is_ok());
 /// ```
 #[derive(Debug, Clone)]
 pub struct RegexQuery {
@@ -57,7 +58,7 @@ pub struct RegexQuery {

 impl RegexQuery {
    /// Creates a new RegexQuery from a given pattern
-    pub fn from_pattern(regex_pattern: &str, field: Field) -> crate::Result<Self> {
+    pub fn from_pattern(regex_pattern: &str, field: Field) -> Result<Self> {
        let regex = Regex::new(&regex_pattern)
            .map_err(|_| TantivyError::InvalidArgument(regex_pattern.to_string()))?;
        Ok(RegexQuery::from_regex(regex, field))
@@ -77,11 +78,7 @@ impl RegexQuery {
 }

 impl Query for RegexQuery {
-    fn weight(
-        &self,
-        _searcher: &Searcher,
-        _scoring_enabled: bool,
-    ) -> crate::Result<Box<dyn Weight>> {
+    fn weight(&self, _searcher: &Searcher, _scoring_enabled: bool) -> Result<Box<dyn Weight>> {
        Ok(Box::new(self.specialized_weight()))
    }
 }
--- a/src/query/term_query/term_query.rs
+++ b/src/query/term_query/term_query.rs
@@ -3,6 +3,7 @@ use crate::query::bm25::BM25Weight;
 use crate::query::Query;
 use crate::query::Weight;
 use crate::schema::IndexRecordOption;
+use crate::Result;
 use crate::Searcher;
 use crate::Term;
 use std::collections::BTreeSet;
@@ -22,39 +23,42 @@ use std::fmt;
 /// use tantivy::collector::{Count, TopDocs};
 /// use tantivy::query::TermQuery;
 /// use tantivy::schema::{Schema, TEXT, IndexRecordOption};
-/// use tantivy::{doc, Index, Term};
-/// # fn test() -> tantivy::Result<()> {
-/// let mut schema_builder = Schema::builder();
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
-/// {
-///     let mut index_writer = index.writer(3_000_000)?;
-///     index_writer.add_document(doc!(
-///         title => "The Name of the Wind",
-///     ));
-///     index_writer.add_document(doc!(
-///         title => "The Diary of Muadib",
-///     ));
-///     index_writer.add_document(doc!(
-///         title => "A Dairy Cow",
-///     ));
-///     index_writer.add_document(doc!(
-///         title => "The Diary of a Young Girl",
-///     ));
-///     index_writer.commit()?;
+/// use tantivy::{doc, Index, Result, Term};
+///
+/// # fn main() { example().unwrap(); }
+/// fn example() -> Result<()> {
+///     let mut schema_builder = Schema::builder();
+///     let title = schema_builder.add_text_field("title", TEXT);
+///     let schema = schema_builder.build();
+///     let index = Index::create_in_ram(schema);
+///     {
+///         let mut index_writer = index.writer(3_000_000)?;
+///         index_writer.add_document(doc!(
+///             title => "The Name of the Wind",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of Muadib",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "A Dairy Cow",
+///         ));
+///         index_writer.add_document(doc!(
+///             title => "The Diary of a Young Girl",
+///         ));
+///         index_writer.commit()?;
+///     }
+///     let reader = index.reader()?;
+///     let searcher = reader.searcher();
+///
+///     let query = TermQuery::new(
+///         Term::from_field_text(title, "diary"),
+///         IndexRecordOption::Basic,
+///     );
+///     let (top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count)).unwrap();
+///     assert_eq!(count, 2);
+///
+///     Ok(())
 /// }
-/// let reader = index.reader()?;
-/// let searcher = reader.searcher();
-/// let query = TermQuery::new(
-///     Term::from_field_text(title, "diary"),
-///     IndexRecordOption::Basic,
-/// );
-/// let (top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count))?;
-/// assert_eq!(count, 2);
-/// Ok(())
-/// # }
-/// # assert!(test().is_ok());
 /// ```
 #[derive(Clone)]
 pub struct TermQuery {
@@ -100,7 +104,7 @@ impl TermQuery {
 }

 impl Query for TermQuery {
-    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> crate::Result<Box<dyn Weight>> {
+    fn weight(&self, searcher: &Searcher, scoring_enabled: bool) -> Result<Box<dyn Weight>> {
        Ok(Box::new(self.specialized_weight(searcher, scoring_enabled)))
    }
    fn query_terms(&self, term_set: &mut BTreeSet<Term>) {
--- a/src/reader/mod.rs
+++ b/src/reader/mod.rs
@@ -7,6 +7,7 @@ use crate::directory::Directory;
 use crate::directory::WatchHandle;
 use crate::directory::META_LOCK;
 use crate::Index;
+use crate::Result;
 use crate::Searcher;
 use crate::SegmentReader;
 use std::sync::Arc;
@@ -61,7 +62,7 @@ impl IndexReaderBuilder {
    /// to open different segment readers. It may take hundreds of milliseconds
    /// of time and it may return an error.
    /// TODO(pmasurel) Use the `TryInto` trait once it is available in stable.
-    pub fn try_into(self) -> crate::Result<IndexReader> {
+    pub fn try_into(self) -> Result<IndexReader> {
        let inner_reader = InnerIndexReader {
            index: self.index,
            num_searchers: self.num_searchers,
@@ -120,14 +121,14 @@ struct InnerIndexReader {
 }

 impl InnerIndexReader {
-    fn reload(&self) -> crate::Result<()> {
+    fn reload(&self) -> Result<()> {
        let segment_readers: Vec<SegmentReader> = {
            let _meta_lock = self.index.directory().acquire_lock(&META_LOCK)?;
            let searchable_segments = self.searchable_segments()?;
            searchable_segments
                .iter()
                .map(SegmentReader::open)
-                .collect::<crate::Result<_>>()?
+                .collect::<Result<_>>()?
        };
        let schema = self.index.schema();
        let searchers = (0..self.num_searchers)
@@ -138,7 +139,7 @@ impl InnerIndexReader {
    }

    /// Returns the list of segments that are searchable
-    fn searchable_segments(&self) -> crate::Result<Vec<Segment>> {
+    fn searchable_segments(&self) -> Result<Vec<Segment>> {
        self.index.searchable_segments()
    }

@@ -161,11 +162,6 @@ pub struct IndexReader {
 }

 impl IndexReader {
-    #[cfg(test)]
-    pub(crate) fn index(&self) -> Index {
-        self.inner.index.clone()
-    }
-
    /// Update searchers so that they reflect the state of the last
    /// `.commit()`.
    ///
@@ -175,7 +171,7 @@ impl IndexReader {
    ///
    /// This automatic reload can take 10s of milliseconds to kick in however, and in unit tests
    /// it can be nice to deterministically force the reload of searchers.
-    pub fn reload(&self) -> crate::Result<()> {
+    pub fn reload(&self) -> Result<()> {
        self.inner.reload()
    }

--- a/src/reader/pool.rs
+++ b/src/reader/pool.rs
@@ -167,7 +167,7 @@ mod tests {

    use super::Pool;
    use super::Queue;
-    use std::{iter, mem};
+    use std::iter;

    #[test]
    fn test_pool() {
@@ -197,67 +197,33 @@ mod tests {
    fn test_pool_dont_panic_on_empty_pop() {
        // When the object pool is exhausted, it shouldn't panic on pop()
        use std::sync::Arc;
-        use std::thread;
+        use std::{thread, time};

        // Wrap the pool in an Arc, same way as its used in `core/index.rs`
-        let pool1 = Arc::new(Pool::new());
+        let pool = Arc::new(Pool::new());
        // clone pools outside the move scope of each new thread
-        let pool2 = Arc::clone(&pool1);
-        let pool3 = Arc::clone(&pool1);
-
+        let pool1 = Arc::clone(&pool);
+        let pool2 = Arc::clone(&pool);
        let elements_for_pool = vec![1, 2];
-        pool1.publish_new_generation(elements_for_pool);
+        pool.publish_new_generation(elements_for_pool);

        let mut threads = vec![];
+        let sleep_dur = time::Duration::from_millis(10);
        // spawn one more thread than there are elements in the pool
-
-        let (start_1_send, start_1_recv) = crossbeam::bounded(0);
-        let (start_2_send, start_2_recv) = crossbeam::bounded(0);
-        let (start_3_send, start_3_recv) = crossbeam::bounded(0);
-
-        let (event_send1, event_recv) = crossbeam::unbounded();
-        let event_send2 = event_send1.clone();
-        let event_send3 = event_send1.clone();
-
        threads.push(thread::spawn(move || {
-            assert_eq!(start_1_recv.recv(), Ok("start"));
+            // leasing to make sure it's not dropped before sleep is called
+            let _leased_searcher = &pool.acquire();
+            thread::sleep(sleep_dur);
+        }));
+        threads.push(thread::spawn(move || {
+            // leasing to make sure it's not dropped before sleep is called
            let _leased_searcher = &pool1.acquire();
-            assert!(event_send1.send("1 acquired").is_ok());
-            assert_eq!(start_1_recv.recv(), Ok("stop"));
-            assert!(event_send1.send("1 stopped").is_ok());
-            mem::drop(_leased_searcher);
+            thread::sleep(sleep_dur);
        }));
-
        threads.push(thread::spawn(move || {
-            assert_eq!(start_2_recv.recv(), Ok("start"));
+            // leasing to make sure it's not dropped before sleep is called
            let _leased_searcher = &pool2.acquire();
-            assert!(event_send2.send("2 acquired").is_ok());
-            assert_eq!(start_2_recv.recv(), Ok("stop"));
-            mem::drop(_leased_searcher);
-            assert!(event_send2.send("2 stopped").is_ok());
+            thread::sleep(sleep_dur);
        }));
-
-        threads.push(thread::spawn(move || {
-            assert_eq!(start_3_recv.recv(), Ok("start"));
-            let _leased_searcher = &pool3.acquire();
-            assert!(event_send3.send("3 acquired").is_ok());
-            assert_eq!(start_3_recv.recv(), Ok("stop"));
-            mem::drop(_leased_searcher);
-            assert!(event_send3.send("3 stopped").is_ok());
-        }));
-
-        assert!(start_1_send.send("start").is_ok());
-        assert_eq!(event_recv.recv(), Ok("1 acquired"));
-        assert!(start_2_send.send("start").is_ok());
-        assert_eq!(event_recv.recv(), Ok("2 acquired"));
-        assert!(start_3_send.send("start").is_ok());
-        assert!(event_recv.try_recv().is_err());
-        assert!(start_1_send.send("stop").is_ok());
-        assert_eq!(event_recv.recv(), Ok("1 stopped"));
-        assert_eq!(event_recv.recv(), Ok("3 acquired"));
-        assert!(start_3_send.send("stop").is_ok());
-        assert_eq!(event_recv.recv(), Ok("3 stopped"));
-        assert!(start_2_send.send("stop").is_ok());
-        assert_eq!(event_recv.recv(), Ok("2 stopped"));
    }
 }
--- a/src/schema/document.rs
+++ b/src/schema/document.rs
@@ -1,7 +1,6 @@
 use super::*;
 use crate::common::BinarySerializable;
 use crate::common::VInt;
-use crate::tokenizer::PreTokenizedString;
 use crate::DateTime;
 use itertools::Itertools;
 use std::io::{self, Read, Write};
@@ -30,8 +29,8 @@ impl From<Vec<FieldValue>> for Document {
 impl PartialEq for Document {
    fn eq(&self, other: &Document) -> bool {
        // super slow, but only here for tests
-        let mut self_field_values: Vec<&_> = self.field_values.iter().collect();
-        let mut other_field_values: Vec<&_> = other.field_values.iter().collect();
+        let mut self_field_values = self.field_values.clone();
+        let mut other_field_values = other.field_values.clone();
        self_field_values.sort();
        other_field_values.sort();
        self_field_values.eq(&other_field_values)
@@ -79,16 +78,6 @@ impl Document {
        self.add(FieldValue::new(field, value));
    }

-    /// Add a pre-tokenized text field.
-    pub fn add_pre_tokenized_text(
-        &mut self,
-        field: Field,
-        pre_tokenized_text: &PreTokenizedString,
-    ) {
-        let value = Value::PreTokStr(pre_tokenized_text.clone());
-        self.add(FieldValue::new(field, value));
-    }
-
    /// Add a u64 field
    pub fn add_u64(&mut self, field: Field, value: u64) {
        self.add(FieldValue::new(field, Value::U64(value)));
@@ -155,21 +144,6 @@ impl Document {
            .find(|field_value| field_value.field() == field)
            .map(FieldValue::value)
    }
-
-    /// Prepares Document for being stored in the document store
-    ///
-    /// Method transforms PreTokenizedString values into String
-    /// values.
-    pub fn prepare_for_store(&mut self) {
-        for field_value in &mut self.field_values {
-            if let Value::PreTokStr(pre_tokenized_text) = field_value.value() {
-                *field_value = FieldValue::new(
-                    field_value.field(),
-                    Value::Str(pre_tokenized_text.text.clone()), //< TODO somehow remove .clone()
-                );
-            }
-        }
-    }
 }

 impl BinarySerializable for Document {
@@ -195,7 +169,6 @@ impl BinarySerializable for Document {
 mod tests {

    use crate::schema::*;
-    use crate::tokenizer::{PreTokenizedString, Token};

    #[test]
    fn test_doc() {
@@ -205,38 +178,4 @@ mod tests {
        doc.add_text(text_field, "My title");
        assert_eq!(doc.field_values().len(), 1);
    }
-
-    #[test]
-    fn test_prepare_for_store() {
-        let mut schema_builder = Schema::builder();
-        let text_field = schema_builder.add_text_field("title", TEXT);
-        let mut doc = Document::default();
-
-        let pre_tokenized_text = PreTokenizedString {
-            text: String::from("A"),
-            tokens: vec![Token {
-                offset_from: 0,
-                offset_to: 1,
-                position: 0,
-                text: String::from("A"),
-                position_length: 1,
-            }],
-        };
-
-        doc.add_pre_tokenized_text(text_field, &pre_tokenized_text);
-        doc.add_text(text_field, "title");
-        doc.prepare_for_store();
-
-        assert_eq!(doc.field_values().len(), 2);
-
-        match doc.field_values()[0].value() {
-            Value::Str(ref text) => assert_eq!(text, "A"),
-            _ => panic!("Incorrect variant of Value"),
-        }
-
-        match doc.field_values()[1].value() {
-            Value::Str(ref text) => assert_eq!(text, "title"),
-            _ => panic!("Incorrect variant of Value"),
-        }
-    }
 }
--- a/src/schema/facet.rs
+++ b/src/schema/facet.rs
@@ -122,11 +122,6 @@ impl Facet {
    pub fn to_path(&self) -> Vec<&str> {
        self.encoded_str().split(|c| c == FACET_SEP_CHAR).collect()
    }
-
-    /// This function is the inverse of Facet::from(&str).
-    pub fn to_path_string(&self) -> String {
-        format!("{}", self.to_string())
-    }
 }

 impl Borrow<str> for Facet {
@@ -270,21 +265,4 @@ mod tests {
        let facet = Facet::from_path(v.iter());
        assert_eq!(facet.to_path(), v);
    }
-
-    #[test]
-    fn test_to_path_string() {
-        let v = ["first", "second", "third/not_fourth"];
-        let facet = Facet::from_path(v.iter());
-        assert_eq!(
-            facet.to_path_string(),
-            String::from("/first/second/third\\/not_fourth")
-        );
-    }
-
-    #[test]
-    fn test_to_path_string_empty() {
-        let v: Vec<&str> = vec![];
-        let facet = Facet::from_path(v.iter());
-        assert_eq!(facet.to_path_string(), "/");
-    }
 }
--- a/src/schema/field.rs
+++ b/src/schema/field.rs
@@ -15,7 +15,6 @@ impl Field {
    }

    /// Returns a u32 identifying uniquely a field within a schema.
-    #[allow(clippy::trivially_copy_pass_by_ref)]
    pub fn field_id(&self) -> u32 {
        self.0
    }
--- a/src/schema/field_type.rs
+++ b/src/schema/field_type.rs
@@ -1,12 +1,11 @@
 use base64::decode;

+use crate::schema::{IntOptions, TextOptions};
+
 use crate::schema::Facet;
 use crate::schema::IndexRecordOption;
 use crate::schema::TextFieldIndexing;
 use crate::schema::Value;
-use crate::schema::{IntOptions, TextOptions};
-use crate::tokenizer::PreTokenizedString;
-use chrono::{FixedOffset, Utc};
 use serde_json::Value as JsonValue;

 /// Possible error that may occur while parsing a field value
@@ -125,20 +124,13 @@ impl FieldType {
    pub fn value_from_json(&self, json: &JsonValue) -> Result<Value, ValueParsingError> {
        match *json {
            JsonValue::String(ref field_text) => match *self {
-                FieldType::Date(_) => {
-                    let dt_with_fixed_tz: chrono::DateTime<FixedOffset> =
-                        chrono::DateTime::parse_from_rfc3339(field_text).map_err(|err|
-                            ValueParsingError::TypeError(format!(
-                                "Failed to parse date from JSON. Expected rfc3339 format, got {}. {:?}",
-                                field_text, err
-                            ))
-                        )?;
-                    Ok(Value::Date(dt_with_fixed_tz.with_timezone(&Utc)))
-                }
                FieldType::Str(_) => Ok(Value::Str(field_text.clone())),
-                FieldType::U64(_) | FieldType::I64(_) | FieldType::F64(_) => Err(
-                    ValueParsingError::TypeError(format!("Expected an integer, got {:?}", json)),
-                ),
+                FieldType::U64(_) | FieldType::I64(_) | FieldType::F64(_) | FieldType::Date(_) => {
+                    Err(ValueParsingError::TypeError(format!(
+                        "Expected an integer, got {:?}",
+                        json
+                    )))
+                }
                FieldType::HierarchicalFacet => Ok(Value::Facet(Facet::from(field_text))),
                FieldType::Bytes => decode(field_text).map(Value::Bytes).map_err(|_| {
                    ValueParsingError::InvalidBase64(format!(
@@ -177,28 +169,6 @@ impl FieldType {
                    Err(ValueParsingError::TypeError(msg))
                }
            },
-            JsonValue::Object(_) => match *self {
-                FieldType::Str(_) => {
-                    if let Ok(tok_str_val) =
-                        serde_json::from_value::<PreTokenizedString>(json.clone())
-                    {
-                        Ok(Value::PreTokStr(tok_str_val))
-                    } else {
-                        let msg = format!(
-                            "Json value {:?} cannot be translated to PreTokenizedString.",
-                            json
-                        );
-                        Err(ValueParsingError::TypeError(msg))
-                    }
-                }
-                _ => {
-                    let msg = format!(
-                        "Json value not supported error {:?}. Expected {:?}",
-                        json, self
-                    );
-                    Err(ValueParsingError::TypeError(msg))
-                }
-            },
            _ => {
                let msg = format!(
                    "Json value not supported error {:?}. Expected {:?}",
@@ -214,37 +184,7 @@ impl FieldType {
 mod tests {
    use super::FieldType;
    use crate::schema::field_type::ValueParsingError;
-    use crate::schema::TextOptions;
    use crate::schema::Value;
-    use crate::schema::{Schema, INDEXED};
-    use crate::tokenizer::{PreTokenizedString, Token};
-    use crate::{DateTime, Document};
-    use chrono::{NaiveDate, NaiveDateTime, NaiveTime, Utc};
-
-    #[test]
-    fn test_deserialize_json_date() {
-        let mut schema_builder = Schema::builder();
-        let date_field = schema_builder.add_date_field("date", INDEXED);
-        let schema = schema_builder.build();
-        let doc_json = r#"{"date": "2019-10-12T07:20:50.52+02:00"}"#;
-        let doc = schema.parse_document(doc_json).unwrap();
-        let date = doc.get_first(date_field).unwrap();
-        assert_eq!(format!("{:?}", date), "Date(2019-10-12T05:20:50.520Z)");
-    }
-
-    #[test]
-    fn test_serialize_json_date() {
-        let mut doc = Document::new();
-        let mut schema_builder = Schema::builder();
-        let date_field = schema_builder.add_date_field("date", INDEXED);
-        let schema = schema_builder.build();
-        let naive_date = NaiveDate::from_ymd(1982, 9, 17);
-        let naive_time = NaiveTime::from_hms(13, 20, 00);
-        let date_time = DateTime::from_utc(NaiveDateTime::new(naive_date, naive_time), Utc);
-        doc.add_date(date_field, &date_time);
-        let doc_json = schema.to_json(&doc);
-        assert_eq!(doc_json, r#"{"date":["1982-09-17T13:20:00+00:00"]}"#);
-    }

    #[test]
    fn test_bytes_value_from_json() {
@@ -265,71 +205,4 @@ mod tests {
            _ => panic!("Expected parse failure for invalid base64"),
        }
    }
-
-    #[test]
-    fn test_pre_tok_str_value_from_json() {
-        let pre_tokenized_string_json = r#"{
-  "text": "The Old Man",
-  "tokens": [
-    {
-      "offset_from": 0,
-      "offset_to": 3,
-      "position": 0,
-      "text": "The",
-      "position_length": 1
-    },
-    {
-      "offset_from": 4,
-      "offset_to": 7,
-      "position": 1,
-      "text": "Old",
-      "position_length": 1
-    },
-    {
-      "offset_from": 8,
-      "offset_to": 11,
-      "position": 2,
-      "text": "Man",
-      "position_length": 1
-    }
-  ]
-}"#;
-
-        let expected_value = Value::PreTokStr(PreTokenizedString {
-            text: String::from("The Old Man"),
-            tokens: vec![
-                Token {
-                    offset_from: 0,
-                    offset_to: 3,
-                    position: 0,
-                    text: String::from("The"),
-                    position_length: 1,
-                },
-                Token {
-                    offset_from: 4,
-                    offset_to: 7,
-                    position: 1,
-                    text: String::from("Old"),
-                    position_length: 1,
-                },
-                Token {
-                    offset_from: 8,
-                    offset_to: 11,
-                    position: 2,
-                    text: String::from("Man"),
-                    position_length: 1,
-                },
-            ],
-        });
-
-        let deserialized_value = FieldType::Str(TextOptions::default())
-            .value_from_json(&serde_json::from_str(pre_tokenized_string_json).unwrap())
-            .unwrap();
-
-        assert_eq!(deserialized_value, expected_value);
-
-        let serialized_value_json = serde_json::to_string_pretty(&expected_value).unwrap();
-
-        assert_eq!(serialized_value_json, pre_tokenized_string_json);
-    }
 }
--- a/src/schema/flags.rs
+++ b/src/schema/flags.rs
@@ -53,7 +53,7 @@ where
    fn bitor(self, head: SchemaFlagList<Head, ()>) -> Self::Output {
        SchemaFlagList {
            head: head.head,
-            tail: self,
+            tail: self.clone(),
        }
    }
 }
--- a/src/schema/mod.rs
+++ b/src/schema/mod.rs
@@ -44,7 +44,7 @@ We can split the problem of generating a search result page into two phases :
  the search results page. (`doc_ids[] -> Document[]`)

 In the first phase, the ability to search for documents by the given field is determined by the
-[`IndexRecordOption`](enum.IndexRecordOption.html) of our
+[`TextIndexingOptions`](enum.TextIndexingOptions.html) of our
 [`TextOptions`](struct.TextOptions.html).

 The effect of each possible setting is described more in detail
--- a/src/schema/schema.rs
+++ b/src/schema/schema.rs
@@ -166,7 +166,7 @@ impl SchemaBuilder {
    }

    /// Adds a field entry to the schema in build.
-    pub fn add_field(&mut self, field_entry: FieldEntry) -> Field {
+    fn add_field(&mut self, field_entry: FieldEntry) -> Field {
        let field = Field::from_field_id(self.fields.len() as u32);
        let field_name = field_entry.name().to_string();
        self.fields.push(field_entry);
@@ -401,7 +401,6 @@ pub enum DocParsingError {
 mod tests {

    use crate::schema::field_type::ValueParsingError;
-    use crate::schema::int_options::Cardinality::SingleValue;
    use crate::schema::schema::DocParsingError::NotJSON;
    use crate::schema::*;
    use matches::{assert_matches, matches};
@@ -716,94 +715,4 @@ mod tests {
            assert_matches!(json_err, Err(NotJSON(_)));
        }
    }
-
-    #[test]
-    pub fn test_schema_add_field() {
-        let mut schema_builder = SchemaBuilder::default();
-        let id_options = TextOptions::default().set_stored().set_indexing_options(
-            TextFieldIndexing::default()
-                .set_tokenizer("raw")
-                .set_index_option(IndexRecordOption::Basic),
-        );
-        let timestamp_options = IntOptions::default()
-            .set_stored()
-            .set_indexed()
-            .set_fast(SingleValue);
-        schema_builder.add_text_field("_id", id_options);
-        schema_builder.add_date_field("_timestamp", timestamp_options);
-
-        let schema_content = r#"[
-  {
-    "name": "text",
-    "type": "text",
-    "options": {
-      "indexing": {
-        "record": "position",
-        "tokenizer": "default"
-      },
-      "stored": false
-    }
-  },
-  {
-    "name": "popularity",
-    "type": "i64",
-    "options": {
-      "indexed": false,
-      "fast": "single",
-      "stored": true
-    }
-  }
-]"#;
-        let tmp_schema: Schema =
-            serde_json::from_str(&schema_content).expect("error while reading json");
-        for (_field, field_entry) in tmp_schema.fields() {
-            schema_builder.add_field(field_entry.clone());
-        }
-
-        let schema = schema_builder.build();
-        let schema_json = serde_json::to_string_pretty(&schema).unwrap();
-        let expected = r#"[
-  {
-    "name": "_id",
-    "type": "text",
-    "options": {
-      "indexing": {
-        "record": "basic",
-        "tokenizer": "raw"
-      },
-      "stored": true
-    }
-  },
-  {
-    "name": "_timestamp",
-    "type": "date",
-    "options": {
-      "indexed": true,
-      "fast": "single",
-      "stored": true
-    }
-  },
-  {
-    "name": "text",
-    "type": "text",
-    "options": {
-      "indexing": {
-        "record": "position",
-        "tokenizer": "default"
-      },
-      "stored": false
-    }
-  },
-  {
-    "name": "popularity",
-    "type": "i64",
-    "options": {
-      "indexed": false,
-      "fast": "single",
-      "stored": true
-    }
-  }
-]"#;
-        assert_eq!(schema_json, expected);
-    }
 }
--- a/src/schema/value.rs
+++ b/src/schema/value.rs
@@ -1,5 +1,4 @@
 use crate::schema::Facet;
-use crate::tokenizer::PreTokenizedString;
 use crate::DateTime;
 use serde::de::Visitor;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -11,8 +10,6 @@ use std::{cmp::Ordering, fmt};
 pub enum Value {
    /// The str type is used for any text information.
    Str(String),
-    /// Pre-tokenized str type,
-    PreTokStr(PreTokenizedString),
    /// Unsigned 64-bits Integer `u64`
    U64(u64),
    /// Signed 64-bits Integer `i64`
@@ -32,7 +29,6 @@ impl Ord for Value {
    fn cmp(&self, other: &Self) -> Ordering {
        match (self, other) {
            (Value::Str(l), Value::Str(r)) => l.cmp(r),
-            (Value::PreTokStr(l), Value::PreTokStr(r)) => l.cmp(r),
            (Value::U64(l), Value::U64(r)) => l.cmp(r),
            (Value::I64(l), Value::I64(r)) => l.cmp(r),
            (Value::Date(l), Value::Date(r)) => l.cmp(r),
@@ -48,8 +44,6 @@ impl Ord for Value {
            }
            (Value::Str(_), _) => Ordering::Less,
            (_, Value::Str(_)) => Ordering::Greater,
-            (Value::PreTokStr(_), _) => Ordering::Less,
-            (_, Value::PreTokStr(_)) => Ordering::Greater,
            (Value::U64(_), _) => Ordering::Less,
            (_, Value::U64(_)) => Ordering::Greater,
            (Value::I64(_), _) => Ordering::Less,
@@ -71,11 +65,10 @@ impl Serialize for Value {
    {
        match *self {
            Value::Str(ref v) => serializer.serialize_str(v),
-            Value::PreTokStr(ref v) => v.serialize(serializer),
            Value::U64(u) => serializer.serialize_u64(u),
            Value::I64(u) => serializer.serialize_i64(u),
            Value::F64(u) => serializer.serialize_f64(u),
-            Value::Date(ref date) => serializer.serialize_str(&date.to_rfc3339()),
+            Value::Date(ref date) => serializer.serialize_i64(date.timestamp()),
            Value::Facet(ref facet) => facet.serialize(serializer),
            Value::Bytes(ref bytes) => serializer.serialize_bytes(bytes),
        }
@@ -96,14 +89,14 @@ impl<'de> Deserialize<'de> for Value {
                formatter.write_str("a string or u32")
            }

-            fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E> {
-                Ok(Value::I64(v))
-            }
-
            fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E> {
                Ok(Value::U64(v))
            }

+            fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E> {
+                Ok(Value::I64(v))
+            }
+
            fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E> {
                Ok(Value::F64(v))
            }
@@ -131,15 +124,6 @@ impl Value {
        }
    }

-    /// Returns the tokenized text, provided the value is of the `PreTokStr` type.
-    /// (Returns None if the value is not of the `PreTokStr` type).
-    pub fn tokenized_text(&self) -> Option<&PreTokenizedString> {
-        match *self {
-            Value::PreTokStr(ref tok_text) => Some(tok_text),
-            _ => None,
-        }
-    }
-
    /// Returns the u64-value, provided the value is of the `U64` type.
    ///
    /// # Panics
@@ -209,8 +193,8 @@ impl From<f64> for Value {
    }
 }

-impl From<crate::DateTime> for Value {
-    fn from(date_time: crate::DateTime) -> Value {
+impl From<DateTime> for Value {
+    fn from(date_time: DateTime) -> Value {
        Value::Date(date_time)
    }
 }
@@ -233,17 +217,10 @@ impl From<Vec<u8>> for Value {
    }
 }

-impl From<PreTokenizedString> for Value {
-    fn from(pretokenized_string: PreTokenizedString) -> Value {
-        Value::PreTokStr(pretokenized_string)
-    }
-}
-
 mod binary_serialize {
    use super::Value;
    use crate::common::{f64_to_u64, u64_to_f64, BinarySerializable};
    use crate::schema::Facet;
-    use crate::tokenizer::PreTokenizedString;
    use chrono::{TimeZone, Utc};
    use std::io::{self, Read, Write};

@@ -254,11 +231,6 @@ mod binary_serialize {
    const BYTES_CODE: u8 = 4;
    const DATE_CODE: u8 = 5;
    const F64_CODE: u8 = 6;
-    const EXT_CODE: u8 = 7;
-
-    // extended types
-
-    const TOK_STR_CODE: u8 = 0;

    impl BinarySerializable for Value {
        fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
@@ -267,18 +239,6 @@ mod binary_serialize {
                    TEXT_CODE.serialize(writer)?;
                    text.serialize(writer)
                }
-                Value::PreTokStr(ref tok_str) => {
-                    EXT_CODE.serialize(writer)?;
-                    TOK_STR_CODE.serialize(writer)?;
-                    if let Ok(text) = serde_json::to_string(tok_str) {
-                        text.serialize(writer)
-                    } else {
-                        Err(io::Error::new(
-                            io::ErrorKind::Other,
-                            "Failed to dump Value::PreTokStr(_) to json.",
-                        ))
-                    }
-                }
                Value::U64(ref val) => {
                    U64_CODE.serialize(writer)?;
                    val.serialize(writer)
@@ -330,30 +290,6 @@ mod binary_serialize {
                }
                HIERARCHICAL_FACET_CODE => Ok(Value::Facet(Facet::deserialize(reader)?)),
                BYTES_CODE => Ok(Value::Bytes(Vec::<u8>::deserialize(reader)?)),
-                EXT_CODE => {
-                    let ext_type_code = u8::deserialize(reader)?;
-                    match ext_type_code {
-                        TOK_STR_CODE => {
-                            let str_val = String::deserialize(reader)?;
-                            if let Ok(value) = serde_json::from_str::<PreTokenizedString>(&str_val)
-                            {
-                                Ok(Value::PreTokStr(value))
-                            } else {
-                                Err(io::Error::new(
-                                    io::ErrorKind::Other,
-                                    "Failed to parse string data as Value::PreTokStr(_).",
-                                ))
-                            }
-                        }
-                        _ => Err(io::Error::new(
-                            io::ErrorKind::InvalidData,
-                            format!(
-                                "No extened field type is associated with code {:?}",
-                                ext_type_code
-                            ),
-                        )),
-                    }
-                }
                _ => Err(io::Error::new(
                    io::ErrorKind::InvalidData,
                    format!("No field type is associated with code {:?}", type_code),
@@ -362,17 +298,3 @@ mod binary_serialize {
        }
    }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::Value;
-    use crate::DateTime;
-    use std::str::FromStr;
-
-    #[test]
-    fn test_serialize_date() {
-        let value = Value::Date(DateTime::from_str("1996-12-20T00:39:57+00:00").unwrap());
-        let serialized_value_json = serde_json::to_string_pretty(&value).unwrap();
-        assert_eq!(serialized_value_json, r#""1996-12-20T00:39:57+00:00""#);
-    }
-}
--- a/src/snippet/mod.rs
+++ b/src/snippet/mod.rs
@@ -1,8 +1,10 @@
 use crate::query::Query;
 use crate::schema::Field;
 use crate::schema::Value;
-use crate::tokenizer::{TextAnalyzer, Token};
+use crate::tokenizer::BoxedTokenizer;
+use crate::tokenizer::{Token, TokenStream};
 use crate::Document;
+use crate::Result;
 use crate::Searcher;
 use htmlescape::encode_minimal;
 use std::cmp::Ordering;
@@ -140,7 +142,7 @@ impl Snippet {
 /// Fragments must be valid in the sense that `&text[fragment.start..fragment.stop]`\
 /// has to be a valid string.
 fn search_fragments<'a>(
-    tokenizer: &TextAnalyzer,
+    tokenizer: &BoxedTokenizer,
    text: &'a str,
    terms: &BTreeMap<String, f32>,
    max_num_chars: usize,
@@ -249,7 +251,7 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str)
 /// ```
 pub struct SnippetGenerator {
    terms_text: BTreeMap<String, f32>,
-    tokenizer: TextAnalyzer,
+    tokenizer: BoxedTokenizer,
    field: Field,
    max_num_chars: usize,
 }
@@ -260,7 +262,7 @@ impl SnippetGenerator {
        searcher: &Searcher,
        query: &dyn Query,
        field: Field,
-    ) -> crate::Result<SnippetGenerator> {
+    ) -> Result<SnippetGenerator> {
        let mut terms = BTreeSet::new();
        query.query_terms(&mut terms);
        let terms_text: BTreeMap<String, f32> = terms
@@ -329,8 +331,9 @@ mod tests {
    use std::collections::BTreeMap;
    use std::iter::Iterator;

-    const TEST_TEXT: &'static str = r#"Rust is a systems programming language sponsored by
-Mozilla which describes it as a "safe, concurrent, practical language", supporting functional and
+    const TEST_TEXT: &'static str =
+        r#"Rust is a systems programming language sponsored by Mozilla which
+describes it as a "safe, concurrent, practical language", supporting functional and
 imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?],
 but its designers intend it to provide better memory safety while still maintaining
 performance.
@@ -345,11 +348,12 @@ Survey in 2016, 2017, and 2018."#;

    #[test]
    fn test_snippet() {
+        let boxed_tokenizer = SimpleTokenizer.into();
        let terms = btreemap! {
            String::from("rust") => 1.0,
            String::from("language") => 0.9
        };
-        let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 100);
+        let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 100);
        assert_eq!(fragments.len(), 7);
        {
            let first = &fragments[0];
@@ -359,24 +363,25 @@ Survey in 2016, 2017, and 2018."#;
        let snippet = select_best_fragment_combination(&fragments[..], &TEST_TEXT);
        assert_eq!(
            snippet.fragments,
-            "Rust is a systems programming language sponsored by\n\
-             Mozilla which describes it as a \"safe"
+            "Rust is a systems programming language sponsored by \
+             Mozilla which\ndescribes it as a \"safe"
        );
        assert_eq!(
            snippet.to_html(),
            "<b>Rust</b> is a systems programming <b>language</b> \
-             sponsored by\nMozilla which describes it as a &quot;safe"
+             sponsored by Mozilla which\ndescribes it as a &quot;safe"
        )
    }

    #[test]
    fn test_snippet_scored_fragment() {
+        let boxed_tokenizer = SimpleTokenizer.into();
        {
            let terms = btreemap! {
                String::from("rust") =>1.0f32,
                String::from("language") => 0.9f32
            };
-            let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 20);
+            let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 20);
            {
                let first = &fragments[0];
                assert_eq!(first.score, 1.0);
@@ -385,12 +390,13 @@ Survey in 2016, 2017, and 2018."#;
            let snippet = select_best_fragment_combination(&fragments[..], &TEST_TEXT);
            assert_eq!(snippet.to_html(), "<b>Rust</b> is a systems")
        }
+        let boxed_tokenizer = SimpleTokenizer.into();
        {
            let terms = btreemap! {
                String::from("rust") =>0.9f32,
                String::from("language") => 1.0f32
            };
-            let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 20);
+            let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 20);
            //assert_eq!(fragments.len(), 7);
            {
                let first = &fragments[0];
@@ -404,12 +410,14 @@ Survey in 2016, 2017, and 2018."#;

    #[test]
    fn test_snippet_in_second_fragment() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
        let text = "a b c d e f g";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("c"), 1.0);

-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);

        assert_eq!(fragments.len(), 1);
        {
@@ -426,12 +434,14 @@ Survey in 2016, 2017, and 2018."#;

    #[test]
    fn test_snippet_with_term_at_the_end_of_fragment() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
        let text = "a b c d e f f g";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("f"), 1.0);

-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);

        assert_eq!(fragments.len(), 2);
        {
@@ -448,13 +458,15 @@ Survey in 2016, 2017, and 2018."#;

    #[test]
    fn test_snippet_with_second_fragment_has_the_highest_score() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
        let text = "a b c d e f g";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("f"), 1.0);
        terms.insert(String::from("a"), 0.9);

-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 7);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 7);

        assert_eq!(fragments.len(), 2);
        {
@@ -471,12 +483,14 @@ Survey in 2016, 2017, and 2018."#;

    #[test]
    fn test_snippet_with_term_not_in_text() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
        let text = "a b c d";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("z"), 1.0);

-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);

        assert_eq!(fragments.len(), 0);

@@ -487,10 +501,12 @@ Survey in 2016, 2017, and 2018."#;

    #[test]
    fn test_snippet_with_no_terms() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
        let text = "a b c d";

        let terms = BTreeMap::new();
-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);
        assert_eq!(fragments.len(), 0);

        let snippet = select_best_fragment_combination(&fragments[..], &text);
--- a/src/store/compression_lz4.rs
+++ b/src/store/compression_lz4.rs
@@ -1,9 +1,6 @@
-use std::io::{self, Read, Write};
+extern crate lz4;

-/// Name of the compression scheme used in the doc store.
-///
-/// This name is appended to the version string of tantivy.
-pub const COMPRESSION: &'static str = "lz4";
+use std::io::{self, Read, Write};

 pub fn compress(uncompressed: &[u8], compressed: &mut Vec<u8>) -> io::Result<()> {
    compressed.clear();
--- a/src/store/compression_snap.rs
+++ b/src/store/compression_snap.rs
@@ -2,11 +2,6 @@ use snap;

 use std::io::{self, Read, Write};

-/// Name of the compression scheme used in the doc store.
-///
-/// This name is appended to the version string of tantivy.
-pub const COMPRESSION: &str = "snappy";
-
 pub fn compress(uncompressed: &[u8], compressed: &mut Vec<u8>) -> io::Result<()> {
    compressed.clear();
    let mut encoder = snap::Writer::new(compressed);
--- a/src/store/mod.rs
+++ b/src/store/mod.rs
@@ -42,16 +42,12 @@ pub use self::writer::StoreWriter;
 #[cfg(feature = "lz4")]
 mod compression_lz4;
 #[cfg(feature = "lz4")]
-pub use self::compression_lz4::COMPRESSION;
-#[cfg(feature = "lz4")]
-use self::compression_lz4::{compress, decompress};
+use self::compression_lz4::*;

 #[cfg(not(feature = "lz4"))]
 mod compression_snap;
 #[cfg(not(feature = "lz4"))]
-pub use self::compression_snap::COMPRESSION;
-#[cfg(not(feature = "lz4"))]
-use self::compression_snap::{compress, decompress};
+use self::compression_snap::*;

 #[cfg(test)]
 pub mod tests {
--- a/src/store/reader.rs
+++ b/src/store/reader.rs
@@ -1,3 +1,5 @@
+use crate::Result;
+
 use super::decompress;
 use super::skiplist::SkipList;
 use crate::common::BinarySerializable;
@@ -73,7 +75,7 @@ impl StoreReader {
    ///
    /// It should not be called to score documents
    /// for instance.
-    pub fn get(&self, doc_id: DocId) -> crate::Result<Document> {
+    pub fn get(&self, doc_id: DocId) -> Result<Document> {
        let (first_doc_id, block_offset) = self.block_offset(doc_id);
        self.read_block(block_offset as usize)?;
        let current_block_mut = self.current_block.borrow_mut();
--- a/src/termdict/mod.rs
+++ b/src/termdict/mod.rs
@@ -38,7 +38,7 @@ mod tests {
    use crate::core::Index;
    use crate::directory::{Directory, RAMDirectory, ReadOnlySource};
    use crate::postings::TermInfo;
-    use crate::schema::{Document, Schema, TEXT};
+    use crate::schema::{Document, FieldType, Schema, TEXT};
    use std::path::PathBuf;
    use std::str;

@@ -52,12 +52,6 @@ mod tests {
        }
    }

-    #[test]
-    fn test_empty_term_dictionary() {
-        let empty = TermDictionary::empty();
-        assert!(empty.stream().next().is_none());
-    }
-
    #[test]
    fn test_term_ordinals() {
        const COUNTRIES: [&'static str; 7] = [
@@ -73,7 +67,9 @@ mod tests {
        let path = PathBuf::from("TermDictionary");
        {
            let write = directory.open_write(&path).unwrap();
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(write).unwrap();
+            let field_type = FieldType::Str(TEXT);
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(write, &field_type).unwrap();
            for term in COUNTRIES.iter() {
                term_dictionary_builder
                    .insert(term.as_bytes(), &make_term_info(0u64))
@@ -97,7 +93,9 @@ mod tests {
        let path = PathBuf::from("TermDictionary");
        {
            let write = directory.open_write(&path).unwrap();
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(write).unwrap();
+            let field_type = FieldType::Str(TEXT);
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(write, &field_type).unwrap();
            term_dictionary_builder
                .insert("abc".as_bytes(), &make_term_info(34u64))
                .unwrap();
@@ -181,8 +179,10 @@ mod tests {
        let ids: Vec<_> = (0u32..10_000u32)
            .map(|i| (format!("doc{:0>6}", i), i))
            .collect();
+        let field_type = FieldType::Str(TEXT);
        let buffer: Vec<u8> = {
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
            for &(ref id, ref i) in &ids {
                term_dictionary_builder
                    .insert(id.as_bytes(), &make_term_info(*i as u64))
@@ -209,8 +209,10 @@ mod tests {

    #[test]
    fn test_stream_high_range_prefix_suffix() {
+        let field_type = FieldType::Str(TEXT);
        let buffer: Vec<u8> = {
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
            // term requires more than 16bits
            term_dictionary_builder
                .insert("abcdefghijklmnopqrstuvwxy", &make_term_info(1))
@@ -242,8 +244,10 @@ mod tests {
        let ids: Vec<_> = (0u32..10_000u32)
            .map(|i| (format!("doc{:0>6}", i), i))
            .collect();
+        let field_type = FieldType::Str(TEXT);
        let buffer: Vec<u8> = {
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
            for &(ref id, ref i) in &ids {
                term_dictionary_builder
                    .insert(id.as_bytes(), &make_term_info(*i as u64))
@@ -309,8 +313,10 @@ mod tests {

    #[test]
    fn test_empty_string() {
+        let field_type = FieldType::Str(TEXT);
        let buffer: Vec<u8> = {
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
            term_dictionary_builder
                .insert(&[], &make_term_info(1 as u64))
                .unwrap();
@@ -331,8 +337,10 @@ mod tests {

    #[test]
    fn test_stream_range_boundaries() {
+        let field_type = FieldType::Str(TEXT);
        let buffer: Vec<u8> = {
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
            for i in 0u8..10u8 {
                let number_arr = [i; 1];
                term_dictionary_builder
@@ -344,91 +352,41 @@ mod tests {
        let source = ReadOnlySource::from(buffer);
        let term_dictionary: TermDictionary = TermDictionary::from_source(&source);

-        let value_list = |mut streamer: TermStreamer<'_>, backwards: bool| {
+        let value_list = |mut streamer: TermStreamer<'_>| {
            let mut res: Vec<u32> = vec![];
            while let Some((_, ref v)) = streamer.next() {
                res.push(v.doc_freq);
            }
-            if backwards {
-                res.reverse();
-            }
            res
        };
-        {
-            let range = term_dictionary.range().backward().into_stream();
-            assert_eq!(
-                value_list(range, true),
-                vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
-            );
-        }
        {
            let range = term_dictionary.range().ge([2u8]).into_stream();
            assert_eq!(
-                value_list(range, false),
-                vec![2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
-            );
-        }
-        {
-            let range = term_dictionary.range().ge([2u8]).backward().into_stream();
-            assert_eq!(
-                value_list(range, true),
+                value_list(range),
                vec![2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
            );
        }
        {
            let range = term_dictionary.range().gt([2u8]).into_stream();
            assert_eq!(
-                value_list(range, false),
-                vec![3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
-            );
-        }
-        {
-            let range = term_dictionary.range().gt([2u8]).backward().into_stream();
-            assert_eq!(
-                value_list(range, true),
+                value_list(range),
                vec![3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
            );
        }
        {
            let range = term_dictionary.range().lt([6u8]).into_stream();
-            assert_eq!(
-                value_list(range, false),
-                vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32]
-            );
-        }
-        {
-            let range = term_dictionary.range().lt([6u8]).backward().into_stream();
-            assert_eq!(
-                value_list(range, true),
-                vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32]
-            );
+            assert_eq!(value_list(range), vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32]);
        }
        {
            let range = term_dictionary.range().le([6u8]).into_stream();
            assert_eq!(
-                value_list(range, false),
-                vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32]
-            );
-        }
-        {
-            let range = term_dictionary.range().le([6u8]).backward().into_stream();
-            assert_eq!(
-                value_list(range, true),
+                value_list(range),
                vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32]
            );
        }
        {
            let range = term_dictionary.range().ge([0u8]).lt([5u8]).into_stream();
-            assert_eq!(value_list(range, false), vec![0u32, 1u32, 2u32, 3u32, 4u32]);
-        }
-        {
-            let range = term_dictionary
-                .range()
-                .ge([0u8])
-                .lt([5u8])
-                .backward()
-                .into_stream();
-            assert_eq!(value_list(range, true), vec![0u32, 1u32, 2u32, 3u32, 4u32]);
+            assert_eq!(value_list(range), vec![0u32, 1u32, 2u32, 3u32, 4u32]);
        }
    }

@@ -450,7 +408,9 @@ mod tests {
        let path = PathBuf::from("TermDictionary");
        {
            let write = directory.open_write(&path).unwrap();
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(write).unwrap();
+            let field_type = FieldType::Str(TEXT);
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(write, &field_type).unwrap();
            for term in COUNTRIES.iter() {
                term_dictionary_builder
                    .insert(term.as_bytes(), &make_term_info(0u64))
--- a/src/termdict/streamer.rs
+++ b/src/termdict/streamer.rs
@@ -51,12 +51,6 @@ where
        self
    }

-    /// Iterate over the range backwards.
-    pub fn backward(mut self) -> Self {
-        self.stream_builder = self.stream_builder.backward();
-        self
-    }
-
    /// Creates the stream corresponding to the range
    /// of terms defined using the `TermStreamerBuilder`.
    pub fn into_stream(self) -> TermStreamer<'a, A> {
--- a/src/termdict/termdict.rs
+++ b/src/termdict/termdict.rs
@@ -4,8 +4,8 @@ use crate::common::BinarySerializable;
 use crate::common::CountingWriter;
 use crate::directory::ReadOnlySource;
 use crate::postings::TermInfo;
+use crate::schema::FieldType;
 use crate::termdict::TermOrdinal;
-use once_cell::sync::Lazy;
 use std::io::{self, Write};
 use tantivy_fst;
 use tantivy_fst::raw::Fst;
@@ -29,7 +29,7 @@ where
    W: Write,
 {
    /// Creates a new `TermDictionaryBuilder`
-    pub fn create(w: W) -> io::Result<Self> {
+    pub fn create(w: W, _field_type: &FieldType) -> io::Result<Self> {
        let fst_builder = tantivy_fst::MapBuilder::new(w).map_err(convert_fst_error)?;
        Ok(TermDictionaryBuilder {
            fst_builder,
@@ -92,14 +92,6 @@ fn open_fst_index(source: ReadOnlySource) -> tantivy_fst::Map<ReadOnlySource> {
    tantivy_fst::Map::from(fst)
 }

-static EMPTY_DATA_SOURCE: Lazy<ReadOnlySource> = Lazy::new(|| {
-    let term_dictionary_data: Vec<u8> = TermDictionaryBuilder::create(Vec::<u8>::new())
-        .expect("Creating a TermDictionaryBuilder in a Vec<u8> should never fail")
-        .finish()
-        .expect("Writing in a Vec<u8> should never fail");
-    ReadOnlySource::from(term_dictionary_data)
-});
-
 /// The term dictionary contains all of the terms in
 /// `tantivy index` in a sorted manner.
 ///
@@ -130,8 +122,14 @@ impl TermDictionary {
    }

    /// Creates an empty term dictionary which contains no terms.
-    pub fn empty() -> Self {
-        TermDictionary::from_source(&*EMPTY_DATA_SOURCE)
+    pub fn empty(field_type: &FieldType) -> Self {
+        let term_dictionary_data: Vec<u8> =
+            TermDictionaryBuilder::create(Vec::<u8>::new(), &field_type)
+                .expect("Creating a TermDictionaryBuilder in a Vec<u8> should never fail")
+                .finish()
+                .expect("Writing in a Vec<u8> should never fail");
+        let source = ReadOnlySource::from(term_dictionary_data);
+        Self::from_source(&source)
    }

    /// Returns the number of terms in the dictionary.
--- a/src/tokenizer/alphanum_only.rs
+++ b/src/tokenizer/alphanum_only.rs
@@ -2,7 +2,9 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let tokenizer = TextAnalyzer::from(RawTokenizer)
+//! # fn main() {
+//!
+//! let tokenizer = RawTokenizer
 //!   .filter(AlphaNumOnlyFilter);
 //!
 //! let mut stream = tokenizer.token_stream("hello there");
@@ -10,7 +12,7 @@
 //! // contains a space
 //! assert!(stream.next().is_none());
 //!
-//! let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+//! let tokenizer = SimpleTokenizer
 //!   .filter(AlphaNumOnlyFilter);
 //!
 //! let mut stream = tokenizer.token_stream("hello there 💣");
@@ -18,31 +20,58 @@
 //! assert!(stream.next().is_some());
 //! // the "emoji" is dropped because its not an alphanum
 //! assert!(stream.next().is_none());
+//! # }
 //! ```
-use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
+use super::{Token, TokenFilter, TokenStream};

 /// `TokenFilter` that removes all tokens that contain non
 /// ascii alphanumeric characters.
 #[derive(Clone)]
 pub struct AlphaNumOnlyFilter;

-pub struct AlphaNumOnlyFilterStream<'a> {
-    tail: BoxTokenStream<'a>,
+pub struct AlphaNumOnlyFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    tail: TailTokenStream,
 }

-impl<'a> AlphaNumOnlyFilterStream<'a> {
+impl<TailTokenStream> AlphaNumOnlyFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
    fn predicate(&self, token: &Token) -> bool {
        token.text.chars().all(|c| c.is_ascii_alphanumeric())
    }
-}

-impl TokenFilter for AlphaNumOnlyFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(AlphaNumOnlyFilterStream { tail: token_stream })
+    fn wrap(tail: TailTokenStream) -> AlphaNumOnlyFilterStream<TailTokenStream> {
+        AlphaNumOnlyFilterStream { tail }
    }
 }

-impl<'a> TokenStream for AlphaNumOnlyFilterStream<'a> {
+impl<TailTokenStream> TokenFilter<TailTokenStream> for AlphaNumOnlyFilter
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = AlphaNumOnlyFilterStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+        AlphaNumOnlyFilterStream::wrap(token_stream)
+    }
+}
+
+impl<TailTokenStream> TokenStream for AlphaNumOnlyFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+
    fn advance(&mut self) -> bool {
        while self.tail.advance() {
            if self.predicate(self.tail.token()) {
@@ -52,12 +81,4 @@ impl<'a> TokenStream for AlphaNumOnlyFilterStream<'a> {

        false
    }
-
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
-    }
 }
--- a/src/tokenizer/ascii_folding_filter.rs
+++ b/src/tokenizer/ascii_folding_filter.rs
@@ -1,4 +1,4 @@
-use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
+use super::{Token, TokenFilter, TokenStream};
 use std::mem;

 /// This class converts alphabetic, numeric, and symbolic Unicode characters
@@ -7,21 +7,26 @@ use std::mem;
 #[derive(Clone)]
 pub struct AsciiFoldingFilter;

-impl TokenFilter for AsciiFoldingFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        From::from(AsciiFoldingFilterTokenStream {
-            tail: token_stream,
-            buffer: String::with_capacity(100),
-        })
+impl<TailTokenStream> TokenFilter<TailTokenStream> for AsciiFoldingFilter
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = AsciiFoldingFilterTokenStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+        AsciiFoldingFilterTokenStream::wrap(token_stream)
    }
 }

-pub struct AsciiFoldingFilterTokenStream<'a> {
+pub struct AsciiFoldingFilterTokenStream<TailTokenStream> {
    buffer: String,
-    tail: BoxTokenStream<'a>,
+    tail: TailTokenStream,
 }

-impl<'a> TokenStream for AsciiFoldingFilterTokenStream<'a> {
+impl<TailTokenStream> TokenStream for AsciiFoldingFilterTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
    fn advance(&mut self) -> bool {
        if !self.tail.advance() {
            return false;
@@ -43,6 +48,18 @@ impl<'a> TokenStream for AsciiFoldingFilterTokenStream<'a> {
    }
 }

+impl<TailTokenStream> AsciiFoldingFilterTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn wrap(tail: TailTokenStream) -> AsciiFoldingFilterTokenStream<TailTokenStream> {
+        AsciiFoldingFilterTokenStream {
+            tail,
+            buffer: String::with_capacity(100),
+        }
+    }
+}
+
 // Returns a string that represents the ascii folded version of
 // the character. If the `char` does not require ascii folding
 // (e.g. simple ASCII chars like `A`) or if the `char`
@@ -1544,7 +1561,8 @@ mod tests {
    use crate::tokenizer::AsciiFoldingFilter;
    use crate::tokenizer::RawTokenizer;
    use crate::tokenizer::SimpleTokenizer;
-    use crate::tokenizer::TextAnalyzer;
+    use crate::tokenizer::TokenStream;
+    use crate::tokenizer::Tokenizer;
    use std::iter;

    #[test]
@@ -1561,7 +1579,7 @@ mod tests {

    fn folding_helper(text: &str) -> Vec<String> {
        let mut tokens = Vec::new();
-        TextAnalyzer::from(SimpleTokenizer)
+        SimpleTokenizer
            .filter(AsciiFoldingFilter)
            .token_stream(text)
            .process(&mut |token| {
@@ -1571,9 +1589,7 @@ mod tests {
    }

    fn folding_using_raw_tokenizer_helper(text: &str) -> String {
-        let mut token_stream = TextAnalyzer::from(RawTokenizer)
-            .filter(AsciiFoldingFilter)
-            .token_stream(text);
+        let mut token_stream = RawTokenizer.filter(AsciiFoldingFilter).token_stream(text);
        token_stream.advance();
        token_stream.token().text.clone()
    }
--- a/src/tokenizer/facet_tokenizer.rs
+++ b/src/tokenizer/facet_tokenizer.rs
@@ -1,4 +1,4 @@
-use super::{BoxTokenStream, Token, TokenStream, Tokenizer};
+use super::{Token, TokenStream, Tokenizer};
 use crate::schema::FACET_SEP_BYTE;

 /// The `FacetTokenizer` process a `Facet` binary representation
@@ -25,14 +25,15 @@ pub struct FacetTokenStream<'a> {
    token: Token,
 }

-impl Tokenizer for FacetTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+impl<'a> Tokenizer<'a> for FacetTokenizer {
+    type TokenStreamImpl = FacetTokenStream<'a>;
+
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
        FacetTokenStream {
            text,
            state: State::RootFacetNotEmitted, //< pos is the first char that has not been processed yet.
            token: Token::default(),
        }
-        .into()
    }
 }

@@ -83,7 +84,7 @@ mod tests {

    use super::FacetTokenizer;
    use crate::schema::Facet;
-    use crate::tokenizer::{Token, Tokenizer};
+    use crate::tokenizer::{Token, TokenStream, Tokenizer};

    #[test]
    fn test_facet_tokenizer() {
--- a/src/tokenizer/lower_caser.rs
+++ b/src/tokenizer/lower_caser.rs
@@ -1,23 +1,24 @@
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;
 use std::mem;

-impl TokenFilter for LowerCaser {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(LowerCaserTokenStream {
-            tail: token_stream,
-            buffer: String::with_capacity(100),
-        })
-    }
-}
-
 /// Token filter that lowercase terms.
 #[derive(Clone)]
 pub struct LowerCaser;

-pub struct LowerCaserTokenStream<'a> {
+impl<TailTokenStream> TokenFilter<TailTokenStream> for LowerCaser
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = LowerCaserTokenStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+        LowerCaserTokenStream::wrap(token_stream)
+    }
+}
+
+pub struct LowerCaserTokenStream<TailTokenStream> {
    buffer: String,
-    tail: BoxTokenStream<'a>,
+    tail: TailTokenStream,
 }

 // writes a lowercased version of text into output.
@@ -30,7 +31,18 @@ fn to_lowercase_unicode(text: &mut String, output: &mut String) {
    }
 }

-impl<'a> TokenStream for LowerCaserTokenStream<'a> {
+impl<TailTokenStream> TokenStream for LowerCaserTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+
    fn advance(&mut self) -> bool {
        if !self.tail.advance() {
            return false;
@@ -44,19 +56,26 @@ impl<'a> TokenStream for LowerCaserTokenStream<'a> {
        }
        true
    }
+}

-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
+impl<TailTokenStream> LowerCaserTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn wrap(tail: TailTokenStream) -> LowerCaserTokenStream<TailTokenStream> {
+        LowerCaserTokenStream {
+            tail,
+            buffer: String::with_capacity(100),
+        }
    }
 }

 #[cfg(test)]
 mod tests {
-    use crate::tokenizer::{LowerCaser, SimpleTokenizer, TextAnalyzer};
+    use crate::tokenizer::LowerCaser;
+    use crate::tokenizer::SimpleTokenizer;
+    use crate::tokenizer::TokenStream;
+    use crate::tokenizer::Tokenizer;

    #[test]
    fn test_to_lower_case() {
@@ -68,9 +87,7 @@ mod tests {

    fn lowercase_helper(text: &str) -> Vec<String> {
        let mut tokens = vec![];
-        let mut token_stream = TextAnalyzer::from(SimpleTokenizer)
-            .filter(LowerCaser)
-            .token_stream(text);
+        let mut token_stream = SimpleTokenizer.filter(LowerCaser).token_stream(text);
        while token_stream.advance() {
            let token_text = token_stream.token().text.clone();
            tokens.push(token_text);
--- a/src/tokenizer/mod.rs
+++ b/src/tokenizer/mod.rs
@@ -7,6 +7,7 @@
 //! ```rust
 //! use tantivy::schema::*;
 //!
+//! # fn main() {
 //! let mut schema_builder = Schema::builder();
 //!
 //! let text_options = TextOptions::default()
@@ -30,6 +31,7 @@
 //! schema_builder.add_text_field("uuid", id_options);
 //!
 //! let schema = schema_builder.build();
+//! # }
 //! ```
 //!
 //! By default, `tantivy` offers the following tokenizers:
@@ -64,10 +66,12 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let en_stem = TextAnalyzer::from(SimpleTokenizer)
+//! # fn main() {
+//! let en_stem = SimpleTokenizer
 //!     .filter(RemoveLongFilter::limit(40))
 //!     .filter(LowerCaser)
 //!     .filter(Stemmer::new(Language::English));
+//! # }
 //! ```
 //!
 //! Once your tokenizer is defined, you need to
@@ -77,12 +81,13 @@
 //! # use tantivy::schema::Schema;
 //! # use tantivy::tokenizer::*;
 //! # use tantivy::Index;
-//! #
-//! let custom_en_tokenizer = SimpleTokenizer;
+//! # fn main() {
+//! # let custom_en_tokenizer = SimpleTokenizer;
 //! # let schema = Schema::builder().build();
 //! let index = Index::create_in_ram(schema);
 //! index.tokenizers()
 //!      .register("custom_en", custom_en_tokenizer);
+//! # }
 //! ```
 //!
 //! If you built your schema programmatically, a complete example
@@ -97,6 +102,7 @@
 //! use tantivy::tokenizer::*;
 //! use tantivy::Index;
 //!
+//! # fn main() {
 //! let mut schema_builder = Schema::builder();
 //! let text_field_indexing = TextFieldIndexing::default()
 //!     .set_tokenizer("custom_en")
@@ -109,12 +115,14 @@
 //! let index = Index::create_in_ram(schema);
 //!
 //! // We need to register our tokenizer :
-//! let custom_en_tokenizer = TextAnalyzer::from(SimpleTokenizer)
+//! let custom_en_tokenizer = SimpleTokenizer
 //!     .filter(RemoveLongFilter::limit(40))
 //!     .filter(LowerCaser);
 //! index
 //!     .tokenizers()
 //!     .register("custom_en", custom_en_tokenizer);
+//! // ...
+//! # }
 //! ```
 //!
 mod alphanum_only;
@@ -128,7 +136,6 @@ mod simple_tokenizer;
 mod stemmer;
 mod stop_word_filter;
 mod token_stream_chain;
-mod tokenized_string;
 mod tokenizer;
 mod tokenizer_manager;

@@ -143,12 +150,9 @@ pub use self::simple_tokenizer::SimpleTokenizer;
 pub use self::stemmer::{Language, Stemmer};
 pub use self::stop_word_filter::StopWordFilter;
 pub(crate) use self::token_stream_chain::TokenStreamChain;
+pub use self::tokenizer::BoxedTokenizer;

-pub use self::tokenized_string::{PreTokenizedStream, PreTokenizedString};
-pub use self::tokenizer::{
-    BoxTokenFilter, BoxTokenStream, TextAnalyzer, Token, TokenFilter, TokenStream, Tokenizer,
-};
-
+pub use self::tokenizer::{Token, TokenFilter, TokenStream, Tokenizer};
 pub use self::tokenizer_manager::TokenizerManager;

 /// Maximum authorized len (in bytes) for a token.
@@ -161,9 +165,9 @@ pub const MAX_TOKEN_LEN: usize = u16::max_value() as usize - 4;
 #[cfg(test)]
 pub mod tests {
    use super::{
-        Language, LowerCaser, RemoveLongFilter, SimpleTokenizer, Stemmer, Token, TokenizerManager,
+        Language, LowerCaser, RemoveLongFilter, SimpleTokenizer, Stemmer, Token, Tokenizer,
+        TokenizerManager,
    };
-    use crate::tokenizer::TextAnalyzer;

    /// This is a function that can be used in tests and doc tests
    /// to assert a token's correctness.
@@ -230,7 +234,7 @@ pub mod tests {
        let tokenizer_manager = TokenizerManager::default();
        tokenizer_manager.register(
            "el_stem",
-            TextAnalyzer::from(SimpleTokenizer)
+            SimpleTokenizer
                .filter(RemoveLongFilter::limit(40))
                .filter(LowerCaser)
                .filter(Stemmer::new(Language::Greek)),
--- a/src/tokenizer/ngram_tokenizer.rs
+++ b/src/tokenizer/ngram_tokenizer.rs
@@ -1,5 +1,4 @@
 use super::{Token, TokenStream, Tokenizer};
-use crate::tokenizer::BoxTokenStream;

 /// Tokenize the text by splitting words into n-grams of the given size(s)
 ///
@@ -32,7 +31,7 @@ use crate::tokenizer::BoxTokenStream;
 ///
 /// ```rust
 /// use tantivy::tokenizer::*;
-///
+/// # fn main() {
 /// let tokenizer = NgramTokenizer::new(2, 3, false);
 /// let mut stream = tokenizer.token_stream("hello");
 /// {
@@ -78,6 +77,7 @@ use crate::tokenizer::BoxTokenStream;
 ///   assert_eq!(token.offset_to, 5);
 /// }
 /// assert!(stream.next().is_none());
+/// # }
 /// ```
 #[derive(Clone)]
 pub struct NgramTokenizer {
@@ -130,9 +130,11 @@ pub struct NgramTokenStream<'a> {
    token: Token,
 }

-impl Tokenizer for NgramTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
-        From::from(NgramTokenStream {
+impl<'a> Tokenizer<'a> for NgramTokenizer {
+    type TokenStreamImpl = NgramTokenStream<'a>;
+
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
+        NgramTokenStream {
            ngram_charidx_iterator: StutteringIterator::new(
                CodepointFrontiers::for_str(text),
                self.min_gram,
@@ -141,7 +143,7 @@ impl Tokenizer for NgramTokenizer {
            prefix_only: self.prefix_only,
            text,
            token: Token::default(),
-        })
+        }
    }
 }

@@ -307,10 +309,10 @@ mod tests {
    use super::NgramTokenizer;
    use super::StutteringIterator;
    use crate::tokenizer::tests::assert_token;
-    use crate::tokenizer::tokenizer::Tokenizer;
-    use crate::tokenizer::{BoxTokenStream, Token};
+    use crate::tokenizer::tokenizer::{TokenStream, Tokenizer};
+    use crate::tokenizer::Token;

-    fn test_helper(mut tokenizer: BoxTokenStream) -> Vec<Token> {
+    fn test_helper<T: TokenStream>(mut tokenizer: T) -> Vec<Token> {
        let mut tokens: Vec<Token> = vec![];
        tokenizer.process(&mut |token: &Token| tokens.push(token.clone()));
        tokens
--- a/src/tokenizer/raw_tokenizer.rs
+++ b/src/tokenizer/raw_tokenizer.rs
@@ -1,5 +1,4 @@
 use super::{Token, TokenStream, Tokenizer};
-use crate::tokenizer::BoxTokenStream;

 /// For each value of the field, emit a single unprocessed token.
 #[derive(Clone)]
@@ -10,8 +9,10 @@ pub struct RawTokenStream {
    has_token: bool,
 }

-impl Tokenizer for RawTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+impl<'a> Tokenizer<'a> for RawTokenizer {
+    type TokenStreamImpl = RawTokenStream;
+
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
        let token = Token {
            offset_from: 0,
            offset_to: text.len(),
@@ -23,7 +24,6 @@ impl Tokenizer for RawTokenizer {
            token,
            has_token: true,
        }
-        .into()
    }
 }

--- a/src/tokenizer/remove_long.rs
+++ b/src/tokenizer/remove_long.rs
@@ -2,7 +2,9 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+//! # fn main() {
+//!
+//! let tokenizer = SimpleTokenizer
 //!   .filter(RemoveLongFilter::limit(5));
 //!
 //! let mut stream = tokenizer.token_stream("toolong nice");
@@ -10,10 +12,10 @@
 //! // out of the token stream.
 //! assert_eq!(stream.next().unwrap().text, "nice");
 //! assert!(stream.next().is_none());
+//! # }
 //! ```
 //!
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;

 /// `RemoveLongFilter` removes tokens that are longer
 /// than a given number of bytes (in UTF-8 representation).
@@ -32,27 +34,56 @@ impl RemoveLongFilter {
    }
 }

-impl<'a> RemoveLongFilterStream<'a> {
+impl<TailTokenStream> RemoveLongFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
    fn predicate(&self, token: &Token) -> bool {
        token.text.len() < self.token_length_limit
    }
-}

-impl TokenFilter for RemoveLongFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(RemoveLongFilterStream {
-            token_length_limit: self.length_limit,
-            tail: token_stream,
-        })
+    fn wrap(
+        token_length_limit: usize,
+        tail: TailTokenStream,
+    ) -> RemoveLongFilterStream<TailTokenStream> {
+        RemoveLongFilterStream {
+            token_length_limit,
+            tail,
+        }
    }
 }

-pub struct RemoveLongFilterStream<'a> {
-    token_length_limit: usize,
-    tail: BoxTokenStream<'a>,
+impl<TailTokenStream> TokenFilter<TailTokenStream> for RemoveLongFilter
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = RemoveLongFilterStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+        RemoveLongFilterStream::wrap(self.length_limit, token_stream)
+    }
 }

-impl<'a> TokenStream for RemoveLongFilterStream<'a> {
+pub struct RemoveLongFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    token_length_limit: usize,
+    tail: TailTokenStream,
+}
+
+impl<TailTokenStream> TokenStream for RemoveLongFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+
    fn advance(&mut self) -> bool {
        while self.tail.advance() {
            if self.predicate(self.tail.token()) {
@@ -61,12 +92,4 @@ impl<'a> TokenStream for RemoveLongFilterStream<'a> {
        }
        false
    }
-
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
-    }
 }
--- a/src/tokenizer/simple_tokenizer.rs
+++ b/src/tokenizer/simple_tokenizer.rs
@@ -1,4 +1,3 @@
-use super::BoxTokenStream;
 use super::{Token, TokenStream, Tokenizer};
 use std::str::CharIndices;

@@ -12,13 +11,15 @@ pub struct SimpleTokenStream<'a> {
    token: Token,
 }

-impl Tokenizer for SimpleTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(SimpleTokenStream {
+impl<'a> Tokenizer<'a> for SimpleTokenizer {
+    type TokenStreamImpl = SimpleTokenStream<'a>;
+
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
+        SimpleTokenStream {
            text,
            chars: text.char_indices(),
            token: Token::default(),
-        })
+        }
    }
 }

--- a/src/tokenizer/stemmer.rs
+++ b/src/tokenizer/stemmer.rs
@@ -1,5 +1,4 @@
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;
 use rust_stemmers::{self, Algorithm};

 /// Available stemmer languages.
@@ -16,7 +15,6 @@ pub enum Language {
    Greek,
    Hungarian,
    Italian,
-    Norwegian,
    Portuguese,
    Romanian,
    Russian,
@@ -40,7 +38,6 @@ impl Language {
            Greek => Algorithm::Greek,
            Hungarian => Algorithm::Hungarian,
            Italian => Algorithm::Italian,
-            Norwegian => Algorithm::Norwegian,
            Portuguese => Algorithm::Portuguese,
            Romanian => Algorithm::Romanian,
            Russian => Algorithm::Russian,
@@ -76,22 +73,38 @@ impl Default for Stemmer {
    }
 }

-impl TokenFilter for Stemmer {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+impl<TailTokenStream> TokenFilter<TailTokenStream> for Stemmer
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = StemmerTokenStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
        let inner_stemmer = rust_stemmers::Stemmer::create(self.stemmer_algorithm);
-        BoxTokenStream::from(StemmerTokenStream {
-            tail: token_stream,
-            stemmer: inner_stemmer,
-        })
+        StemmerTokenStream::wrap(inner_stemmer, token_stream)
    }
 }

-pub struct StemmerTokenStream<'a> {
-    tail: BoxTokenStream<'a>,
+pub struct StemmerTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    tail: TailTokenStream,
    stemmer: rust_stemmers::Stemmer,
 }

-impl<'a> TokenStream for StemmerTokenStream<'a> {
+impl<TailTokenStream> TokenStream for StemmerTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+
    fn advance(&mut self) -> bool {
        if !self.tail.advance() {
            return false;
@@ -102,12 +115,16 @@ impl<'a> TokenStream for StemmerTokenStream<'a> {
        self.token_mut().text.push_str(&stemmed_str);
        true
    }
+}

-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
+impl<TailTokenStream> StemmerTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn wrap(
+        stemmer: rust_stemmers::Stemmer,
+        tail: TailTokenStream,
+    ) -> StemmerTokenStream<TailTokenStream> {
+        StemmerTokenStream { tail, stemmer }
    }
 }
--- a/Show More
+++ b/Show More