Added psuedocode for poll implementation

* Removed generics in tokenizers

* lowercaser

* Added TokenizerExt

* Introducing BoxedTokenizer

* Introducing BoxXXXXX helper struct

* Closes #762.

* Introducing a TextAnalyzer

CHANGELOG.md

@@ -1,3 +1,27 @@
+Tantivy 0.12.0
+======================
+- Removing static dispatch in tokenizers for simplicity. (#762)
+
+## How to update?
+
+Crates relying on custom tokenizer, or registering tokenizer in the manager will require some 
+minor changes. Check https://github.com/tantivy-search/tantivy/blob/master/examples/custom_tokenizer.rs
+to check for some code sample.
+
+Tantivy 0.11.3
+=======================
+- Fixed DateTime as a fast field (#735)
+
+Tantivy 0.11.2
+=======================
+- The future returned by `IndexWriter::merge` does not borrow `self` mutably anymore (#732)
+- Exposing a constructor for `WatchHandle` (#731)
+
+Tantivy 0.11.1
+=====================
+- Bug fix #729
+
+
 Tantivy 0.11.0
 =====================
 
@@ -9,6 +33,7 @@ Tantivy 0.11.0
 - API change around `Box<BoxableTokenizer>`. See detail in #629
 - Avoid rebuilding Regex automaton whenever a regex query is reused. #639 (@brainlock)
 - Add footer with some metadata to index files. #605 (@fdb-hiroshima)
+- Add a method to check the compatibility of the footer in the index with the running version of tantivy (@petr-tik)
 - TopDocs collector: ensure stable sorting on equal score. #671 (@brainlock)
 - Added handling of pre-tokenized text fields (#642), which will enable users to
   load tokens created outside tantivy. See usage in examples/pre_tokenized_text. (@kkoziara)
@@ -16,10 +41,11 @@ Tantivy 0.11.0
 
 ## How to update?
 
+- The index format is changed. You are required to reindex your data to use tantivy 0.11. 
 - `Box<dyn BoxableTokenizer>` has been replaced by a `BoxedTokenizer` struct.
 - Regex are now compiled when the `RegexQuery` instance is built. As a result, it can now return
 an error and handling the `Result` is required.
-
+- `tantivy::version()` now returns a `Version` object. This object implements `ToString()`
 
 Tantivy 0.10.2
 =====================

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.11.0"
+version = "0.11.3"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -18,7 +18,7 @@ byteorder = "1.0"
 crc32fast = "1.2.0"
 once_cell = "1.0"
 regex ={version = "1.3.0", default-features = false, features = ["std"]}
-tantivy-fst = "0.1"
+tantivy-fst = "0.2"
 memmap = {version = "0.7", optional=true}
 lz4 = {version="1.20", optional=true}
 snap = {version="0.2"}
@@ -33,17 +33,16 @@ fs2={version="0.4", optional=true}
 itertools = "0.8"
 levenshtein_automata = {version="0.1", features=["fst_automaton"]}
 notify = {version="4", optional=true}
-bit-set = "0.5"
 uuid = { version = "0.8", features = ["v4", "serde"] }
 crossbeam = "0.7"
 futures = {version = "0.3",  features=["thread-pool"] }
 owning_ref = "0.4"
 stable_deref_trait = "1.0.0"
-rust-stemmers = "1.1"
+rust-stemmers = "1.2"
 downcast-rs = { version="1.0" }
-tantivy-query-grammar = { path="./query-grammar" }
+tantivy-query-grammar = { version="0.11", path="./query-grammar" }
 bitpacking = {version="0.8", default-features = false, features=["bitpacker4x"]}
-census = "0.3" 
+census = "0.4"
 fnv = "1.0.6"
 owned-read = "0.4"
 failure = "0.1"

examples/faceted_search.rs

@@ -13,63 +13,100 @@
 // ---
 // Importing tantivy...
 use tantivy::collector::FacetCollector;
-use tantivy::query::AllQuery;
+use tantivy::query::{AllQuery, TermQuery};
 use tantivy::schema::*;
 use tantivy::{doc, Index};
-use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {
-    // Let's create a temporary directory for the
-    // sake of this example
-    let index_path = TempDir::new()?;
+    // Let's create a temporary directory for the sake of this example
     let mut schema_builder = Schema::builder();
 
-    schema_builder.add_text_field("name", TEXT | STORED);
-
-    // this is our faceted field
-    schema_builder.add_facet_field("tags");
+    let name = schema_builder.add_text_field("felin_name", TEXT | STORED);
+    // this is our faceted field: its scientific classification
+    let classification = schema_builder.add_facet_field("classification");
 
     let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema);
 
-    let index = Index::create_in_dir(&index_path, schema.clone())?;
-
-    let mut index_writer = index.writer(50_000_000)?;
-
-    let name = schema.get_field("name").unwrap();
-    let tags = schema.get_field("tags").unwrap();
+    let mut index_writer = index.writer(30_000_000)?;
 
     // For convenience, tantivy also comes with a macro to
     // reduce the boilerplate above.
     index_writer.add_document(doc!(
-        name => "the ditch",
-        tags => Facet::from("/pools/north")
+        name => "Cat",
+        classification => Facet::from("/Felidae/Felinae/Felis")
     ));
-
     index_writer.add_document(doc!(
-        name => "little stacey",
-        tags => Facet::from("/pools/south")
+        name => "Canada lynx",
+        classification => Facet::from("/Felidae/Felinae/Lynx")
+    ));
+    index_writer.add_document(doc!(
+        name => "Cheetah",
+        classification => Facet::from("/Felidae/Felinae/Acinonyx")
+    ));
+    index_writer.add_document(doc!(
+        name => "Tiger",
+        classification => Facet::from("/Felidae/Pantherinae/Panthera")
+    ));
+    index_writer.add_document(doc!(
+        name => "Lion",
+        classification => Facet::from("/Felidae/Pantherinae/Panthera")
+    ));
+    index_writer.add_document(doc!(
+        name => "Jaguar",
+        classification => Facet::from("/Felidae/Pantherinae/Panthera")
+    ));
+    index_writer.add_document(doc!(
+        name => "Sunda clouded leopard",
+        classification => Facet::from("/Felidae/Pantherinae/Neofelis")
+    ));
+    index_writer.add_document(doc!(
+        name => "Fossa",
+        classification => Facet::from("/Eupleridae/Cryptoprocta")
     ));
-
     index_writer.commit()?;
 
     let reader = index.reader()?;
-
     let searcher = reader.searcher();
+    {
+        let mut facet_collector = FacetCollector::for_field(classification);
+        facet_collector.add_facet("/Felidae");
+        let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
+        // This lists all of the facet counts, right below "/Felidae".
+        let facets: Vec<(&Facet, u64)> = facet_counts.get("/Felidae").collect();
+        assert_eq!(
+            facets,
+            vec![
+                (&Facet::from("/Felidae/Felinae"), 3),
+                (&Facet::from("/Felidae/Pantherinae"), 4),
+            ]
+        );
+    }
 
-    let mut facet_collector = FacetCollector::for_field(tags);
-    facet_collector.add_facet("/pools");
+    // Facets are also searchable.
+    //
+    // For instance a common UI pattern is to allow the user someone to click on a facet link
+    // (e.g: `Pantherinae`) to drill down and filter the current result set with this subfacet.
+    //
+    // The search would then look as follows.
 
-    let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
-
-    // This lists all of the facet counts
-    let facets: Vec<(&Facet, u64)> = facet_counts.get("/pools").collect();
-    assert_eq!(
-        facets,
-        vec![
-            (&Facet::from("/pools/north"), 1),
-            (&Facet::from("/pools/south"), 1),
-        ]
-    );
+    // Check the reference doc for different ways to create a `Facet` object.
+    {
+        let facet = Facet::from_text("/Felidae/Pantherinae");
+        let facet_term = Term::from_facet(classification, &facet);
+        let facet_term_query = TermQuery::new(facet_term, IndexRecordOption::Basic);
+        let mut facet_collector = FacetCollector::for_field(classification);
+        facet_collector.add_facet("/Felidae/Pantherinae");
+        let facet_counts = searcher.search(&facet_term_query, &facet_collector)?;
+        let facets: Vec<(&Facet, u64)> = facet_counts.get("/Felidae/Pantherinae").collect();
+        assert_eq!(
+            facets,
+            vec![
+                (&Facet::from("/Felidae/Pantherinae/Neofelis"), 1),
+                (&Facet::from("/Felidae/Pantherinae/Panthera"), 3),
+            ]
+        );
+    }
 
     Ok(())
 }

examples/pre_tokenized_text.rs

@@ -9,7 +9,7 @@
 // - import tokenized text straight from json,
 // - perform a search on documents with pre-tokenized text
 
-use tantivy::tokenizer::{PreTokenizedString, SimpleTokenizer, Token, TokenStream, Tokenizer};
+use tantivy::tokenizer::{PreTokenizedString, SimpleTokenizer, Token, Tokenizer};
 
 use tantivy::collector::{Count, TopDocs};
 use tantivy::query::TermQuery;
@@ -65,8 +65,7 @@ fn main() -> tantivy::Result<()> {
         tokens: pre_tokenize_text(body_text),
     };
 
-    // Now lets create a document and add our `PreTokenizedString` using
-    // `add_pre_tokenized_text` method of `Document`
+    // Now lets create a document and add our `PreTokenizedString`
     let old_man_doc = doc!(title => title_tok, body => body_tok);
 
     // ... now let's just add it to the IndexWriter
@@ -114,6 +113,9 @@ fn main() -> tantivy::Result<()> {
 
     assert_eq!(count, 2);
 
+    // Now let's print out the results.
+    // Note that the tokens are not stored along with the original text
+    // in the document store
     for (_score, doc_address) in top_docs {
         let retrieved_doc = searcher.doc(doc_address)?;
         println!("Document: {}", schema.to_json(&retrieved_doc));

examples/stop_words.rs

@@ -50,7 +50,7 @@ fn main() -> tantivy::Result<()> {
 
     // This tokenizer lowers all of the text (to help with stop word matching)
     // then removes all instances of `the` and `and` from the corpus
-    let tokenizer = SimpleTokenizer
+    let tokenizer = TextAnalyzer::from(SimpleTokenizer)
         .filter(LowerCaser)
         .filter(StopWordFilter::remove(vec![
             "the".to_string(),

query-grammar/README.md

@@ -0,0 +1,3 @@
+# Tantivy Query Grammar
+
+This crate is used by tantivy to parse queries.

src/collector/count_collector.rs

@@ -13,44 +13,29 @@ use crate::SegmentReader;
 /// use tantivy::collector::Count;
 /// use tantivy::query::QueryParser;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Result};
+/// use tantivy::{doc, Index};
 ///
-/// # fn main() { example().unwrap(); }
-/// fn example() -> Result<()> {
-///     let mut schema_builder = Schema::builder();
-///     let title = schema_builder.add_text_field("title", TEXT);
-///     let schema = schema_builder.build();
-///     let index = Index::create_in_ram(schema);
-///     {
-///         let mut index_writer = index.writer(3_000_000)?;
-///         index_writer.add_document(doc!(
-///             title => "The Name of the Wind",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of Muadib",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "A Dairy Cow",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of a Young Girl",
-///         ));
-///         index_writer.commit().unwrap();
-///     }
+/// let mut schema_builder = Schema::builder();
+/// let title = schema_builder.add_text_field("title", TEXT);
+/// let schema = schema_builder.build();
+/// let index = Index::create_in_ram(schema);
 ///
-///     let reader = index.reader()?;
-///     let searcher = reader.searcher();
+/// let mut index_writer = index.writer(3_000_000).unwrap();
+/// index_writer.add_document(doc!(title => "The Name of the Wind"));
+/// index_writer.add_document(doc!(title => "The Diary of Muadib"));
+/// index_writer.add_document(doc!(title => "A Dairy Cow"));
+/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"));
+/// assert!(index_writer.commit().is_ok());
 ///
-///     {
-///         let query_parser = QueryParser::for_index(&index, vec![title]);
-///         let query = query_parser.parse_query("diary")?;
-///         let count = searcher.search(&query, &Count).unwrap();
+/// let reader = index.reader().unwrap();
+/// let searcher = reader.searcher();
 ///
-///         assert_eq!(count, 2);
-///     }
+/// // Here comes the important part
+/// let query_parser = QueryParser::for_index(&index, vec![title]);
+/// let query = query_parser.parse_query("diary").unwrap();
+/// let count = searcher.search(&query, &Count).unwrap();
 ///
-///     Ok(())
-/// }
+/// assert_eq!(count, 2);
 /// ```
 pub struct Count;
 

src/collector/facet_collector.rs

@@ -86,7 +86,6 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 /// use tantivy::schema::{Facet, Schema, TEXT};
 /// use tantivy::{doc, Index, Result};
 ///
-/// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {
 ///     let mut schema_builder = Schema::builder();
 ///
@@ -127,7 +126,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///     let searcher = reader.searcher();
 ///
 ///     {
-///			let mut facet_collector = FacetCollector::for_field(facet);
+///         let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/lang");
 ///         facet_collector.add_facet("/category");
 ///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
@@ -143,7 +142,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///     }
 ///
 ///     {
-///			let mut facet_collector = FacetCollector::for_field(facet);
+///         let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/category/fiction");
 ///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
@@ -158,8 +157,8 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///         ]);
 ///     }
 ///
-///    {
-///			let mut facet_collector = FacetCollector::for_field(facet);
+///     {
+///         let mut facet_collector = FacetCollector::for_field(facet);
 ///         facet_collector.add_facet("/category/fiction");
 ///         let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
 ///
@@ -172,6 +171,7 @@ fn facet_depth(facet_bytes: &[u8]) -> usize {
 ///
 ///     Ok(())
 /// }
+/// # assert!(example().is_ok());
 /// ```
 pub struct FacetCollector {
     field: Field,
@@ -452,9 +452,11 @@ impl FacetCounts {
 #[cfg(test)]
 mod tests {
     use super::{FacetCollector, FacetCounts};
+    use crate::collector::Count;
     use crate::core::Index;
-    use crate::query::AllQuery;
-    use crate::schema::{Document, Facet, Field, Schema};
+    use crate::query::{AllQuery, QueryParser, TermQuery};
+    use crate::schema::{Document, Facet, Field, IndexRecordOption, Schema};
+    use crate::Term;
     use rand::distributions::Uniform;
     use rand::prelude::SliceRandom;
     use rand::{thread_rng, Rng};
@@ -544,6 +546,56 @@ mod tests {
         assert_eq!(facets[0].1, 1);
     }
 
+    #[test]
+    fn test_doc_search_by_facet() {
+        let mut schema_builder = Schema::builder();
+        let facet_field = schema_builder.add_facet_field("facet");
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.add_document(doc!(
+            facet_field => Facet::from_text(&"/A/A"),
+        ));
+        index_writer.add_document(doc!(
+            facet_field => Facet::from_text(&"/A/B"),
+        ));
+        index_writer.add_document(doc!(
+            facet_field => Facet::from_text(&"/A/C/A"),
+        ));
+        index_writer.add_document(doc!(
+            facet_field => Facet::from_text(&"/D/C/A"),
+        ));
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        assert_eq!(searcher.num_docs(), 4);
+
+        let count_facet = |facet_str: &str| {
+            let term = Term::from_facet(facet_field, &Facet::from_text(facet_str));
+            searcher
+                .search(&TermQuery::new(term, IndexRecordOption::Basic), &Count)
+                .unwrap()
+        };
+
+        assert_eq!(count_facet("/"), 4);
+        assert_eq!(count_facet("/A"), 3);
+        assert_eq!(count_facet("/A/B"), 1);
+        assert_eq!(count_facet("/A/C"), 1);
+        assert_eq!(count_facet("/A/C/A"), 1);
+        assert_eq!(count_facet("/C/A"), 0);
+        {
+            let query_parser = QueryParser::for_index(&index, vec![]);
+            {
+                let query = query_parser.parse_query("facet:/A/B").unwrap();
+                assert_eq!(1, searcher.search(&query, &Count).unwrap());
+            }
+            {
+                let query = query_parser.parse_query("facet:/A").unwrap();
+                assert_eq!(3, searcher.search(&query, &Count).unwrap());
+            }
+        }
+    }
+
     #[test]
     fn test_non_used_facet_collector() {
         let mut facet_collector = FacetCollector::for_field(Field::from_field_id(0));

src/collector/multi_collector.rs

@@ -108,49 +108,35 @@ impl<TFruit: Fruit> FruitHandle<TFruit> {
 /// use tantivy::collector::{Count, TopDocs, MultiCollector};
 /// use tantivy::query::QueryParser;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Result};
+/// use tantivy::{doc, Index};
 ///
-/// # fn main() { example().unwrap(); }
-/// fn example() -> Result<()> {
-///     let mut schema_builder = Schema::builder();
-///     let title = schema_builder.add_text_field("title", TEXT);
-///     let schema = schema_builder.build();
-///     let index = Index::create_in_ram(schema);
-///     {
-///         let mut index_writer = index.writer(3_000_000)?;
-///         index_writer.add_document(doc!(
-///             title => "The Name of the Wind",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of Muadib",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "A Dairy Cow",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of a Young Girl",
-///         ));
-///         index_writer.commit().unwrap();
-///     }
+/// let mut schema_builder = Schema::builder();
+/// let title = schema_builder.add_text_field("title", TEXT);
+/// let schema = schema_builder.build();
+/// let index = Index::create_in_ram(schema);
 ///
-///     let reader = index.reader()?;
-///     let searcher = reader.searcher();
+/// let mut index_writer = index.writer(3_000_000).unwrap();
+/// index_writer.add_document(doc!(title => "The Name of the Wind"));
+/// index_writer.add_document(doc!(title => "The Diary of Muadib"));
+/// index_writer.add_document(doc!(title => "A Dairy Cow"));
+/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"));
+/// assert!(index_writer.commit().is_ok());
 ///
-///     let mut collectors = MultiCollector::new();
-///     let top_docs_handle = collectors.add_collector(TopDocs::with_limit(2));
-///     let count_handle = collectors.add_collector(Count);
-///     let query_parser = QueryParser::for_index(&index, vec![title]);
-///     let query = query_parser.parse_query("diary")?;
-///     let mut multi_fruit = searcher.search(&query, &collectors)?;
+/// let reader = index.reader().unwrap();
+/// let searcher = reader.searcher();
 ///
-///     let count = count_handle.extract(&mut multi_fruit);
-///     let top_docs = top_docs_handle.extract(&mut multi_fruit);
+/// let mut collectors = MultiCollector::new();
+/// let top_docs_handle = collectors.add_collector(TopDocs::with_limit(2));
+/// let count_handle = collectors.add_collector(Count);
+/// let query_parser = QueryParser::for_index(&index, vec![title]);
+/// let query = query_parser.parse_query("diary").unwrap();
+/// let mut multi_fruit = searcher.search(&query, &collectors).unwrap();
 ///
-///     # assert_eq!(count, 2);
-///     # assert_eq!(top_docs.len(), 2);
+/// let count = count_handle.extract(&mut multi_fruit);
+/// let top_docs = top_docs_handle.extract(&mut multi_fruit);
 ///
-///     Ok(())
-/// }
+/// assert_eq!(count, 2);
+/// assert_eq!(top_docs.len(), 2);
 /// ```
 #[allow(clippy::type_complexity)]
 #[derive(Default)]

src/collector/top_score_collector.rs

@@ -6,6 +6,7 @@ use crate::collector::tweak_score_top_collector::TweakedScoreTopCollector;
 use crate::collector::{
     CustomScorer, CustomSegmentScorer, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector,
 };
+use crate::fastfield::FastFieldReader;
 use crate::schema::Field;
 use crate::DocAddress;
 use crate::DocId;
@@ -29,43 +30,29 @@ use std::fmt;
 /// use tantivy::collector::TopDocs;
 /// use tantivy::query::QueryParser;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, DocAddress, Index, Result};
+/// use tantivy::{doc, DocAddress, Index};
 ///
-/// # fn main() { example().unwrap(); }
-/// fn example() -> Result<()> {
-///     let mut schema_builder = Schema::builder();
-///     let title = schema_builder.add_text_field("title", TEXT);
-///     let schema = schema_builder.build();
-///     let index = Index::create_in_ram(schema);
-///     {
-///         let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
-///         index_writer.add_document(doc!(
-///             title => "The Name of the Wind",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of Muadib",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "A Dairy Cow",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of a Young Girl",
-///         ));
-///         index_writer.commit().unwrap();
-///     }
+/// let mut schema_builder = Schema::builder();
+/// let title = schema_builder.add_text_field("title", TEXT);
+/// let schema = schema_builder.build();
+/// let index = Index::create_in_ram(schema);
 ///
-///     let reader = index.reader()?;
-///     let searcher = reader.searcher();
+/// let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+/// index_writer.add_document(doc!(title => "The Name of the Wind"));
+/// index_writer.add_document(doc!(title => "The Diary of Muadib"));
+/// index_writer.add_document(doc!(title => "A Dairy Cow"));
+/// index_writer.add_document(doc!(title => "The Diary of a Young Girl"));
+/// assert!(index_writer.commit().is_ok());
 ///
-///     let query_parser = QueryParser::for_index(&index, vec![title]);
-///     let query = query_parser.parse_query("diary")?;
-///     let top_docs = searcher.search(&query, &TopDocs::with_limit(2))?;
+/// let reader = index.reader().unwrap();
+/// let searcher = reader.searcher();
 ///
-///     assert_eq!(&top_docs[0], &(0.7261542, DocAddress(0, 1)));
-///     assert_eq!(&top_docs[1], &(0.6099695, DocAddress(0, 3)));
+/// let query_parser = QueryParser::for_index(&index, vec![title]);
+/// let query = query_parser.parse_query("diary").unwrap();
+/// let top_docs = searcher.search(&query, &TopDocs::with_limit(2)).unwrap();
 ///
-///     Ok(())
-/// }
+/// assert_eq!(&top_docs[0], &(0.7261542, DocAddress(0, 1)));
+/// assert_eq!(&top_docs[1], &(0.6099695, DocAddress(0, 3)));
 /// ```
 pub struct TopDocs(TopCollector<Score>);
 
@@ -75,6 +62,34 @@ impl fmt::Debug for TopDocs {
     }
 }
 
+struct ScorerByFastFieldReader {
+    ff_reader: FastFieldReader<u64>,
+}
+
+impl CustomSegmentScorer<u64> for ScorerByFastFieldReader {
+    fn score(&self, doc: DocId) -> u64 {
+        self.ff_reader.get_u64(u64::from(doc))
+    }
+}
+
+struct ScorerByField {
+    field: Field,
+}
+
+impl CustomScorer<u64> for ScorerByField {
+    type Child = ScorerByFastFieldReader;
+
+    fn segment_scorer(&self, segment_reader: &SegmentReader) -> crate::Result<Self::Child> {
+        let ff_reader = segment_reader
+            .fast_fields()
+            .u64(self.field)
+            .ok_or_else(|| {
+                crate::Error::SchemaError(format!("Field requested is not a i64/u64 fast field."))
+            })?;
+        Ok(ScorerByFastFieldReader { ff_reader })
+    }
+}
+
 impl TopDocs {
     /// Creates a top score collector, with a number of documents equal to "limit".
     ///
@@ -102,15 +117,12 @@ impl TopDocs {
     /// #  
     /// #   let index = Index::create_in_ram(schema);
     /// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
-    /// #   index_writer.add_document(doc!(
-    /// #       title => "The Name of the Wind",
-    /// #       rating => 92u64,
-    /// #   ));
+    /// #   index_writer.add_document(doc!(title => "The Name of the Wind", rating => 92u64));
     /// #   index_writer.add_document(doc!(title => "The Diary of Muadib", rating => 97u64));
     /// #   index_writer.add_document(doc!(title => "A Dairy Cow", rating => 63u64));
     /// #   index_writer.add_document(doc!(title => "The Diary of a Young Girl", rating => 80u64));
-    /// #   index_writer.commit()?;
-    /// #   let reader = index.reader()?;
+    /// #   assert!(index_writer.commit().is_ok());
+    /// #   let reader = index.reader().unwrap();
     /// #   let query = QueryParser::for_index(&index, vec![title]).parse_query("diary")?;
     /// #   let top_docs = docs_sorted_by_rating(&reader.searcher(), &query, rating)?;
     /// #   assert_eq!(top_docs,
@@ -160,14 +172,7 @@ impl TopDocs {
         self,
         field: Field,
     ) -> impl Collector<Fruit = Vec<(u64, DocAddress)>> {
-        self.custom_score(move |segment_reader: &SegmentReader| {
-            let ff_reader = segment_reader
-                .fast_fields()
-                .u64(field)
-                .expect("Field requested is not a i64/u64 fast field.");
-            //TODO error message missmatch actual behavior for i64
-            move |doc: DocId| ff_reader.get(doc)
-        })
+        self.custom_score(ScorerByField { field })
     }
 
     /// Ranks the documents using a custom score.
@@ -202,27 +207,33 @@ impl TopDocs {
     /// use tantivy::collector::TopDocs;
     /// use tantivy::schema::Field;
     ///
-    /// # fn create_schema() -> Schema {
-    /// #    let mut schema_builder = Schema::builder();
-    /// #    schema_builder.add_text_field("product_name", TEXT);
-    /// #    schema_builder.add_u64_field("popularity", FAST);
-    /// #    schema_builder.build()
-    /// # }
-    /// #
-    /// # fn main() -> tantivy::Result<()> {
-    /// #   let schema = create_schema();
-    /// #   let index = Index::create_in_ram(schema);
-    /// #   let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
-    /// #   let product_name = index.schema().get_field("product_name").unwrap();
-    /// #   
+    /// fn create_schema() -> Schema {
+    ///    let mut schema_builder = Schema::builder();
+    ///    schema_builder.add_text_field("product_name", TEXT);
+    ///    schema_builder.add_u64_field("popularity", FAST);
+    ///    schema_builder.build()
+    /// }
+    ///
+    /// fn create_index() -> tantivy::Result<Index> {
+    ///   let schema = create_schema();
+    ///   let index = Index::create_in_ram(schema);
+    ///   let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
+    ///   let product_name = index.schema().get_field("product_name").unwrap();
+    ///   let popularity: Field = index.schema().get_field("popularity").unwrap();
+    ///   index_writer.add_document(doc!(product_name => "The Diary of Muadib", popularity => 1u64));
+    ///   index_writer.add_document(doc!(product_name => "A Dairy Cow", popularity => 10u64));
+    ///   index_writer.add_document(doc!(product_name => "The Diary of a Young Girl", popularity => 15u64));
+    ///   index_writer.commit()?;
+    ///   Ok(index)
+    /// }
+    ///
+    /// let index = create_index().unwrap();
+    /// let product_name = index.schema().get_field("product_name").unwrap();
     /// let popularity: Field = index.schema().get_field("popularity").unwrap();
-    /// #   index_writer.add_document(doc!(product_name => "The Diary of Muadib", popularity => 1u64));
-    /// #   index_writer.add_document(doc!(product_name => "A Dairy Cow", popularity => 10u64));
-    /// #   index_writer.add_document(doc!(product_name => "The Diary of a Young Girl", popularity => 15u64));
-    /// #   index_writer.commit()?;
-    /// // ...
-    /// # let user_query = "diary";
-    /// # let query = QueryParser::for_index(&index, vec![product_name]).parse_query(user_query)?;
+    ///
+    /// let user_query_str = "diary";
+    /// let query_parser = QueryParser::for_index(&index, vec![product_name]);
+    /// let query = query_parser.parse_query(user_query_str).unwrap();
     ///
     /// // This is where we build our collector with our custom score.
     /// let top_docs_by_custom_score = TopDocs
@@ -249,15 +260,12 @@ impl TopDocs {
     ///                 popularity_boost_score * original_score
     ///             }
     ///           });
-    /// # let reader = index.reader()?;
-    /// # let searcher = reader.searcher();
+    /// let reader = index.reader().unwrap();
+    /// let searcher = reader.searcher();
     /// // ... and here are our documents. Note this is a simple vec.
     /// // The `Score` in the pair is our tweaked score.
     /// let resulting_docs: Vec<(Score, DocAddress)> =
-    ///      searcher.search(&*query, &top_docs_by_custom_score)?;
-    ///
-    /// # Ok(())
-    /// # }
+    ///      searcher.search(&query, &top_docs_by_custom_score).unwrap();
     /// ```
     ///
     /// # See also
@@ -586,7 +594,6 @@ mod tests {
     }
 
     #[test]
-    #[should_panic(expected = "Field requested is not a i64/u64 fast field")]
     fn test_field_not_fast_field() {
         let mut schema_builder = Schema::builder();
         let title = schema_builder.add_text_field(TITLE, TEXT);
@@ -601,7 +608,12 @@ mod tests {
         let searcher = index.reader().unwrap().searcher();
         let segment = searcher.segment_reader(0);
         let top_collector = TopDocs::with_limit(4).order_by_u64_field(size);
-        assert!(top_collector.for_segment(0, segment).is_ok());
+        let err = top_collector.for_segment(0, segment);
+        if let Err(crate::Error::SchemaError(msg)) = err {
+            assert_eq!(msg, "Field requested is not a i64/u64 fast field.");
+        } else {
+            assert!(false);
+        }
     }
 
     fn index(

src/core/index.rs

@@ -20,8 +20,7 @@ use crate::reader::IndexReaderBuilder;
 use crate::schema::Field;
 use crate::schema::FieldType;
 use crate::schema::Schema;
-use crate::tokenizer::BoxedTokenizer;
-use crate::tokenizer::TokenizerManager;
+use crate::tokenizer::{TextAnalyzer, TokenizerManager};
 use crate::IndexWriter;
 use crate::Result;
 use num_cpus;
@@ -173,11 +172,11 @@ impl Index {
     }
 
     /// Helper to access the tokenizer associated to a specific field.
-    pub fn tokenizer_for_field(&self, field: Field) -> Result<BoxedTokenizer> {
+    pub fn tokenizer_for_field(&self, field: Field) -> Result<TextAnalyzer> {
         let field_entry = self.schema.get_field_entry(field);
         let field_type = field_entry.field_type();
         let tokenizer_manager: &TokenizerManager = self.tokenizers();
-        let tokenizer_name_opt: Option<BoxedTokenizer> = match field_type {
+        let tokenizer_name_opt: Option<TextAnalyzer> = match field_type {
             FieldType::Str(text_options) => text_options
                 .get_indexing_options()
                 .map(|text_indexing_options| text_indexing_options.tokenizer().to_string())

src/directory/directory.rs

@@ -119,7 +119,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     /// Specifically, subsequent writes or flushes should
     /// have no effect on the returned `ReadOnlySource` object.
     ///
-    /// You should only use this to read files create with [`open_write`]
+    /// You should only use this to read files create with [Directory::open_write].
     fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError>;
 
     /// Removes a file
@@ -160,7 +160,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     ///
     /// This should only be used for small files.
     ///
-    /// You should only use this to read files create with [`atomic_write`]
+    /// You should only use this to read files create with [Directory::atomic_write].
     fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError>;
 
     /// Atomically replace the content of a file with data.
@@ -197,7 +197,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     /// Registers a callback that will be called whenever a change on the `meta.json`
     /// using the `atomic_write` API is detected.
     ///
-    /// The behavior when using `.watch()` on a file using `.open_write(...)` is, on the other
+    /// The behavior when using `.watch()` on a file using [Directory::open_write] is, on the other
     /// hand, undefined.
     ///
     /// The file will be watched for the lifetime of the returned `WatchHandle`. The caller is

src/directory/error.rs

@@ -1,3 +1,4 @@
+use crate::Version;
 use std::error::Error as StdError;
 use std::fmt;
 use std::io;
@@ -156,6 +157,65 @@ impl StdError for OpenWriteError {
     }
 }
 
+/// Type of index incompatibility between the library and the index found on disk
+/// Used to catch and provide a hint to solve this incompatibility issue
+pub enum Incompatibility {
+    /// This library cannot decompress the index found on disk
+    CompressionMismatch {
+        /// Compression algorithm used by the current version of tantivy
+        library_compression_format: String,
+        /// Compression algorithm that was used to serialise the index
+        index_compression_format: String,
+    },
+    /// The index format found on disk isn't supported by this version of the library
+    IndexMismatch {
+        /// Version used by the library
+        library_version: Version,
+        /// Version the index was built with
+        index_version: Version,
+    },
+}
+
+impl fmt::Debug for Incompatibility {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
+        match self {
+            Incompatibility::CompressionMismatch {
+                library_compression_format,
+                index_compression_format,
+            } => {
+                let err = format!(
+                    "Library was compiled with {:?} compression, index was compressed with {:?}",
+                    library_compression_format, index_compression_format
+                );
+                let advice = format!(
+                    "Change the feature flag to {:?} and rebuild the library",
+                    index_compression_format
+                );
+                write!(f, "{}. {}", err, advice)?;
+            }
+            Incompatibility::IndexMismatch {
+                library_version,
+                index_version,
+            } => {
+                let err = format!(
+                    "Library version: {}, index version: {}",
+                    library_version.index_format_version, index_version.index_format_version
+                );
+                // TODO make a more useful error message
+                // include the version range that supports this index_format_version
+                let advice = format!(
+                    "Change tantivy to a version compatible with index format {} (e.g. {}.{}.x) \
+                     and rebuild your project.",
+                    index_version.index_format_version, index_version.major, index_version.minor
+                );
+                write!(f, "{}. {}", err, advice)?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
 /// Error that may occur when accessing a file read
 #[derive(Debug)]
 pub enum OpenReadError {
@@ -164,6 +224,8 @@ pub enum OpenReadError {
     /// Any kind of IO error that happens when
     /// interacting with the underlying IO device.
     IOError(IOError),
+    /// This library doesn't support the index version found on disk
+    IncompatibleIndex(Incompatibility),
 }
 
 impl From<IOError> for OpenReadError {
@@ -183,19 +245,9 @@ impl fmt::Display for OpenReadError {
                 "an io error occurred while opening a file for reading: '{}'",
                 err
             ),
-        }
-    }
-}
-
-impl StdError for OpenReadError {
-    fn description(&self) -> &str {
-        "error occurred while opening a file for reading"
-    }
-
-    fn cause(&self) -> Option<&dyn StdError> {
-        match *self {
-            OpenReadError::FileDoesNotExist(_) => None,
-            OpenReadError::IOError(ref err) => Some(err),
+            OpenReadError::IncompatibleIndex(ref footer) => {
+                write!(f, "Incompatible index format: {:?}", footer)
+            }
         }
     }
 }
@@ -216,6 +268,12 @@ impl From<IOError> for DeleteError {
     }
 }
 
+impl From<Incompatibility> for OpenReadError {
+    fn from(incompatibility: Incompatibility) -> Self {
+        OpenReadError::IncompatibleIndex(incompatibility)
+    }
+}
+
 impl fmt::Display for DeleteError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {

src/directory/footer.rs

@@ -1,181 +1,175 @@
+use crate::common::{BinarySerializable, CountingWriter, FixedSize, VInt};
+use crate::directory::error::Incompatibility;
 use crate::directory::read_only_source::ReadOnlySource;
 use crate::directory::{AntiCallToken, TerminatingWrite};
-use byteorder::{ByteOrder, LittleEndian};
+use crate::Version;
+use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
 use crc32fast::Hasher;
 use std::io;
 use std::io::Write;
 
-const COMMON_FOOTER_SIZE: usize = 4 * 5;
-
 type CrcHashU32 = u32;
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct Footer {
-    pub tantivy_version: (u32, u32, u32),
+    pub version: Version,
     pub meta: String,
     pub versioned_footer: VersionedFooter,
 }
 
+/// Serialises the footer to a byte-array
+/// - versioned_footer_len : 4 bytes
+///-  versioned_footer: variable bytes
+/// - meta_len: 4 bytes
+/// - meta: variable bytes
+/// - version_len: 4 bytes
+/// - version json: variable bytes
+impl BinarySerializable for Footer {
+    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
+        BinarySerializable::serialize(&self.versioned_footer, writer)?;
+        BinarySerializable::serialize(&self.meta, writer)?;
+        let version_string =
+            serde_json::to_string(&self.version).map_err(|_err| io::ErrorKind::InvalidInput)?;
+        BinarySerializable::serialize(&version_string, writer)?;
+        Ok(())
+    }
+
+    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
+        let versioned_footer = VersionedFooter::deserialize(reader)?;
+        let meta = String::deserialize(reader)?;
+        let version_json = String::deserialize(reader)?;
+        let version = serde_json::from_str(&version_json)?;
+        Ok(Footer {
+            version,
+            meta,
+            versioned_footer,
+        })
+    }
+}
+
 impl Footer {
     pub fn new(versioned_footer: VersionedFooter) -> Self {
-        let tantivy_version = (
-            env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap(),
-            env!("CARGO_PKG_VERSION_MINOR").parse().unwrap(),
-            env!("CARGO_PKG_VERSION_PATCH").parse().unwrap(),
-        );
+        let version = crate::VERSION.clone();
+        let meta = version.to_string();
         Footer {
-            tantivy_version,
-            meta: format!(
-                "tantivy v{}.{}.{}, index_format v{}",
-                tantivy_version.0,
-                tantivy_version.1,
-                tantivy_version.2,
-                versioned_footer.version()
-            ),
+            version,
+            meta,
             versioned_footer,
         }
     }
 
-    /// Serialises the footer to a byte-array
-    /// [      versioned_footer     |     meta      |    common_footer ]
-    /// [           0..8            |     8..32     |        32..52    ]
-    pub fn to_bytes(&self) -> Vec<u8> {
-        let mut res = self.versioned_footer.to_bytes();
-        res.extend_from_slice(self.meta.as_bytes());
-        let len = res.len();
-        res.resize(len + COMMON_FOOTER_SIZE, 0);
-        let mut common_footer = &mut res[len..];
-        LittleEndian::write_u32(&mut common_footer, self.meta.len() as u32);
-        LittleEndian::write_u32(&mut common_footer[4..], self.tantivy_version.0);
-        LittleEndian::write_u32(&mut common_footer[8..], self.tantivy_version.1);
-        LittleEndian::write_u32(&mut common_footer[12..], self.tantivy_version.2);
-        LittleEndian::write_u32(&mut common_footer[16..], (len + COMMON_FOOTER_SIZE) as u32);
-        res
-    }
-
-    pub fn from_bytes(data: &[u8]) -> Result<Self, io::Error> {
-        let len = data.len();
-        if len < COMMON_FOOTER_SIZE + 4 {
-            // 4 bytes for index version, stored in versioned footer
-            return Err(io::Error::new(
-                    io::ErrorKind::UnexpectedEof,
-                    format!("File corrupted. The footer len must be over 24, while the entire file len is {}", len)
-                    )
-            );
-        }
-
-        let size = LittleEndian::read_u32(&data[len - 4..]) as usize;
-        if len < size as usize {
-            return Err(io::Error::new(
-                io::ErrorKind::UnexpectedEof,
-                format!(
-                    "The footer len is {}, while the entire file len is {}. \
-                     Your index is either corrupted or was built using a tantivy version\
-                     anterior to 0.11.",
-                    size, len
-                ),
-            ));
-        }
-        let footer = &data[len - size as usize..];
-        let meta_len = LittleEndian::read_u32(&footer[size - COMMON_FOOTER_SIZE..]) as usize;
-        let tantivy_major = LittleEndian::read_u32(&footer[size - 16..]);
-        let tantivy_minor = LittleEndian::read_u32(&footer[size - 12..]);
-        let tantivy_patch = LittleEndian::read_u32(&footer[size - 8..]);
-        Ok(Footer {
-            tantivy_version: (tantivy_major, tantivy_minor, tantivy_patch),
-            meta: String::from_utf8_lossy(
-                &footer[size - meta_len - COMMON_FOOTER_SIZE..size - COMMON_FOOTER_SIZE],
-            )
-            .into_owned(),
-            versioned_footer: VersionedFooter::from_bytes(
-                &footer[..size - meta_len - COMMON_FOOTER_SIZE],
-            )?,
-        })
+    pub fn append_footer<W: io::Write>(&self, mut write: &mut W) -> io::Result<()> {
+        let mut counting_write = CountingWriter::wrap(&mut write);
+        self.serialize(&mut counting_write)?;
+        let written_len = counting_write.written_bytes();
+        write.write_u32::<LittleEndian>(written_len as u32)?;
+        Ok(())
     }
 
     pub fn extract_footer(source: ReadOnlySource) -> Result<(Footer, ReadOnlySource), io::Error> {
-        let footer = Footer::from_bytes(source.as_slice())?;
-        let reader = source.slice_to(source.as_slice().len() - footer.size());
-        Ok((footer, reader))
+        if source.len() < 4 {
+            return Err(io::Error::new(
+                io::ErrorKind::UnexpectedEof,
+                format!(
+                    "File corrupted. The file is smaller than 4 bytes (len={}).",
+                    source.len()
+                ),
+            ));
+        }
+        let (body_footer, footer_len_bytes) = source.split_from_end(u32::SIZE_IN_BYTES);
+        let footer_len = LittleEndian::read_u32(footer_len_bytes.as_slice()) as usize;
+        let body_len = body_footer.len() - footer_len;
+        let (body, footer_data) = body_footer.split(body_len);
+        let mut cursor = footer_data.as_slice();
+        let footer = Footer::deserialize(&mut cursor)?;
+        Ok((footer, body))
     }
 
-    pub fn size(&self) -> usize {
-        self.versioned_footer.size() as usize + self.meta.len() + COMMON_FOOTER_SIZE
+    /// Confirms that the index will be read correctly by this version of tantivy
+    /// Has to be called after `extract_footer` to make sure it's not accessing uninitialised memory
+    pub fn is_compatible(&self) -> Result<(), Incompatibility> {
+        let library_version = crate::version();
+        match &self.versioned_footer {
+            VersionedFooter::V1 {
+                crc32: _crc,
+                store_compression: compression,
+            } => {
+                if &library_version.store_compression != compression {
+                    return Err(Incompatibility::CompressionMismatch {
+                        library_compression_format: library_version.store_compression.to_string(),
+                        index_compression_format: compression.to_string(),
+                    });
+                }
+                Ok(())
+            }
+            VersionedFooter::UnknownVersion => Err(Incompatibility::IndexMismatch {
+                library_version: library_version.clone(),
+                index_version: self.version.clone(),
+            }),
+        }
     }
 }
 
 /// Footer that includes a crc32 hash that enables us to checksum files in the index
 #[derive(Debug, Clone, PartialEq)]
 pub enum VersionedFooter {
-    UnknownVersion { version: u32, size: u32 },
-    V0(CrcHashU32), // crc
+    UnknownVersion,
+    V1 {
+        crc32: CrcHashU32,
+        store_compression: String,
+    },
+}
+
+impl BinarySerializable for VersionedFooter {
+    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
+        let mut buf = Vec::new();
+        match self {
+            VersionedFooter::V1 {
+                crc32,
+                store_compression: compression,
+            } => {
+                // Serializes a valid `VersionedFooter` or panics if the version is unknown
+                // [   version    |   crc_hash  | compression_mode ]
+                // [    0..4      |     4..8    |     variable     ]
+                BinarySerializable::serialize(&1u32, &mut buf)?;
+                BinarySerializable::serialize(crc32, &mut buf)?;
+                BinarySerializable::serialize(compression, &mut buf)?;
+            }
+            VersionedFooter::UnknownVersion => {
+                return Err(io::Error::new(
+                    io::ErrorKind::InvalidInput,
+                    "Cannot serialize an unknown versioned footer ",
+                ));
+            }
+        }
+        BinarySerializable::serialize(&VInt(buf.len() as u64), writer)?;
+        writer.write_all(&buf[..])?;
+        Ok(())
+    }
+
+    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
+        let len = VInt::deserialize(reader)?.0 as usize;
+        let mut buf = vec![0u8; len];
+        reader.read_exact(&mut buf[..])?;
+        let mut cursor = &buf[..];
+        let version = u32::deserialize(&mut cursor)?;
+        if version == 1 {
+            let crc32 = u32::deserialize(&mut cursor)?;
+            let compression = String::deserialize(&mut cursor)?;
+            Ok(VersionedFooter::V1 {
+                crc32,
+                store_compression: compression,
+            })
+        } else {
+            Ok(VersionedFooter::UnknownVersion)
+        }
+    }
 }
 
 impl VersionedFooter {
-    /// Serializes a valid `VersionedFooter` or panics if the version is unknown
-    /// [   version    |   crc_hash  ]
-    /// [      0..4    |     4..8    ]
-    pub fn to_bytes(&self) -> Vec<u8> {
-        match self {
-            VersionedFooter::V0(crc) => {
-                let mut buf = [0u8; 8];
-                LittleEndian::write_u32(&mut buf[0..4], 0);
-                LittleEndian::write_u32(&mut buf[4..8], *crc);
-                buf.to_vec()
-            }
-            VersionedFooter::UnknownVersion { .. } => {
-                panic!("Unsupported index should never get serialized");
-            }
-        }
-    }
-
-    pub fn from_bytes(footer: &[u8]) -> Result<Self, io::Error> {
-        assert!(footer.len() >= 4);
-        if footer.len() < 4 {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidData,
-                "Footer should be more than 4 bytes.",
-            ));
-        }
-        let version = LittleEndian::read_u32(footer);
-        match version {
-            // the first 4 bytes should be zeroed out thus returning a `0`
-            0 => {
-                if footer.len() != 8 {
-                    return Err(io::Error::new(
-                        io::ErrorKind::UnexpectedEof,
-                        format!(
-                            "File corrupted. The versioned footer len is {}, while it should be 8",
-                            footer.len()
-                        ),
-                    ));
-                }
-                Ok(VersionedFooter::V0(LittleEndian::read_u32(&footer[4..])))
-            }
-            version => Ok(VersionedFooter::UnknownVersion {
-                version,
-                size: footer.len() as u32,
-            }),
-        }
-    }
-
-    pub fn size(&self) -> u32 {
-        match self {
-            VersionedFooter::V0(_) => 8,
-            VersionedFooter::UnknownVersion { size, .. } => *size,
-        }
-    }
-
-    pub fn version(&self) -> u32 {
-        match self {
-            VersionedFooter::V0(_) => 0,
-            VersionedFooter::UnknownVersion { version, .. } => *version,
-        }
-    }
-
     pub fn crc(&self) -> Option<CrcHashU32> {
         match self {
-            VersionedFooter::V0(crc) => Some(*crc),
+            VersionedFooter::V1 { crc32, .. } => Some(*crc32),
             VersionedFooter::UnknownVersion { .. } => None,
         }
     }
@@ -211,10 +205,13 @@ impl<W: TerminatingWrite> Write for FooterProxy<W> {
 
 impl<W: TerminatingWrite> TerminatingWrite for FooterProxy<W> {
     fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()> {
-        let crc = self.hasher.take().unwrap().finalize();
-        let footer = Footer::new(VersionedFooter::V0(crc)).to_bytes();
+        let crc32 = self.hasher.take().unwrap().finalize();
+        let footer = Footer::new(VersionedFooter::V1 {
+            crc32,
+            store_compression: crate::store::COMPRESSION.to_string(),
+        });
         let mut writer = self.writer.take().unwrap();
-        writer.write_all(&footer)?;
+        footer.append_footer(&mut writer)?;
         writer.terminate()
     }
 }
@@ -222,56 +219,121 @@ impl<W: TerminatingWrite> TerminatingWrite for FooterProxy<W> {
 #[cfg(test)]
 mod tests {
 
+    use super::CrcHashU32;
+    use super::FooterProxy;
+    use crate::common::BinarySerializable;
     use crate::directory::footer::{Footer, VersionedFooter};
+    use crate::directory::TerminatingWrite;
+    use byteorder::{ByteOrder, LittleEndian};
     use regex::Regex;
 
+    #[test]
+    fn test_versioned_footer() {
+        let mut vec = Vec::new();
+        let footer_proxy = FooterProxy::new(&mut vec);
+        assert!(footer_proxy.terminate().is_ok());
+        assert_eq!(vec.len(), 167);
+        let footer = Footer::deserialize(&mut &vec[..]).unwrap();
+        if let VersionedFooter::V1 {
+            crc32: _,
+            store_compression,
+        } = footer.versioned_footer
+        {
+            assert_eq!(store_compression, crate::store::COMPRESSION);
+        } else {
+            panic!("Versioned footer should be V1.");
+        }
+        assert_eq!(&footer.version, crate::version());
+    }
+
     #[test]
     fn test_serialize_deserialize_footer() {
-        let crc = 123456;
-        let footer = Footer::new(VersionedFooter::V0(crc));
-        let footer_bytes = footer.to_bytes();
-        assert_eq!(Footer::from_bytes(&footer_bytes).unwrap(), footer);
+        let mut buffer = Vec::new();
+        let crc32 = 123456u32;
+        let footer: Footer = Footer::new(VersionedFooter::V1 {
+            crc32,
+            store_compression: "lz4".to_string(),
+        });
+        footer.serialize(&mut buffer).unwrap();
+        let footer_deser = Footer::deserialize(&mut &buffer[..]).unwrap();
+        assert_eq!(footer_deser, footer);
     }
 
     #[test]
     fn footer_length() {
-        // test to make sure the ascii art in the doc-strings is correct
-        let crc = 1111111 as u32;
-        let versioned_footer = VersionedFooter::V0(crc);
-        assert_eq!(versioned_footer.size(), 8);
+        let crc32 = 1111111u32;
+        let versioned_footer = VersionedFooter::V1 {
+            crc32,
+            store_compression: "lz4".to_string(),
+        };
+        let mut buf = Vec::new();
+        versioned_footer.serialize(&mut buf).unwrap();
+        assert_eq!(buf.len(), 13);
         let footer = Footer::new(versioned_footer);
         let regex_ptn = Regex::new(
             "tantivy v[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.{0,10}, index_format v[0-9]{1,5}",
         )
         .unwrap();
-        assert!(regex_ptn.find(&footer.meta).is_some());
+        assert!(regex_ptn.is_match(&footer.meta));
     }
 
     #[test]
     fn versioned_footer_from_bytes() {
-        use byteorder::{ByteOrder, LittleEndian};
-        let v_footer_bytes = vec![0, 0, 0, 0, 12, 35, 89, 18];
-        let versioned_footer = VersionedFooter::from_bytes(&v_footer_bytes).unwrap();
-        let expected_versioned_footer =
-            VersionedFooter::V0(LittleEndian::read_u32(&[12, 35, 89, 18]));
-        assert_eq!(versioned_footer, expected_versioned_footer);
-
-        assert_eq!(versioned_footer.to_bytes(), v_footer_bytes);
-    }
-
-    #[should_panic(expected = "Unsupported index should never get serialized")]
-    #[test]
-    fn versioned_footer_panic() {
-        use byteorder::{ByteOrder, LittleEndian};
-        let v_footer_bytes = vec![1; 8];
-        let versioned_footer = VersionedFooter::from_bytes(&v_footer_bytes).unwrap();
-        let expected_version = LittleEndian::read_u32(&[1, 1, 1, 1]);
-        let expected_versioned_footer = VersionedFooter::UnknownVersion {
-            version: expected_version,
-            size: v_footer_bytes.len() as u32,
+        let v_footer_bytes = vec![
+            // versionned footer length
+            12 | 128,
+            // index format version
+            1,
+            0,
+            0,
+            0,
+            // crc 32
+            12,
+            35,
+            89,
+            18,
+            // compression format
+            3 | 128,
+            b'l',
+            b'z',
+            b'4',
+        ];
+        let mut cursor = &v_footer_bytes[..];
+        let versioned_footer = VersionedFooter::deserialize(&mut cursor).unwrap();
+        assert!(cursor.is_empty());
+        let expected_crc: u32 = LittleEndian::read_u32(&v_footer_bytes[5..9]) as CrcHashU32;
+        let expected_versioned_footer: VersionedFooter = VersionedFooter::V1 {
+            crc32: expected_crc,
+            store_compression: "lz4".to_string(),
         };
         assert_eq!(versioned_footer, expected_versioned_footer);
+        let mut buffer = Vec::new();
+        assert!(versioned_footer.serialize(&mut buffer).is_ok());
+        assert_eq!(&v_footer_bytes[..], &buffer[..]);
+    }
 
-        versioned_footer.to_bytes();
+    #[test]
+    fn versioned_footer_panic() {
+        let v_footer_bytes = vec![6u8 | 128u8, 3u8, 0u8, 0u8, 1u8, 0u8, 0u8];
+        let mut b = &v_footer_bytes[..];
+        let versioned_footer = VersionedFooter::deserialize(&mut b).unwrap();
+        assert!(b.is_empty());
+        let expected_versioned_footer = VersionedFooter::UnknownVersion;
+        assert_eq!(versioned_footer, expected_versioned_footer);
+        let mut buf = Vec::new();
+        assert!(versioned_footer.serialize(&mut buf).is_err());
+    }
+
+    #[test]
+    #[cfg(not(feature = "lz4"))]
+    fn compression_mismatch() {
+        let crc32 = 1111111u32;
+        let versioned_footer = VersionedFooter::V1 {
+            crc32,
+            store_compression: "lz4".to_string(),
+        };
+        let footer = Footer::new(versioned_footer);
+        let res = footer.is_compatible();
+        assert!(res.is_err());
     }
 }

src/directory/managed_directory.rs

@@ -9,7 +9,7 @@ use crate::directory::{ReadOnlySource, WritePtr};
 use crate::directory::{WatchCallback, WatchHandle};
 use crate::error::DataCorruption;
 use crate::Directory;
-use crate::Result;
+
 use crc32fast::Hasher;
 use serde_json;
 use std::collections::HashSet;
@@ -65,7 +65,7 @@ fn save_managed_paths(
 
 impl ManagedDirectory {
     /// Wraps a directory as managed directory.
-    pub fn wrap<Dir: Directory>(directory: Dir) -> Result<ManagedDirectory> {
+    pub fn wrap<Dir: Directory>(directory: Dir) -> crate::Result<ManagedDirectory> {
         match directory.atomic_read(&MANAGED_FILEPATH) {
             Ok(data) => {
                 let managed_files_json = String::from_utf8_lossy(&data);
@@ -88,6 +88,11 @@ impl ManagedDirectory {
                 meta_informations: Arc::default(),
             }),
             Err(OpenReadError::IOError(e)) => Err(From::from(e)),
+            Err(OpenReadError::IncompatibleIndex(incompatibility)) => {
+                // For the moment, this should never happen  `meta.json`
+                // do not have any footer and cannot detect incompatibility.
+                Err(crate::TantivyError::IncompatibleIndex(incompatibility))
+            }
         }
     }
 
@@ -261,8 +266,9 @@ impl ManagedDirectory {
 impl Directory for ManagedDirectory {
     fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
         let read_only_source = self.directory.open_read(path)?;
-        let (_footer, reader) = Footer::extract_footer(read_only_source)
+        let (footer, reader) = Footer::extract_footer(read_only_source)
             .map_err(|err| IOError::with_path(path.to_path_buf(), err))?;
+        footer.is_compatible()?;
         Ok(reader)
     }
 
@@ -409,6 +415,8 @@ mod tests_mmap_specific {
         write.write_all(&[3u8, 4u8, 5u8]).unwrap();
         write.terminate().unwrap();
 
+        let read_source = managed_directory.open_read(test_path2).unwrap();
+        assert_eq!(read_source.as_slice(), &[3u8, 4u8, 5u8]);
         assert!(managed_directory.list_damaged().unwrap().is_empty());
 
         let mut corrupted_path = tempdir_path.clone();

src/directory/mmap_directory.rs

@@ -131,22 +131,29 @@ impl MmapCache {
         }
         self.cache.remove(full_path);
         self.counters.miss += 1;
-        Ok(if let Some(mmap) = open_mmap(full_path)? {
+        let mmap_opt = open_mmap(full_path)?;
+        Ok(mmap_opt.map(|mmap| {
             let mmap_arc: Arc<BoxedData> = Arc::new(Box::new(mmap));
             let mmap_weak = Arc::downgrade(&mmap_arc);
             self.cache.insert(full_path.to_owned(), mmap_weak);
-            Some(mmap_arc)
-        } else {
-            None
-        })
+            mmap_arc
+        }))
     }
 }
 
+pub enum WatcherMode {
+    Event,
+    Poll
+}
+
 struct WatcherWrapper {
     _watcher: Mutex<notify::RecommendedWatcher>,
     watcher_router: Arc<WatchCallbackList>,
+    watcher_mode: WatcherMode,
 }
 
+
+
 impl WatcherWrapper {
     pub fn new(path: &Path) -> Result<Self, OpenDirectoryError> {
         let (tx, watcher_recv): (Sender<RawEvent>, Receiver<RawEvent>) = channel();
@@ -164,33 +171,57 @@ impl WatcherWrapper {
             })?;
         let watcher_router: Arc<WatchCallbackList> = Default::default();
         let watcher_router_clone = watcher_router.clone();
+        let path_clone = path.clone();
+        let meta_path = path_clone.join(*META_FILEPATH);
         thread::Builder::new()
             .name("meta-file-watch-thread".to_string())
             .spawn(move || {
+                let mut old_content = String::new();
+                let mode = WatcherMode::Event;
                 loop {
-                    match watcher_recv.recv().map(|evt| evt.path) {
-                        Ok(Some(changed_path)) => {
-                            // ... Actually subject to false positive.
-                            // We might want to be more accurate than this at one point.
-                            if let Some(filename) = changed_path.file_name() {
-                                if filename == *META_FILEPATH {
-                                    let _ = watcher_router_clone.broadcast();
+                    match mode {
+                        WatcherMode::Event => {
+                            match watcher_recv.recv().map(|evt| evt.path) {
+                                Ok(Some(changed_path)) => {
+                                    // ... Actually subject to false positive.
+                                    // We might want to be more accurate than this at one point.
+                                    if let Some(filename) = changed_path.file_name() {
+                                        if filename == *META_FILEPATH {
+                                            let _ = watcher_router_clone.broadcast();
+                                        }
+                                    }
+                                }
+                                Ok(None) => {
+                                    // not an event we are interested in.
+                                }
+                                Err(_e) => {
+                                    // the watch send channel was dropped
+                                    break;
                                 }
                             }
                         }
-                        Ok(None) => {
-                            // not an event we are interested in.
+                        WatcherMode::Poll => {
+                            let mut file = match File::open(&meta_path) {
+                                Err(why) => panic!("open: nope"),
+                                Ok(file) => file,
+                            };
+                            let mut new_content = String::new();
+                            match file.read_to_string(&mut new_content) {
+                                Err(why) => panic!("read: nope"),
+                                Ok(_) => {},
+                            }
+                            if old_content != new_content {
+                                let _ = watcher_router_clone.broadcast();
+                                old_content = new_content;
+                            }
                         }
-                        Err(_e) => {
-                            // the watch send channel was dropped
-                            break;
-                        }
-                    }
+                    };
                 }
             })?;
         Ok(WatcherWrapper {
             _watcher: Mutex::new(watcher),
             watcher_router,
+            watcher_mode: WatcherMode::Event,
         })
     }
 

src/directory/mod.rs

@@ -23,11 +23,9 @@ pub use self::directory::{Directory, DirectoryClone};
 pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};
 pub use self::ram_directory::RAMDirectory;
 pub use self::read_only_source::ReadOnlySource;
-pub(crate) use self::watch_event_router::WatchCallbackList;
-pub use self::watch_event_router::{WatchCallback, WatchHandle};
+pub use self::watch_event_router::{WatchCallback, WatchCallbackList, WatchHandle};
 use std::io::{self, BufWriter, Write};
 use std::path::PathBuf;
-
 /// Outcome of the Garbage collection
 pub struct GarbageCollectionResult {
     /// List of files that were deleted in this cycle
@@ -48,6 +46,9 @@ pub use self::mmap_directory::MmapDirectory;
 pub use self::managed_directory::ManagedDirectory;
 
 /// Struct used to prevent from calling [`terminate_ref`](trait.TerminatingWrite#method.terminate_ref) directly
+///
+/// The point is that while the type is public, it cannot be built by anyone
+/// outside of this module.
 pub struct AntiCallToken(());
 
 /// Trait used to indicate when no more write need to be done on a writer
@@ -78,6 +79,13 @@ impl<W: TerminatingWrite> TerminatingWrite for BufWriter<W> {
     }
 }
 
+#[cfg(test)]
+impl<'a> TerminatingWrite for &'a mut Vec<u8> {
+    fn terminate_ref(&mut self, _a: AntiCallToken) -> io::Result<()> {
+        self.flush()
+    }
+}
+
 /// Write object for Directory.
 ///
 /// `WritePtr` are required to implement both Write

src/directory/ram_directory.rs

@@ -191,7 +191,7 @@ impl Directory for RAMDirectory {
         // Reserve the path to prevent calls to .write() to succeed.
         self.fs.write().unwrap().write(path_buf.clone(), &[]);
 
-        let mut vec_writer = VecWriter::new(path_buf.clone(), self.clone());
+        let mut vec_writer = VecWriter::new(path_buf, self.clone());
         vec_writer.write_all(data)?;
         vec_writer.flush()?;
         if path == Path::new(&*META_FILEPATH) {

src/directory/read_only_source.rs

@@ -70,6 +70,12 @@ impl ReadOnlySource {
         (left, right)
     }
 
+    /// Splits into 2 `ReadOnlySource`, at the offset `end - right_len`.
+    pub fn split_from_end(self, right_len: usize) -> (ReadOnlySource, ReadOnlySource) {
+        let left_len = self.len() - right_len;
+        self.split(left_len)
+    }
+
     /// Creates a ReadOnlySource that is just a
     /// view over a slice of the data.
     ///

src/directory/watch_event_router.rs

@@ -24,13 +24,20 @@ pub struct WatchCallbackList {
 #[derive(Clone)]
 pub struct WatchHandle(Arc<WatchCallback>);
 
+impl WatchHandle {
+    /// Create a WatchHandle handle.
+    pub fn new(watch_callback: Arc<WatchCallback>) -> WatchHandle {
+        WatchHandle(watch_callback)
+    }
+}
+
 impl WatchCallbackList {
     /// Suscribes a new callback and returns a handle that controls the lifetime of the callback.
     pub fn subscribe(&self, watch_callback: WatchCallback) -> WatchHandle {
         let watch_callback_arc = Arc::new(watch_callback);
         let watch_callback_weak = Arc::downgrade(&watch_callback_arc);
         self.router.write().unwrap().push(watch_callback_weak);
-        WatchHandle(watch_callback_arc)
+        WatchHandle::new(watch_callback_arc)
     }
 
     fn list_callback(&self) -> Vec<Arc<WatchCallback>> {

src/error.rs

@@ -2,8 +2,8 @@
 
 use std::io;
 
-use crate::directory::error::LockError;
 use crate::directory::error::{IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
+use crate::directory::error::{Incompatibility, LockError};
 use crate::fastfield::FastFieldNotAvailableError;
 use crate::query;
 use crate::schema;
@@ -80,6 +80,9 @@ pub enum TantivyError {
     /// System error. (e.g.: We failed spawning a new thread)
     #[fail(display = "System error.'{}'", _0)]
     SystemError(String),
+    /// Index incompatible with current version of tantivy
+    #[fail(display = "{:?}", _0)]
+    IncompatibleIndex(Incompatibility),
 }
 
 impl From<DataCorruption> for TantivyError {
@@ -129,6 +132,9 @@ impl From<OpenReadError> for TantivyError {
         match error {
             OpenReadError::FileDoesNotExist(filepath) => TantivyError::PathDoesNotExist(filepath),
             OpenReadError::IOError(io_error) => TantivyError::IOError(io_error),
+            OpenReadError::IncompatibleIndex(incompatibility) => {
+                TantivyError::IncompatibleIndex(incompatibility)
+            }
         }
     }
 }

src/fastfield/delete.rs

@@ -1,9 +1,8 @@
-use crate::common::HasLen;
+use crate::common::{BitSet, HasLen};
 use crate::directory::ReadOnlySource;
 use crate::directory::WritePtr;
 use crate::space_usage::ByteCount;
 use crate::DocId;
-use bit_set::BitSet;
 use std::io;
 use std::io::Write;
 
@@ -17,7 +16,7 @@ pub fn write_delete_bitset(
 ) -> io::Result<()> {
     let mut byte = 0u8;
     let mut shift = 0u8;
-    for doc in 0..(max_doc as usize) {
+    for doc in 0..max_doc {
         if delete_bitset.contains(doc) {
             byte |= 1 << shift;
         }
@@ -32,7 +31,7 @@ pub fn write_delete_bitset(
     if max_doc % 8 > 0 {
         writer.write_all(&[byte])?;
     }
-    writer.flush()
+    Ok(())
 }
 
 /// Set of deleted `DocId`s.
@@ -86,7 +85,6 @@ impl HasLen for DeleteBitSet {
 mod tests {
     use super::*;
     use crate::directory::*;
-    use bit_set::BitSet;
     use std::path::PathBuf;
 
     fn test_delete_bitset_helper(bitset: &BitSet, max_doc: u32) {
@@ -95,27 +93,26 @@ mod tests {
         {
             let mut writer = directory.open_write(&*test_path).unwrap();
             write_delete_bitset(bitset, max_doc, &mut writer).unwrap();
+            writer.terminate().unwrap();
         }
-        {
-            let source = directory.open_read(&test_path).unwrap();
-            let delete_bitset = DeleteBitSet::open(source);
-            for doc in 0..max_doc as usize {
-                assert_eq!(bitset.contains(doc), delete_bitset.is_deleted(doc as DocId));
-            }
-            assert_eq!(delete_bitset.len(), bitset.len());
+        let source = directory.open_read(&test_path).unwrap();
+        let delete_bitset = DeleteBitSet::open(source);
+        for doc in 0..max_doc {
+            assert_eq!(bitset.contains(doc), delete_bitset.is_deleted(doc as DocId));
         }
+        assert_eq!(delete_bitset.len(), bitset.len());
     }
 
     #[test]
     fn test_delete_bitset() {
         {
-            let mut bitset = BitSet::with_capacity(10);
+            let mut bitset = BitSet::with_max_value(10);
             bitset.insert(1);
             bitset.insert(9);
             test_delete_bitset_helper(&bitset, 10);
         }
         {
-            let mut bitset = BitSet::with_capacity(8);
+            let mut bitset = BitSet::with_max_value(8);
             bitset.insert(1);
             bitset.insert(2);
             bitset.insert(3);

src/fastfield/mod.rs

@@ -33,6 +33,7 @@ pub use self::reader::FastFieldReader;
 pub use self::readers::FastFieldReaders;
 pub use self::serializer::FastFieldSerializer;
 pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
+use crate::chrono::{NaiveDateTime, Utc};
 use crate::common;
 use crate::schema::Cardinality;
 use crate::schema::FieldType;
@@ -49,7 +50,7 @@ mod serializer;
 mod writer;
 
 /// Trait for types that are allowed for fast fields: (u64, i64 and f64).
-pub trait FastValue: Default + Clone + Copy + Send + Sync + PartialOrd {
+pub trait FastValue: Clone + Copy + Send + Sync + PartialOrd {
     /// Converts a value from u64
     ///
     /// Internally all fast field values are encoded as u64.
@@ -69,6 +70,12 @@ pub trait FastValue: Default + Clone + Copy + Send + Sync + PartialOrd {
     /// Cast value to `u64`.
     /// The value is just reinterpreted in memory.
     fn as_u64(&self) -> u64;
+
+    /// Build a default value. This default value is never used, so the value does not
+    /// really matter.
+    fn make_zero() -> Self {
+        Self::from_u64(0i64.to_u64())
+    }
 }
 
 impl FastValue for u64 {
@@ -135,11 +142,34 @@ impl FastValue for f64 {
     }
 }
 
+impl FastValue for crate::DateTime {
+    fn from_u64(timestamp_u64: u64) -> Self {
+        let timestamp_i64 = i64::from_u64(timestamp_u64);
+        crate::DateTime::from_utc(NaiveDateTime::from_timestamp(timestamp_i64, 0), Utc)
+    }
+
+    fn to_u64(&self) -> u64 {
+        self.timestamp().to_u64()
+    }
+
+    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
+        match *field_type {
+            FieldType::Date(ref integer_options) => integer_options.get_fastfield_cardinality(),
+            _ => None,
+        }
+    }
+
+    fn as_u64(&self) -> u64 {
+        self.timestamp().as_u64()
+    }
+}
+
 fn value_to_u64(value: &Value) -> u64 {
     match *value {
         Value::U64(ref val) => *val,
         Value::I64(ref val) => common::i64_to_u64(*val),
         Value::F64(ref val) => common::f64_to_u64(*val),
+        Value::Date(ref datetime) => common::i64_to_u64(datetime.timestamp()),
         _ => panic!("Expected a u64/i64/f64 field, got {:?} ", value),
     }
 }
@@ -151,10 +181,12 @@ mod tests {
     use crate::common::CompositeFile;
     use crate::directory::{Directory, RAMDirectory, WritePtr};
     use crate::fastfield::FastFieldReader;
-    use crate::schema::Document;
+    use crate::merge_policy::NoMergePolicy;
     use crate::schema::Field;
     use crate::schema::Schema;
     use crate::schema::FAST;
+    use crate::schema::{Document, IntOptions};
+    use crate::{Index, SegmentId, SegmentReader};
     use once_cell::sync::Lazy;
     use rand::prelude::SliceRandom;
     use rand::rngs::StdRng;
@@ -178,6 +210,12 @@ mod tests {
         assert_eq!(test_fastfield.get(2), 300);
     }
 
+    #[test]
+    pub fn test_fastfield_i64_u64() {
+        let datetime = crate::DateTime::from_utc(NaiveDateTime::from_timestamp(0i64, 0), Utc);
+        assert_eq!(i64::from_u64(datetime.to_u64()), 0i64);
+    }
+
     #[test]
     fn test_intfastfield_small() {
         let path = Path::new("test");
@@ -429,6 +467,93 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn test_merge_missing_date_fast_field() {
+        let mut schema_builder = Schema::builder();
+        let date_field = schema_builder.add_date_field("date", FAST);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.set_merge_policy(Box::new(NoMergePolicy));
+        index_writer.add_document(doc!(date_field =>crate::chrono::prelude::Utc::now()));
+        index_writer.commit().unwrap();
+        index_writer.add_document(doc!());
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+        let segment_ids: Vec<SegmentId> = reader
+            .searcher()
+            .segment_readers()
+            .iter()
+            .map(SegmentReader::segment_id)
+            .collect();
+        assert_eq!(segment_ids.len(), 2);
+        let merge_future = index_writer.merge(&segment_ids[..]);
+        let merge_res = futures::executor::block_on(merge_future);
+        assert!(merge_res.is_ok());
+        assert!(reader.reload().is_ok());
+        assert_eq!(reader.searcher().segment_readers().len(), 1);
+    }
+
+    #[test]
+    fn test_default_datetime() {
+        assert_eq!(crate::DateTime::make_zero().timestamp(), 0i64);
+    }
+
+    #[test]
+    fn test_datefastfield() {
+        use crate::fastfield::FastValue;
+        let mut schema_builder = Schema::builder();
+        let date_field = schema_builder.add_date_field("date", FAST);
+        let multi_date_field = schema_builder.add_date_field(
+            "multi_date",
+            IntOptions::default().set_fast(Cardinality::MultiValues),
+        );
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.set_merge_policy(Box::new(NoMergePolicy));
+        index_writer.add_document(doc!(
+            date_field => crate::DateTime::from_u64(1i64.to_u64()),
+            multi_date_field => crate::DateTime::from_u64(2i64.to_u64()),
+            multi_date_field => crate::DateTime::from_u64(3i64.to_u64())
+        ));
+        index_writer.add_document(doc!(
+            date_field => crate::DateTime::from_u64(4i64.to_u64())
+        ));
+        index_writer.add_document(doc!(
+            multi_date_field => crate::DateTime::from_u64(5i64.to_u64()),
+            multi_date_field => crate::DateTime::from_u64(6i64.to_u64())
+        ));
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        assert_eq!(searcher.segment_readers().len(), 1);
+        let segment_reader = searcher.segment_reader(0);
+        let fast_fields = segment_reader.fast_fields();
+        let date_fast_field = fast_fields.date(date_field).unwrap();
+        let dates_fast_field = fast_fields.dates(multi_date_field).unwrap();
+        let mut dates = vec![];
+        {
+            assert_eq!(date_fast_field.get(0u32).timestamp(), 1i64);
+            dates_fast_field.get_vals(0u32, &mut dates);
+            assert_eq!(dates.len(), 2);
+            assert_eq!(dates[0].timestamp(), 2i64);
+            assert_eq!(dates[1].timestamp(), 3i64);
+        }
+        {
+            assert_eq!(date_fast_field.get(1u32).timestamp(), 4i64);
+            dates_fast_field.get_vals(1u32, &mut dates);
+            assert!(dates.is_empty());
+        }
+        {
+            assert_eq!(date_fast_field.get(2u32).timestamp(), 0i64);
+            dates_fast_field.get_vals(2u32, &mut dates);
+            assert_eq!(dates.len(), 2);
+            assert_eq!(dates[0].timestamp(), 5i64);
+            assert_eq!(dates[1].timestamp(), 6i64);
+        }
+    }
 }
 
 #[cfg(all(test, feature = "unstable"))]

src/fastfield/multivalued/reader.rs

@@ -45,7 +45,7 @@ impl<Item: FastValue> MultiValueIntFastFieldReader<Item> {
     pub fn get_vals(&self, doc: DocId, vals: &mut Vec<Item>) {
         let (start, stop) = self.range(doc);
         let len = (stop - start) as usize;
-        vals.resize(len, Item::default());
+        vals.resize(len, Item::make_zero());
         self.vals_reader.get_range_u64(start, &mut vals[..]);
     }
 

src/fastfield/readers.rs

@@ -15,9 +15,11 @@ pub struct FastFieldReaders {
     fast_field_i64: HashMap<Field, FastFieldReader<i64>>,
     fast_field_u64: HashMap<Field, FastFieldReader<u64>>,
     fast_field_f64: HashMap<Field, FastFieldReader<f64>>,
+    fast_field_date: HashMap<Field, FastFieldReader<crate::DateTime>>,
     fast_field_i64s: HashMap<Field, MultiValueIntFastFieldReader<i64>>,
     fast_field_u64s: HashMap<Field, MultiValueIntFastFieldReader<u64>>,
     fast_field_f64s: HashMap<Field, MultiValueIntFastFieldReader<f64>>,
+    fast_field_dates: HashMap<Field, MultiValueIntFastFieldReader<crate::DateTime>>,
     fast_bytes: HashMap<Field, BytesFastFieldReader>,
     fast_fields_composite: CompositeFile,
 }
@@ -26,6 +28,7 @@ enum FastType {
     I64,
     U64,
     F64,
+    Date,
 }
 
 fn type_and_cardinality(field_type: &FieldType) -> Option<(FastType, Cardinality)> {
@@ -39,6 +42,9 @@ fn type_and_cardinality(field_type: &FieldType) -> Option<(FastType, Cardinality
         FieldType::F64(options) => options
             .get_fastfield_cardinality()
             .map(|cardinality| (FastType::F64, cardinality)),
+        FieldType::Date(options) => options
+            .get_fastfield_cardinality()
+            .map(|cardinality| (FastType::Date, cardinality)),
         FieldType::HierarchicalFacet => Some((FastType::U64, Cardinality::MultiValues)),
         _ => None,
     }
@@ -53,9 +59,11 @@ impl FastFieldReaders {
             fast_field_i64: Default::default(),
             fast_field_u64: Default::default(),
             fast_field_f64: Default::default(),
+            fast_field_date: Default::default(),
             fast_field_i64s: Default::default(),
             fast_field_u64s: Default::default(),
             fast_field_f64s: Default::default(),
+            fast_field_dates: Default::default(),
             fast_bytes: Default::default(),
             fast_fields_composite: fast_fields_composite.clone(),
         };
@@ -95,6 +103,12 @@ impl FastFieldReaders {
                                         FastFieldReader::open(fast_field_data.clone()),
                                     );
                                 }
+                                FastType::Date => {
+                                    fast_field_readers.fast_field_date.insert(
+                                        field,
+                                        FastFieldReader::open(fast_field_data.clone()),
+                                    );
+                                }
                             }
                         } else {
                             return Err(From::from(FastFieldNotAvailableError::new(field_entry)));
@@ -130,6 +144,14 @@ impl FastFieldReaders {
                                         .fast_field_f64s
                                         .insert(field, multivalued_int_fast_field);
                                 }
+                                FastType::Date => {
+                                    let vals_reader = FastFieldReader::open(fast_field_data);
+                                    let multivalued_int_fast_field =
+                                        MultiValueIntFastFieldReader::open(idx_reader, vals_reader);
+                                    fast_field_readers
+                                        .fast_field_dates
+                                        .insert(field, multivalued_int_fast_field);
+                                }
                             }
                         } else {
                             return Err(From::from(FastFieldNotAvailableError::new(field_entry)));
@@ -156,8 +178,6 @@ impl FastFieldReaders {
     /// If the field is a i64-fast field, return the associated u64 reader. Values are
     /// mapped from i64 to u64 using a (well the, it is unique) monotonic mapping.    ///
     ///
-    ///TODO should it also be lenient with f64?
-    ///
     /// This method is useful when merging segment reader.
     pub(crate) fn u64_lenient(&self, field: Field) -> Option<FastFieldReader<u64>> {
         if let Some(u64_ff_reader) = self.u64(field) {
@@ -166,6 +186,12 @@ impl FastFieldReaders {
         if let Some(i64_ff_reader) = self.i64(field) {
             return Some(i64_ff_reader.into_u64_reader());
         }
+        if let Some(f64_ff_reader) = self.f64(field) {
+            return Some(f64_ff_reader.into_u64_reader());
+        }
+        if let Some(date_ff_reader) = self.date(field) {
+            return Some(date_ff_reader.into_u64_reader());
+        }
         None
     }
 
@@ -176,6 +202,13 @@ impl FastFieldReaders {
         self.fast_field_i64.get(&field).cloned()
     }
 
+    /// Returns the `i64` fast field reader reader associated to `field`.
+    ///
+    /// If `field` is not a i64 fast field, this method returns `None`.
+    pub fn date(&self, field: Field) -> Option<FastFieldReader<crate::DateTime>> {
+        self.fast_field_date.get(&field).cloned()
+    }
+
     /// Returns the `f64` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a f64 fast field, this method returns `None`.
@@ -202,6 +235,9 @@ impl FastFieldReaders {
         if let Some(i64s_ff_reader) = self.i64s(field) {
             return Some(i64s_ff_reader.into_u64s_reader());
         }
+        if let Some(f64s_ff_reader) = self.f64s(field) {
+            return Some(f64s_ff_reader.into_u64s_reader());
+        }
         None
     }
 
@@ -219,6 +255,13 @@ impl FastFieldReaders {
         self.fast_field_f64s.get(&field).cloned()
     }
 
+    /// Returns a `crate::DateTime` multi-valued fast field reader reader associated to `field`.
+    ///
+    /// If `field` is not a `crate::DateTime` multi-valued fast field, this method returns `None`.
+    pub fn dates(&self, field: Field) -> Option<MultiValueIntFastFieldReader<crate::DateTime>> {
+        self.fast_field_dates.get(&field).cloned()
+    }
+
     /// Returns the `bytes` fast field reader associated to `field`.
     ///
     /// If `field` is not a bytes fast field, returns `None`.

src/fastfield/writer.rs

@@ -4,7 +4,7 @@ use crate::common::BinarySerializable;
 use crate::common::VInt;
 use crate::fastfield::{BytesFastFieldWriter, FastFieldSerializer};
 use crate::postings::UnorderedTermId;
-use crate::schema::{Cardinality, Document, Field, FieldType, Schema};
+use crate::schema::{Cardinality, Document, Field, FieldEntry, FieldType, Schema};
 use crate::termdict::TermOrdinal;
 use fnv::FnvHashMap;
 use std::collections::HashMap;
@@ -17,6 +17,14 @@ pub struct FastFieldsWriter {
     bytes_value_writers: Vec<BytesFastFieldWriter>,
 }
 
+fn fast_field_default_value(field_entry: &FieldEntry) -> u64 {
+    match *field_entry.field_type() {
+        FieldType::I64(_) | FieldType::Date(_) => common::i64_to_u64(0i64),
+        FieldType::F64(_) => common::f64_to_u64(0.0f64),
+        _ => 0u64,
+    }
+}
+
 impl FastFieldsWriter {
     /// Create all `FastFieldWriter` required by the schema.
     pub fn from_schema(schema: &Schema) -> FastFieldsWriter {
@@ -25,18 +33,15 @@ impl FastFieldsWriter {
         let mut bytes_value_writers = Vec::new();
 
         for (field, field_entry) in schema.fields() {
-            let default_value = match *field_entry.field_type() {
-                FieldType::I64(_) => common::i64_to_u64(0i64),
-                FieldType::F64(_) => common::f64_to_u64(0.0f64),
-                _ => 0u64,
-            };
             match *field_entry.field_type() {
                 FieldType::I64(ref int_options)
                 | FieldType::U64(ref int_options)
-                | FieldType::F64(ref int_options) => {
+                | FieldType::F64(ref int_options)
+                | FieldType::Date(ref int_options) => {
                     match int_options.get_fastfield_cardinality() {
                         Some(Cardinality::SingleValue) => {
                             let mut fast_field_writer = IntFastFieldWriter::new(field);
+                            let default_value = fast_field_default_value(field_entry);
                             fast_field_writer.set_val_if_missing(default_value);
                             single_value_writers.push(fast_field_writer);
                         }

src/indexer/delete_queue.rs

@@ -2,7 +2,7 @@ use super::operation::DeleteOperation;
 use crate::Opstamp;
 use std::mem;
 use std::ops::DerefMut;
-use std::sync::{Arc, RwLock};
+use std::sync::{Arc, RwLock, Weak};
 
 // The DeleteQueue is similar in conceptually to a multiple
 // consumer single producer broadcast channel.
@@ -14,14 +14,15 @@ use std::sync::{Arc, RwLock};
 //
 // New consumer can be created in two ways
 // - calling `delete_queue.cursor()` returns a cursor, that
-//   will include all future delete operation (and no past operations).
+//   will include all future delete operation (and some or none
+//   of the past operations... The client is in charge of checking the opstamps.).
 // - cloning an existing cursor returns a new cursor, that
 //   is at the exact same position, and can now advance independently
 //   from the original cursor.
 #[derive(Default)]
 struct InnerDeleteQueue {
     writer: Vec<DeleteOperation>,
-    last_block: Option<Arc<Block>>,
+    last_block: Weak<Block>,
 }
 
 #[derive(Clone)]
@@ -32,21 +33,31 @@ pub struct DeleteQueue {
 impl DeleteQueue {
     // Creates a new delete queue.
     pub fn new() -> DeleteQueue {
-        let delete_queue = DeleteQueue {
+        DeleteQueue {
             inner: Arc::default(),
-        };
-
-        let next_block = NextBlock::from(delete_queue.clone());
-
-        {
-            let mut delete_queue_wlock = delete_queue.inner.write().unwrap();
-            delete_queue_wlock.last_block = Some(Arc::new(Block {
-                operations: Arc::default(),
-                next: next_block,
-            }));
         }
+    }
 
-        delete_queue
+    fn get_last_block(&self) -> Arc<Block> {
+        {
+            // try get the last block with simply acquiring the read lock.
+            let rlock = self.inner.read().unwrap();
+            if let Some(block) = rlock.last_block.upgrade() {
+                return block;
+            }
+        }
+        // It failed. Let's double check after acquiring the write, as someone could have called
+        // `get_last_block` right after we released the rlock.
+        let mut wlock = self.inner.write().unwrap();
+        if let Some(block) = wlock.last_block.upgrade() {
+            return block;
+        }
+        let block = Arc::new(Block {
+            operations: Arc::default(),
+            next: NextBlock::from(self.clone()),
+        });
+        wlock.last_block = Arc::downgrade(&block);
+        block
     }
 
     // Creates a new cursor that makes it possible to
@@ -54,17 +65,7 @@ impl DeleteQueue {
     //
     // Past delete operations are not accessible.
     pub fn cursor(&self) -> DeleteCursor {
-        let last_block = self
-            .inner
-            .read()
-            .expect("Read lock poisoned when opening delete queue cursor")
-            .last_block
-            .clone()
-            .expect(
-                "Failed to unwrap last_block. This should never happen
-                as the Option<> is only here to make
-                initialization possible",
-            );
+        let last_block = self.get_last_block();
         let operations_len = last_block.operations.len();
         DeleteCursor {
             block: last_block,
@@ -100,23 +101,19 @@ impl DeleteQueue {
             .write()
             .expect("Failed to acquire write lock on delete queue writer");
 
-        let delete_operations;
-        {
-            let writer: &mut Vec<DeleteOperation> = &mut self_wlock.writer;
-            if writer.is_empty() {
-                return None;
-            }
-            delete_operations = mem::replace(writer, vec![]);
+        if self_wlock.writer.is_empty() {
+            return None;
         }
 
-        let next_block = NextBlock::from(self.clone());
-        {
-            self_wlock.last_block = Some(Arc::new(Block {
-                operations: Arc::new(delete_operations),
-                next: next_block,
-            }));
-        }
-        self_wlock.last_block.clone()
+        let delete_operations = mem::replace(&mut self_wlock.writer, vec![]);
+
+        let new_block = Arc::new(Block {
+            operations: Arc::new(delete_operations.into_boxed_slice()),
+            next: NextBlock::from(self.clone()),
+        });
+
+        self_wlock.last_block = Arc::downgrade(&new_block);
+        Some(new_block)
     }
 }
 
@@ -170,7 +167,7 @@ impl NextBlock {
 }
 
 struct Block {
-    operations: Arc<Vec<DeleteOperation>>,
+    operations: Arc<Box<[DeleteOperation]>>,
     next: NextBlock,
 }
 

src/indexer/index_writer.rs

@@ -1,6 +1,7 @@
 use super::operation::{AddOperation, UserOperation};
 use super::segment_updater::SegmentUpdater;
 use super::PreparedCommit;
+use crate::common::BitSet;
 use crate::core::Index;
 use crate::core::Segment;
 use crate::core::SegmentComponent;
@@ -23,7 +24,6 @@ use crate::schema::Document;
 use crate::schema::IndexRecordOption;
 use crate::schema::Term;
 use crate::Opstamp;
-use bit_set::BitSet;
 use crossbeam::channel;
 use futures::executor::block_on;
 use futures::future::Future;
@@ -115,7 +115,7 @@ fn compute_deleted_bitset(
             while docset.advance() {
                 let deleted_doc = docset.doc();
                 if deleted_doc < limit_doc {
-                    delete_bitset.insert(deleted_doc as usize);
+                    delete_bitset.insert(deleted_doc);
                     might_have_changed = true;
                 }
             }
@@ -126,51 +126,60 @@ fn compute_deleted_bitset(
     Ok(might_have_changed)
 }
 
-/// Advance delete for the given segment up
-/// to the target opstamp.
+/// Advance delete for the given segment up to the target opstamp.
+///
+/// Note that there are no guarantee that the resulting `segment_entry` delete_opstamp
+/// is `==` target_opstamp.
+/// For instance, there was no delete operation between the state of the `segment_entry` and
+/// the `target_opstamp`, `segment_entry` is not updated.
 pub(crate) fn advance_deletes(
     mut segment: Segment,
     segment_entry: &mut SegmentEntry,
     target_opstamp: Opstamp,
 ) -> crate::Result<()> {
-    {
-        if segment_entry.meta().delete_opstamp() == Some(target_opstamp) {
-            // We are already up-to-date here.
-            return Ok(());
-        }
+    if segment_entry.meta().delete_opstamp() == Some(target_opstamp) {
+        // We are already up-to-date here.
+        return Ok(());
+    }
 
-        let segment_reader = SegmentReader::open(&segment)?;
+    if segment_entry.delete_bitset().is_none() && segment_entry.delete_cursor().get().is_none() {
+        // There has been no `DeleteOperation` between the segment status and `target_opstamp`.
+        return Ok(());
+    }
 
-        let max_doc = segment_reader.max_doc();
-        let mut delete_bitset: BitSet = match segment_entry.delete_bitset() {
-            Some(previous_delete_bitset) => (*previous_delete_bitset).clone(),
-            None => BitSet::with_capacity(max_doc as usize),
-        };
+    let segment_reader = SegmentReader::open(&segment)?;
 
-        let delete_cursor = segment_entry.delete_cursor();
-        compute_deleted_bitset(
-            &mut delete_bitset,
-            &segment_reader,
-            delete_cursor,
-            &DocToOpstampMapping::None,
-            target_opstamp,
-        )?;
+    let max_doc = segment_reader.max_doc();
+    let mut delete_bitset: BitSet = match segment_entry.delete_bitset() {
+        Some(previous_delete_bitset) => (*previous_delete_bitset).clone(),
+        None => BitSet::with_max_value(max_doc),
+    };
 
-        // TODO optimize
+    compute_deleted_bitset(
+        &mut delete_bitset,
+        &segment_reader,
+        segment_entry.delete_cursor(),
+        &DocToOpstampMapping::None,
+        target_opstamp,
+    )?;
+
+    // TODO optimize
+    if let Some(seg_delete_bitset) = segment_reader.delete_bitset() {
         for doc in 0u32..max_doc {
-            if segment_reader.is_deleted(doc) {
-                delete_bitset.insert(doc as usize);
+            if seg_delete_bitset.is_deleted(doc) {
+                delete_bitset.insert(doc);
             }
         }
-
-        let num_deleted_docs = delete_bitset.len();
-        if num_deleted_docs > 0 {
-            segment = segment.with_delete_meta(num_deleted_docs as u32, target_opstamp);
-            let mut delete_file = segment.open_write(SegmentComponent::DELETE)?;
-            write_delete_bitset(&delete_bitset, max_doc, &mut delete_file)?;
-            delete_file.terminate()?;
-        }
     }
+
+    let num_deleted_docs = delete_bitset.len();
+    if num_deleted_docs > 0 {
+        segment = segment.with_delete_meta(num_deleted_docs as u32, target_opstamp);
+        let mut delete_file = segment.open_write(SegmentComponent::DELETE)?;
+        write_delete_bitset(&delete_bitset, max_doc, &mut delete_file)?;
+        delete_file.terminate()?;
+    }
+
     segment_entry.set_meta(segment.meta().clone());
     Ok(())
 }
@@ -236,7 +245,7 @@ fn apply_deletes(
     mut delete_cursor: &mut DeleteCursor,
     doc_opstamps: &[Opstamp],
     last_docstamp: Opstamp,
-) -> crate::Result<Option<BitSet<u32>>> {
+) -> crate::Result<Option<BitSet>> {
     if delete_cursor.get().is_none() {
         // if there are no delete operation in the queue, no need
         // to even open the segment.
@@ -246,7 +255,7 @@ fn apply_deletes(
     let doc_to_opstamps = DocToOpstampMapping::from(doc_opstamps);
 
     let max_doc = segment.meta().max_doc();
-    let mut deleted_bitset = BitSet::with_capacity(max_doc as usize);
+    let mut deleted_bitset = BitSet::with_max_value(max_doc);
     let may_have_deletes = compute_deleted_bitset(
         &mut deleted_bitset,
         &segment_reader,
@@ -330,12 +339,17 @@ impl IndexWriter {
         Ok(index_writer)
     }
 
+    fn drop_sender(&mut self) {
+        let (sender, _receiver) = channel::bounded(1);
+        mem::replace(&mut self.operation_sender, sender);
+    }
+
     /// If there are some merging threads, blocks until they all finish their work and
     /// then drop the `IndexWriter`.
     pub fn wait_merging_threads(mut self) -> crate::Result<()> {
         // this will stop the indexing thread,
         // dropping the last reference to the segment_updater.
-        drop(self.operation_sender);
+        self.drop_sender();
 
         let former_workers_handles = mem::replace(&mut self.workers_join_handle, vec![]);
         for join_handle in former_workers_handles {
@@ -346,7 +360,6 @@ impl IndexWriter {
                     TantivyError::ErrorInThread("Error in indexing worker thread.".into())
                 })?;
         }
-        drop(self.workers_join_handle);
 
         let result = self
             .segment_updater
@@ -501,9 +514,13 @@ impl IndexWriter {
     /// Merges a given list of segments
     ///
     /// `segment_ids` is required to be non-empty.
-    pub async fn merge(&mut self, segment_ids: &[SegmentId]) -> crate::Result<SegmentMeta> {
+    pub fn merge(
+        &mut self,
+        segment_ids: &[SegmentId],
+    ) -> impl Future<Output = crate::Result<SegmentMeta>> {
         let merge_operation = self.segment_updater.make_merge_operation(segment_ids);
-        self.segment_updater.start_merge(merge_operation)?.await
+        let segment_updater = self.segment_updater.clone();
+        async move { segment_updater.start_merge(merge_operation)?.await }
     }
 
     /// Closes the current document channel send.
@@ -747,6 +764,16 @@ impl IndexWriter {
     }
 }
 
+impl Drop for IndexWriter {
+    fn drop(&mut self) {
+        self.segment_updater.kill();
+        self.drop_sender();
+        for work in self.workers_join_handle.drain(..) {
+            let _ = work.join();
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
 
@@ -756,7 +783,7 @@ mod tests {
     use crate::error::*;
     use crate::indexer::NoMergePolicy;
     use crate::query::TermQuery;
-    use crate::schema::{self, IndexRecordOption};
+    use crate::schema::{self, IndexRecordOption, STRING};
     use crate::Index;
     use crate::ReloadPolicy;
     use crate::Term;
@@ -1181,4 +1208,16 @@ mod tests {
         assert!(clear_again.is_ok());
         assert!(commit_again.is_ok());
     }
+
+    #[test]
+    fn test_index_doc_missing_field() {
+        let mut schema_builder = schema::Schema::builder();
+        let idfield = schema_builder.add_text_field("id", STRING);
+        schema_builder.add_text_field("optfield", STRING);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.add_document(doc!(idfield=>"myid"));
+        let commit = index_writer.commit();
+        assert!(commit.is_ok());
+    }
 }

src/indexer/merger.rs

@@ -1492,4 +1492,46 @@ mod tests {
             assert_eq!(&vals, &[20]);
         }
     }
+
+    #[test]
+    fn merges_f64_fast_fields_correctly() -> crate::Result<()> {
+        let mut builder = schema::SchemaBuilder::new();
+
+        let fast_multi = IntOptions::default().set_fast(Cardinality::MultiValues);
+
+        let field = builder.add_f64_field("f64", schema::FAST);
+        let multi_field = builder.add_f64_field("f64s", fast_multi);
+
+        let index = Index::create_in_ram(builder.build());
+
+        let mut writer = index.writer_with_num_threads(1, 3_000_000)?;
+
+        // Make sure we'll attempt to merge every created segment
+        let mut policy = crate::indexer::LogMergePolicy::default();
+        policy.set_min_merge_size(2);
+        writer.set_merge_policy(Box::new(policy));
+
+        for i in 0..100 {
+            let mut doc = Document::new();
+            doc.add_f64(field, 42.0);
+
+            doc.add_f64(multi_field, 0.24);
+            doc.add_f64(multi_field, 0.27);
+
+            writer.add_document(doc);
+
+            if i % 5 == 0 {
+                writer.commit()?;
+            }
+        }
+
+        writer.commit()?;
+        writer.wait_merging_threads()?;
+
+        // If a merging thread fails, we should end up with more
+        // than one segment here
+        assert_eq!(1, index.searchable_segments()?.len());
+
+        Ok(())
+    }
 }

src/indexer/mod.rs

@@ -33,6 +33,7 @@ pub type DefaultMergePolicy = LogMergePolicy;
 mod tests {
     use crate::schema::{self, Schema};
     use crate::{Index, Term};
+
     #[test]
     fn test_advance_delete_bug() {
         let mut schema_builder = Schema::builder();

src/indexer/operation.rs

@@ -19,6 +19,8 @@ pub struct AddOperation {
 /// UserOperation is an enum type that encapsulates other operation types.
 #[derive(Eq, PartialEq, Debug)]
 pub enum UserOperation {
+    /// Add operation
     Add(Document),
+    /// Delete operation
     Delete(Term),
 }

src/indexer/segment_entry.rs

@@ -1,7 +1,7 @@
+use crate::common::BitSet;
 use crate::core::SegmentId;
 use crate::core::SegmentMeta;
 use crate::indexer::delete_queue::DeleteCursor;
-use bit_set::BitSet;
 use std::fmt;
 
 /// A segment entry describes the state of

src/indexer/segment_updater.rs

@@ -138,7 +138,7 @@ fn merge(
 
     let segment_meta = index.new_segment_meta(merged_segment.id(), num_docs);
 
-    Ok(SegmentEntry::new(segment_meta.clone(), delete_cursor, None))
+    Ok(SegmentEntry::new(segment_meta, delete_cursor, None))
 }
 
 pub(crate) struct InnerSegmentUpdater {
@@ -210,9 +210,15 @@ impl SegmentUpdater {
         f: F,
     ) -> impl Future<Output = crate::Result<T>> {
         let (sender, receiver) = oneshot::channel();
-        self.pool.spawn_ok(async move {
-            let _ = sender.send(f.await);
-        });
+        if self.is_alive() {
+            self.pool.spawn_ok(async move {
+                let _ = sender.send(f.await);
+            });
+        } else {
+            let _ = sender.send(Err(crate::TantivyError::SystemError(
+                "Segment updater killed".to_string(),
+            )));
+        }
         receiver.unwrap_or_else(|_| {
             let err_msg =
                 "A segment_updater future did not success. This should never happen.".to_string();
@@ -324,13 +330,11 @@ impl SegmentUpdater {
     ) -> impl Future<Output = crate::Result<()>> {
         let segment_updater: SegmentUpdater = self.clone();
         self.schedule_future(async move {
-            if segment_updater.is_alive() {
-                let segment_entries = segment_updater.purge_deletes(opstamp)?;
-                segment_updater.segment_manager.commit(segment_entries);
-                segment_updater.save_metas(opstamp, payload)?;
-                let _ = garbage_collect_files(segment_updater.clone()).await;
-                segment_updater.consider_merge_options().await;
-            }
+            let segment_entries = segment_updater.purge_deletes(opstamp)?;
+            segment_updater.segment_manager.commit(segment_entries);
+            segment_updater.save_metas(opstamp, payload)?;
+            let _ = garbage_collect_files(segment_updater.clone()).await;
+            segment_updater.consider_merge_options().await;
             Ok(())
         })
     }
@@ -526,11 +530,7 @@ impl SegmentUpdater {
     /// Obsolete files will eventually be cleaned up
     /// by the directory garbage collector.
     pub fn wait_merging_thread(&self) -> crate::Result<()> {
-        for merge_operation in self.merge_operations.changes_iter() {
-            if merge_operation.is_empty() {
-                break;
-            }
-        }
+        self.merge_operations.wait_until_empty();
         Ok(())
     }
 }

src/indexer/segment_writer.rs

@@ -11,10 +11,9 @@ use crate::schema::Schema;
 use crate::schema::Term;
 use crate::schema::Value;
 use crate::schema::{Field, FieldEntry};
-use crate::tokenizer::BoxedTokenizer;
-use crate::tokenizer::FacetTokenizer;
-use crate::tokenizer::PreTokenizedStream;
-use crate::tokenizer::{TokenStream, TokenStreamChain, Tokenizer};
+use crate::tokenizer::{BoxTokenStream, PreTokenizedStream};
+use crate::tokenizer::{FacetTokenizer, TextAnalyzer};
+use crate::tokenizer::{TokenStreamChain, Tokenizer};
 use crate::DocId;
 use crate::Opstamp;
 use crate::Result;
@@ -50,7 +49,7 @@ pub struct SegmentWriter {
     fast_field_writers: FastFieldsWriter,
     fieldnorms_writer: FieldNormsWriter,
     doc_opstamps: Vec<Opstamp>,
-    tokenizers: Vec<Option<BoxedTokenizer>>,
+    tokenizers: Vec<Option<TextAnalyzer>>,
 }
 
 impl SegmentWriter {
@@ -159,7 +158,7 @@ impl SegmentWriter {
                     }
                 }
                 FieldType::Str(_) => {
-                    let mut token_streams: Vec<Box<dyn TokenStream>> = vec![];
+                    let mut token_streams: Vec<BoxTokenStream> = vec![];
                     let mut offsets = vec![];
                     let mut total_offset = 0;
 
@@ -172,7 +171,7 @@ impl SegmentWriter {
                                 }
 
                                 token_streams
-                                    .push(Box::new(PreTokenizedStream::from(tok_str.clone())));
+                                    .push(PreTokenizedStream::from(tok_str.clone()).into());
                             }
                             Value::Str(ref text) => {
                                 if let Some(ref mut tokenizer) =
@@ -191,8 +190,7 @@ impl SegmentWriter {
                     let num_tokens = if token_streams.is_empty() {
                         0
                     } else {
-                        let mut token_stream: Box<dyn TokenStream> =
-                            Box::new(TokenStreamChain::new(offsets, token_streams));
+                        let mut token_stream = TokenStreamChain::new(offsets, token_streams);
                         self.multifield_postings
                             .index_text(doc_id, field, &mut token_stream)
                     };
@@ -249,6 +247,7 @@ impl SegmentWriter {
             }
         }
         doc.filter_fields(|field| schema.get_field_entry(field).is_stored());
+        doc.prepare_for_store();
         let doc_writer = self.segment_serializer.get_store_writer();
         doc_writer.store(&doc)?;
         self.max_doc += 1;

src/indexer/stamper.rs

@@ -1,18 +1,76 @@
 use crate::Opstamp;
 use std::ops::Range;
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
+#[cfg(not(target_arch = "arm"))]
+mod atomic_impl {
+
+    use crate::Opstamp;
+    use std::sync::atomic::{AtomicU64, Ordering};
+
+    #[derive(Default)]
+    pub struct AtomicU64Wrapper(AtomicU64);
+
+    impl AtomicU64Wrapper {
+        pub fn new(first_opstamp: Opstamp) -> AtomicU64Wrapper {
+            AtomicU64Wrapper(AtomicU64::new(first_opstamp as u64))
+        }
+
+        pub fn fetch_add(&self, val: u64, order: Ordering) -> u64 {
+            self.0.fetch_add(val as u64, order) as u64
+        }
+
+        pub fn revert(&self, val: u64, order: Ordering) -> u64 {
+            self.0.store(val, order);
+            val
+        }
+    }
+}
+
+#[cfg(target_arch = "arm")]
+mod atomic_impl {
+
+    use crate::Opstamp;
+    /// Under other architecture, we rely on a mutex.
+    use std::sync::atomic::Ordering;
+    use std::sync::RwLock;
+
+    #[derive(Default)]
+    pub struct AtomicU64Wrapper(RwLock<u64>);
+
+    impl AtomicU64Wrapper {
+        pub fn new(first_opstamp: Opstamp) -> AtomicU64Wrapper {
+            AtomicU64Wrapper(RwLock::new(first_opstamp))
+        }
+
+        pub fn fetch_add(&self, incr: u64, _order: Ordering) -> u64 {
+            let mut lock = self.0.write().unwrap();
+            let previous_val = *lock;
+            *lock = previous_val + incr;
+            previous_val
+        }
+
+        pub fn revert(&self, val: u64, _order: Ordering) -> u64 {
+            let mut lock = self.0.write().unwrap();
+            *lock = val;
+            val
+        }
+    }
+}
+
+use self::atomic_impl::AtomicU64Wrapper;
+
 /// Stamper provides Opstamps, which is just an auto-increment id to label
 /// an operation.
 ///
 /// Cloning does not "fork" the stamp generation. The stamper actually wraps an `Arc`.
 #[derive(Clone, Default)]
-pub struct Stamper(Arc<AtomicU64>);
+pub struct Stamper(Arc<AtomicU64Wrapper>);
 
 impl Stamper {
     pub fn new(first_opstamp: Opstamp) -> Stamper {
-        Stamper(Arc::new(AtomicU64::new(first_opstamp)))
+        Stamper(Arc::new(AtomicU64Wrapper::new(first_opstamp)))
     }
 
     pub fn stamp(&self) -> Opstamp {
@@ -31,8 +89,7 @@ impl Stamper {
 
     /// Reverts the stamper to a given `Opstamp` value and returns it
     pub fn revert(&self, to_opstamp: Opstamp) -> Opstamp {
-        self.0.store(to_opstamp, Ordering::SeqCst);
-        to_opstamp
+        self.0.revert(to_opstamp, Ordering::SeqCst)
     }
 }
 

src/lib.rs

@@ -160,21 +160,68 @@ pub use self::snippet::{Snippet, SnippetGenerator};
 
 mod docset;
 pub use self::docset::{DocSet, SkipResult};
-
 pub use crate::common::{f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
 pub use crate::core::SegmentComponent;
 pub use crate::core::{Index, IndexMeta, Searcher, Segment, SegmentId, SegmentMeta};
 pub use crate::core::{InvertedIndexReader, SegmentReader};
 pub use crate::directory::Directory;
+pub use crate::indexer::operation::UserOperation;
 pub use crate::indexer::IndexWriter;
 pub use crate::postings::Postings;
 pub use crate::reader::LeasedItem;
 pub use crate::schema::{Document, Term};
+use std::fmt;
 
-/// Expose the current version of tantivy, as well
-/// whether it was compiled with the simd compression.
-pub fn version() -> &'static str {
-    env!("CARGO_PKG_VERSION")
+use once_cell::sync::Lazy;
+
+/// Index format version.
+const INDEX_FORMAT_VERSION: u32 = 1;
+
+/// Structure version for the index.
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Version {
+    major: u32,
+    minor: u32,
+    patch: u32,
+    index_format_version: u32,
+    store_compression: String,
+}
+
+impl fmt::Debug for Version {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.to_string())
+    }
+}
+
+static VERSION: Lazy<Version> = Lazy::new(|| Version {
+    major: env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap(),
+    minor: env!("CARGO_PKG_VERSION_MINOR").parse().unwrap(),
+    patch: env!("CARGO_PKG_VERSION_PATCH").parse().unwrap(),
+    index_format_version: INDEX_FORMAT_VERSION,
+    store_compression: crate::store::COMPRESSION.to_string(),
+});
+
+impl ToString for Version {
+    fn to_string(&self) -> String {
+        format!(
+            "tantivy v{}.{}.{}, index_format v{}, store_compression: {}",
+            self.major, self.minor, self.patch, self.index_format_version, self.store_compression
+        )
+    }
+}
+
+static VERSION_STRING: Lazy<String> = Lazy::new(|| VERSION.to_string());
+
+/// Expose the current version of tantivy as found in Cargo.toml during compilation.
+/// eg. "0.11.0" as well as the compression scheme used in the docstore.
+pub fn version() -> &'static Version {
+    &VERSION
+}
+
+/// Exposes the complete version of tantivy as found in Cargo.toml during compilation as a string.
+/// eg. "tantivy v0.11.0, index_format v1, store_compression: lz4".
+pub fn version_string() -> &'static str {
+    VERSION_STRING.as_str()
 }
 
 /// Defines tantivy's merging strategy
@@ -287,6 +334,18 @@ mod tests {
         sample_with_seed(n, ratio, 4)
     }
 
+    #[test]
+    #[cfg(not(feature = "lz4"))]
+    fn test_version_string() {
+        use regex::Regex;
+        let regex_ptn = Regex::new(
+            "tantivy v[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.{0,10}, index_format v[0-9]{1,5}",
+        )
+        .unwrap();
+        let version = super::version().to_string();
+        assert!(regex_ptn.find(&version).is_some());
+    }
+
     #[test]
     #[cfg(feature = "mmap")]
     fn test_indexing() {
@@ -882,4 +941,73 @@ mod tests {
             assert_eq!(fast_field_reader.get(0), 4f64)
         }
     }
+
+    // motivated by #729
+    #[test]
+    fn test_update_via_delete_insert() {
+        use crate::collector::Count;
+        use crate::indexer::NoMergePolicy;
+        use crate::query::AllQuery;
+        use crate::SegmentId;
+        use futures::executor::block_on;
+
+        const DOC_COUNT: u64 = 2u64;
+
+        let mut schema_builder = SchemaBuilder::default();
+        let id = schema_builder.add_u64_field("id", INDEXED);
+        let schema = schema_builder.build();
+
+        let index = Index::create_in_ram(schema.clone());
+        let index_reader = index.reader().unwrap();
+
+        let mut index_writer = index.writer(3_000_000).unwrap();
+        index_writer.set_merge_policy(Box::new(NoMergePolicy));
+
+        for doc_id in 0u64..DOC_COUNT {
+            index_writer.add_document(doc!(id => doc_id));
+        }
+        index_writer.commit().unwrap();
+
+        index_reader.reload().unwrap();
+        let searcher = index_reader.searcher();
+
+        assert_eq!(
+            searcher.search(&AllQuery, &Count).unwrap(),
+            DOC_COUNT as usize
+        );
+
+        // update the 10 elements by deleting and re-adding
+        for doc_id in 0u64..DOC_COUNT {
+            index_writer.delete_term(Term::from_field_u64(id, doc_id));
+            index_writer.commit().unwrap();
+            index_reader.reload().unwrap();
+            let doc = doc!(id =>  doc_id);
+            index_writer.add_document(doc);
+            index_writer.commit().unwrap();
+            index_reader.reload().unwrap();
+            let searcher = index_reader.searcher();
+            // The number of document should be stable.
+            assert_eq!(
+                searcher.search(&AllQuery, &Count).unwrap(),
+                DOC_COUNT as usize
+            );
+        }
+
+        index_reader.reload().unwrap();
+        let searcher = index_reader.searcher();
+        let segment_ids: Vec<SegmentId> = searcher
+            .segment_readers()
+            .into_iter()
+            .map(|reader| reader.segment_id())
+            .collect();
+        block_on(index_writer.merge(&segment_ids)).unwrap();
+
+        index_reader.reload().unwrap();
+        let searcher = index_reader.searcher();
+
+        assert_eq!(
+            searcher.search(&AllQuery, &Count).unwrap(),
+            DOC_COUNT as usize
+        );
+    }
 }

src/macros.rs

@@ -35,9 +35,9 @@
 /// let likes = schema_builder.add_u64_field("num_u64", FAST);
 /// let schema = schema_builder.build();
 /// let doc = doc!(
-/// 	title => "Life Aquatic",
-/// 	author => "Wes Anderson",
-/// 	likes => 4u64
+///     title => "Life Aquatic",
+///     author => "Wes Anderson",
+///     likes => 4u64
 /// );
 /// # }
 /// ```

src/positions/reader.rs

@@ -36,11 +36,10 @@ struct Positions {
 
 impl Positions {
     pub fn new(position_source: ReadOnlySource, skip_source: ReadOnlySource) -> Positions {
-        let skip_len = skip_source.len();
-        let (body, footer) = skip_source.split(skip_len - u32::SIZE_IN_BYTES);
+        let (body, footer) = skip_source.split_from_end(u32::SIZE_IN_BYTES);
         let num_long_skips = u32::deserialize(&mut footer.as_slice()).expect("Index corrupted");
-        let body_split = body.len() - u64::SIZE_IN_BYTES * (num_long_skips as usize);
-        let (skip_source, long_skip_source) = body.split(body_split);
+        let (skip_source, long_skip_source) =
+            body.split_from_end(u64::SIZE_IN_BYTES * (num_long_skips as usize));
         Positions {
             bit_packer: BitPacker4x::new(),
             skip_source,

src/query/exclude.rs

@@ -54,21 +54,21 @@ where
         match self.excluding_state {
             State::ExcludeOne(excluded_doc) => {
                 if doc == excluded_doc {
-                    false
-                } else if excluded_doc > doc {
-                    true
-                } else {
-                    match self.excluding_docset.skip_next(doc) {
-                        SkipResult::OverStep => {
-                            self.excluding_state = State::ExcludeOne(self.excluding_docset.doc());
-                            true
-                        }
-                        SkipResult::End => {
-                            self.excluding_state = State::Finished;
-                            true
-                        }
-                        SkipResult::Reached => false,
+                    return false;
+                }
+                if excluded_doc > doc {
+                    return true;
+                }
+                match self.excluding_docset.skip_next(doc) {
+                    SkipResult::OverStep => {
+                        self.excluding_state = State::ExcludeOne(self.excluding_docset.doc());
+                        true
                     }
+                    SkipResult::End => {
+                        self.excluding_state = State::Finished;
+                        true
+                    }
+                    SkipResult::Reached => false,
                 }
             }
             State::Finished => true,

src/query/fuzzy_query.rs

@@ -33,7 +33,6 @@ static LEV_BUILDER: Lazy<HashMap<(u8, bool), LevenshteinAutomatonBuilder>> = Laz
 /// use tantivy::schema::{Schema, TEXT};
 /// use tantivy::{doc, Index, Result, Term};
 ///
-/// # fn main() { example().unwrap(); }
 /// fn example() -> Result<()> {
 ///     let mut schema_builder = Schema::builder();
 ///     let title = schema_builder.add_text_field("title", TEXT);
@@ -59,7 +58,6 @@ static LEV_BUILDER: Lazy<HashMap<(u8, bool), LevenshteinAutomatonBuilder>> = Laz
 ///     let searcher = reader.searcher();
 ///
 ///     {
-///
 ///         let term = Term::from_field_text(title, "Diary");
 ///         let query = FuzzyTermQuery::new(term, 1, true);
 ///         let (top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count)).unwrap();
@@ -69,6 +67,7 @@ static LEV_BUILDER: Lazy<HashMap<(u8, bool), LevenshteinAutomatonBuilder>> = Laz
 ///
 ///     Ok(())
 /// }
+/// # assert!(example().is_ok());
 /// ```
 #[derive(Debug, Clone)]
 pub struct FuzzyTermQuery {

src/query/phrase_query/phrase_scorer.rs

@@ -4,6 +4,7 @@ use crate::postings::Postings;
 use crate::query::bm25::BM25Weight;
 use crate::query::{Intersection, Scorer};
 use crate::DocId;
+use std::cmp::Ordering;
 
 struct PostingsWithOffset<TPostings> {
     offset: u32,
@@ -59,12 +60,16 @@ fn intersection_exists(left: &[u32], right: &[u32]) -> bool {
     while left_i < left.len() && right_i < right.len() {
         let left_val = left[left_i];
         let right_val = right[right_i];
-        if left_val < right_val {
-            left_i += 1;
-        } else if right_val < left_val {
-            right_i += 1;
-        } else {
-            return true;
+        match left_val.cmp(&right_val) {
+            Ordering::Less => {
+                left_i += 1;
+            }
+            Ordering::Equal => {
+                return true;
+            }
+            Ordering::Greater => {
+                right_i += 1;
+            }
         }
     }
     false
@@ -77,14 +82,18 @@ fn intersection_count(left: &[u32], right: &[u32]) -> usize {
     while left_i < left.len() && right_i < right.len() {
         let left_val = left[left_i];
         let right_val = right[right_i];
-        if left_val < right_val {
-            left_i += 1;
-        } else if right_val < left_val {
-            right_i += 1;
-        } else {
-            count += 1;
-            left_i += 1;
-            right_i += 1;
+        match left_val.cmp(&right_val) {
+            Ordering::Less => {
+                left_i += 1;
+            }
+            Ordering::Equal => {
+                count += 1;
+                left_i += 1;
+                right_i += 1;
+            }
+            Ordering::Greater => {
+                right_i += 1;
+            }
         }
     }
     count
@@ -103,15 +112,19 @@ fn intersection(left: &mut [u32], right: &[u32]) -> usize {
     while left_i < left_len && right_i < right_len {
         let left_val = left[left_i];
         let right_val = right[right_i];
-        if left_val < right_val {
-            left_i += 1;
-        } else if right_val < left_val {
-            right_i += 1;
-        } else {
-            left[count] = left_val;
-            count += 1;
-            left_i += 1;
-            right_i += 1;
+        match left_val.cmp(&right_val) {
+            Ordering::Less => {
+                left_i += 1;
+            }
+            Ordering::Equal => {
+                left[count] = left_val;
+                count += 1;
+                left_i += 1;
+                right_i += 1;
+            }
+            Ordering::Greater => {
+                right_i += 1;
+            }
         }
     }
     count

src/query/query_parser/query_parser.rs

@@ -8,7 +8,7 @@ use crate::query::PhraseQuery;
 use crate::query::Query;
 use crate::query::RangeQuery;
 use crate::query::TermQuery;
-use crate::schema::IndexRecordOption;
+use crate::schema::{Facet, IndexRecordOption};
 use crate::schema::{Field, Schema};
 use crate::schema::{FieldType, Term};
 use crate::tokenizer::TokenizerManager;
@@ -319,7 +319,10 @@ impl QueryParser {
                     ))
                 }
             }
-            FieldType::HierarchicalFacet => Ok(vec![(0, Term::from_field_text(field, phrase))]),
+            FieldType::HierarchicalFacet => {
+                let facet = Facet::from_text(phrase);
+                Ok(vec![(0, Term::from_field_text(field, facet.encoded_str()))])
+            }
             FieldType::Bytes => {
                 let field_name = self.schema.get_field_name(field).to_string();
                 Err(QueryParserError::FieldNotIndexed(field_name))
@@ -530,7 +533,7 @@ mod test {
     use crate::schema::{IndexRecordOption, TextFieldIndexing, TextOptions};
     use crate::schema::{Schema, Term, INDEXED, STORED, STRING, TEXT};
     use crate::tokenizer::{
-        LowerCaser, SimpleTokenizer, StopWordFilter, Tokenizer, TokenizerManager,
+        LowerCaser, SimpleTokenizer, StopWordFilter, TextAnalyzer, TokenizerManager,
     };
     use crate::Index;
     use matches::assert_matches;
@@ -554,12 +557,13 @@ mod test {
         schema_builder.add_text_field("with_stop_words", text_options);
         schema_builder.add_date_field("date", INDEXED);
         schema_builder.add_f64_field("float", INDEXED);
+        schema_builder.add_facet_field("facet");
         let schema = schema_builder.build();
         let default_fields = vec![title, text];
         let tokenizer_manager = TokenizerManager::default();
         tokenizer_manager.register(
             "en_with_stop_words",
-            SimpleTokenizer
+            TextAnalyzer::from(SimpleTokenizer)
                 .filter(LowerCaser)
                 .filter(StopWordFilter::remove(vec!["the".to_string()])),
         );
@@ -588,9 +592,13 @@ mod test {
     }
 
     #[test]
-    pub fn test_parse_query_simple() {
+    pub fn test_parse_query_facet() {
         let query_parser = make_query_parser();
-        assert!(query_parser.parse_query("toto").is_ok());
+        let query = query_parser.parse_query("facet:/root/branch/leaf").unwrap();
+        assert_eq!(
+            format!("{:?}", query),
+            "TermQuery(Term(field=11,bytes=[114, 111, 111, 116, 0, 98, 114, 97, 110, 99, 104, 0, 108, 101, 97, 102]))"
+        );
     }
 
     #[test]

src/query/range_query.rs

@@ -38,41 +38,33 @@ fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
 /// # Example
 ///
 /// ```rust
-/// # use tantivy::collector::Count;
-/// # use tantivy::query::RangeQuery;
-/// # use tantivy::schema::{Schema, INDEXED};
-/// # use tantivy::{doc, Index, Result};
-/// #
-/// # fn run() -> Result<()> {
-/// #     let mut schema_builder = Schema::builder();
-/// #     let year_field = schema_builder.add_u64_field("year", INDEXED);
-/// #     let schema = schema_builder.build();
-/// #
-/// #     let index = Index::create_in_ram(schema);
-/// #     {
-/// #         let mut index_writer = index.writer_with_num_threads(1, 6_000_000).unwrap();
-/// #         for year in 1950u64..2017u64 {
-/// #             let num_docs_within_year = 10 + (year - 1950) * (year - 1950);
-/// #             for _ in 0..num_docs_within_year {
-/// #                 index_writer.add_document(doc!(year_field => year));
-/// #             }
-/// #         }
-/// #         index_writer.commit().unwrap();
-/// #     }
-/// #   let reader = index.reader()?;
+/// use tantivy::collector::Count;
+/// use tantivy::query::RangeQuery;
+/// use tantivy::schema::{Schema, INDEXED};
+/// use tantivy::{doc, Index};
+/// # fn test() -> tantivy::Result<()> {
+/// let mut schema_builder = Schema::builder();
+/// let year_field = schema_builder.add_u64_field("year", INDEXED);
+/// let schema = schema_builder.build();
+///
+/// let index = Index::create_in_ram(schema);
+/// let mut index_writer = index.writer_with_num_threads(1, 6_000_000)?;
+/// for year in 1950u64..2017u64 {
+///     let num_docs_within_year = 10 + (year - 1950) * (year - 1950);
+///     for _ in 0..num_docs_within_year {
+///       index_writer.add_document(doc!(year_field => year));
+///     }
+/// }
+/// index_writer.commit()?;
+///
+/// let reader = index.reader()?;
 /// let searcher = reader.searcher();
-///
 /// let docs_in_the_sixties = RangeQuery::new_u64(year_field, 1960..1970);
-///
 /// let num_60s_books = searcher.search(&docs_in_the_sixties, &Count)?;
-///
-/// #     assert_eq!(num_60s_books, 2285);
-/// #     Ok(())
-/// # }
-/// #
-/// # fn main() {
-/// #   run().unwrap()
+/// assert_eq!(num_60s_books, 2285);
+/// Ok(())
 /// # }
+/// # assert!(test().is_ok());
 /// ```
 #[derive(Clone, Debug)]
 pub struct RangeQuery {

src/query/regex_query.rs

@@ -15,40 +15,40 @@ use tantivy_fst::Regex;
 /// use tantivy::collector::Count;
 /// use tantivy::query::RegexQuery;
 /// use tantivy::schema::{Schema, TEXT};
-/// use tantivy::{doc, Index, Result, Term};
+/// use tantivy::{doc, Index, Term};
 ///
-/// # fn main() { example().unwrap(); }
-/// fn example() -> Result<()> {
-///     let mut schema_builder = Schema::builder();
-///     let title = schema_builder.add_text_field("title", TEXT);
-///     let schema = schema_builder.build();
-///     let index = Index::create_in_ram(schema);
-///     {
-///         let mut index_writer = index.writer(3_000_000)?;
-///         index_writer.add_document(doc!(
-///             title => "The Name of the Wind",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of Muadib",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "A Dairy Cow",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of a Young Girl",
-///         ));
-///         index_writer.commit().unwrap();
-///     }
-///
-///     let reader = index.reader()?;
-///     let searcher = reader.searcher();
-///
-///     let term = Term::from_field_text(title, "Diary");
-///     let query = RegexQuery::from_pattern("d[ai]{2}ry", title)?;
-///     let count = searcher.search(&query, &Count)?;
-///     assert_eq!(count, 3);
-///     Ok(())
+/// # fn test() -> tantivy::Result<()> {
+/// let mut schema_builder = Schema::builder();
+/// let title = schema_builder.add_text_field("title", TEXT);
+/// let schema = schema_builder.build();
+/// let index = Index::create_in_ram(schema);
+/// {
+///     let mut index_writer = index.writer(3_000_000)?;
+///     index_writer.add_document(doc!(
+///         title => "The Name of the Wind",
+///     ));
+///     index_writer.add_document(doc!(
+///         title => "The Diary of Muadib",
+///     ));
+///     index_writer.add_document(doc!(
+///         title => "A Dairy Cow",
+///     ));
+///     index_writer.add_document(doc!(
+///         title => "The Diary of a Young Girl",
+///     ));
+///     index_writer.commit().unwrap();
 /// }
+///
+/// let reader = index.reader()?;
+/// let searcher = reader.searcher();
+///
+/// let term = Term::from_field_text(title, "Diary");
+/// let query = RegexQuery::from_pattern("d[ai]{2}ry", title)?;
+/// let count = searcher.search(&query, &Count)?;
+/// assert_eq!(count, 3);
+/// Ok(())
+/// # }
+/// # assert!(test().is_ok());
 /// ```
 #[derive(Debug, Clone)]
 pub struct RegexQuery {

src/query/term_query/term_query.rs

@@ -23,42 +23,39 @@ use std::fmt;
 /// use tantivy::collector::{Count, TopDocs};
 /// use tantivy::query::TermQuery;
 /// use tantivy::schema::{Schema, TEXT, IndexRecordOption};
-/// use tantivy::{doc, Index, Result, Term};
-///
-/// # fn main() { example().unwrap(); }
-/// fn example() -> Result<()> {
-///     let mut schema_builder = Schema::builder();
-///     let title = schema_builder.add_text_field("title", TEXT);
-///     let schema = schema_builder.build();
-///     let index = Index::create_in_ram(schema);
-///     {
-///         let mut index_writer = index.writer(3_000_000)?;
-///         index_writer.add_document(doc!(
-///             title => "The Name of the Wind",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of Muadib",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "A Dairy Cow",
-///         ));
-///         index_writer.add_document(doc!(
-///             title => "The Diary of a Young Girl",
-///         ));
-///         index_writer.commit()?;
-///     }
-///     let reader = index.reader()?;
-///     let searcher = reader.searcher();
-///
-///     let query = TermQuery::new(
-///         Term::from_field_text(title, "diary"),
-///         IndexRecordOption::Basic,
-///     );
-///     let (top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count)).unwrap();
-///     assert_eq!(count, 2);
-///
-///     Ok(())
+/// use tantivy::{doc, Index, Term};
+/// # fn test() -> tantivy::Result<()> {
+/// let mut schema_builder = Schema::builder();
+/// let title = schema_builder.add_text_field("title", TEXT);
+/// let schema = schema_builder.build();
+/// let index = Index::create_in_ram(schema);
+/// {
+///     let mut index_writer = index.writer(3_000_000)?;
+///     index_writer.add_document(doc!(
+///         title => "The Name of the Wind",
+///     ));
+///     index_writer.add_document(doc!(
+///         title => "The Diary of Muadib",
+///     ));
+///     index_writer.add_document(doc!(
+///         title => "A Dairy Cow",
+///     ));
+///     index_writer.add_document(doc!(
+///         title => "The Diary of a Young Girl",
+///     ));
+///     index_writer.commit()?;
 /// }
+/// let reader = index.reader()?;
+/// let searcher = reader.searcher();
+/// let query = TermQuery::new(
+///     Term::from_field_text(title, "diary"),
+///     IndexRecordOption::Basic,
+/// );
+/// let (top_docs, count) = searcher.search(&query, &(TopDocs::with_limit(2), Count))?;
+/// assert_eq!(count, 2);
+/// Ok(())
+/// # }
+/// # assert!(test().is_ok());
 /// ```
 #[derive(Clone)]
 pub struct TermQuery {

src/reader/mod.rs

@@ -11,6 +11,7 @@ use crate::Result;
 use crate::Searcher;
 use crate::SegmentReader;
 use std::sync::Arc;
+use std::thread;
 
 /// Defines when a new version of the index should be reloaded.
 ///

src/schema/document.rs

@@ -155,6 +155,21 @@ impl Document {
             .find(|field_value| field_value.field() == field)
             .map(FieldValue::value)
     }
+
+    /// Prepares Document for being stored in the document store
+    ///
+    /// Method transforms PreTokenizedString values into String
+    /// values.
+    pub fn prepare_for_store(&mut self) {
+        for field_value in &mut self.field_values {
+            if let Value::PreTokStr(pre_tokenized_text) = field_value.value() {
+                *field_value = FieldValue::new(
+                    field_value.field(),
+                    Value::Str(pre_tokenized_text.text.clone()), //< TODO somehow remove .clone()
+                );
+            }
+        }
+    }
 }
 
 impl BinarySerializable for Document {
@@ -180,6 +195,7 @@ impl BinarySerializable for Document {
 mod tests {
 
     use crate::schema::*;
+    use crate::tokenizer::{PreTokenizedString, Token};
 
     #[test]
     fn test_doc() {
@@ -189,4 +205,38 @@ mod tests {
         doc.add_text(text_field, "My title");
         assert_eq!(doc.field_values().len(), 1);
     }
+
+    #[test]
+    fn test_prepare_for_store() {
+        let mut schema_builder = Schema::builder();
+        let text_field = schema_builder.add_text_field("title", TEXT);
+        let mut doc = Document::default();
+
+        let pre_tokenized_text = PreTokenizedString {
+            text: String::from("A"),
+            tokens: vec![Token {
+                offset_from: 0,
+                offset_to: 1,
+                position: 0,
+                text: String::from("A"),
+                position_length: 1,
+            }],
+        };
+
+        doc.add_pre_tokenized_text(text_field, &pre_tokenized_text);
+        doc.add_text(text_field, "title");
+        doc.prepare_for_store();
+
+        assert_eq!(doc.field_values().len(), 2);
+
+        match doc.field_values()[0].value() {
+            Value::Str(ref text) => assert_eq!(text, "A"),
+            _ => panic!("Incorrect variant of Value"),
+        }
+
+        match doc.field_values()[1].value() {
+            Value::Str(ref text) => assert_eq!(text, "title"),
+            _ => panic!("Incorrect variant of Value"),
+        }
+    }
 }

src/schema/field_type.rs

@@ -6,6 +6,7 @@ use crate::schema::TextFieldIndexing;
 use crate::schema::Value;
 use crate::schema::{IntOptions, TextOptions};
 use crate::tokenizer::PreTokenizedString;
+use chrono::{FixedOffset, Utc};
 use serde_json::Value as JsonValue;
 
 /// Possible error that may occur while parsing a field value
@@ -124,13 +125,20 @@ impl FieldType {
     pub fn value_from_json(&self, json: &JsonValue) -> Result<Value, ValueParsingError> {
         match *json {
             JsonValue::String(ref field_text) => match *self {
-                FieldType::Str(_) => Ok(Value::Str(field_text.clone())),
-                FieldType::U64(_) | FieldType::I64(_) | FieldType::F64(_) | FieldType::Date(_) => {
-                    Err(ValueParsingError::TypeError(format!(
-                        "Expected an integer, got {:?}",
-                        json
-                    )))
+                FieldType::Date(_) => {
+                    let dt_with_fixed_tz: chrono::DateTime<FixedOffset> =
+                        chrono::DateTime::parse_from_rfc3339(field_text).map_err(|err|
+                            ValueParsingError::TypeError(format!(
+                                "Failed to parse date from JSON. Expected rfc3339 format, got {}. {:?}",
+                                field_text, err
+                            ))
+                        )?;
+                    Ok(Value::Date(dt_with_fixed_tz.with_timezone(&Utc)))
                 }
+                FieldType::Str(_) => Ok(Value::Str(field_text.clone())),
+                FieldType::U64(_) | FieldType::I64(_) | FieldType::F64(_) => Err(
+                    ValueParsingError::TypeError(format!("Expected an integer, got {:?}", json)),
+                ),
                 FieldType::HierarchicalFacet => Ok(Value::Facet(Facet::from(field_text))),
                 FieldType::Bytes => decode(field_text).map(Value::Bytes).map_err(|_| {
                     ValueParsingError::InvalidBase64(format!(
@@ -208,7 +216,35 @@ mod tests {
     use crate::schema::field_type::ValueParsingError;
     use crate::schema::TextOptions;
     use crate::schema::Value;
+    use crate::schema::{Schema, INDEXED};
     use crate::tokenizer::{PreTokenizedString, Token};
+    use crate::{DateTime, Document};
+    use chrono::{NaiveDate, NaiveDateTime, NaiveTime, Utc};
+
+    #[test]
+    fn test_deserialize_json_date() {
+        let mut schema_builder = Schema::builder();
+        let date_field = schema_builder.add_date_field("date", INDEXED);
+        let schema = schema_builder.build();
+        let doc_json = r#"{"date": "2019-10-12T07:20:50.52+02:00"}"#;
+        let doc = schema.parse_document(doc_json).unwrap();
+        let date = doc.get_first(date_field).unwrap();
+        assert_eq!(format!("{:?}", date), "Date(2019-10-12T05:20:50.520Z)");
+    }
+
+    #[test]
+    fn test_serialize_json_date() {
+        let mut doc = Document::new();
+        let mut schema_builder = Schema::builder();
+        let date_field = schema_builder.add_date_field("date", INDEXED);
+        let schema = schema_builder.build();
+        let naive_date = NaiveDate::from_ymd(1982, 9, 17);
+        let naive_time = NaiveTime::from_hms(13, 20, 00);
+        let date_time = DateTime::from_utc(NaiveDateTime::new(naive_date, naive_time), Utc);
+        doc.add_date(date_field, &date_time);
+        let doc_json = schema.to_json(&doc);
+        assert_eq!(doc_json, r#"{"date":["1982-09-17T13:20:00+00:00"]}"#);
+    }
 
     #[test]
     fn test_bytes_value_from_json() {

src/schema/flags.rs

@@ -53,7 +53,7 @@ where
     fn bitor(self, head: SchemaFlagList<Head, ()>) -> Self::Output {
         SchemaFlagList {
             head: head.head,
-            tail: self.clone(),
+            tail: self,
         }
     }
 }

src/schema/mod.rs

@@ -44,7 +44,7 @@ We can split the problem of generating a search result page into two phases :
   the search results page. (`doc_ids[] -> Document[]`)
 
 In the first phase, the ability to search for documents by the given field is determined by the
-[`TextIndexingOptions`](enum.TextIndexingOptions.html) of our
+[`IndexRecordOption`](enum.IndexRecordOption.html) of our
 [`TextOptions`](struct.TextOptions.html).
 
 The effect of each possible setting is described more in detail

src/schema/schema.rs

@@ -166,7 +166,7 @@ impl SchemaBuilder {
     }
 
     /// Adds a field entry to the schema in build.
-    fn add_field(&mut self, field_entry: FieldEntry) -> Field {
+    pub fn add_field(&mut self, field_entry: FieldEntry) -> Field {
         let field = Field::from_field_id(self.fields.len() as u32);
         let field_name = field_entry.name().to_string();
         self.fields.push(field_entry);
@@ -401,6 +401,7 @@ pub enum DocParsingError {
 mod tests {
 
     use crate::schema::field_type::ValueParsingError;
+    use crate::schema::int_options::Cardinality::SingleValue;
     use crate::schema::schema::DocParsingError::NotJSON;
     use crate::schema::*;
     use matches::{assert_matches, matches};
@@ -715,4 +716,94 @@ mod tests {
             assert_matches!(json_err, Err(NotJSON(_)));
         }
     }
+
+    #[test]
+    pub fn test_schema_add_field() {
+        let mut schema_builder = SchemaBuilder::default();
+        let id_options = TextOptions::default().set_stored().set_indexing_options(
+            TextFieldIndexing::default()
+                .set_tokenizer("raw")
+                .set_index_option(IndexRecordOption::Basic),
+        );
+        let timestamp_options = IntOptions::default()
+            .set_stored()
+            .set_indexed()
+            .set_fast(SingleValue);
+        schema_builder.add_text_field("_id", id_options);
+        schema_builder.add_date_field("_timestamp", timestamp_options);
+
+        let schema_content = r#"[
+  {
+    "name": "text",
+    "type": "text",
+    "options": {
+      "indexing": {
+        "record": "position",
+        "tokenizer": "default"
+      },
+      "stored": false
+    }
+  },
+  {
+    "name": "popularity",
+    "type": "i64",
+    "options": {
+      "indexed": false,
+      "fast": "single",
+      "stored": true
+    }
+  }
+]"#;
+        let tmp_schema: Schema =
+            serde_json::from_str(&schema_content).expect("error while reading json");
+        for (_field, field_entry) in tmp_schema.fields() {
+            schema_builder.add_field(field_entry.clone());
+        }
+
+        let schema = schema_builder.build();
+        let schema_json = serde_json::to_string_pretty(&schema).unwrap();
+        let expected = r#"[
+  {
+    "name": "_id",
+    "type": "text",
+    "options": {
+      "indexing": {
+        "record": "basic",
+        "tokenizer": "raw"
+      },
+      "stored": true
+    }
+  },
+  {
+    "name": "_timestamp",
+    "type": "date",
+    "options": {
+      "indexed": true,
+      "fast": "single",
+      "stored": true
+    }
+  },
+  {
+    "name": "text",
+    "type": "text",
+    "options": {
+      "indexing": {
+        "record": "position",
+        "tokenizer": "default"
+      },
+      "stored": false
+    }
+  },
+  {
+    "name": "popularity",
+    "type": "i64",
+    "options": {
+      "indexed": false,
+      "fast": "single",
+      "stored": true
+    }
+  }
+]"#;
+        assert_eq!(schema_json, expected);
+    }
 }

src/schema/value.rs

@@ -75,7 +75,7 @@ impl Serialize for Value {
             Value::U64(u) => serializer.serialize_u64(u),
             Value::I64(u) => serializer.serialize_i64(u),
             Value::F64(u) => serializer.serialize_f64(u),
-            Value::Date(ref date) => serializer.serialize_i64(date.timestamp()),
+            Value::Date(ref date) => serializer.serialize_str(&date.to_rfc3339()),
             Value::Facet(ref facet) => facet.serialize(serializer),
             Value::Bytes(ref bytes) => serializer.serialize_bytes(bytes),
         }
@@ -96,14 +96,14 @@ impl<'de> Deserialize<'de> for Value {
                 formatter.write_str("a string or u32")
             }
 
-            fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E> {
-                Ok(Value::U64(v))
-            }
-
             fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E> {
                 Ok(Value::I64(v))
             }
 
+            fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E> {
+                Ok(Value::U64(v))
+            }
+
             fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E> {
                 Ok(Value::F64(v))
             }
@@ -209,8 +209,8 @@ impl From<f64> for Value {
     }
 }
 
-impl From<DateTime> for Value {
-    fn from(date_time: DateTime) -> Value {
+impl From<crate::DateTime> for Value {
+    fn from(date_time: crate::DateTime) -> Value {
         Value::Date(date_time)
     }
 }
@@ -362,3 +362,17 @@ mod binary_serialize {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::Value;
+    use crate::DateTime;
+    use std::str::FromStr;
+
+    #[test]
+    fn test_serialize_date() {
+        let value = Value::Date(DateTime::from_str("1996-12-20T00:39:57+00:00").unwrap());
+        let serialized_value_json = serde_json::to_string_pretty(&value).unwrap();
+        assert_eq!(serialized_value_json, r#""1996-12-20T00:39:57+00:00""#);
+    }
+}

src/snippet/mod.rs

@@ -1,8 +1,7 @@
 use crate::query::Query;
 use crate::schema::Field;
 use crate::schema::Value;
-use crate::tokenizer::BoxedTokenizer;
-use crate::tokenizer::{Token, TokenStream};
+use crate::tokenizer::{TextAnalyzer, Token};
 use crate::Document;
 use crate::Result;
 use crate::Searcher;
@@ -142,7 +141,7 @@ impl Snippet {
 /// Fragments must be valid in the sense that `&text[fragment.start..fragment.stop]`\
 /// has to be a valid string.
 fn search_fragments<'a>(
-    tokenizer: &BoxedTokenizer,
+    tokenizer: &TextAnalyzer,
     text: &'a str,
     terms: &BTreeMap<String, f32>,
     max_num_chars: usize,
@@ -251,7 +250,7 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str)
 /// ```
 pub struct SnippetGenerator {
     terms_text: BTreeMap<String, f32>,
-    tokenizer: BoxedTokenizer,
+    tokenizer: TextAnalyzer,
     field: Field,
     max_num_chars: usize,
 }
@@ -347,12 +346,11 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet() {
-        let boxed_tokenizer = SimpleTokenizer.into();
         let terms = btreemap! {
             String::from("rust") => 1.0,
             String::from("language") => 0.9
         };
-        let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 100);
+        let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 100);
         assert_eq!(fragments.len(), 7);
         {
             let first = &fragments[0];
@@ -374,13 +372,12 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_scored_fragment() {
-        let boxed_tokenizer = SimpleTokenizer.into();
         {
             let terms = btreemap! {
                 String::from("rust") =>1.0f32,
                 String::from("language") => 0.9f32
             };
-            let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 20);
+            let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 20);
             {
                 let first = &fragments[0];
                 assert_eq!(first.score, 1.0);
@@ -389,13 +386,12 @@ Survey in 2016, 2017, and 2018."#;
             let snippet = select_best_fragment_combination(&fragments[..], &TEST_TEXT);
             assert_eq!(snippet.to_html(), "<b>Rust</b> is a systems")
         }
-        let boxed_tokenizer = SimpleTokenizer.into();
         {
             let terms = btreemap! {
                 String::from("rust") =>0.9f32,
                 String::from("language") => 1.0f32
             };
-            let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 20);
+            let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 20);
             //assert_eq!(fragments.len(), 7);
             {
                 let first = &fragments[0];
@@ -409,14 +405,12 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_in_second_fragment() {
-        let boxed_tokenizer = SimpleTokenizer.into();
-
         let text = "a b c d e f g";
 
         let mut terms = BTreeMap::new();
         terms.insert(String::from("c"), 1.0);
 
-        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);
+        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
 
         assert_eq!(fragments.len(), 1);
         {
@@ -433,14 +427,12 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_with_term_at_the_end_of_fragment() {
-        let boxed_tokenizer = SimpleTokenizer.into();
-
         let text = "a b c d e f f g";
 
         let mut terms = BTreeMap::new();
         terms.insert(String::from("f"), 1.0);
 
-        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);
+        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
 
         assert_eq!(fragments.len(), 2);
         {
@@ -457,15 +449,13 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_with_second_fragment_has_the_highest_score() {
-        let boxed_tokenizer = SimpleTokenizer.into();
-
         let text = "a b c d e f g";
 
         let mut terms = BTreeMap::new();
         terms.insert(String::from("f"), 1.0);
         terms.insert(String::from("a"), 0.9);
 
-        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 7);
+        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 7);
 
         assert_eq!(fragments.len(), 2);
         {
@@ -482,14 +472,12 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_with_term_not_in_text() {
-        let boxed_tokenizer = SimpleTokenizer.into();
-
         let text = "a b c d";
 
         let mut terms = BTreeMap::new();
         terms.insert(String::from("z"), 1.0);
 
-        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);
+        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
 
         assert_eq!(fragments.len(), 0);
 
@@ -500,12 +488,10 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_with_no_terms() {
-        let boxed_tokenizer = SimpleTokenizer.into();
-
         let text = "a b c d";
 
         let terms = BTreeMap::new();
-        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);
+        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
         assert_eq!(fragments.len(), 0);
 
         let snippet = select_best_fragment_combination(&fragments[..], &text);

src/store/compression_lz4.rs

@@ -1,7 +1,10 @@
-extern crate lz4;
-
 use std::io::{self, Read, Write};
 
+/// Name of the compression scheme used in the doc store.
+///
+/// This name is appended to the version string of tantivy.
+pub const COMPRESSION: &'static str = "lz4";
+
 pub fn compress(uncompressed: &[u8], compressed: &mut Vec<u8>) -> io::Result<()> {
     compressed.clear();
     let mut encoder = lz4::EncoderBuilder::new().build(compressed)?;

src/store/compression_snap.rs

@@ -2,6 +2,11 @@ use snap;
 
 use std::io::{self, Read, Write};
 
+/// Name of the compression scheme used in the doc store.
+///
+/// This name is appended to the version string of tantivy.
+pub const COMPRESSION: &str = "snappy";
+
 pub fn compress(uncompressed: &[u8], compressed: &mut Vec<u8>) -> io::Result<()> {
     compressed.clear();
     let mut encoder = snap::Writer::new(compressed);

src/store/mod.rs

@@ -42,12 +42,16 @@ pub use self::writer::StoreWriter;
 #[cfg(feature = "lz4")]
 mod compression_lz4;
 #[cfg(feature = "lz4")]
-use self::compression_lz4::*;
+pub use self::compression_lz4::COMPRESSION;
+#[cfg(feature = "lz4")]
+use self::compression_lz4::{compress, decompress};
 
 #[cfg(not(feature = "lz4"))]
 mod compression_snap;
 #[cfg(not(feature = "lz4"))]
-use self::compression_snap::*;
+pub use self::compression_snap::COMPRESSION;
+#[cfg(not(feature = "lz4"))]
+use self::compression_snap::{compress, decompress};
 
 #[cfg(test)]
 pub mod tests {

src/tokenizer/alphanum_only.rs

@@ -2,9 +2,7 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! # fn main() {
-//!
-//! let tokenizer = RawTokenizer
+//! let tokenizer = TextAnalyzer::from(RawTokenizer)
 //!   .filter(AlphaNumOnlyFilter);
 //!
 //! let mut stream = tokenizer.token_stream("hello there");
@@ -12,7 +10,7 @@
 //! // contains a space
 //! assert!(stream.next().is_none());
 //!
-//! let tokenizer = SimpleTokenizer
+//! let tokenizer = TextAnalyzer::from(SimpleTokenizer)
 //!   .filter(AlphaNumOnlyFilter);
 //!
 //! let mut stream = tokenizer.token_stream("hello there 💣");
@@ -20,58 +18,31 @@
 //! assert!(stream.next().is_some());
 //! // the "emoji" is dropped because its not an alphanum
 //! assert!(stream.next().is_none());
-//! # }
 //! ```
-use super::{Token, TokenFilter, TokenStream};
+use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
 
 /// `TokenFilter` that removes all tokens that contain non
 /// ascii alphanumeric characters.
 #[derive(Clone)]
 pub struct AlphaNumOnlyFilter;
 
-pub struct AlphaNumOnlyFilterStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
-    tail: TailTokenStream,
+pub struct AlphaNumOnlyFilterStream<'a> {
+    tail: BoxTokenStream<'a>,
 }
 
-impl<TailTokenStream> AlphaNumOnlyFilterStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
+impl<'a> AlphaNumOnlyFilterStream<'a> {
     fn predicate(&self, token: &Token) -> bool {
         token.text.chars().all(|c| c.is_ascii_alphanumeric())
     }
+}
 
-    fn wrap(tail: TailTokenStream) -> AlphaNumOnlyFilterStream<TailTokenStream> {
-        AlphaNumOnlyFilterStream { tail }
+impl TokenFilter for AlphaNumOnlyFilter {
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(AlphaNumOnlyFilterStream { tail: token_stream })
     }
 }
 
-impl<TailTokenStream> TokenFilter<TailTokenStream> for AlphaNumOnlyFilter
-where
-    TailTokenStream: TokenStream,
-{
-    type ResultTokenStream = AlphaNumOnlyFilterStream<TailTokenStream>;
-
-    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
-        AlphaNumOnlyFilterStream::wrap(token_stream)
-    }
-}
-
-impl<TailTokenStream> TokenStream for AlphaNumOnlyFilterStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
-    }
-
+impl<'a> TokenStream for AlphaNumOnlyFilterStream<'a> {
     fn advance(&mut self) -> bool {
         while self.tail.advance() {
             if self.predicate(self.tail.token()) {
@@ -81,4 +52,12 @@ where
 
         false
     }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
 }

src/tokenizer/ascii_folding_filter.rs

@@ -1,4 +1,4 @@
-use super::{Token, TokenFilter, TokenStream};
+use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
 use std::mem;
 
 /// This class converts alphabetic, numeric, and symbolic Unicode characters
@@ -7,26 +7,21 @@ use std::mem;
 #[derive(Clone)]
 pub struct AsciiFoldingFilter;
 
-impl<TailTokenStream> TokenFilter<TailTokenStream> for AsciiFoldingFilter
-where
-    TailTokenStream: TokenStream,
-{
-    type ResultTokenStream = AsciiFoldingFilterTokenStream<TailTokenStream>;
-
-    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
-        AsciiFoldingFilterTokenStream::wrap(token_stream)
+impl TokenFilter for AsciiFoldingFilter {
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        From::from(AsciiFoldingFilterTokenStream {
+            tail: token_stream,
+            buffer: String::with_capacity(100),
+        })
     }
 }
 
-pub struct AsciiFoldingFilterTokenStream<TailTokenStream> {
+pub struct AsciiFoldingFilterTokenStream<'a> {
     buffer: String,
-    tail: TailTokenStream,
+    tail: BoxTokenStream<'a>,
 }
 
-impl<TailTokenStream> TokenStream for AsciiFoldingFilterTokenStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
+impl<'a> TokenStream for AsciiFoldingFilterTokenStream<'a> {
     fn advance(&mut self) -> bool {
         if !self.tail.advance() {
             return false;
@@ -48,18 +43,6 @@ where
     }
 }
 
-impl<TailTokenStream> AsciiFoldingFilterTokenStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
-    fn wrap(tail: TailTokenStream) -> AsciiFoldingFilterTokenStream<TailTokenStream> {
-        AsciiFoldingFilterTokenStream {
-            tail,
-            buffer: String::with_capacity(100),
-        }
-    }
-}
-
 // Returns a string that represents the ascii folded version of
 // the character. If the `char` does not require ascii folding
 // (e.g. simple ASCII chars like `A`) or if the `char`
@@ -1561,8 +1544,7 @@ mod tests {
     use crate::tokenizer::AsciiFoldingFilter;
     use crate::tokenizer::RawTokenizer;
     use crate::tokenizer::SimpleTokenizer;
-    use crate::tokenizer::TokenStream;
-    use crate::tokenizer::Tokenizer;
+    use crate::tokenizer::TextAnalyzer;
     use std::iter;
 
     #[test]
@@ -1579,7 +1561,7 @@ mod tests {
 
     fn folding_helper(text: &str) -> Vec<String> {
         let mut tokens = Vec::new();
-        SimpleTokenizer
+        TextAnalyzer::from(SimpleTokenizer)
             .filter(AsciiFoldingFilter)
             .token_stream(text)
             .process(&mut |token| {
@@ -1589,7 +1571,9 @@ mod tests {
     }
 
     fn folding_using_raw_tokenizer_helper(text: &str) -> String {
-        let mut token_stream = RawTokenizer.filter(AsciiFoldingFilter).token_stream(text);
+        let mut token_stream = TextAnalyzer::from(RawTokenizer)
+            .filter(AsciiFoldingFilter)
+            .token_stream(text);
         token_stream.advance();
         token_stream.token().text.clone()
     }

src/tokenizer/facet_tokenizer.rs

@@ -1,4 +1,4 @@
-use super::{Token, TokenStream, Tokenizer};
+use super::{BoxTokenStream, Token, TokenStream, Tokenizer};
 use crate::schema::FACET_SEP_BYTE;
 
 /// The `FacetTokenizer` process a `Facet` binary representation
@@ -25,15 +25,14 @@ pub struct FacetTokenStream<'a> {
     token: Token,
 }
 
-impl<'a> Tokenizer<'a> for FacetTokenizer {
-    type TokenStreamImpl = FacetTokenStream<'a>;
-
-    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
+impl Tokenizer for FacetTokenizer {
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
         FacetTokenStream {
             text,
             state: State::RootFacetNotEmitted, //< pos is the first char that has not been processed yet.
             token: Token::default(),
         }
+        .into()
     }
 }
 
@@ -84,7 +83,7 @@ mod tests {
 
     use super::FacetTokenizer;
     use crate::schema::Facet;
-    use crate::tokenizer::{Token, TokenStream, Tokenizer};
+    use crate::tokenizer::{Token, Tokenizer};
 
     #[test]
     fn test_facet_tokenizer() {

src/tokenizer/lower_caser.rs

@@ -1,24 +1,23 @@
 use super::{Token, TokenFilter, TokenStream};
+use crate::tokenizer::BoxTokenStream;
 use std::mem;
 
+impl TokenFilter for LowerCaser {
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(LowerCaserTokenStream {
+            tail: token_stream,
+            buffer: String::with_capacity(100),
+        })
+    }
+}
+
 /// Token filter that lowercase terms.
 #[derive(Clone)]
 pub struct LowerCaser;
 
-impl<TailTokenStream> TokenFilter<TailTokenStream> for LowerCaser
-where
-    TailTokenStream: TokenStream,
-{
-    type ResultTokenStream = LowerCaserTokenStream<TailTokenStream>;
-
-    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
-        LowerCaserTokenStream::wrap(token_stream)
-    }
-}
-
-pub struct LowerCaserTokenStream<TailTokenStream> {
+pub struct LowerCaserTokenStream<'a> {
     buffer: String,
-    tail: TailTokenStream,
+    tail: BoxTokenStream<'a>,
 }
 
 // writes a lowercased version of text into output.
@@ -31,18 +30,7 @@ fn to_lowercase_unicode(text: &mut String, output: &mut String) {
     }
 }
 
-impl<TailTokenStream> TokenStream for LowerCaserTokenStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
-    }
-
+impl<'a> TokenStream for LowerCaserTokenStream<'a> {
     fn advance(&mut self) -> bool {
         if !self.tail.advance() {
             return false;
@@ -56,26 +44,19 @@ where
         }
         true
     }
-}
 
-impl<TailTokenStream> LowerCaserTokenStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
-    fn wrap(tail: TailTokenStream) -> LowerCaserTokenStream<TailTokenStream> {
-        LowerCaserTokenStream {
-            tail,
-            buffer: String::with_capacity(100),
-        }
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use crate::tokenizer::LowerCaser;
-    use crate::tokenizer::SimpleTokenizer;
-    use crate::tokenizer::TokenStream;
-    use crate::tokenizer::Tokenizer;
+    use crate::tokenizer::{LowerCaser, SimpleTokenizer, TextAnalyzer};
 
     #[test]
     fn test_to_lower_case() {
@@ -87,7 +68,9 @@ mod tests {
 
     fn lowercase_helper(text: &str) -> Vec<String> {
         let mut tokens = vec![];
-        let mut token_stream = SimpleTokenizer.filter(LowerCaser).token_stream(text);
+        let mut token_stream = TextAnalyzer::from(SimpleTokenizer)
+            .filter(LowerCaser)
+            .token_stream(text);
         while token_stream.advance() {
             let token_text = token_stream.token().text.clone();
             tokens.push(token_text);

src/tokenizer/mod.rs

@@ -7,7 +7,6 @@
 //! ```rust
 //! use tantivy::schema::*;
 //!
-//! # fn main() {
 //! let mut schema_builder = Schema::builder();
 //!
 //! let text_options = TextOptions::default()
@@ -31,7 +30,6 @@
 //! schema_builder.add_text_field("uuid", id_options);
 //!
 //! let schema = schema_builder.build();
-//! # }
 //! ```
 //!
 //! By default, `tantivy` offers the following tokenizers:
@@ -66,12 +64,10 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! # fn main() {
-//! let en_stem = SimpleTokenizer
+//! let en_stem = TextAnalyzer::from(SimpleTokenizer)
 //!     .filter(RemoveLongFilter::limit(40))
 //!     .filter(LowerCaser)
 //!     .filter(Stemmer::new(Language::English));
-//! # }
 //! ```
 //!
 //! Once your tokenizer is defined, you need to
@@ -81,13 +77,12 @@
 //! # use tantivy::schema::Schema;
 //! # use tantivy::tokenizer::*;
 //! # use tantivy::Index;
-//! # fn main() {
-//! # let custom_en_tokenizer = SimpleTokenizer;
+//! #
+//! let custom_en_tokenizer = SimpleTokenizer;
 //! # let schema = Schema::builder().build();
 //! let index = Index::create_in_ram(schema);
 //! index.tokenizers()
 //!      .register("custom_en", custom_en_tokenizer);
-//! # }
 //! ```
 //!
 //! If you built your schema programmatically, a complete example
@@ -102,7 +97,6 @@
 //! use tantivy::tokenizer::*;
 //! use tantivy::Index;
 //!
-//! # fn main() {
 //! let mut schema_builder = Schema::builder();
 //! let text_field_indexing = TextFieldIndexing::default()
 //!     .set_tokenizer("custom_en")
@@ -115,14 +109,12 @@
 //! let index = Index::create_in_ram(schema);
 //!
 //! // We need to register our tokenizer :
-//! let custom_en_tokenizer = SimpleTokenizer
+//! let custom_en_tokenizer = TextAnalyzer::from(SimpleTokenizer)
 //!     .filter(RemoveLongFilter::limit(40))
 //!     .filter(LowerCaser);
 //! index
 //!     .tokenizers()
 //!     .register("custom_en", custom_en_tokenizer);
-//! // ...
-//! # }
 //! ```
 //!
 mod alphanum_only;
@@ -151,10 +143,11 @@ pub use self::simple_tokenizer::SimpleTokenizer;
 pub use self::stemmer::{Language, Stemmer};
 pub use self::stop_word_filter::StopWordFilter;
 pub(crate) use self::token_stream_chain::TokenStreamChain;
-pub use self::tokenizer::BoxedTokenizer;
 
 pub use self::tokenized_string::{PreTokenizedStream, PreTokenizedString};
-pub use self::tokenizer::{Token, TokenFilter, TokenStream, Tokenizer};
+pub use self::tokenizer::{
+    BoxTokenFilter, BoxTokenStream, TextAnalyzer, Token, TokenFilter, TokenStream, Tokenizer,
+};
 
 pub use self::tokenizer_manager::TokenizerManager;
 
@@ -168,9 +161,9 @@ pub const MAX_TOKEN_LEN: usize = u16::max_value() as usize - 4;
 #[cfg(test)]
 pub mod tests {
     use super::{
-        Language, LowerCaser, RemoveLongFilter, SimpleTokenizer, Stemmer, Token, Tokenizer,
-        TokenizerManager,
+        Language, LowerCaser, RemoveLongFilter, SimpleTokenizer, Stemmer, Token, TokenizerManager,
     };
+    use crate::tokenizer::TextAnalyzer;
 
     /// This is a function that can be used in tests and doc tests
     /// to assert a token's correctness.
@@ -237,7 +230,7 @@ pub mod tests {
         let tokenizer_manager = TokenizerManager::default();
         tokenizer_manager.register(
             "el_stem",
-            SimpleTokenizer
+            TextAnalyzer::from(SimpleTokenizer)
                 .filter(RemoveLongFilter::limit(40))
                 .filter(LowerCaser)
                 .filter(Stemmer::new(Language::Greek)),

src/tokenizer/ngram_tokenizer.rs

@@ -1,4 +1,5 @@
 use super::{Token, TokenStream, Tokenizer};
+use crate::tokenizer::BoxTokenStream;
 
 /// Tokenize the text by splitting words into n-grams of the given size(s)
 ///
@@ -31,7 +32,7 @@ use super::{Token, TokenStream, Tokenizer};
 ///
 /// ```rust
 /// use tantivy::tokenizer::*;
-/// # fn main() {
+///
 /// let tokenizer = NgramTokenizer::new(2, 3, false);
 /// let mut stream = tokenizer.token_stream("hello");
 /// {
@@ -77,7 +78,6 @@ use super::{Token, TokenStream, Tokenizer};
 ///   assert_eq!(token.offset_to, 5);
 /// }
 /// assert!(stream.next().is_none());
-/// # }
 /// ```
 #[derive(Clone)]
 pub struct NgramTokenizer {
@@ -130,11 +130,9 @@ pub struct NgramTokenStream<'a> {
     token: Token,
 }
 
-impl<'a> Tokenizer<'a> for NgramTokenizer {
-    type TokenStreamImpl = NgramTokenStream<'a>;
-
-    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
-        NgramTokenStream {
+impl Tokenizer for NgramTokenizer {
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+        From::from(NgramTokenStream {
             ngram_charidx_iterator: StutteringIterator::new(
                 CodepointFrontiers::for_str(text),
                 self.min_gram,
@@ -143,7 +141,7 @@ impl<'a> Tokenizer<'a> for NgramTokenizer {
             prefix_only: self.prefix_only,
             text,
             token: Token::default(),
-        }
+        })
     }
 }
 
@@ -309,10 +307,10 @@ mod tests {
     use super::NgramTokenizer;
     use super::StutteringIterator;
     use crate::tokenizer::tests::assert_token;
-    use crate::tokenizer::tokenizer::{TokenStream, Tokenizer};
-    use crate::tokenizer::Token;
+    use crate::tokenizer::tokenizer::Tokenizer;
+    use crate::tokenizer::{BoxTokenStream, Token};
 
-    fn test_helper<T: TokenStream>(mut tokenizer: T) -> Vec<Token> {
+    fn test_helper(mut tokenizer: BoxTokenStream) -> Vec<Token> {
         let mut tokens: Vec<Token> = vec![];
         tokenizer.process(&mut |token: &Token| tokens.push(token.clone()));
         tokens

src/tokenizer/raw_tokenizer.rs

@@ -1,4 +1,5 @@
 use super::{Token, TokenStream, Tokenizer};
+use crate::tokenizer::BoxTokenStream;
 
 /// For each value of the field, emit a single unprocessed token.
 #[derive(Clone)]
@@ -9,10 +10,8 @@ pub struct RawTokenStream {
     has_token: bool,
 }
 
-impl<'a> Tokenizer<'a> for RawTokenizer {
-    type TokenStreamImpl = RawTokenStream;
-
-    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
+impl Tokenizer for RawTokenizer {
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
         let token = Token {
             offset_from: 0,
             offset_to: text.len(),
@@ -24,6 +23,7 @@ impl<'a> Tokenizer<'a> for RawTokenizer {
             token,
             has_token: true,
         }
+        .into()
     }
 }
 

src/tokenizer/remove_long.rs

@@ -2,9 +2,7 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! # fn main() {
-//!
-//! let tokenizer = SimpleTokenizer
+//! let tokenizer = TextAnalyzer::from(SimpleTokenizer)
 //!   .filter(RemoveLongFilter::limit(5));
 //!
 //! let mut stream = tokenizer.token_stream("toolong nice");
@@ -12,10 +10,10 @@
 //! // out of the token stream.
 //! assert_eq!(stream.next().unwrap().text, "nice");
 //! assert!(stream.next().is_none());
-//! # }
 //! ```
 //!
 use super::{Token, TokenFilter, TokenStream};
+use crate::tokenizer::BoxTokenStream;
 
 /// `RemoveLongFilter` removes tokens that are longer
 /// than a given number of bytes (in UTF-8 representation).
@@ -34,56 +32,27 @@ impl RemoveLongFilter {
     }
 }
 
-impl<TailTokenStream> RemoveLongFilterStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
+impl<'a> RemoveLongFilterStream<'a> {
     fn predicate(&self, token: &Token) -> bool {
         token.text.len() < self.token_length_limit
     }
+}
 
-    fn wrap(
-        token_length_limit: usize,
-        tail: TailTokenStream,
-    ) -> RemoveLongFilterStream<TailTokenStream> {
-        RemoveLongFilterStream {
-            token_length_limit,
-            tail,
-        }
+impl TokenFilter for RemoveLongFilter {
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(RemoveLongFilterStream {
+            token_length_limit: self.length_limit,
+            tail: token_stream,
+        })
     }
 }
 
-impl<TailTokenStream> TokenFilter<TailTokenStream> for RemoveLongFilter
-where
-    TailTokenStream: TokenStream,
-{
-    type ResultTokenStream = RemoveLongFilterStream<TailTokenStream>;
-
-    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
-        RemoveLongFilterStream::wrap(self.length_limit, token_stream)
-    }
-}
-
-pub struct RemoveLongFilterStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
+pub struct RemoveLongFilterStream<'a> {
     token_length_limit: usize,
-    tail: TailTokenStream,
+    tail: BoxTokenStream<'a>,
 }
 
-impl<TailTokenStream> TokenStream for RemoveLongFilterStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
-    }
-
+impl<'a> TokenStream for RemoveLongFilterStream<'a> {
     fn advance(&mut self) -> bool {
         while self.tail.advance() {
             if self.predicate(self.tail.token()) {
@@ -92,4 +61,12 @@ where
         }
         false
     }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
 }

src/tokenizer/simple_tokenizer.rs

@@ -1,3 +1,4 @@
+use super::BoxTokenStream;
 use super::{Token, TokenStream, Tokenizer};
 use std::str::CharIndices;
 
@@ -11,15 +12,13 @@ pub struct SimpleTokenStream<'a> {
     token: Token,
 }
 
-impl<'a> Tokenizer<'a> for SimpleTokenizer {
-    type TokenStreamImpl = SimpleTokenStream<'a>;
-
-    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
-        SimpleTokenStream {
+impl Tokenizer for SimpleTokenizer {
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(SimpleTokenStream {
             text,
             chars: text.char_indices(),
             token: Token::default(),
-        }
+        })
     }
 }
 

src/tokenizer/stemmer.rs

@@ -1,4 +1,5 @@
 use super::{Token, TokenFilter, TokenStream};
+use crate::tokenizer::BoxTokenStream;
 use rust_stemmers::{self, Algorithm};
 
 /// Available stemmer languages.
@@ -15,6 +16,7 @@ pub enum Language {
     Greek,
     Hungarian,
     Italian,
+    Norwegian,
     Portuguese,
     Romanian,
     Russian,
@@ -38,6 +40,7 @@ impl Language {
             Greek => Algorithm::Greek,
             Hungarian => Algorithm::Hungarian,
             Italian => Algorithm::Italian,
+            Norwegian => Algorithm::Norwegian,
             Portuguese => Algorithm::Portuguese,
             Romanian => Algorithm::Romanian,
             Russian => Algorithm::Russian,
@@ -73,38 +76,22 @@ impl Default for Stemmer {
     }
 }
 
-impl<TailTokenStream> TokenFilter<TailTokenStream> for Stemmer
-where
-    TailTokenStream: TokenStream,
-{
-    type ResultTokenStream = StemmerTokenStream<TailTokenStream>;
-
-    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+impl TokenFilter for Stemmer {
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
         let inner_stemmer = rust_stemmers::Stemmer::create(self.stemmer_algorithm);
-        StemmerTokenStream::wrap(inner_stemmer, token_stream)
+        BoxTokenStream::from(StemmerTokenStream {
+            tail: token_stream,
+            stemmer: inner_stemmer,
+        })
     }
 }
 
-pub struct StemmerTokenStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
-    tail: TailTokenStream,
+pub struct StemmerTokenStream<'a> {
+    tail: BoxTokenStream<'a>,
     stemmer: rust_stemmers::Stemmer,
 }
 
-impl<TailTokenStream> TokenStream for StemmerTokenStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
-    }
-
+impl<'a> TokenStream for StemmerTokenStream<'a> {
     fn advance(&mut self) -> bool {
         if !self.tail.advance() {
             return false;
@@ -115,16 +102,12 @@ where
         self.token_mut().text.push_str(&stemmed_str);
         true
     }
-}
 
-impl<TailTokenStream> StemmerTokenStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
-    fn wrap(
-        stemmer: rust_stemmers::Stemmer,
-        tail: TailTokenStream,
-    ) -> StemmerTokenStream<TailTokenStream> {
-        StemmerTokenStream { tail, stemmer }
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
     }
 }

src/tokenizer/stop_word_filter.rs

@@ -2,17 +2,16 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! # fn main() {
-//! let tokenizer = SimpleTokenizer
+//! let tokenizer = TextAnalyzer::from(SimpleTokenizer)
 //!   .filter(StopWordFilter::remove(vec!["the".to_string(), "is".to_string()]));
 //!
 //! let mut stream = tokenizer.token_stream("the fox is crafty");
 //! assert_eq!(stream.next().unwrap().text, "fox");
 //! assert_eq!(stream.next().unwrap().text, "crafty");
 //! assert!(stream.next().is_none());
-//! # }
 //! ```
 use super::{Token, TokenFilter, TokenStream};
+use crate::tokenizer::BoxTokenStream;
 use fnv::FnvHasher;
 use std::collections::HashSet;
 use std::hash::BuildHasherDefault;
@@ -46,57 +45,31 @@ impl StopWordFilter {
             "there", "these", "they", "this", "to", "was", "will", "with",
         ];
 
-        StopWordFilter::remove(words.iter().map(|s| s.to_string()).collect())
+        StopWordFilter::remove(words.iter().map(|&s| s.to_string()).collect())
     }
 }
 
-pub struct StopWordFilterStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
+pub struct StopWordFilterStream<'a> {
     words: StopWordHashSet,
-    tail: TailTokenStream,
+    tail: BoxTokenStream<'a>,
 }
 
-impl<TailTokenStream> TokenFilter<TailTokenStream> for StopWordFilter
-where
-    TailTokenStream: TokenStream,
-{
-    type ResultTokenStream = StopWordFilterStream<TailTokenStream>;
-
-    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
-        StopWordFilterStream::wrap(self.words.clone(), token_stream)
+impl TokenFilter for StopWordFilter {
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(StopWordFilterStream {
+            words: self.words.clone(),
+            tail: token_stream,
+        })
     }
 }
 
-impl<TailTokenStream> StopWordFilterStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
+impl<'a> StopWordFilterStream<'a> {
     fn predicate(&self, token: &Token) -> bool {
         !self.words.contains(&token.text)
     }
-
-    fn wrap(
-        words: StopWordHashSet,
-        tail: TailTokenStream,
-    ) -> StopWordFilterStream<TailTokenStream> {
-        StopWordFilterStream { words, tail }
-    }
 }
 
-impl<TailTokenStream> TokenStream for StopWordFilterStream<TailTokenStream>
-where
-    TailTokenStream: TokenStream,
-{
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
-    }
-
+impl<'a> TokenStream for StopWordFilterStream<'a> {
     fn advance(&mut self) -> bool {
         while self.tail.advance() {
             if self.predicate(self.tail.token()) {
@@ -105,6 +78,14 @@ where
         }
         false
     }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
 }
 
 impl Default for StopWordFilter {

src/tokenizer/token_stream_chain.rs

@@ -1,23 +1,21 @@
-use crate::tokenizer::{Token, TokenStream};
+use crate::tokenizer::{BoxTokenStream, Token, TokenStream};
+use std::ops::DerefMut;
 
 const POSITION_GAP: usize = 2;
 
-pub(crate) struct TokenStreamChain<TTokenStream: TokenStream> {
+pub(crate) struct TokenStreamChain<'a> {
     offsets: Vec<usize>,
-    token_streams: Vec<TTokenStream>,
+    token_streams: Vec<BoxTokenStream<'a>>,
     position_shift: usize,
     stream_idx: usize,
     token: Token,
 }
 
-impl<'a, TTokenStream> TokenStreamChain<TTokenStream>
-where
-    TTokenStream: TokenStream,
-{
+impl<'a> TokenStreamChain<'a> {
     pub fn new(
         offsets: Vec<usize>,
-        token_streams: Vec<TTokenStream>,
-    ) -> TokenStreamChain<TTokenStream> {
+        token_streams: Vec<BoxTokenStream<'a>>,
+    ) -> TokenStreamChain<'a> {
         TokenStreamChain {
             offsets,
             stream_idx: 0,
@@ -28,13 +26,10 @@ where
     }
 }
 
-impl<'a, TTokenStream> TokenStream for TokenStreamChain<TTokenStream>
-where
-    TTokenStream: TokenStream,
-{
+impl<'a> TokenStream for TokenStreamChain<'a> {
     fn advance(&mut self) -> bool {
         while self.stream_idx < self.token_streams.len() {
-            let token_stream = &mut self.token_streams[self.stream_idx];
+            let token_stream = self.token_streams[self.stream_idx].deref_mut();
             if token_stream.advance() {
                 let token = token_stream.token();
                 let offset_offset = self.offsets[self.stream_idx];

src/tokenizer/tokenized_string.rs

@@ -1,4 +1,4 @@
-use crate::tokenizer::{Token, TokenStream, TokenStreamChain};
+use crate::tokenizer::{BoxTokenStream, Token, TokenStream, TokenStreamChain};
 use std::cmp::Ordering;
 
 /// Struct representing pre-tokenized text
@@ -41,9 +41,9 @@ impl PreTokenizedStream {
     /// Creates a TokenStream from PreTokenizedString array
     pub fn chain_tokenized_strings<'a>(
         tok_strings: &'a [&'a PreTokenizedString],
-    ) -> Box<dyn TokenStream + 'a> {
+    ) -> BoxTokenStream {
         if tok_strings.len() == 1 {
-            Box::new(PreTokenizedStream::from((*tok_strings[0]).clone()))
+            PreTokenizedStream::from((*tok_strings[0]).clone()).into()
         } else {
             let mut offsets = vec![];
             let mut total_offset = 0;
@@ -53,11 +53,12 @@ impl PreTokenizedStream {
                     total_offset += last_token.offset_to;
                 }
             }
-            let token_streams: Vec<_> = tok_strings
+            // TODO remove the string cloning.
+            let token_streams: Vec<BoxTokenStream<'static>> = tok_strings
                 .iter()
-                .map(|tok_string| PreTokenizedStream::from((*tok_string).clone()))
+                .map(|&tok_string| PreTokenizedStream::from((*tok_string).clone()).into())
                 .collect();
-            Box::new(TokenStreamChain::new(offsets, token_streams))
+            TokenStreamChain::new(offsets, token_streams).into()
         }
     }
 }

src/tokenizer/tokenizer.rs

@@ -2,6 +2,7 @@ use crate::tokenizer::TokenStreamChain;
 /// The tokenizer module contains all of the tools used to process
 /// text in `tantivy`.
 use std::borrow::{Borrow, BorrowMut};
+use std::ops::{Deref, DerefMut};
 
 /// Token
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
@@ -33,20 +34,31 @@ impl Default for Token {
     }
 }
 
-/// `Tokenizer` are in charge of splitting text into a stream of token
-/// before indexing.
+/// `TextAnalyzer` tokenizes an input text into tokens and modifies the resulting `TokenStream`.
 ///
-/// See the [module documentation](./index.html) for more detail.
-///
-/// # Warning
-///
-/// This API may change to use associated types.
-pub trait Tokenizer<'a>: Sized + Clone {
-    /// Type associated to the resulting tokenstream tokenstream.
-    type TokenStreamImpl: TokenStream;
+/// It simply wraps a `Tokenizer` and a list of `TokenFilter` that are applied sequentially.
+pub struct TextAnalyzer {
+    tokenizer: Box<dyn Tokenizer>,
+    token_filters: Vec<BoxTokenFilter>,
+}
 
-    /// Creates a token stream for a given `str`.
-    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl;
+impl<T: Tokenizer> From<T> for TextAnalyzer {
+    fn from(tokenizer: T) -> Self {
+        TextAnalyzer::new(tokenizer, Vec::new())
+    }
+}
+
+impl TextAnalyzer {
+    /// Creates a new `TextAnalyzer` given a tokenizer and a vector of `BoxTokenFilter`.
+    ///
+    /// When creating a `TextAnalyzer` from a `Tokenizer` alone, prefer using
+    /// `TextAnalyzer::from(tokenizer)`.
+    pub fn new<T: Tokenizer>(tokenizer: T, token_filters: Vec<BoxTokenFilter>) -> TextAnalyzer {
+        TextAnalyzer {
+            tokenizer: Box::new(tokenizer),
+            token_filters,
+        }
+    }
 
     /// Appends a token filter to the current tokenizer.
     ///
@@ -58,92 +70,26 @@ pub trait Tokenizer<'a>: Sized + Clone {
     /// ```rust
     /// use tantivy::tokenizer::*;
     ///
-    /// # fn main() {
-    /// let en_stem = SimpleTokenizer
+    /// let en_stem = TextAnalyzer::from(SimpleTokenizer)
     ///     .filter(RemoveLongFilter::limit(40))
     ///     .filter(LowerCaser)
     ///     .filter(Stemmer::default());
-    /// # }
     /// ```
     ///
-    fn filter<NewFilter>(self, new_filter: NewFilter) -> ChainTokenizer<NewFilter, Self>
-    where
-        NewFilter: TokenFilter<<Self as Tokenizer<'a>>::TokenStreamImpl>,
-    {
-        ChainTokenizer {
-            head: new_filter,
-            tail: self,
-        }
-    }
-}
-
-/// A boxed tokenizer
-trait BoxedTokenizerTrait: Send + Sync {
-    /// Tokenize a `&str`
-    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a>;
-
-    /// Tokenize an array`&str`
-    ///
-    /// The resulting `TokenStream` is equivalent to what would be obtained if the &str were
-    /// one concatenated `&str`, with an artificial position gap of `2` between the different fields
-    /// to prevent accidental `PhraseQuery` to match accross two terms.
-    fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<dyn TokenStream + 'b>;
-
-    /// Return a boxed clone of the tokenizer
-    fn boxed_clone(&self) -> BoxedTokenizer;
-}
-
-/// A boxed tokenizer
-pub struct BoxedTokenizer(Box<dyn BoxedTokenizerTrait>);
-
-impl<T> From<T> for BoxedTokenizer
-where
-    T: 'static + Send + Sync + for<'a> Tokenizer<'a>,
-{
-    fn from(tokenizer: T) -> BoxedTokenizer {
-        BoxedTokenizer(Box::new(BoxableTokenizer(tokenizer)))
-    }
-}
-
-impl BoxedTokenizer {
-    /// Tokenize a `&str`
-    pub fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
-        self.0.token_stream(text)
+    pub fn filter<F: Into<BoxTokenFilter>>(mut self, token_filter: F) -> Self {
+        self.token_filters.push(token_filter.into());
+        self
     }
 
     /// Tokenize an array`&str`
     ///
-    /// The resulting `TokenStream` is equivalent to what would be obtained if the &str were
+    /// The resulting `BoxTokenStream` is equivalent to what would be obtained if the &str were
     /// one concatenated `&str`, with an artificial position gap of `2` between the different fields
     /// to prevent accidental `PhraseQuery` to match accross two terms.
-    pub fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<dyn TokenStream + 'b> {
-        self.0.token_stream_texts(texts)
-    }
-}
-
-impl Clone for BoxedTokenizer {
-    fn clone(&self) -> BoxedTokenizer {
-        self.0.boxed_clone()
-    }
-}
-
-#[derive(Clone)]
-struct BoxableTokenizer<A>(A)
-where
-    A: for<'a> Tokenizer<'a> + Send + Sync;
-
-impl<A> BoxedTokenizerTrait for BoxableTokenizer<A>
-where
-    A: 'static + Send + Sync + for<'a> Tokenizer<'a>,
-{
-    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
-        Box::new(self.0.token_stream(text))
-    }
-
-    fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<dyn TokenStream + 'b> {
+    pub fn token_stream_texts<'a>(&self, texts: &'a [&'a str]) -> BoxTokenStream<'a> {
         assert!(!texts.is_empty());
         if texts.len() == 1 {
-            Box::new(self.0.token_stream(texts[0]))
+            self.token_stream(texts[0])
         } else {
             let mut offsets = vec![];
             let mut total_offset = 0;
@@ -151,34 +97,124 @@ where
                 offsets.push(total_offset);
                 total_offset += text.len();
             }
-            let token_streams: Vec<_> =
-                texts.iter().map(|text| self.0.token_stream(text)).collect();
-            Box::new(TokenStreamChain::new(offsets, token_streams))
+            let token_streams: Vec<BoxTokenStream<'a>> = texts
+                .iter()
+                .cloned()
+                .map(|text| self.token_stream(text))
+                .collect();
+            From::from(TokenStreamChain::new(offsets, token_streams))
         }
     }
 
-    fn boxed_clone(&self) -> BoxedTokenizer {
-        self.0.clone().into()
+    /// Creates a token stream for a given `str`.
+    pub fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+        let mut token_stream = self.tokenizer.token_stream(text);
+        for token_filter in &self.token_filters {
+            token_stream = token_filter.transform(token_stream);
+        }
+        token_stream
     }
 }
 
-impl<'b> TokenStream for Box<dyn TokenStream + 'b> {
+impl Clone for TextAnalyzer {
+    fn clone(&self) -> Self {
+        TextAnalyzer {
+            tokenizer: self.tokenizer.box_clone(),
+            token_filters: self
+                .token_filters
+                .iter()
+                .map(|token_filter| token_filter.box_clone())
+                .collect(),
+        }
+    }
+}
+
+/// `Tokenizer` are in charge of splitting text into a stream of token
+/// before indexing.
+///
+/// See the [module documentation](./index.html) for more detail.
+///
+/// # Warning
+///
+/// This API may change to use associated types.
+pub trait Tokenizer: 'static + Send + Sync + TokenizerClone {
+    /// Creates a token stream for a given `str`.
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a>;
+}
+
+pub trait TokenizerClone {
+    fn box_clone(&self) -> Box<dyn Tokenizer>;
+}
+
+impl<T: Tokenizer + Clone> TokenizerClone for T {
+    fn box_clone(&self) -> Box<dyn Tokenizer> {
+        Box::new(self.clone())
+    }
+}
+
+impl<'a> TokenStream for Box<dyn TokenStream + 'a> {
     fn advance(&mut self) -> bool {
         let token_stream: &mut dyn TokenStream = self.borrow_mut();
         token_stream.advance()
     }
 
-    fn token(&self) -> &Token {
-        let token_stream: &dyn TokenStream = self.borrow();
+    fn token<'b>(&'b self) -> &'b Token {
+        let token_stream: &'b (dyn TokenStream + 'a) = self.borrow();
         token_stream.token()
     }
 
-    fn token_mut(&mut self) -> &mut Token {
-        let token_stream: &mut dyn TokenStream = self.borrow_mut();
+    fn token_mut<'b>(&'b mut self) -> &'b mut Token {
+        let token_stream: &'b mut (dyn TokenStream + 'a) = self.borrow_mut();
         token_stream.token_mut()
     }
 }
 
+/// Simple wrapper of `Box<dyn TokenStream + 'a>`.
+///
+/// See `TokenStream` for more information.
+pub struct BoxTokenStream<'a>(Box<dyn TokenStream + 'a>);
+
+impl<'a, T> From<T> for BoxTokenStream<'a>
+where
+    T: TokenStream + 'a,
+{
+    fn from(token_stream: T) -> BoxTokenStream<'a> {
+        BoxTokenStream(Box::new(token_stream))
+    }
+}
+
+impl<'a> Deref for BoxTokenStream<'a> {
+    type Target = dyn TokenStream + 'a;
+
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}
+impl<'a> DerefMut for BoxTokenStream<'a> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut *self.0
+    }
+}
+
+/// Simple wrapper of `Box<dyn TokenFilter + 'a>`.
+///
+/// See `TokenStream` for more information.
+pub struct BoxTokenFilter(Box<dyn TokenFilter>);
+
+impl Deref for BoxTokenFilter {
+    type Target = dyn TokenFilter;
+
+    fn deref(&self) -> &dyn TokenFilter {
+        &*self.0
+    }
+}
+
+impl<T: TokenFilter> From<T> for BoxTokenFilter {
+    fn from(tokenizer: T) -> BoxTokenFilter {
+        BoxTokenFilter(Box::new(tokenizer))
+    }
+}
+
 /// `TokenStream` is the result of the tokenization.
 ///
 /// It consists consumable stream of `Token`s.
@@ -188,8 +224,7 @@ impl<'b> TokenStream for Box<dyn TokenStream + 'b> {
 /// ```
 /// use tantivy::tokenizer::*;
 ///
-/// # fn main() {
-/// let tokenizer = SimpleTokenizer
+/// let tokenizer = TextAnalyzer::from(SimpleTokenizer)
 ///        .filter(RemoveLongFilter::limit(40))
 ///        .filter(LowerCaser);
 /// let mut token_stream = tokenizer.token_stream("Hello, happy tax payer");
@@ -207,7 +242,6 @@ impl<'b> TokenStream for Box<dyn TokenStream + 'b> {
 ///     assert_eq!(token.offset_to, 12);
 ///     assert_eq!(token.position, 1);
 /// }
-/// # }
 /// ```
 ///
 pub trait TokenStream {
@@ -227,17 +261,15 @@ pub trait TokenStream {
     /// and `.token()`.
     ///
     /// ```
-    /// # use tantivy::tokenizer::*;
-    /// #
-    /// # fn main() {
-    /// # let tokenizer = SimpleTokenizer
-    /// #       .filter(RemoveLongFilter::limit(40))
-    /// #       .filter(LowerCaser);
+    /// use tantivy::tokenizer::*;
+    ///
+    /// let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+    ///       .filter(RemoveLongFilter::limit(40))
+    ///       .filter(LowerCaser);
     /// let mut token_stream = tokenizer.token_stream("Hello, happy tax payer");
     /// while let Some(token) = token_stream.next() {
     ///     println!("Token {:?}", token.text);
     /// }
-    /// # }
     /// ```
     fn next(&mut self) -> Option<&Token> {
         if self.advance() {
@@ -249,6 +281,8 @@ pub trait TokenStream {
 
     /// Helper function to consume the entire `TokenStream`
     /// and push the tokens to a sink function.
+    ///
+    /// Remove this.
     fn process(&mut self, sink: &mut dyn FnMut(&Token)) -> u32 {
         let mut num_tokens_pushed = 0u32;
         while self.advance() {
@@ -259,33 +293,20 @@ pub trait TokenStream {
     }
 }
 
-#[derive(Clone)]
-pub struct ChainTokenizer<HeadTokenFilterFactory, TailTokenizer> {
-    head: HeadTokenFilterFactory,
-    tail: TailTokenizer,
-}
-
-impl<'a, HeadTokenFilterFactory, TailTokenizer> Tokenizer<'a>
-    for ChainTokenizer<HeadTokenFilterFactory, TailTokenizer>
-where
-    HeadTokenFilterFactory: TokenFilter<TailTokenizer::TokenStreamImpl>,
-    TailTokenizer: Tokenizer<'a>,
-{
-    type TokenStreamImpl = HeadTokenFilterFactory::ResultTokenStream;
-
-    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
-        let tail_token_stream = self.tail.token_stream(text);
-        self.head.transform(tail_token_stream)
-    }
+pub trait TokenFilterClone {
+    fn box_clone(&self) -> BoxTokenFilter;
 }
 
 /// Trait for the pluggable components of `Tokenizer`s.
-pub trait TokenFilter<TailTokenStream: TokenStream>: Clone {
-    /// The resulting `TokenStream` type.
-    type ResultTokenStream: TokenStream;
-
+pub trait TokenFilter: 'static + Send + Sync + TokenFilterClone {
     /// Wraps a token stream and returns the modified one.
-    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream;
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a>;
+}
+
+impl<T: TokenFilter + Clone> TokenFilterClone for T {
+    fn box_clone(&self) -> BoxTokenFilter {
+        BoxTokenFilter::from(self.clone())
+    }
 }
 
 #[cfg(test)]

src/tokenizer/tokenizer_manager.rs

@@ -1,11 +1,10 @@
 use crate::tokenizer::stemmer::Language;
-use crate::tokenizer::BoxedTokenizer;
+use crate::tokenizer::tokenizer::TextAnalyzer;
 use crate::tokenizer::LowerCaser;
 use crate::tokenizer::RawTokenizer;
 use crate::tokenizer::RemoveLongFilter;
 use crate::tokenizer::SimpleTokenizer;
 use crate::tokenizer::Stemmer;
-use crate::tokenizer::Tokenizer;
 use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
 
@@ -23,16 +22,16 @@ use std::sync::{Arc, RwLock};
 ///  search engine.
 #[derive(Clone)]
 pub struct TokenizerManager {
-    tokenizers: Arc<RwLock<HashMap<String, BoxedTokenizer>>>,
+    tokenizers: Arc<RwLock<HashMap<String, TextAnalyzer>>>,
 }
 
 impl TokenizerManager {
     /// Registers a new tokenizer associated with a given name.
-    pub fn register<A>(&self, tokenizer_name: &str, tokenizer: A)
+    pub fn register<T>(&self, tokenizer_name: &str, tokenizer: T)
     where
-        A: Into<BoxedTokenizer>,
+        TextAnalyzer: From<T>,
     {
-        let boxed_tokenizer = tokenizer.into();
+        let boxed_tokenizer: TextAnalyzer = TextAnalyzer::from(tokenizer);
         self.tokenizers
             .write()
             .expect("Acquiring the lock should never fail")
@@ -40,7 +39,7 @@ impl TokenizerManager {
     }
 
     /// Accessing a tokenizer given its name.
-    pub fn get(&self, tokenizer_name: &str) -> Option<BoxedTokenizer> {
+    pub fn get(&self, tokenizer_name: &str) -> Option<TextAnalyzer> {
         self.tokenizers
             .read()
             .expect("Acquiring the lock should never fail")
@@ -62,13 +61,13 @@ impl Default for TokenizerManager {
         manager.register("raw", RawTokenizer);
         manager.register(
             "default",
-            SimpleTokenizer
+            TextAnalyzer::from(SimpleTokenizer)
                 .filter(RemoveLongFilter::limit(40))
                 .filter(LowerCaser),
         );
         manager.register(
             "en_stem",
-            SimpleTokenizer
+            TextAnalyzer::from(SimpleTokenizer)
                 .filter(RemoveLongFilter::limit(40))
                 .filter(LowerCaser)
                 .filter(Stemmer::new(Language::English)),