Updated changelog

Leave a defensive versioned footer check to make sure we add new logic handling
when we add possible footer versions

Restricted VersionedFooter::from_bytes to be used inside the crate only

remove a half-baked test

CHANGELOG.md

@@ -1,30 +1,3 @@
-Tantivy 0.12.0
-======================
-- Removing static dispatch in tokenizers for simplicity. (#762)
-- Added backward iteration for `TermDictionary` stream. (@halvorboe)
-- Fixed a performance issue when searching for the posting lists of a missing term (@audunhalland)
-- Added a configurable maximum number of docs (10M by default) for a segment to be considered for merge (@hntd187, landed by @halvorboe #713) 
-
-## How to update?
-
-Crates relying on custom tokenizer, or registering tokenizer in the manager will require some 
-minor changes. Check https://github.com/tantivy-search/tantivy/blob/master/examples/custom_tokenizer.rs
-to check for some code sample.
-
-Tantivy 0.11.3
-=======================
-- Fixed DateTime as a fast field (#735)
-
-Tantivy 0.11.2
-=======================
-- The future returned by `IndexWriter::merge` does not borrow `self` mutably anymore (#732)
-- Exposing a constructor for `WatchHandle` (#731)
-
-Tantivy 0.11.1
-=====================
-- Bug fix #729
-
-
 Tantivy 0.11.0
 =====================
 
@@ -36,7 +9,6 @@ Tantivy 0.11.0
 - API change around `Box<BoxableTokenizer>`. See detail in #629
 - Avoid rebuilding Regex automaton whenever a regex query is reused. #639 (@brainlock)
 - Add footer with some metadata to index files. #605 (@fdb-hiroshima)
-- Add a method to check the compatibility of the footer in the index with the running version of tantivy (@petr-tik)
 - TopDocs collector: ensure stable sorting on equal score. #671 (@brainlock)
 - Added handling of pre-tokenized text fields (#642), which will enable users to
   load tokens created outside tantivy. See usage in examples/pre_tokenized_text. (@kkoziara)

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.12.0"
+version = "0.11.0"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -18,7 +18,7 @@ byteorder = "1.0"
 crc32fast = "1.2.0"
 once_cell = "1.0"
 regex ={version = "1.3.0", default-features = false, features = ["std"]}
-tantivy-fst = "0.2.1"
+tantivy-fst = "0.1"
 memmap = {version = "0.7", optional=true}
 lz4 = {version="1.20", optional=true}
 snap = {version="0.2"}
@@ -33,6 +33,7 @@ fs2={version="0.4", optional=true}
 itertools = "0.8"
 levenshtein_automata = {version="0.1", features=["fst_automaton"]}
 notify = {version="4", optional=true}
+bit-set = "0.5"
 uuid = { version = "0.8", features = ["v4", "serde"] }
 crossbeam = "0.7"
 futures = {version = "0.3",  features=["thread-pool"] }
@@ -40,7 +41,7 @@ owning_ref = "0.4"
 stable_deref_trait = "1.0.0"
 rust-stemmers = "1.2"
 downcast-rs = { version="1.0" }
-tantivy-query-grammar = { version="0.11", path="./query-grammar" }
+tantivy-query-grammar = { path="./query-grammar" }
 bitpacking = {version="0.8", default-features = false, features=["bitpacker4x"]}
 census = "0.4"
 fnv = "1.0.6"
@@ -60,6 +61,7 @@ winapi = "0.3"
 rand = "0.7"
 maplit = "1"
 matches = "0.1.8"
+time = "0.1.42"
 
 [dev-dependencies.fail]
 version = "0.3"

examples/faceted_search.rs

@@ -13,100 +13,63 @@
 // ---
 // Importing tantivy...
 use tantivy::collector::FacetCollector;
-use tantivy::query::{AllQuery, TermQuery};
+use tantivy::query::AllQuery;
 use tantivy::schema::*;
 use tantivy::{doc, Index};
+use tempfile::TempDir;
 
 fn main() -> tantivy::Result<()> {
-    // Let's create a temporary directory for the sake of this example
+    // Let's create a temporary directory for the
+    // sake of this example
+    let index_path = TempDir::new()?;
     let mut schema_builder = Schema::builder();
 
-    let name = schema_builder.add_text_field("felin_name", TEXT | STORED);
-    // this is our faceted field: its scientific classification
-    let classification = schema_builder.add_facet_field("classification");
+    schema_builder.add_text_field("name", TEXT | STORED);
+
+    // this is our faceted field
+    schema_builder.add_facet_field("tags");
 
     let schema = schema_builder.build();
-    let index = Index::create_in_ram(schema);
 
-    let mut index_writer = index.writer(30_000_000)?;
+    let index = Index::create_in_dir(&index_path, schema.clone())?;
+
+    let mut index_writer = index.writer(50_000_000)?;
+
+    let name = schema.get_field("name").unwrap();
+    let tags = schema.get_field("tags").unwrap();
 
     // For convenience, tantivy also comes with a macro to
     // reduce the boilerplate above.
     index_writer.add_document(doc!(
-        name => "Cat",
-        classification => Facet::from("/Felidae/Felinae/Felis")
+        name => "the ditch",
+        tags => Facet::from("/pools/north")
     ));
+
     index_writer.add_document(doc!(
-        name => "Canada lynx",
-        classification => Facet::from("/Felidae/Felinae/Lynx")
-    ));
-    index_writer.add_document(doc!(
-        name => "Cheetah",
-        classification => Facet::from("/Felidae/Felinae/Acinonyx")
-    ));
-    index_writer.add_document(doc!(
-        name => "Tiger",
-        classification => Facet::from("/Felidae/Pantherinae/Panthera")
-    ));
-    index_writer.add_document(doc!(
-        name => "Lion",
-        classification => Facet::from("/Felidae/Pantherinae/Panthera")
-    ));
-    index_writer.add_document(doc!(
-        name => "Jaguar",
-        classification => Facet::from("/Felidae/Pantherinae/Panthera")
-    ));
-    index_writer.add_document(doc!(
-        name => "Sunda clouded leopard",
-        classification => Facet::from("/Felidae/Pantherinae/Neofelis")
-    ));
-    index_writer.add_document(doc!(
-        name => "Fossa",
-        classification => Facet::from("/Eupleridae/Cryptoprocta")
+        name => "little stacey",
+        tags => Facet::from("/pools/south")
     ));
+
     index_writer.commit()?;
 
     let reader = index.reader()?;
+
     let searcher = reader.searcher();
-    {
-        let mut facet_collector = FacetCollector::for_field(classification);
-        facet_collector.add_facet("/Felidae");
-        let facet_counts = searcher.search(&AllQuery, &facet_collector)?;
-        // This lists all of the facet counts, right below "/Felidae".
-        let facets: Vec<(&Facet, u64)> = facet_counts.get("/Felidae").collect();
-        assert_eq!(
-            facets,
-            vec![
-                (&Facet::from("/Felidae/Felinae"), 3),
-                (&Facet::from("/Felidae/Pantherinae"), 4),
-            ]
-        );
-    }
 
-    // Facets are also searchable.
-    //
-    // For instance a common UI pattern is to allow the user someone to click on a facet link
-    // (e.g: `Pantherinae`) to drill down and filter the current result set with this subfacet.
-    //
-    // The search would then look as follows.
+    let mut facet_collector = FacetCollector::for_field(tags);
+    facet_collector.add_facet("/pools");
 
-    // Check the reference doc for different ways to create a `Facet` object.
-    {
-        let facet = Facet::from_text("/Felidae/Pantherinae");
-        let facet_term = Term::from_facet(classification, &facet);
-        let facet_term_query = TermQuery::new(facet_term, IndexRecordOption::Basic);
-        let mut facet_collector = FacetCollector::for_field(classification);
-        facet_collector.add_facet("/Felidae/Pantherinae");
-        let facet_counts = searcher.search(&facet_term_query, &facet_collector)?;
-        let facets: Vec<(&Facet, u64)> = facet_counts.get("/Felidae/Pantherinae").collect();
-        assert_eq!(
-            facets,
-            vec![
-                (&Facet::from("/Felidae/Pantherinae/Neofelis"), 1),
-                (&Facet::from("/Felidae/Pantherinae/Panthera"), 3),
-            ]
-        );
-    }
+    let facet_counts = searcher.search(&AllQuery, &facet_collector).unwrap();
+
+    // This lists all of the facet counts
+    let facets: Vec<(&Facet, u64)> = facet_counts.get("/pools").collect();
+    assert_eq!(
+        facets,
+        vec![
+            (&Facet::from("/pools/north"), 1),
+            (&Facet::from("/pools/south"), 1),
+        ]
+    );
 
     Ok(())
 }

examples/pre_tokenized_text.rs

@@ -9,7 +9,7 @@
 // - import tokenized text straight from json,
 // - perform a search on documents with pre-tokenized text
 
-use tantivy::tokenizer::{PreTokenizedString, SimpleTokenizer, Token, Tokenizer};
+use tantivy::tokenizer::{PreTokenizedString, SimpleTokenizer, Token, TokenStream, Tokenizer};
 
 use tantivy::collector::{Count, TopDocs};
 use tantivy::query::TermQuery;

examples/stop_words.rs

@@ -50,7 +50,7 @@ fn main() -> tantivy::Result<()> {
 
     // This tokenizer lowers all of the text (to help with stop word matching)
     // then removes all instances of `the` and `and` from the corpus
-    let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+    let tokenizer = SimpleTokenizer
         .filter(LowerCaser)
         .filter(StopWordFilter::remove(vec![
             "the".to_string(),

query-grammar/README.md

@@ -1,3 +0,0 @@
-# Tantivy Query Grammar
-
-This crate is used by tantivy to parse queries.

src/collector/facet_collector.rs

@@ -452,11 +452,9 @@ impl FacetCounts {
 #[cfg(test)]
 mod tests {
     use super::{FacetCollector, FacetCounts};
-    use crate::collector::Count;
     use crate::core::Index;
-    use crate::query::{AllQuery, QueryParser, TermQuery};
-    use crate::schema::{Document, Facet, Field, IndexRecordOption, Schema};
-    use crate::Term;
+    use crate::query::AllQuery;
+    use crate::schema::{Document, Facet, Field, Schema};
     use rand::distributions::Uniform;
     use rand::prelude::SliceRandom;
     use rand::{thread_rng, Rng};
@@ -546,56 +544,6 @@ mod tests {
         assert_eq!(facets[0].1, 1);
     }
 
-    #[test]
-    fn test_doc_search_by_facet() {
-        let mut schema_builder = Schema::builder();
-        let facet_field = schema_builder.add_facet_field("facet");
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-        index_writer.add_document(doc!(
-            facet_field => Facet::from_text(&"/A/A"),
-        ));
-        index_writer.add_document(doc!(
-            facet_field => Facet::from_text(&"/A/B"),
-        ));
-        index_writer.add_document(doc!(
-            facet_field => Facet::from_text(&"/A/C/A"),
-        ));
-        index_writer.add_document(doc!(
-            facet_field => Facet::from_text(&"/D/C/A"),
-        ));
-        index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-        let searcher = reader.searcher();
-        assert_eq!(searcher.num_docs(), 4);
-
-        let count_facet = |facet_str: &str| {
-            let term = Term::from_facet(facet_field, &Facet::from_text(facet_str));
-            searcher
-                .search(&TermQuery::new(term, IndexRecordOption::Basic), &Count)
-                .unwrap()
-        };
-
-        assert_eq!(count_facet("/"), 4);
-        assert_eq!(count_facet("/A"), 3);
-        assert_eq!(count_facet("/A/B"), 1);
-        assert_eq!(count_facet("/A/C"), 1);
-        assert_eq!(count_facet("/A/C/A"), 1);
-        assert_eq!(count_facet("/C/A"), 0);
-        {
-            let query_parser = QueryParser::for_index(&index, vec![]);
-            {
-                let query = query_parser.parse_query("facet:/A/B").unwrap();
-                assert_eq!(1, searcher.search(&query, &Count).unwrap());
-            }
-            {
-                let query = query_parser.parse_query("facet:/A").unwrap();
-                assert_eq!(3, searcher.search(&query, &Count).unwrap());
-            }
-        }
-    }
-
     #[test]
     fn test_non_used_facet_collector() {
         let mut facet_collector = FacetCollector::for_field(Field::from_field_id(0));

src/collector/top_score_collector.rs

@@ -6,7 +6,6 @@ use crate::collector::tweak_score_top_collector::TweakedScoreTopCollector;
 use crate::collector::{
     CustomScorer, CustomSegmentScorer, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector,
 };
-use crate::fastfield::FastFieldReader;
 use crate::schema::Field;
 use crate::DocAddress;
 use crate::DocId;
@@ -62,34 +61,6 @@ impl fmt::Debug for TopDocs {
     }
 }
 
-struct ScorerByFastFieldReader {
-    ff_reader: FastFieldReader<u64>,
-}
-
-impl CustomSegmentScorer<u64> for ScorerByFastFieldReader {
-    fn score(&self, doc: DocId) -> u64 {
-        self.ff_reader.get_u64(u64::from(doc))
-    }
-}
-
-struct ScorerByField {
-    field: Field,
-}
-
-impl CustomScorer<u64> for ScorerByField {
-    type Child = ScorerByFastFieldReader;
-
-    fn segment_scorer(&self, segment_reader: &SegmentReader) -> crate::Result<Self::Child> {
-        let ff_reader = segment_reader
-            .fast_fields()
-            .u64(self.field)
-            .ok_or_else(|| {
-                crate::Error::SchemaError(format!("Field requested is not a i64/u64 fast field."))
-            })?;
-        Ok(ScorerByFastFieldReader { ff_reader })
-    }
-}
-
 impl TopDocs {
     /// Creates a top score collector, with a number of documents equal to "limit".
     ///
@@ -172,7 +143,14 @@ impl TopDocs {
         self,
         field: Field,
     ) -> impl Collector<Fruit = Vec<(u64, DocAddress)>> {
-        self.custom_score(ScorerByField { field })
+        self.custom_score(move |segment_reader: &SegmentReader| {
+            let ff_reader = segment_reader
+                .fast_fields()
+                .u64(field)
+                .expect("Field requested is not a i64/u64 fast field.");
+            //TODO error message missmatch actual behavior for i64
+            move |doc: DocId| ff_reader.get(doc)
+        })
     }
 
     /// Ranks the documents using a custom score.
@@ -594,6 +572,7 @@ mod tests {
     }
 
     #[test]
+    #[should_panic(expected = "Field requested is not a i64/u64 fast field")]
     fn test_field_not_fast_field() {
         let mut schema_builder = Schema::builder();
         let title = schema_builder.add_text_field(TITLE, TEXT);
@@ -608,12 +587,7 @@ mod tests {
         let searcher = index.reader().unwrap().searcher();
         let segment = searcher.segment_reader(0);
         let top_collector = TopDocs::with_limit(4).order_by_u64_field(size);
-        let err = top_collector.for_segment(0, segment);
-        if let Err(crate::Error::SchemaError(msg)) = err {
-            assert_eq!(msg, "Field requested is not a i64/u64 fast field.");
-        } else {
-            assert!(false);
-        }
+        assert!(top_collector.for_segment(0, segment).is_ok());
     }
 
     fn index(

src/core/executor.rs

@@ -10,9 +10,7 @@ use rayon::{ThreadPool, ThreadPoolBuilder};
 /// API of a dependency, knowing it might conflict with a different version
 /// used by the client. Second, we may stop using rayon in the future.
 pub enum Executor {
-    /// Single thread variant of an Executor
     SingleThread,
-    /// Thread pool variant of an Executor
     ThreadPool(ThreadPool),
 }
 
@@ -22,7 +20,7 @@ impl Executor {
         Executor::SingleThread
     }
 
-    /// Creates an Executor that dispatches the tasks in a thread pool.
+    // Creates an Executor that dispatches the tasks in a thread pool.
     pub fn multi_thread(num_threads: usize, prefix: &'static str) -> Result<Executor> {
         let pool = ThreadPoolBuilder::new()
             .num_threads(num_threads)
@@ -31,10 +29,10 @@ impl Executor {
         Ok(Executor::ThreadPool(pool))
     }
 
-    /// Perform a map in the thread pool.
-    ///
-    /// Regardless of the executor (`SingleThread` or `ThreadPool`), panics in the task
-    /// will propagate to the caller.
+    // Perform a map in the thread pool.
+    //
+    // Regardless of the executor (`SingleThread` or `ThreadPool`), panics in the task
+    // will propagate to the caller.
     pub fn map<
         A: Send,
         R: Send,

src/core/index.rs

@@ -20,7 +20,8 @@ use crate::reader::IndexReaderBuilder;
 use crate::schema::Field;
 use crate::schema::FieldType;
 use crate::schema::Schema;
-use crate::tokenizer::{TextAnalyzer, TokenizerManager};
+use crate::tokenizer::BoxedTokenizer;
+use crate::tokenizer::TokenizerManager;
 use crate::IndexWriter;
 use crate::Result;
 use num_cpus;
@@ -172,11 +173,11 @@ impl Index {
     }
 
     /// Helper to access the tokenizer associated to a specific field.
-    pub fn tokenizer_for_field(&self, field: Field) -> Result<TextAnalyzer> {
+    pub fn tokenizer_for_field(&self, field: Field) -> Result<BoxedTokenizer> {
         let field_entry = self.schema.get_field_entry(field);
         let field_type = field_entry.field_type();
         let tokenizer_manager: &TokenizerManager = self.tokenizers();
-        let tokenizer_name_opt: Option<TextAnalyzer> = match field_type {
+        let tokenizer_name_opt: Option<BoxedTokenizer> = match field_type {
             FieldType::Str(text_options) => text_options
                 .get_indexing_options()
                 .map(|text_indexing_options| text_indexing_options.tokenizer().to_string())

src/core/inverted_index_reader.rs

@@ -60,7 +60,7 @@ impl InvertedIndexReader {
             .get_index_record_option()
             .unwrap_or(IndexRecordOption::Basic);
         InvertedIndexReader {
-            termdict: TermDictionary::empty(),
+            termdict: TermDictionary::empty(&field_type),
             postings_source: ReadOnlySource::empty(),
             positions_source: ReadOnlySource::empty(),
             positions_idx_source: ReadOnlySource::empty(),

src/directory/directory.rs

@@ -119,7 +119,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     /// Specifically, subsequent writes or flushes should
     /// have no effect on the returned `ReadOnlySource` object.
     ///
-    /// You should only use this to read files create with [Directory::open_write].
+    /// You should only use this to read files create with [`open_write`]
     fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError>;
 
     /// Removes a file
@@ -160,7 +160,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     ///
     /// This should only be used for small files.
     ///
-    /// You should only use this to read files create with [Directory::atomic_write].
+    /// You should only use this to read files create with [`atomic_write`]
     fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError>;
 
     /// Atomically replace the content of a file with data.
@@ -197,7 +197,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     /// Registers a callback that will be called whenever a change on the `meta.json`
     /// using the `atomic_write` API is detected.
     ///
-    /// The behavior when using `.watch()` on a file using [Directory::open_write] is, on the other
+    /// The behavior when using `.watch()` on a file using `.open_write(...)` is, on the other
     /// hand, undefined.
     ///
     /// The file will be watched for the lifetime of the returned `WatchHandle`. The caller is

src/directory/error.rs

@@ -1,4 +1,4 @@
-use crate::Version;
+use crate::directory::footer::Footer;
 use std::error::Error as StdError;
 use std::fmt;
 use std::io;
@@ -157,65 +157,6 @@ impl StdError for OpenWriteError {
     }
 }
 
-/// Type of index incompatibility between the library and the index found on disk
-/// Used to catch and provide a hint to solve this incompatibility issue
-pub enum Incompatibility {
-    /// This library cannot decompress the index found on disk
-    CompressionMismatch {
-        /// Compression algorithm used by the current version of tantivy
-        library_compression_format: String,
-        /// Compression algorithm that was used to serialise the index
-        index_compression_format: String,
-    },
-    /// The index format found on disk isn't supported by this version of the library
-    IndexMismatch {
-        /// Version used by the library
-        library_version: Version,
-        /// Version the index was built with
-        index_version: Version,
-    },
-}
-
-impl fmt::Debug for Incompatibility {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
-        match self {
-            Incompatibility::CompressionMismatch {
-                library_compression_format,
-                index_compression_format,
-            } => {
-                let err = format!(
-                    "Library was compiled with {:?} compression, index was compressed with {:?}",
-                    library_compression_format, index_compression_format
-                );
-                let advice = format!(
-                    "Change the feature flag to {:?} and rebuild the library",
-                    index_compression_format
-                );
-                write!(f, "{}. {}", err, advice)?;
-            }
-            Incompatibility::IndexMismatch {
-                library_version,
-                index_version,
-            } => {
-                let err = format!(
-                    "Library version: {}, index version: {}",
-                    library_version.index_format_version, index_version.index_format_version
-                );
-                // TODO make a more useful error message
-                // include the version range that supports this index_format_version
-                let advice = format!(
-                    "Change tantivy to a version compatible with index format {} (e.g. {}.{}.x) \
-                     and rebuild your project.",
-                    index_version.index_format_version, index_version.major, index_version.minor
-                );
-                write!(f, "{}. {}", err, advice)?;
-            }
-        }
-
-        Ok(())
-    }
-}
-
 /// Error that may occur when accessing a file read
 #[derive(Debug)]
 pub enum OpenReadError {
@@ -224,8 +165,8 @@ pub enum OpenReadError {
     /// Any kind of IO error that happens when
     /// interacting with the underlying IO device.
     IOError(IOError),
-    /// This library doesn't support the index version found on disk
-    IncompatibleIndex(Incompatibility),
+    /// The version of tantivy trying to read the index doesn't support its format
+    IncompatibleIndex(Footer),
 }
 
 impl From<IOError> for OpenReadError {
@@ -252,6 +193,20 @@ impl fmt::Display for OpenReadError {
     }
 }
 
+impl StdError for OpenReadError {
+    fn description(&self) -> &str {
+        "error occurred while opening a file for reading"
+    }
+
+    fn cause(&self) -> Option<&dyn StdError> {
+        match *self {
+            OpenReadError::FileDoesNotExist(_) => None,
+            OpenReadError::IOError(ref err) => Some(err),
+            OpenReadError::IncompatibleIndex(_) => None,
+        }
+    }
+}
+
 /// Error that may occur when trying to delete a file
 #[derive(Debug)]
 pub enum DeleteError {
@@ -268,12 +223,6 @@ impl From<IOError> for DeleteError {
     }
 }
 
-impl From<Incompatibility> for OpenReadError {
-    fn from(incompatibility: Incompatibility) -> Self {
-        OpenReadError::IncompatibleIndex(incompatibility)
-    }
-}
-
 impl fmt::Display for DeleteError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {

src/directory/footer.rs

@@ -1,5 +1,4 @@
 use crate::common::{BinarySerializable, CountingWriter, FixedSize, VInt};
-use crate::directory::error::Incompatibility;
 use crate::directory::read_only_source::ReadOnlySource;
 use crate::directory::{AntiCallToken, TerminatingWrite};
 use crate::Version;
@@ -18,11 +17,11 @@ pub struct Footer {
 }
 
 /// Serialises the footer to a byte-array
-/// - versioned_footer_len : 4 bytes
+/// - versioned_footer_len : 4bytes
 ///-  versioned_footer: variable bytes
 /// - meta_len: 4 bytes
 /// - meta: variable bytes
-/// - version_len: 4 bytes
+/// - version_len: 4bytes
 /// - version json: variable bytes
 impl BinarySerializable for Footer {
     fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
@@ -87,25 +86,16 @@ impl Footer {
 
     /// Confirms that the index will be read correctly by this version of tantivy
     /// Has to be called after `extract_footer` to make sure it's not accessing uninitialised memory
-    pub fn is_compatible(&self) -> Result<(), Incompatibility> {
-        let library_version = crate::version();
+    pub fn is_compatible(&self) -> bool {
+        let version = &*crate::VERSION;
         match &self.versioned_footer {
             VersionedFooter::V1 {
-                crc32: _crc,
-                store_compression: compression,
+                crc32: _,
+                compression,
             } => {
-                if &library_version.store_compression != compression {
-                    return Err(Incompatibility::CompressionMismatch {
-                        library_compression_format: library_version.store_compression.to_string(),
-                        index_compression_format: compression.to_string(),
-                    });
-                }
-                Ok(())
+                return compression == &version.store_compression;
             }
-            VersionedFooter::UnknownVersion => Err(Incompatibility::IndexMismatch {
-                library_version: library_version.clone(),
-                index_version: self.version.clone(),
-            }),
+            VersionedFooter::UnknownVersion { version: _ } => false,
         }
     }
 }
@@ -113,32 +103,31 @@ impl Footer {
 /// Footer that includes a crc32 hash that enables us to checksum files in the index
 #[derive(Debug, Clone, PartialEq)]
 pub enum VersionedFooter {
-    UnknownVersion,
+    UnknownVersion {
+        version: u32,
+    },
     V1 {
         crc32: CrcHashU32,
-        store_compression: String,
+        compression: String,
     },
 }
 
 impl BinarySerializable for VersionedFooter {
     fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
         let mut buf = Vec::new();
+        BinarySerializable::serialize(&self.version(), &mut buf)?;
         match self {
-            VersionedFooter::V1 {
-                crc32,
-                store_compression: compression,
-            } => {
+            VersionedFooter::V1 { crc32, compression } => {
                 // Serializes a valid `VersionedFooter` or panics if the version is unknown
                 // [   version    |   crc_hash  | compression_mode ]
-                // [    0..4      |     4..8    |     variable     ]
-                BinarySerializable::serialize(&1u32, &mut buf)?;
+                // [    0..4      |     4..8    |  variable ]
                 BinarySerializable::serialize(crc32, &mut buf)?;
                 BinarySerializable::serialize(compression, &mut buf)?;
             }
-            VersionedFooter::UnknownVersion => {
+            VersionedFooter::UnknownVersion { version: _ } => {
                 return Err(io::Error::new(
                     io::ErrorKind::InvalidInput,
-                    "Cannot serialize an unknown versioned footer ",
+                    "Cannot serialize an unknown versionned footer ",
                 ));
             }
         }
@@ -156,20 +145,30 @@ impl BinarySerializable for VersionedFooter {
         if version == 1 {
             let crc32 = u32::deserialize(&mut cursor)?;
             let compression = String::deserialize(&mut cursor)?;
-            Ok(VersionedFooter::V1 {
-                crc32,
-                store_compression: compression,
-            })
+            Ok(VersionedFooter::V1 { crc32, compression })
         } else {
-            Ok(VersionedFooter::UnknownVersion)
+            Ok(VersionedFooter::UnknownVersion { version })
         }
     }
 }
 
 impl VersionedFooter {
+    pub fn version(&self) -> u32 {
+        match self {
+            VersionedFooter::V1 {
+                crc32: _,
+                compression: _,
+            } => 1u32,
+            VersionedFooter::UnknownVersion { version, .. } => *version,
+        }
+    }
+
     pub fn crc(&self) -> Option<CrcHashU32> {
         match self {
-            VersionedFooter::V1 { crc32, .. } => Some(*crc32),
+            VersionedFooter::V1 {
+                crc32,
+                compression: _,
+            } => Some(*crc32),
             VersionedFooter::UnknownVersion { .. } => None,
         }
     }
@@ -208,7 +207,7 @@ impl<W: TerminatingWrite> TerminatingWrite for FooterProxy<W> {
         let crc32 = self.hasher.take().unwrap().finalize();
         let footer = Footer::new(VersionedFooter::V1 {
             crc32,
-            store_compression: crate::store::COMPRESSION.to_string(),
+            compression: crate::store::COMPRESSION.to_string(),
         });
         let mut writer = self.writer.take().unwrap();
         footer.append_footer(&mut writer)?;
@@ -228,21 +227,16 @@ mod tests {
     use regex::Regex;
 
     #[test]
-    fn test_versioned_footer() {
+    fn test_footer_version() {
         let mut vec = Vec::new();
         let footer_proxy = FooterProxy::new(&mut vec);
         assert!(footer_proxy.terminate().is_ok());
         assert_eq!(vec.len(), 167);
         let footer = Footer::deserialize(&mut &vec[..]).unwrap();
-        if let VersionedFooter::V1 {
-            crc32: _,
-            store_compression,
-        } = footer.versioned_footer
-        {
-            assert_eq!(store_compression, crate::store::COMPRESSION);
-        } else {
-            panic!("Versioned footer should be V1.");
-        }
+        assert_eq!(
+            footer.versioned_footer.version(),
+            crate::INDEX_FORMAT_VERSION
+        );
         assert_eq!(&footer.version, crate::version());
     }
 
@@ -252,7 +246,7 @@ mod tests {
         let crc32 = 123456u32;
         let footer: Footer = Footer::new(VersionedFooter::V1 {
             crc32,
-            store_compression: "lz4".to_string(),
+            compression: "lz4".to_string(),
         });
         footer.serialize(&mut buffer).unwrap();
         let footer_deser = Footer::deserialize(&mut &buffer[..]).unwrap();
@@ -261,10 +255,11 @@ mod tests {
 
     #[test]
     fn footer_length() {
+        // test to make sure the ascii art in the doc-strings is correct
         let crc32 = 1111111u32;
         let versioned_footer = VersionedFooter::V1 {
             crc32,
-            store_compression: "lz4".to_string(),
+            compression: "lz4".to_string(),
         };
         let mut buf = Vec::new();
         versioned_footer.serialize(&mut buf).unwrap();
@@ -304,7 +299,7 @@ mod tests {
         let expected_crc: u32 = LittleEndian::read_u32(&v_footer_bytes[5..9]) as CrcHashU32;
         let expected_versioned_footer: VersionedFooter = VersionedFooter::V1 {
             crc32: expected_crc,
-            store_compression: "lz4".to_string(),
+            compression: "lz4".to_string(),
         };
         assert_eq!(versioned_footer, expected_versioned_footer);
         let mut buffer = Vec::new();
@@ -318,22 +313,11 @@ mod tests {
         let mut b = &v_footer_bytes[..];
         let versioned_footer = VersionedFooter::deserialize(&mut b).unwrap();
         assert!(b.is_empty());
-        let expected_versioned_footer = VersionedFooter::UnknownVersion;
+        let expected_versioned_footer = VersionedFooter::UnknownVersion {
+            version: 16_777_219u32,
+        };
         assert_eq!(versioned_footer, expected_versioned_footer);
         let mut buf = Vec::new();
         assert!(versioned_footer.serialize(&mut buf).is_err());
     }
-
-    #[test]
-    #[cfg(not(feature = "lz4"))]
-    fn compression_mismatch() {
-        let crc32 = 1111111u32;
-        let versioned_footer = VersionedFooter::V1 {
-            crc32,
-            store_compression: "lz4".to_string(),
-        };
-        let footer = Footer::new(versioned_footer);
-        let res = footer.is_compatible();
-        assert!(res.is_err());
-    }
 }

src/directory/managed_directory.rs

@@ -9,6 +9,7 @@ use crate::directory::{ReadOnlySource, WritePtr};
 use crate::directory::{WatchCallback, WatchHandle};
 use crate::error::DataCorruption;
 use crate::Directory;
+use crate::Result;
 
 use crc32fast::Hasher;
 use serde_json;
@@ -65,7 +66,7 @@ fn save_managed_paths(
 
 impl ManagedDirectory {
     /// Wraps a directory as managed directory.
-    pub fn wrap<Dir: Directory>(directory: Dir) -> crate::Result<ManagedDirectory> {
+    pub fn wrap<Dir: Directory>(directory: Dir) -> Result<ManagedDirectory> {
         match directory.atomic_read(&MANAGED_FILEPATH) {
             Ok(data) => {
                 let managed_files_json = String::from_utf8_lossy(&data);
@@ -88,10 +89,8 @@ impl ManagedDirectory {
                 meta_informations: Arc::default(),
             }),
             Err(OpenReadError::IOError(e)) => Err(From::from(e)),
-            Err(OpenReadError::IncompatibleIndex(incompatibility)) => {
-                // For the moment, this should never happen  `meta.json`
-                // do not have any footer and cannot detect incompatibility.
-                Err(crate::TantivyError::IncompatibleIndex(incompatibility))
+            Err(OpenReadError::IncompatibleIndex(footer)) => {
+                Err(crate::Error::IncompatibleIndex(format!("{:?}", footer)))
             }
         }
     }
@@ -268,7 +267,9 @@ impl Directory for ManagedDirectory {
         let read_only_source = self.directory.open_read(path)?;
         let (footer, reader) = Footer::extract_footer(read_only_source)
             .map_err(|err| IOError::with_path(path.to_path_buf(), err))?;
-        footer.is_compatible()?;
+        if !footer.is_compatible() {
+            return Err(OpenReadError::IncompatibleIndex(footer));
+        }
         Ok(reader)
     }
 

src/directory/mmap_directory.rs

@@ -131,13 +131,14 @@ impl MmapCache {
         }
         self.cache.remove(full_path);
         self.counters.miss += 1;
-        let mmap_opt = open_mmap(full_path)?;
-        Ok(mmap_opt.map(|mmap| {
+        Ok(if let Some(mmap) = open_mmap(full_path)? {
             let mmap_arc: Arc<BoxedData> = Arc::new(Box::new(mmap));
             let mmap_weak = Arc::downgrade(&mmap_arc);
             self.cache.insert(full_path.to_owned(), mmap_weak);
-            mmap_arc
-        }))
+            Some(mmap_arc)
+        } else {
+            None
+        })
     }
 }
 

src/directory/mod.rs

@@ -23,9 +23,11 @@ pub use self::directory::{Directory, DirectoryClone};
 pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};
 pub use self::ram_directory::RAMDirectory;
 pub use self::read_only_source::ReadOnlySource;
-pub use self::watch_event_router::{WatchCallback, WatchCallbackList, WatchHandle};
+pub(crate) use self::watch_event_router::WatchCallbackList;
+pub use self::watch_event_router::{WatchCallback, WatchHandle};
 use std::io::{self, BufWriter, Write};
 use std::path::PathBuf;
+
 /// Outcome of the Garbage collection
 pub struct GarbageCollectionResult {
     /// List of files that were deleted in this cycle
@@ -46,9 +48,6 @@ pub use self::mmap_directory::MmapDirectory;
 pub use self::managed_directory::ManagedDirectory;
 
 /// Struct used to prevent from calling [`terminate_ref`](trait.TerminatingWrite#method.terminate_ref) directly
-///
-/// The point is that while the type is public, it cannot be built by anyone
-/// outside of this module.
 pub struct AntiCallToken(());
 
 /// Trait used to indicate when no more write need to be done on a writer

src/directory/watch_event_router.rs

@@ -24,20 +24,13 @@ pub struct WatchCallbackList {
 #[derive(Clone)]
 pub struct WatchHandle(Arc<WatchCallback>);
 
-impl WatchHandle {
-    /// Create a WatchHandle handle.
-    pub fn new(watch_callback: Arc<WatchCallback>) -> WatchHandle {
-        WatchHandle(watch_callback)
-    }
-}
-
 impl WatchCallbackList {
     /// Suscribes a new callback and returns a handle that controls the lifetime of the callback.
     pub fn subscribe(&self, watch_callback: WatchCallback) -> WatchHandle {
         let watch_callback_arc = Arc::new(watch_callback);
         let watch_callback_weak = Arc::downgrade(&watch_callback_arc);
         self.router.write().unwrap().push(watch_callback_weak);
-        WatchHandle::new(watch_callback_arc)
+        WatchHandle(watch_callback_arc)
     }
 
     fn list_callback(&self) -> Vec<Arc<WatchCallback>> {

src/error.rs

@@ -2,8 +2,8 @@
 
 use std::io;
 
+use crate::directory::error::LockError;
 use crate::directory::error::{IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
-use crate::directory::error::{Incompatibility, LockError};
 use crate::fastfield::FastFieldNotAvailableError;
 use crate::query;
 use crate::schema;
@@ -81,8 +81,11 @@ pub enum TantivyError {
     #[fail(display = "System error.'{}'", _0)]
     SystemError(String),
     /// Index incompatible with current version of tantivy
-    #[fail(display = "{:?}", _0)]
-    IncompatibleIndex(Incompatibility),
+    #[fail(
+        display = "Current version of tantivy is incompatible with index version: '{}'",
+        _0
+    )]
+    IncompatibleIndex(String),
 }
 
 impl From<DataCorruption> for TantivyError {
@@ -132,8 +135,8 @@ impl From<OpenReadError> for TantivyError {
         match error {
             OpenReadError::FileDoesNotExist(filepath) => TantivyError::PathDoesNotExist(filepath),
             OpenReadError::IOError(io_error) => TantivyError::IOError(io_error),
-            OpenReadError::IncompatibleIndex(incompatibility) => {
-                TantivyError::IncompatibleIndex(incompatibility)
+            OpenReadError::IncompatibleIndex(tantivy_err) => {
+                TantivyError::IncompatibleIndex(format!("{:?}", tantivy_err))
             }
         }
     }

src/fastfield/delete.rs

@@ -1,8 +1,9 @@
-use crate::common::{BitSet, HasLen};
+use crate::common::HasLen;
 use crate::directory::ReadOnlySource;
 use crate::directory::WritePtr;
 use crate::space_usage::ByteCount;
 use crate::DocId;
+use bit_set::BitSet;
 use std::io;
 use std::io::Write;
 
@@ -16,7 +17,7 @@ pub fn write_delete_bitset(
 ) -> io::Result<()> {
     let mut byte = 0u8;
     let mut shift = 0u8;
-    for doc in 0..max_doc {
+    for doc in 0..(max_doc as usize) {
         if delete_bitset.contains(doc) {
             byte |= 1 << shift;
         }
@@ -31,7 +32,7 @@ pub fn write_delete_bitset(
     if max_doc % 8 > 0 {
         writer.write_all(&[byte])?;
     }
-    Ok(())
+    writer.flush()
 }
 
 /// Set of deleted `DocId`s.
@@ -85,6 +86,7 @@ impl HasLen for DeleteBitSet {
 mod tests {
     use super::*;
     use crate::directory::*;
+    use bit_set::BitSet;
     use std::path::PathBuf;
 
     fn test_delete_bitset_helper(bitset: &BitSet, max_doc: u32) {
@@ -93,26 +95,27 @@ mod tests {
         {
             let mut writer = directory.open_write(&*test_path).unwrap();
             write_delete_bitset(bitset, max_doc, &mut writer).unwrap();
-            writer.terminate().unwrap();
         }
-        let source = directory.open_read(&test_path).unwrap();
-        let delete_bitset = DeleteBitSet::open(source);
-        for doc in 0..max_doc {
-            assert_eq!(bitset.contains(doc), delete_bitset.is_deleted(doc as DocId));
+        {
+            let source = directory.open_read(&test_path).unwrap();
+            let delete_bitset = DeleteBitSet::open(source);
+            for doc in 0..max_doc as usize {
+                assert_eq!(bitset.contains(doc), delete_bitset.is_deleted(doc as DocId));
+            }
+            assert_eq!(delete_bitset.len(), bitset.len());
         }
-        assert_eq!(delete_bitset.len(), bitset.len());
     }
 
     #[test]
     fn test_delete_bitset() {
         {
-            let mut bitset = BitSet::with_max_value(10);
+            let mut bitset = BitSet::with_capacity(10);
             bitset.insert(1);
             bitset.insert(9);
             test_delete_bitset_helper(&bitset, 10);
         }
         {
-            let mut bitset = BitSet::with_max_value(8);
+            let mut bitset = BitSet::with_capacity(8);
             bitset.insert(1);
             bitset.insert(2);
             bitset.insert(3);

src/fastfield/mod.rs

@@ -33,7 +33,6 @@ pub use self::reader::FastFieldReader;
 pub use self::readers::FastFieldReaders;
 pub use self::serializer::FastFieldSerializer;
 pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
-use crate::chrono::{NaiveDateTime, Utc};
 use crate::common;
 use crate::schema::Cardinality;
 use crate::schema::FieldType;
@@ -50,7 +49,7 @@ mod serializer;
 mod writer;
 
 /// Trait for types that are allowed for fast fields: (u64, i64 and f64).
-pub trait FastValue: Clone + Copy + Send + Sync + PartialOrd {
+pub trait FastValue: Default + Clone + Copy + Send + Sync + PartialOrd {
     /// Converts a value from u64
     ///
     /// Internally all fast field values are encoded as u64.
@@ -70,12 +69,6 @@ pub trait FastValue: Clone + Copy + Send + Sync + PartialOrd {
     /// Cast value to `u64`.
     /// The value is just reinterpreted in memory.
     fn as_u64(&self) -> u64;
-
-    /// Build a default value. This default value is never used, so the value does not
-    /// really matter.
-    fn make_zero() -> Self {
-        Self::from_u64(0i64.to_u64())
-    }
 }
 
 impl FastValue for u64 {
@@ -142,34 +135,11 @@ impl FastValue for f64 {
     }
 }
 
-impl FastValue for crate::DateTime {
-    fn from_u64(timestamp_u64: u64) -> Self {
-        let timestamp_i64 = i64::from_u64(timestamp_u64);
-        crate::DateTime::from_utc(NaiveDateTime::from_timestamp(timestamp_i64, 0), Utc)
-    }
-
-    fn to_u64(&self) -> u64 {
-        self.timestamp().to_u64()
-    }
-
-    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
-        match *field_type {
-            FieldType::Date(ref integer_options) => integer_options.get_fastfield_cardinality(),
-            _ => None,
-        }
-    }
-
-    fn as_u64(&self) -> u64 {
-        self.timestamp().as_u64()
-    }
-}
-
 fn value_to_u64(value: &Value) -> u64 {
     match *value {
         Value::U64(ref val) => *val,
         Value::I64(ref val) => common::i64_to_u64(*val),
         Value::F64(ref val) => common::f64_to_u64(*val),
-        Value::Date(ref datetime) => common::i64_to_u64(datetime.timestamp()),
         _ => panic!("Expected a u64/i64/f64 field, got {:?} ", value),
     }
 }
@@ -181,12 +151,10 @@ mod tests {
     use crate::common::CompositeFile;
     use crate::directory::{Directory, RAMDirectory, WritePtr};
     use crate::fastfield::FastFieldReader;
-    use crate::merge_policy::NoMergePolicy;
+    use crate::schema::Document;
     use crate::schema::Field;
     use crate::schema::Schema;
     use crate::schema::FAST;
-    use crate::schema::{Document, IntOptions};
-    use crate::{Index, SegmentId, SegmentReader};
     use once_cell::sync::Lazy;
     use rand::prelude::SliceRandom;
     use rand::rngs::StdRng;
@@ -210,12 +178,6 @@ mod tests {
         assert_eq!(test_fastfield.get(2), 300);
     }
 
-    #[test]
-    pub fn test_fastfield_i64_u64() {
-        let datetime = crate::DateTime::from_utc(NaiveDateTime::from_timestamp(0i64, 0), Utc);
-        assert_eq!(i64::from_u64(datetime.to_u64()), 0i64);
-    }
-
     #[test]
     fn test_intfastfield_small() {
         let path = Path::new("test");
@@ -467,93 +429,6 @@ mod tests {
             }
         }
     }
-
-    #[test]
-    fn test_merge_missing_date_fast_field() {
-        let mut schema_builder = Schema::builder();
-        let date_field = schema_builder.add_date_field("date", FAST);
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-        index_writer.set_merge_policy(Box::new(NoMergePolicy));
-        index_writer.add_document(doc!(date_field =>crate::chrono::prelude::Utc::now()));
-        index_writer.commit().unwrap();
-        index_writer.add_document(doc!());
-        index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-        let segment_ids: Vec<SegmentId> = reader
-            .searcher()
-            .segment_readers()
-            .iter()
-            .map(SegmentReader::segment_id)
-            .collect();
-        assert_eq!(segment_ids.len(), 2);
-        let merge_future = index_writer.merge(&segment_ids[..]);
-        let merge_res = futures::executor::block_on(merge_future);
-        assert!(merge_res.is_ok());
-        assert!(reader.reload().is_ok());
-        assert_eq!(reader.searcher().segment_readers().len(), 1);
-    }
-
-    #[test]
-    fn test_default_datetime() {
-        assert_eq!(crate::DateTime::make_zero().timestamp(), 0i64);
-    }
-
-    #[test]
-    fn test_datefastfield() {
-        use crate::fastfield::FastValue;
-        let mut schema_builder = Schema::builder();
-        let date_field = schema_builder.add_date_field("date", FAST);
-        let multi_date_field = schema_builder.add_date_field(
-            "multi_date",
-            IntOptions::default().set_fast(Cardinality::MultiValues),
-        );
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-        index_writer.set_merge_policy(Box::new(NoMergePolicy));
-        index_writer.add_document(doc!(
-            date_field => crate::DateTime::from_u64(1i64.to_u64()),
-            multi_date_field => crate::DateTime::from_u64(2i64.to_u64()),
-            multi_date_field => crate::DateTime::from_u64(3i64.to_u64())
-        ));
-        index_writer.add_document(doc!(
-            date_field => crate::DateTime::from_u64(4i64.to_u64())
-        ));
-        index_writer.add_document(doc!(
-            multi_date_field => crate::DateTime::from_u64(5i64.to_u64()),
-            multi_date_field => crate::DateTime::from_u64(6i64.to_u64())
-        ));
-        index_writer.commit().unwrap();
-        let reader = index.reader().unwrap();
-        let searcher = reader.searcher();
-        assert_eq!(searcher.segment_readers().len(), 1);
-        let segment_reader = searcher.segment_reader(0);
-        let fast_fields = segment_reader.fast_fields();
-        let date_fast_field = fast_fields.date(date_field).unwrap();
-        let dates_fast_field = fast_fields.dates(multi_date_field).unwrap();
-        let mut dates = vec![];
-        {
-            assert_eq!(date_fast_field.get(0u32).timestamp(), 1i64);
-            dates_fast_field.get_vals(0u32, &mut dates);
-            assert_eq!(dates.len(), 2);
-            assert_eq!(dates[0].timestamp(), 2i64);
-            assert_eq!(dates[1].timestamp(), 3i64);
-        }
-        {
-            assert_eq!(date_fast_field.get(1u32).timestamp(), 4i64);
-            dates_fast_field.get_vals(1u32, &mut dates);
-            assert!(dates.is_empty());
-        }
-        {
-            assert_eq!(date_fast_field.get(2u32).timestamp(), 0i64);
-            dates_fast_field.get_vals(2u32, &mut dates);
-            assert_eq!(dates.len(), 2);
-            assert_eq!(dates[0].timestamp(), 5i64);
-            assert_eq!(dates[1].timestamp(), 6i64);
-        }
-    }
 }
 
 #[cfg(all(test, feature = "unstable"))]

src/fastfield/multivalued/mod.rs

@@ -7,6 +7,9 @@ pub use self::writer::MultiValueIntFastFieldWriter;
 #[cfg(test)]
 mod tests {
 
+    use time;
+
+    use self::time::Duration;
     use crate::collector::TopDocs;
     use crate::query::QueryParser;
     use crate::schema::Cardinality;
@@ -14,7 +17,6 @@ mod tests {
     use crate::schema::IntOptions;
     use crate::schema::Schema;
     use crate::Index;
-    use chrono::Duration;
 
     #[test]
     fn test_multivalued_u64() {

src/fastfield/multivalued/reader.rs

@@ -45,7 +45,7 @@ impl<Item: FastValue> MultiValueIntFastFieldReader<Item> {
     pub fn get_vals(&self, doc: DocId, vals: &mut Vec<Item>) {
         let (start, stop) = self.range(doc);
         let len = (stop - start) as usize;
-        vals.resize(len, Item::make_zero());
+        vals.resize(len, Item::default());
         self.vals_reader.get_range_u64(start, &mut vals[..]);
     }
 

src/fastfield/readers.rs

@@ -15,11 +15,9 @@ pub struct FastFieldReaders {
     fast_field_i64: HashMap<Field, FastFieldReader<i64>>,
     fast_field_u64: HashMap<Field, FastFieldReader<u64>>,
     fast_field_f64: HashMap<Field, FastFieldReader<f64>>,
-    fast_field_date: HashMap<Field, FastFieldReader<crate::DateTime>>,
     fast_field_i64s: HashMap<Field, MultiValueIntFastFieldReader<i64>>,
     fast_field_u64s: HashMap<Field, MultiValueIntFastFieldReader<u64>>,
     fast_field_f64s: HashMap<Field, MultiValueIntFastFieldReader<f64>>,
-    fast_field_dates: HashMap<Field, MultiValueIntFastFieldReader<crate::DateTime>>,
     fast_bytes: HashMap<Field, BytesFastFieldReader>,
     fast_fields_composite: CompositeFile,
 }
@@ -28,7 +26,6 @@ enum FastType {
     I64,
     U64,
     F64,
-    Date,
 }
 
 fn type_and_cardinality(field_type: &FieldType) -> Option<(FastType, Cardinality)> {
@@ -42,9 +39,6 @@ fn type_and_cardinality(field_type: &FieldType) -> Option<(FastType, Cardinality
         FieldType::F64(options) => options
             .get_fastfield_cardinality()
             .map(|cardinality| (FastType::F64, cardinality)),
-        FieldType::Date(options) => options
-            .get_fastfield_cardinality()
-            .map(|cardinality| (FastType::Date, cardinality)),
         FieldType::HierarchicalFacet => Some((FastType::U64, Cardinality::MultiValues)),
         _ => None,
     }
@@ -59,11 +53,9 @@ impl FastFieldReaders {
             fast_field_i64: Default::default(),
             fast_field_u64: Default::default(),
             fast_field_f64: Default::default(),
-            fast_field_date: Default::default(),
             fast_field_i64s: Default::default(),
             fast_field_u64s: Default::default(),
             fast_field_f64s: Default::default(),
-            fast_field_dates: Default::default(),
             fast_bytes: Default::default(),
             fast_fields_composite: fast_fields_composite.clone(),
         };
@@ -103,12 +95,6 @@ impl FastFieldReaders {
                                         FastFieldReader::open(fast_field_data.clone()),
                                     );
                                 }
-                                FastType::Date => {
-                                    fast_field_readers.fast_field_date.insert(
-                                        field,
-                                        FastFieldReader::open(fast_field_data.clone()),
-                                    );
-                                }
                             }
                         } else {
                             return Err(From::from(FastFieldNotAvailableError::new(field_entry)));
@@ -144,14 +130,6 @@ impl FastFieldReaders {
                                         .fast_field_f64s
                                         .insert(field, multivalued_int_fast_field);
                                 }
-                                FastType::Date => {
-                                    let vals_reader = FastFieldReader::open(fast_field_data);
-                                    let multivalued_int_fast_field =
-                                        MultiValueIntFastFieldReader::open(idx_reader, vals_reader);
-                                    fast_field_readers
-                                        .fast_field_dates
-                                        .insert(field, multivalued_int_fast_field);
-                                }
                             }
                         } else {
                             return Err(From::from(FastFieldNotAvailableError::new(field_entry)));
@@ -178,6 +156,8 @@ impl FastFieldReaders {
     /// If the field is a i64-fast field, return the associated u64 reader. Values are
     /// mapped from i64 to u64 using a (well the, it is unique) monotonic mapping.    ///
     ///
+    ///TODO should it also be lenient with f64?
+    ///
     /// This method is useful when merging segment reader.
     pub(crate) fn u64_lenient(&self, field: Field) -> Option<FastFieldReader<u64>> {
         if let Some(u64_ff_reader) = self.u64(field) {
@@ -186,12 +166,6 @@ impl FastFieldReaders {
         if let Some(i64_ff_reader) = self.i64(field) {
             return Some(i64_ff_reader.into_u64_reader());
         }
-        if let Some(f64_ff_reader) = self.f64(field) {
-            return Some(f64_ff_reader.into_u64_reader());
-        }
-        if let Some(date_ff_reader) = self.date(field) {
-            return Some(date_ff_reader.into_u64_reader());
-        }
         None
     }
 
@@ -202,13 +176,6 @@ impl FastFieldReaders {
         self.fast_field_i64.get(&field).cloned()
     }
 
-    /// Returns the `i64` fast field reader reader associated to `field`.
-    ///
-    /// If `field` is not a i64 fast field, this method returns `None`.
-    pub fn date(&self, field: Field) -> Option<FastFieldReader<crate::DateTime>> {
-        self.fast_field_date.get(&field).cloned()
-    }
-
     /// Returns the `f64` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a f64 fast field, this method returns `None`.
@@ -235,9 +202,6 @@ impl FastFieldReaders {
         if let Some(i64s_ff_reader) = self.i64s(field) {
             return Some(i64s_ff_reader.into_u64s_reader());
         }
-        if let Some(f64s_ff_reader) = self.f64s(field) {
-            return Some(f64s_ff_reader.into_u64s_reader());
-        }
         None
     }
 
@@ -255,13 +219,6 @@ impl FastFieldReaders {
         self.fast_field_f64s.get(&field).cloned()
     }
 
-    /// Returns a `crate::DateTime` multi-valued fast field reader reader associated to `field`.
-    ///
-    /// If `field` is not a `crate::DateTime` multi-valued fast field, this method returns `None`.
-    pub fn dates(&self, field: Field) -> Option<MultiValueIntFastFieldReader<crate::DateTime>> {
-        self.fast_field_dates.get(&field).cloned()
-    }
-
     /// Returns the `bytes` fast field reader associated to `field`.
     ///
     /// If `field` is not a bytes fast field, returns `None`.

src/fastfield/writer.rs

@@ -4,7 +4,7 @@ use crate::common::BinarySerializable;
 use crate::common::VInt;
 use crate::fastfield::{BytesFastFieldWriter, FastFieldSerializer};
 use crate::postings::UnorderedTermId;
-use crate::schema::{Cardinality, Document, Field, FieldEntry, FieldType, Schema};
+use crate::schema::{Cardinality, Document, Field, FieldType, Schema};
 use crate::termdict::TermOrdinal;
 use fnv::FnvHashMap;
 use std::collections::HashMap;
@@ -17,14 +17,6 @@ pub struct FastFieldsWriter {
     bytes_value_writers: Vec<BytesFastFieldWriter>,
 }
 
-fn fast_field_default_value(field_entry: &FieldEntry) -> u64 {
-    match *field_entry.field_type() {
-        FieldType::I64(_) | FieldType::Date(_) => common::i64_to_u64(0i64),
-        FieldType::F64(_) => common::f64_to_u64(0.0f64),
-        _ => 0u64,
-    }
-}
-
 impl FastFieldsWriter {
     /// Create all `FastFieldWriter` required by the schema.
     pub fn from_schema(schema: &Schema) -> FastFieldsWriter {
@@ -33,15 +25,18 @@ impl FastFieldsWriter {
         let mut bytes_value_writers = Vec::new();
 
         for (field, field_entry) in schema.fields() {
+            let default_value = match *field_entry.field_type() {
+                FieldType::I64(_) => common::i64_to_u64(0i64),
+                FieldType::F64(_) => common::f64_to_u64(0.0f64),
+                _ => 0u64,
+            };
             match *field_entry.field_type() {
                 FieldType::I64(ref int_options)
                 | FieldType::U64(ref int_options)
-                | FieldType::F64(ref int_options)
-                | FieldType::Date(ref int_options) => {
+                | FieldType::F64(ref int_options) => {
                     match int_options.get_fastfield_cardinality() {
                         Some(Cardinality::SingleValue) => {
                             let mut fast_field_writer = IntFastFieldWriter::new(field);
-                            let default_value = fast_field_default_value(field_entry);
                             fast_field_writer.set_val_if_missing(default_value);
                             single_value_writers.push(fast_field_writer);
                         }

src/indexer/index_writer.rs

@@ -1,7 +1,6 @@
 use super::operation::{AddOperation, UserOperation};
 use super::segment_updater::SegmentUpdater;
 use super::PreparedCommit;
-use crate::common::BitSet;
 use crate::core::Index;
 use crate::core::Segment;
 use crate::core::SegmentComponent;
@@ -24,6 +23,7 @@ use crate::schema::Document;
 use crate::schema::IndexRecordOption;
 use crate::schema::Term;
 use crate::Opstamp;
+use bit_set::BitSet;
 use crossbeam::channel;
 use futures::executor::block_on;
 use futures::future::Future;
@@ -115,7 +115,7 @@ fn compute_deleted_bitset(
             while docset.advance() {
                 let deleted_doc = docset.doc();
                 if deleted_doc < limit_doc {
-                    delete_bitset.insert(deleted_doc);
+                    delete_bitset.insert(deleted_doc as usize);
                     might_have_changed = true;
                 }
             }
@@ -126,60 +126,51 @@ fn compute_deleted_bitset(
     Ok(might_have_changed)
 }
 
-/// Advance delete for the given segment up to the target opstamp.
-///
-/// Note that there are no guarantee that the resulting `segment_entry` delete_opstamp
-/// is `==` target_opstamp.
-/// For instance, there was no delete operation between the state of the `segment_entry` and
-/// the `target_opstamp`, `segment_entry` is not updated.
+/// Advance delete for the given segment up
+/// to the target opstamp.
 pub(crate) fn advance_deletes(
     mut segment: Segment,
     segment_entry: &mut SegmentEntry,
     target_opstamp: Opstamp,
 ) -> crate::Result<()> {
-    if segment_entry.meta().delete_opstamp() == Some(target_opstamp) {
-        // We are already up-to-date here.
-        return Ok(());
-    }
+    {
+        if segment_entry.meta().delete_opstamp() == Some(target_opstamp) {
+            // We are already up-to-date here.
+            return Ok(());
+        }
 
-    if segment_entry.delete_bitset().is_none() && segment_entry.delete_cursor().get().is_none() {
-        // There has been no `DeleteOperation` between the segment status and `target_opstamp`.
-        return Ok(());
-    }
+        let segment_reader = SegmentReader::open(&segment)?;
 
-    let segment_reader = SegmentReader::open(&segment)?;
+        let max_doc = segment_reader.max_doc();
+        let mut delete_bitset: BitSet = match segment_entry.delete_bitset() {
+            Some(previous_delete_bitset) => (*previous_delete_bitset).clone(),
+            None => BitSet::with_capacity(max_doc as usize),
+        };
 
-    let max_doc = segment_reader.max_doc();
-    let mut delete_bitset: BitSet = match segment_entry.delete_bitset() {
-        Some(previous_delete_bitset) => (*previous_delete_bitset).clone(),
-        None => BitSet::with_max_value(max_doc),
-    };
+        let delete_cursor = segment_entry.delete_cursor();
+        compute_deleted_bitset(
+            &mut delete_bitset,
+            &segment_reader,
+            delete_cursor,
+            &DocToOpstampMapping::None,
+            target_opstamp,
+        )?;
 
-    compute_deleted_bitset(
-        &mut delete_bitset,
-        &segment_reader,
-        segment_entry.delete_cursor(),
-        &DocToOpstampMapping::None,
-        target_opstamp,
-    )?;
-
-    // TODO optimize
-    if let Some(seg_delete_bitset) = segment_reader.delete_bitset() {
+        // TODO optimize
         for doc in 0u32..max_doc {
-            if seg_delete_bitset.is_deleted(doc) {
-                delete_bitset.insert(doc);
+            if segment_reader.is_deleted(doc) {
+                delete_bitset.insert(doc as usize);
             }
         }
-    }
 
-    let num_deleted_docs = delete_bitset.len();
-    if num_deleted_docs > 0 {
-        segment = segment.with_delete_meta(num_deleted_docs as u32, target_opstamp);
-        let mut delete_file = segment.open_write(SegmentComponent::DELETE)?;
-        write_delete_bitset(&delete_bitset, max_doc, &mut delete_file)?;
-        delete_file.terminate()?;
+        let num_deleted_docs = delete_bitset.len();
+        if num_deleted_docs > 0 {
+            segment = segment.with_delete_meta(num_deleted_docs as u32, target_opstamp);
+            let mut delete_file = segment.open_write(SegmentComponent::DELETE)?;
+            write_delete_bitset(&delete_bitset, max_doc, &mut delete_file)?;
+            delete_file.terminate()?;
+        }
     }
-
     segment_entry.set_meta(segment.meta().clone());
     Ok(())
 }
@@ -245,7 +236,7 @@ fn apply_deletes(
     mut delete_cursor: &mut DeleteCursor,
     doc_opstamps: &[Opstamp],
     last_docstamp: Opstamp,
-) -> crate::Result<Option<BitSet>> {
+) -> crate::Result<Option<BitSet<u32>>> {
     if delete_cursor.get().is_none() {
         // if there are no delete operation in the queue, no need
         // to even open the segment.
@@ -255,7 +246,7 @@ fn apply_deletes(
     let doc_to_opstamps = DocToOpstampMapping::from(doc_opstamps);
 
     let max_doc = segment.meta().max_doc();
-    let mut deleted_bitset = BitSet::with_max_value(max_doc);
+    let mut deleted_bitset = BitSet::with_capacity(max_doc as usize);
     let may_have_deletes = compute_deleted_bitset(
         &mut deleted_bitset,
         &segment_reader,
@@ -514,13 +505,9 @@ impl IndexWriter {
     /// Merges a given list of segments
     ///
     /// `segment_ids` is required to be non-empty.
-    pub fn merge(
-        &mut self,
-        segment_ids: &[SegmentId],
-    ) -> impl Future<Output = crate::Result<SegmentMeta>> {
+    pub async fn merge(&mut self, segment_ids: &[SegmentId]) -> crate::Result<SegmentMeta> {
         let merge_operation = self.segment_updater.make_merge_operation(segment_ids);
-        let segment_updater = self.segment_updater.clone();
-        async move { segment_updater.start_merge(merge_operation)?.await }
+        self.segment_updater.start_merge(merge_operation)?.await
     }
 
     /// Closes the current document channel send.
@@ -897,7 +884,7 @@ mod tests {
         let index_writer = index.writer(3_000_000).unwrap();
         assert_eq!(
             format!("{:?}", index_writer.get_merge_policy()),
-            "LogMergePolicy { min_merge_size: 8, max_merge_size: 10000000, min_layer_size: 10000, \
+            "LogMergePolicy { min_merge_size: 8, min_layer_size: 10000, \
              level_log_size: 0.75 }"
         );
         let merge_policy = Box::new(NoMergePolicy::default());

src/indexer/log_merge_policy.rs

@@ -6,14 +6,12 @@ use std::f64;
 const DEFAULT_LEVEL_LOG_SIZE: f64 = 0.75;
 const DEFAULT_MIN_LAYER_SIZE: u32 = 10_000;
 const DEFAULT_MIN_MERGE_SIZE: usize = 8;
-const DEFAULT_MAX_MERGE_SIZE: usize = 10_000_000;
 
 /// `LogMergePolicy` tries tries to merge segments that have a similar number of
 /// documents.
 #[derive(Debug, Clone)]
 pub struct LogMergePolicy {
     min_merge_size: usize,
-    max_merge_size: usize,
     min_layer_size: u32,
     level_log_size: f64,
 }
@@ -28,12 +26,6 @@ impl LogMergePolicy {
         self.min_merge_size = min_merge_size;
     }
 
-    /// Set the maximum number docs in a segment for it to be considered for
-    /// merging.
-    pub fn set_max_merge_size(&mut self, max_merge_size: usize) {
-        self.max_merge_size = max_merge_size;
-    }
-
     /// Set the minimum segment size under which all segment belong
     /// to the same level.
     pub fn set_min_layer_size(&mut self, min_layer_size: u32) {
@@ -61,7 +53,6 @@ impl MergePolicy for LogMergePolicy {
         let mut size_sorted_tuples = segments
             .iter()
             .map(SegmentMeta::num_docs)
-            .filter(|s| s <= &(self.max_merge_size as u32))
             .enumerate()
             .collect::<Vec<(usize, u32)>>();
 
@@ -95,7 +86,6 @@ impl Default for LogMergePolicy {
     fn default() -> LogMergePolicy {
         LogMergePolicy {
             min_merge_size: DEFAULT_MIN_MERGE_SIZE,
-            max_merge_size: DEFAULT_MAX_MERGE_SIZE,
             min_layer_size: DEFAULT_MIN_LAYER_SIZE,
             level_log_size: DEFAULT_LEVEL_LOG_SIZE,
         }
@@ -114,7 +104,6 @@ mod tests {
     fn test_merge_policy() -> LogMergePolicy {
         let mut log_merge_policy = LogMergePolicy::default();
         log_merge_policy.set_min_merge_size(3);
-        log_merge_policy.set_max_merge_size(100_000);
         log_merge_policy.set_min_layer_size(2);
         log_merge_policy
     }
@@ -152,11 +141,11 @@ mod tests {
             create_random_segment_meta(10),
             create_random_segment_meta(10),
             create_random_segment_meta(10),
-            create_random_segment_meta(1_000),
-            create_random_segment_meta(1_000),
-            create_random_segment_meta(1_000),
-            create_random_segment_meta(10_000),
-            create_random_segment_meta(10_000),
+            create_random_segment_meta(1000),
+            create_random_segment_meta(1000),
+            create_random_segment_meta(1000),
+            create_random_segment_meta(10000),
+            create_random_segment_meta(10000),
             create_random_segment_meta(10),
             create_random_segment_meta(10),
             create_random_segment_meta(10),
@@ -193,19 +182,4 @@ mod tests {
         let result_list = test_merge_policy().compute_merge_candidates(&test_input);
         assert_eq!(result_list.len(), 1);
     }
-
-    #[test]
-    fn test_large_merge_segments() {
-        let test_input = vec![
-            create_random_segment_meta(1_000_000),
-            create_random_segment_meta(100_001),
-            create_random_segment_meta(100_000),
-            create_random_segment_meta(100_000),
-            create_random_segment_meta(100_000),
-        ];
-        let result_list = test_merge_policy().compute_merge_candidates(&test_input);
-        // Do not include large segments
-        assert_eq!(result_list.len(), 1);
-        assert_eq!(result_list[0].0.len(), 3)
-    }
 }

src/indexer/merger.rs

@@ -1492,46 +1492,4 @@ mod tests {
             assert_eq!(&vals, &[20]);
         }
     }
-
-    #[test]
-    fn merges_f64_fast_fields_correctly() -> crate::Result<()> {
-        let mut builder = schema::SchemaBuilder::new();
-
-        let fast_multi = IntOptions::default().set_fast(Cardinality::MultiValues);
-
-        let field = builder.add_f64_field("f64", schema::FAST);
-        let multi_field = builder.add_f64_field("f64s", fast_multi);
-
-        let index = Index::create_in_ram(builder.build());
-
-        let mut writer = index.writer_with_num_threads(1, 3_000_000)?;
-
-        // Make sure we'll attempt to merge every created segment
-        let mut policy = crate::indexer::LogMergePolicy::default();
-        policy.set_min_merge_size(2);
-        writer.set_merge_policy(Box::new(policy));
-
-        for i in 0..100 {
-            let mut doc = Document::new();
-            doc.add_f64(field, 42.0);
-
-            doc.add_f64(multi_field, 0.24);
-            doc.add_f64(multi_field, 0.27);
-
-            writer.add_document(doc);
-
-            if i % 5 == 0 {
-                writer.commit()?;
-            }
-        }
-
-        writer.commit()?;
-        writer.wait_merging_threads()?;
-
-        // If a merging thread fails, we should end up with more
-        // than one segment here
-        assert_eq!(1, index.searchable_segments()?.len());
-
-        Ok(())
-    }
 }

src/indexer/mod.rs

@@ -33,7 +33,6 @@ pub type DefaultMergePolicy = LogMergePolicy;
 mod tests {
     use crate::schema::{self, Schema};
     use crate::{Index, Term};
-
     #[test]
     fn test_advance_delete_bug() {
         let mut schema_builder = Schema::builder();

src/indexer/operation.rs

@@ -19,8 +19,6 @@ pub struct AddOperation {
 /// UserOperation is an enum type that encapsulates other operation types.
 #[derive(Eq, PartialEq, Debug)]
 pub enum UserOperation {
-    /// Add operation
     Add(Document),
-    /// Delete operation
     Delete(Term),
 }

src/indexer/segment_entry.rs

@@ -1,7 +1,7 @@
-use crate::common::BitSet;
 use crate::core::SegmentId;
 use crate::core::SegmentMeta;
 use crate::indexer::delete_queue::DeleteCursor;
+use bit_set::BitSet;
 use std::fmt;
 
 /// A segment entry describes the state of

src/indexer/segment_updater.rs

@@ -214,10 +214,6 @@ impl SegmentUpdater {
             self.pool.spawn_ok(async move {
                 let _ = sender.send(f.await);
             });
-        } else {
-            let _ = sender.send(Err(crate::TantivyError::SystemError(
-                "Segment updater killed".to_string(),
-            )));
         }
         receiver.unwrap_or_else(|_| {
             let err_msg =
@@ -330,11 +326,13 @@ impl SegmentUpdater {
     ) -> impl Future<Output = crate::Result<()>> {
         let segment_updater: SegmentUpdater = self.clone();
         self.schedule_future(async move {
-            let segment_entries = segment_updater.purge_deletes(opstamp)?;
-            segment_updater.segment_manager.commit(segment_entries);
-            segment_updater.save_metas(opstamp, payload)?;
-            let _ = garbage_collect_files(segment_updater.clone()).await;
-            segment_updater.consider_merge_options().await;
+            if segment_updater.is_alive() {
+                let segment_entries = segment_updater.purge_deletes(opstamp)?;
+                segment_updater.segment_manager.commit(segment_entries);
+                segment_updater.save_metas(opstamp, payload)?;
+                let _ = garbage_collect_files(segment_updater.clone()).await;
+                segment_updater.consider_merge_options().await;
+            }
             Ok(())
         })
     }

src/indexer/segment_writer.rs

@@ -11,9 +11,10 @@ use crate::schema::Schema;
 use crate::schema::Term;
 use crate::schema::Value;
 use crate::schema::{Field, FieldEntry};
-use crate::tokenizer::{BoxTokenStream, PreTokenizedStream};
-use crate::tokenizer::{FacetTokenizer, TextAnalyzer};
-use crate::tokenizer::{TokenStreamChain, Tokenizer};
+use crate::tokenizer::BoxedTokenizer;
+use crate::tokenizer::FacetTokenizer;
+use crate::tokenizer::PreTokenizedStream;
+use crate::tokenizer::{TokenStream, TokenStreamChain, Tokenizer};
 use crate::DocId;
 use crate::Opstamp;
 use crate::Result;
@@ -49,7 +50,7 @@ pub struct SegmentWriter {
     fast_field_writers: FastFieldsWriter,
     fieldnorms_writer: FieldNormsWriter,
     doc_opstamps: Vec<Opstamp>,
-    tokenizers: Vec<Option<TextAnalyzer>>,
+    tokenizers: Vec<Option<BoxedTokenizer>>,
 }
 
 impl SegmentWriter {
@@ -158,7 +159,7 @@ impl SegmentWriter {
                     }
                 }
                 FieldType::Str(_) => {
-                    let mut token_streams: Vec<BoxTokenStream> = vec![];
+                    let mut token_streams: Vec<Box<dyn TokenStream>> = vec![];
                     let mut offsets = vec![];
                     let mut total_offset = 0;
 
@@ -171,7 +172,7 @@ impl SegmentWriter {
                                 }
 
                                 token_streams
-                                    .push(PreTokenizedStream::from(tok_str.clone()).into());
+                                    .push(Box::new(PreTokenizedStream::from(tok_str.clone())));
                             }
                             Value::Str(ref text) => {
                                 if let Some(ref mut tokenizer) =
@@ -190,7 +191,8 @@ impl SegmentWriter {
                     let num_tokens = if token_streams.is_empty() {
                         0
                     } else {
-                        let mut token_stream = TokenStreamChain::new(offsets, token_streams);
+                        let mut token_stream: Box<dyn TokenStream> =
+                            Box::new(TokenStreamChain::new(offsets, token_streams));
                         self.multifield_postings
                             .index_text(doc_id, field, &mut token_stream)
                     };

src/indexer/stamper.rs

@@ -1,76 +1,18 @@
 use crate::Opstamp;
 use std::ops::Range;
-use std::sync::atomic::Ordering;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
 
-#[cfg(not(target_arch = "arm"))]
-mod atomic_impl {
-
-    use crate::Opstamp;
-    use std::sync::atomic::{AtomicU64, Ordering};
-
-    #[derive(Default)]
-    pub struct AtomicU64Wrapper(AtomicU64);
-
-    impl AtomicU64Wrapper {
-        pub fn new(first_opstamp: Opstamp) -> AtomicU64Wrapper {
-            AtomicU64Wrapper(AtomicU64::new(first_opstamp as u64))
-        }
-
-        pub fn fetch_add(&self, val: u64, order: Ordering) -> u64 {
-            self.0.fetch_add(val as u64, order) as u64
-        }
-
-        pub fn revert(&self, val: u64, order: Ordering) -> u64 {
-            self.0.store(val, order);
-            val
-        }
-    }
-}
-
-#[cfg(target_arch = "arm")]
-mod atomic_impl {
-
-    use crate::Opstamp;
-    /// Under other architecture, we rely on a mutex.
-    use std::sync::atomic::Ordering;
-    use std::sync::RwLock;
-
-    #[derive(Default)]
-    pub struct AtomicU64Wrapper(RwLock<u64>);
-
-    impl AtomicU64Wrapper {
-        pub fn new(first_opstamp: Opstamp) -> AtomicU64Wrapper {
-            AtomicU64Wrapper(RwLock::new(first_opstamp))
-        }
-
-        pub fn fetch_add(&self, incr: u64, _order: Ordering) -> u64 {
-            let mut lock = self.0.write().unwrap();
-            let previous_val = *lock;
-            *lock = previous_val + incr;
-            previous_val
-        }
-
-        pub fn revert(&self, val: u64, _order: Ordering) -> u64 {
-            let mut lock = self.0.write().unwrap();
-            *lock = val;
-            val
-        }
-    }
-}
-
-use self::atomic_impl::AtomicU64Wrapper;
-
 /// Stamper provides Opstamps, which is just an auto-increment id to label
 /// an operation.
 ///
 /// Cloning does not "fork" the stamp generation. The stamper actually wraps an `Arc`.
 #[derive(Clone, Default)]
-pub struct Stamper(Arc<AtomicU64Wrapper>);
+pub struct Stamper(Arc<AtomicU64>);
 
 impl Stamper {
     pub fn new(first_opstamp: Opstamp) -> Stamper {
-        Stamper(Arc::new(AtomicU64Wrapper::new(first_opstamp)))
+        Stamper(Arc::new(AtomicU64::new(first_opstamp)))
     }
 
     pub fn stamp(&self) -> Opstamp {
@@ -89,7 +31,8 @@ impl Stamper {
 
     /// Reverts the stamper to a given `Opstamp` value and returns it
     pub fn revert(&self, to_opstamp: Opstamp) -> Opstamp {
-        self.0.revert(to_opstamp, Ordering::SeqCst)
+        self.0.store(to_opstamp, Ordering::SeqCst);
+        to_opstamp
     }
 }
 

src/lib.rs

@@ -161,11 +161,10 @@ pub use self::snippet::{Snippet, SnippetGenerator};
 mod docset;
 pub use self::docset::{DocSet, SkipResult};
 pub use crate::common::{f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
-pub use crate::core::{Executor, SegmentComponent};
+pub use crate::core::SegmentComponent;
 pub use crate::core::{Index, IndexMeta, Searcher, Segment, SegmentId, SegmentMeta};
 pub use crate::core::{InvertedIndexReader, SegmentReader};
 pub use crate::directory::Directory;
-pub use crate::indexer::operation::UserOperation;
 pub use crate::indexer::IndexWriter;
 pub use crate::postings::Postings;
 pub use crate::reader::LeasedItem;
@@ -174,7 +173,6 @@ use std::fmt;
 
 use once_cell::sync::Lazy;
 
-/// Index format version.
 const INDEX_FORMAT_VERSION: u32 = 1;
 
 /// Structure version for the index.
@@ -941,73 +939,4 @@ mod tests {
             assert_eq!(fast_field_reader.get(0), 4f64)
         }
     }
-
-    // motivated by #729
-    #[test]
-    fn test_update_via_delete_insert() {
-        use crate::collector::Count;
-        use crate::indexer::NoMergePolicy;
-        use crate::query::AllQuery;
-        use crate::SegmentId;
-        use futures::executor::block_on;
-
-        const DOC_COUNT: u64 = 2u64;
-
-        let mut schema_builder = SchemaBuilder::default();
-        let id = schema_builder.add_u64_field("id", INDEXED);
-        let schema = schema_builder.build();
-
-        let index = Index::create_in_ram(schema.clone());
-        let index_reader = index.reader().unwrap();
-
-        let mut index_writer = index.writer(3_000_000).unwrap();
-        index_writer.set_merge_policy(Box::new(NoMergePolicy));
-
-        for doc_id in 0u64..DOC_COUNT {
-            index_writer.add_document(doc!(id => doc_id));
-        }
-        index_writer.commit().unwrap();
-
-        index_reader.reload().unwrap();
-        let searcher = index_reader.searcher();
-
-        assert_eq!(
-            searcher.search(&AllQuery, &Count).unwrap(),
-            DOC_COUNT as usize
-        );
-
-        // update the 10 elements by deleting and re-adding
-        for doc_id in 0u64..DOC_COUNT {
-            index_writer.delete_term(Term::from_field_u64(id, doc_id));
-            index_writer.commit().unwrap();
-            index_reader.reload().unwrap();
-            let doc = doc!(id =>  doc_id);
-            index_writer.add_document(doc);
-            index_writer.commit().unwrap();
-            index_reader.reload().unwrap();
-            let searcher = index_reader.searcher();
-            // The number of document should be stable.
-            assert_eq!(
-                searcher.search(&AllQuery, &Count).unwrap(),
-                DOC_COUNT as usize
-            );
-        }
-
-        index_reader.reload().unwrap();
-        let searcher = index_reader.searcher();
-        let segment_ids: Vec<SegmentId> = searcher
-            .segment_readers()
-            .into_iter()
-            .map(|reader| reader.segment_id())
-            .collect();
-        block_on(index_writer.merge(&segment_ids)).unwrap();
-
-        index_reader.reload().unwrap();
-        let searcher = index_reader.searcher();
-
-        assert_eq!(
-            searcher.search(&AllQuery, &Count).unwrap(),
-            DOC_COUNT as usize
-        );
-    }
 }

src/postings/serializer.rs

@@ -148,7 +148,8 @@ impl<'a> FieldSerializer<'a> {
             }
             _ => (false, false),
         };
-        let term_dictionary_builder = TermDictionaryBuilder::create(term_dictionary_write)?;
+        let term_dictionary_builder =
+            TermDictionaryBuilder::create(term_dictionary_write, &field_type)?;
         let postings_serializer =
             PostingsSerializer::new(postings_write, term_freq_enabled, position_enabled);
         let positions_serializer_opt = if position_enabled {

src/query/automaton_weight.rs

@@ -15,7 +15,6 @@ use tantivy_fst::Automaton;
 pub struct AutomatonWeight<A> {
     field: Field,
     automaton: Arc<A>,
-    boost: f32,
 }
 
 impl<A> AutomatonWeight<A>
@@ -27,15 +26,9 @@ where
         AutomatonWeight {
             field,
             automaton: automaton.into(),
-            boost: 1.0,
         }
     }
 
-    /// Boost the scorer by the given factor.
-    pub fn boost_by(self, boost: f32) -> Self {
-        Self { boost, ..self }
-    }
-
     fn automaton_stream<'a>(&'a self, term_dict: &'a TermDictionary) -> TermStreamer<'a, &'a A> {
         let automaton: &A = &*self.automaton;
         let term_stream_builder = term_dict.search(automaton);
@@ -65,7 +58,7 @@ where
             }
         }
         let doc_bitset = BitSetDocSet::from(doc_bitset);
-        Ok(Box::new(ConstScorer::with_score(doc_bitset, self.boost)))
+        Ok(Box::new(ConstScorer::new(doc_bitset)))
     }
 
     fn explain(&self, reader: &SegmentReader, doc: DocId) -> Result<Explanation> {

src/query/bm25.rs

@@ -34,7 +34,7 @@ pub struct BM25Weight {
 }
 
 impl BM25Weight {
-    pub fn for_terms(searcher: &Searcher, terms: &[Term], boost: f32) -> BM25Weight {
+    pub fn for_terms(searcher: &Searcher, terms: &[Term]) -> BM25Weight {
         assert!(!terms.is_empty(), "BM25 requires at least one term");
         let field = terms[0].field();
         for term in &terms[1..] {
@@ -75,11 +75,11 @@ impl BM25Weight {
                 .sum::<f32>();
             idf_explain = Explanation::new("idf", idf);
         }
-        BM25Weight::new(idf_explain, average_fieldnorm, boost)
+        BM25Weight::new(idf_explain, average_fieldnorm)
     }
 
-    fn new(idf_explain: Explanation, average_fieldnorm: f32, boost: f32) -> BM25Weight {
-        let weight = idf_explain.value() * (1f32 + K1) * boost;
+    fn new(idf_explain: Explanation, average_fieldnorm: f32) -> BM25Weight {
+        let weight = idf_explain.value() * (1f32 + K1);
         BM25Weight {
             idf_explain,
             weight,

src/query/fuzzy_query.rs

@@ -79,7 +79,6 @@ pub struct FuzzyTermQuery {
     transposition_cost_one: bool,
     ///
     prefix: bool,
-    boost: f32,
 }
 
 impl FuzzyTermQuery {
@@ -90,7 +89,6 @@ impl FuzzyTermQuery {
             distance,
             transposition_cost_one,
             prefix: false,
-            boost: 1.0,
         }
     }
 
@@ -101,22 +99,16 @@ impl FuzzyTermQuery {
             distance,
             transposition_cost_one,
             prefix: true,
-            boost: 1.0,
         }
     }
 
-    /// Boost the query score by the given factor.
-    pub fn boost_by(self, boost: f32) -> Self {
-        Self { boost, ..self }
-    }
-
     fn specialized_weight(&self) -> Result<AutomatonWeight<DFA>> {
         // LEV_BUILDER is a HashMap, whose `get` method returns an Option
         match LEV_BUILDER.get(&(self.distance, false)) {
             // Unwrap the option and build the Ok(AutomatonWeight)
             Some(automaton_builder) => {
                 let automaton = automaton_builder.build_dfa(self.term.text());
-                Ok(AutomatonWeight::new(self.term.field(), automaton).boost_by(self.boost))
+                Ok(AutomatonWeight::new(self.term.field(), automaton))
             }
             None => Err(InvalidArgument(format!(
                 "Levenshtein distance of {} is not allowed. Choose a value in the {:?} range",

src/query/phrase_query/phrase_query.rs

@@ -27,7 +27,6 @@ use std::collections::BTreeSet;
 pub struct PhraseQuery {
     field: Field,
     phrase_terms: Vec<(usize, Term)>,
-    boost: f32,
 }
 
 impl PhraseQuery {
@@ -58,15 +57,9 @@ impl PhraseQuery {
         PhraseQuery {
             field,
             phrase_terms: terms,
-            boost: 1.0,
         }
     }
 
-    /// Boost the query score by the given factor.
-    pub fn boost_by(self, boost: f32) -> Self {
-        Self { boost, ..self }
-    }
-
     /// The `Field` this `PhraseQuery` is targeting.
     pub fn field(&self) -> Field {
         self.field
@@ -104,7 +97,7 @@ impl PhraseQuery {
             )));
         }
         let terms = self.phrase_terms();
-        let bm25_weight = BM25Weight::for_terms(searcher, &terms, self.boost);
+        let bm25_weight = BM25Weight::for_terms(searcher, &terms);
         Ok(PhraseWeight::new(
             self.phrase_terms.clone(),
             bm25_weight,

src/query/query_parser/query_parser.rs

@@ -8,7 +8,7 @@ use crate::query::PhraseQuery;
 use crate::query::Query;
 use crate::query::RangeQuery;
 use crate::query::TermQuery;
-use crate::schema::{Facet, IndexRecordOption};
+use crate::schema::IndexRecordOption;
 use crate::schema::{Field, Schema};
 use crate::schema::{FieldType, Term};
 use crate::tokenizer::TokenizerManager;
@@ -319,10 +319,7 @@ impl QueryParser {
                     ))
                 }
             }
-            FieldType::HierarchicalFacet => {
-                let facet = Facet::from_text(phrase);
-                Ok(vec![(0, Term::from_field_text(field, facet.encoded_str()))])
-            }
+            FieldType::HierarchicalFacet => Ok(vec![(0, Term::from_field_text(field, phrase))]),
             FieldType::Bytes => {
                 let field_name = self.schema.get_field_name(field).to_string();
                 Err(QueryParserError::FieldNotIndexed(field_name))
@@ -533,7 +530,7 @@ mod test {
     use crate::schema::{IndexRecordOption, TextFieldIndexing, TextOptions};
     use crate::schema::{Schema, Term, INDEXED, STORED, STRING, TEXT};
     use crate::tokenizer::{
-        LowerCaser, SimpleTokenizer, StopWordFilter, TextAnalyzer, TokenizerManager,
+        LowerCaser, SimpleTokenizer, StopWordFilter, Tokenizer, TokenizerManager,
     };
     use crate::Index;
     use matches::assert_matches;
@@ -557,13 +554,12 @@ mod test {
         schema_builder.add_text_field("with_stop_words", text_options);
         schema_builder.add_date_field("date", INDEXED);
         schema_builder.add_f64_field("float", INDEXED);
-        schema_builder.add_facet_field("facet");
         let schema = schema_builder.build();
         let default_fields = vec![title, text];
         let tokenizer_manager = TokenizerManager::default();
         tokenizer_manager.register(
             "en_with_stop_words",
-            TextAnalyzer::from(SimpleTokenizer)
+            SimpleTokenizer
                 .filter(LowerCaser)
                 .filter(StopWordFilter::remove(vec!["the".to_string()])),
         );
@@ -592,13 +588,9 @@ mod test {
     }
 
     #[test]
-    pub fn test_parse_query_facet() {
+    pub fn test_parse_query_simple() {
         let query_parser = make_query_parser();
-        let query = query_parser.parse_query("facet:/root/branch/leaf").unwrap();
-        assert_eq!(
-            format!("{:?}", query),
-            "TermQuery(Term(field=11,bytes=[114, 111, 111, 116, 0, 98, 114, 97, 110, 99, 104, 0, 108, 101, 97, 102]))"
-        );
+        assert!(query_parser.parse_query("toto").is_ok());
     }
 
     #[test]

src/query/regex_query.rs

@@ -54,7 +54,6 @@ use tantivy_fst::Regex;
 pub struct RegexQuery {
     regex: Arc<Regex>,
     field: Field,
-    boost: f32,
 }
 
 impl RegexQuery {
@@ -70,17 +69,11 @@ impl RegexQuery {
         RegexQuery {
             regex: regex.into(),
             field,
-            boost: 1.0,
         }
     }
 
-    /// Boost the query score by the given factor.
-    pub fn boost_by(self, boost: f32) -> Self {
-        Self { boost, ..self }
-    }
-
     fn specialized_weight(&self) -> AutomatonWeight<Regex> {
-        AutomatonWeight::new(self.field, self.regex.clone()).boost_by(self.boost)
+        AutomatonWeight::new(self.field, self.regex.clone())
     }
 }
 

src/query/scorer.rs

@@ -56,11 +56,6 @@ impl<TDocSet: DocSet> ConstScorer<TDocSet> {
         }
     }
 
-    /// Creates a new `ConstScorer` with a custom score value
-    pub fn with_score(docset: TDocSet, score: f32) -> ConstScorer<TDocSet> {
-        ConstScorer { docset, score }
-    }
-
     /// Sets the constant score to a different value.
     pub fn set_score(&mut self, score: Score) {
         self.score = score;

src/query/term_query/mod.rs

@@ -45,35 +45,6 @@ mod tests {
         assert_eq!(term_scorer.score(), 0.28768212);
     }
 
-    #[test]
-    pub fn test_term_query_boost_by() {
-        let mut schema_builder = Schema::builder();
-        let text_field = schema_builder.add_text_field("text", STRING);
-        let schema = schema_builder.build();
-        let index = Index::create_in_ram(schema);
-        {
-            // writing the segment
-            let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
-            {
-                let doc = doc!(text_field => "a");
-                index_writer.add_document(doc);
-            }
-            assert!(index_writer.commit().is_ok());
-        }
-        let searcher = index.reader().unwrap().searcher();
-        let term_query = TermQuery::new(
-            Term::from_field_text(text_field, "a"),
-            IndexRecordOption::Basic,
-        )
-        .boost_by(42.0);
-        let term_weight = term_query.weight(&searcher, true).unwrap();
-        let segment_reader = searcher.segment_reader(0);
-        let mut term_scorer = term_weight.scorer(segment_reader).unwrap();
-        assert!(term_scorer.advance());
-        assert_eq!(term_scorer.doc(), 0);
-        assert_nearly_equals(0.28768212 * 42.0, term_scorer.score());
-    }
-
     #[test]
     pub fn test_term_weight() {
         let mut schema_builder = Schema::builder();

src/query/term_query/term_query.rs

@@ -61,7 +61,6 @@ use std::fmt;
 pub struct TermQuery {
     term: Term,
     index_record_option: IndexRecordOption,
-    boost: f32,
 }
 
 impl fmt::Debug for TermQuery {
@@ -76,15 +75,9 @@ impl TermQuery {
         TermQuery {
             term,
             index_record_option: segment_postings_options,
-            boost: 1.0,
         }
     }
 
-    /// Boost the query score by the given factor.
-    pub fn boost_by(self, boost: f32) -> Self {
-        Self { boost, ..self }
-    }
-
     /// The `Term` this query is built out of.
     pub fn term(&self) -> &Term {
         &self.term
@@ -97,7 +90,7 @@ impl TermQuery {
     /// This is useful for optimization purpose.
     pub fn specialized_weight(&self, searcher: &Searcher, scoring_enabled: bool) -> TermWeight {
         let term = self.term.clone();
-        let bm25_weight = BM25Weight::for_terms(searcher, &[term], self.boost);
+        let bm25_weight = BM25Weight::for_terms(searcher, &[term]);
         let index_record_option = if scoring_enabled {
             self.index_record_option
         } else {

src/schema/facet.rs

@@ -122,11 +122,6 @@ impl Facet {
     pub fn to_path(&self) -> Vec<&str> {
         self.encoded_str().split(|c| c == FACET_SEP_CHAR).collect()
     }
-
-    /// This function is the inverse of Facet::from(&str).
-    pub fn to_path_string(&self) -> String {
-        format!("{}", self.to_string())
-    }
 }
 
 impl Borrow<str> for Facet {
@@ -270,21 +265,4 @@ mod tests {
         let facet = Facet::from_path(v.iter());
         assert_eq!(facet.to_path(), v);
     }
-
-    #[test]
-    fn test_to_path_string() {
-        let v = ["first", "second", "third/not_fourth"];
-        let facet = Facet::from_path(v.iter());
-        assert_eq!(
-            facet.to_path_string(),
-            String::from("/first/second/third\\/not_fourth")
-        );
-    }
-
-    #[test]
-    fn test_to_path_string_empty() {
-        let v: Vec<&str> = vec![];
-        let facet = Facet::from_path(v.iter());
-        assert_eq!(facet.to_path_string(), "/");
-    }
 }

src/schema/schema.rs

@@ -166,7 +166,7 @@ impl SchemaBuilder {
     }
 
     /// Adds a field entry to the schema in build.
-    pub fn add_field(&mut self, field_entry: FieldEntry) -> Field {
+    fn add_field(&mut self, field_entry: FieldEntry) -> Field {
         let field = Field::from_field_id(self.fields.len() as u32);
         let field_name = field_entry.name().to_string();
         self.fields.push(field_entry);
@@ -401,7 +401,6 @@ pub enum DocParsingError {
 mod tests {
 
     use crate::schema::field_type::ValueParsingError;
-    use crate::schema::int_options::Cardinality::SingleValue;
     use crate::schema::schema::DocParsingError::NotJSON;
     use crate::schema::*;
     use matches::{assert_matches, matches};
@@ -716,94 +715,4 @@ mod tests {
             assert_matches!(json_err, Err(NotJSON(_)));
         }
     }
-
-    #[test]
-    pub fn test_schema_add_field() {
-        let mut schema_builder = SchemaBuilder::default();
-        let id_options = TextOptions::default().set_stored().set_indexing_options(
-            TextFieldIndexing::default()
-                .set_tokenizer("raw")
-                .set_index_option(IndexRecordOption::Basic),
-        );
-        let timestamp_options = IntOptions::default()
-            .set_stored()
-            .set_indexed()
-            .set_fast(SingleValue);
-        schema_builder.add_text_field("_id", id_options);
-        schema_builder.add_date_field("_timestamp", timestamp_options);
-
-        let schema_content = r#"[
-  {
-    "name": "text",
-    "type": "text",
-    "options": {
-      "indexing": {
-        "record": "position",
-        "tokenizer": "default"
-      },
-      "stored": false
-    }
-  },
-  {
-    "name": "popularity",
-    "type": "i64",
-    "options": {
-      "indexed": false,
-      "fast": "single",
-      "stored": true
-    }
-  }
-]"#;
-        let tmp_schema: Schema =
-            serde_json::from_str(&schema_content).expect("error while reading json");
-        for (_field, field_entry) in tmp_schema.fields() {
-            schema_builder.add_field(field_entry.clone());
-        }
-
-        let schema = schema_builder.build();
-        let schema_json = serde_json::to_string_pretty(&schema).unwrap();
-        let expected = r#"[
-  {
-    "name": "_id",
-    "type": "text",
-    "options": {
-      "indexing": {
-        "record": "basic",
-        "tokenizer": "raw"
-      },
-      "stored": true
-    }
-  },
-  {
-    "name": "_timestamp",
-    "type": "date",
-    "options": {
-      "indexed": true,
-      "fast": "single",
-      "stored": true
-    }
-  },
-  {
-    "name": "text",
-    "type": "text",
-    "options": {
-      "indexing": {
-        "record": "position",
-        "tokenizer": "default"
-      },
-      "stored": false
-    }
-  },
-  {
-    "name": "popularity",
-    "type": "i64",
-    "options": {
-      "indexed": false,
-      "fast": "single",
-      "stored": true
-    }
-  }
-]"#;
-        assert_eq!(schema_json, expected);
-    }
 }

src/schema/value.rs

@@ -209,8 +209,8 @@ impl From<f64> for Value {
     }
 }
 
-impl From<crate::DateTime> for Value {
-    fn from(date_time: crate::DateTime) -> Value {
+impl From<DateTime> for Value {
+    fn from(date_time: DateTime) -> Value {
         Value::Date(date_time)
     }
 }

src/snippet/mod.rs

@@ -1,7 +1,8 @@
 use crate::query::Query;
 use crate::schema::Field;
 use crate::schema::Value;
-use crate::tokenizer::{TextAnalyzer, Token};
+use crate::tokenizer::BoxedTokenizer;
+use crate::tokenizer::{Token, TokenStream};
 use crate::Document;
 use crate::Result;
 use crate::Searcher;
@@ -141,7 +142,7 @@ impl Snippet {
 /// Fragments must be valid in the sense that `&text[fragment.start..fragment.stop]`\
 /// has to be a valid string.
 fn search_fragments<'a>(
-    tokenizer: &TextAnalyzer,
+    tokenizer: &BoxedTokenizer,
     text: &'a str,
     terms: &BTreeMap<String, f32>,
     max_num_chars: usize,
@@ -250,7 +251,7 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str)
 /// ```
 pub struct SnippetGenerator {
     terms_text: BTreeMap<String, f32>,
-    tokenizer: TextAnalyzer,
+    tokenizer: BoxedTokenizer,
     field: Field,
     max_num_chars: usize,
 }
@@ -346,11 +347,12 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet() {
+        let boxed_tokenizer = SimpleTokenizer.into();
         let terms = btreemap! {
             String::from("rust") => 1.0,
             String::from("language") => 0.9
         };
-        let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 100);
+        let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 100);
         assert_eq!(fragments.len(), 7);
         {
             let first = &fragments[0];
@@ -372,12 +374,13 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_scored_fragment() {
+        let boxed_tokenizer = SimpleTokenizer.into();
         {
             let terms = btreemap! {
                 String::from("rust") =>1.0f32,
                 String::from("language") => 0.9f32
             };
-            let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 20);
+            let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 20);
             {
                 let first = &fragments[0];
                 assert_eq!(first.score, 1.0);
@@ -386,12 +389,13 @@ Survey in 2016, 2017, and 2018."#;
             let snippet = select_best_fragment_combination(&fragments[..], &TEST_TEXT);
             assert_eq!(snippet.to_html(), "<b>Rust</b> is a systems")
         }
+        let boxed_tokenizer = SimpleTokenizer.into();
         {
             let terms = btreemap! {
                 String::from("rust") =>0.9f32,
                 String::from("language") => 1.0f32
             };
-            let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 20);
+            let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 20);
             //assert_eq!(fragments.len(), 7);
             {
                 let first = &fragments[0];
@@ -405,12 +409,14 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_in_second_fragment() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
         let text = "a b c d e f g";
 
         let mut terms = BTreeMap::new();
         terms.insert(String::from("c"), 1.0);
 
-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);
 
         assert_eq!(fragments.len(), 1);
         {
@@ -427,12 +433,14 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_with_term_at_the_end_of_fragment() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
         let text = "a b c d e f f g";
 
         let mut terms = BTreeMap::new();
         terms.insert(String::from("f"), 1.0);
 
-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);
 
         assert_eq!(fragments.len(), 2);
         {
@@ -449,13 +457,15 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_with_second_fragment_has_the_highest_score() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
         let text = "a b c d e f g";
 
         let mut terms = BTreeMap::new();
         terms.insert(String::from("f"), 1.0);
         terms.insert(String::from("a"), 0.9);
 
-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 7);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 7);
 
         assert_eq!(fragments.len(), 2);
         {
@@ -472,12 +482,14 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_with_term_not_in_text() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
         let text = "a b c d";
 
         let mut terms = BTreeMap::new();
         terms.insert(String::from("z"), 1.0);
 
-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);
 
         assert_eq!(fragments.len(), 0);
 
@@ -488,10 +500,12 @@ Survey in 2016, 2017, and 2018."#;
 
     #[test]
     fn test_snippet_with_no_terms() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
         let text = "a b c d";
 
         let terms = BTreeMap::new();
-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);
         assert_eq!(fragments.len(), 0);
 
         let snippet = select_best_fragment_combination(&fragments[..], &text);

src/store/compression_snap.rs

@@ -5,7 +5,7 @@ use std::io::{self, Read, Write};
 /// Name of the compression scheme used in the doc store.
 ///
 /// This name is appended to the version string of tantivy.
-pub const COMPRESSION: &str = "snappy";
+pub const COMPRESSION: &'static str = "snappy";
 
 pub fn compress(uncompressed: &[u8], compressed: &mut Vec<u8>) -> io::Result<()> {
     compressed.clear();

src/termdict/mod.rs

@@ -38,7 +38,7 @@ mod tests {
     use crate::core::Index;
     use crate::directory::{Directory, RAMDirectory, ReadOnlySource};
     use crate::postings::TermInfo;
-    use crate::schema::{Document, Schema, TEXT};
+    use crate::schema::{Document, FieldType, Schema, TEXT};
     use std::path::PathBuf;
     use std::str;
 
@@ -52,12 +52,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_empty_term_dictionary() {
-        let empty = TermDictionary::empty();
-        assert!(empty.stream().next().is_none());
-    }
-
     #[test]
     fn test_term_ordinals() {
         const COUNTRIES: [&'static str; 7] = [
@@ -73,7 +67,9 @@ mod tests {
         let path = PathBuf::from("TermDictionary");
         {
             let write = directory.open_write(&path).unwrap();
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(write).unwrap();
+            let field_type = FieldType::Str(TEXT);
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(write, &field_type).unwrap();
             for term in COUNTRIES.iter() {
                 term_dictionary_builder
                     .insert(term.as_bytes(), &make_term_info(0u64))
@@ -97,7 +93,9 @@ mod tests {
         let path = PathBuf::from("TermDictionary");
         {
             let write = directory.open_write(&path).unwrap();
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(write).unwrap();
+            let field_type = FieldType::Str(TEXT);
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(write, &field_type).unwrap();
             term_dictionary_builder
                 .insert("abc".as_bytes(), &make_term_info(34u64))
                 .unwrap();
@@ -181,8 +179,10 @@ mod tests {
         let ids: Vec<_> = (0u32..10_000u32)
             .map(|i| (format!("doc{:0>6}", i), i))
             .collect();
+        let field_type = FieldType::Str(TEXT);
         let buffer: Vec<u8> = {
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
             for &(ref id, ref i) in &ids {
                 term_dictionary_builder
                     .insert(id.as_bytes(), &make_term_info(*i as u64))
@@ -209,8 +209,10 @@ mod tests {
 
     #[test]
     fn test_stream_high_range_prefix_suffix() {
+        let field_type = FieldType::Str(TEXT);
         let buffer: Vec<u8> = {
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
             // term requires more than 16bits
             term_dictionary_builder
                 .insert("abcdefghijklmnopqrstuvwxy", &make_term_info(1))
@@ -242,8 +244,10 @@ mod tests {
         let ids: Vec<_> = (0u32..10_000u32)
             .map(|i| (format!("doc{:0>6}", i), i))
             .collect();
+        let field_type = FieldType::Str(TEXT);
         let buffer: Vec<u8> = {
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
             for &(ref id, ref i) in &ids {
                 term_dictionary_builder
                     .insert(id.as_bytes(), &make_term_info(*i as u64))
@@ -309,8 +313,10 @@ mod tests {
 
     #[test]
     fn test_empty_string() {
+        let field_type = FieldType::Str(TEXT);
         let buffer: Vec<u8> = {
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
             term_dictionary_builder
                 .insert(&[], &make_term_info(1 as u64))
                 .unwrap();
@@ -331,8 +337,10 @@ mod tests {
 
     #[test]
     fn test_stream_range_boundaries() {
+        let field_type = FieldType::Str(TEXT);
         let buffer: Vec<u8> = {
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(vec![], &field_type).unwrap();
             for i in 0u8..10u8 {
                 let number_arr = [i; 1];
                 term_dictionary_builder
@@ -344,91 +352,41 @@ mod tests {
         let source = ReadOnlySource::from(buffer);
         let term_dictionary: TermDictionary = TermDictionary::from_source(&source);
 
-        let value_list = |mut streamer: TermStreamer<'_>, backwards: bool| {
+        let value_list = |mut streamer: TermStreamer<'_>| {
             let mut res: Vec<u32> = vec![];
             while let Some((_, ref v)) = streamer.next() {
                 res.push(v.doc_freq);
             }
-            if backwards {
-                res.reverse();
-            }
             res
         };
-        {
-            let range = term_dictionary.range().backward().into_stream();
-            assert_eq!(
-                value_list(range, true),
-                vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
-            );
-        }
         {
             let range = term_dictionary.range().ge([2u8]).into_stream();
             assert_eq!(
-                value_list(range, false),
-                vec![2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
-            );
-        }
-        {
-            let range = term_dictionary.range().ge([2u8]).backward().into_stream();
-            assert_eq!(
-                value_list(range, true),
+                value_list(range),
                 vec![2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
             );
         }
         {
             let range = term_dictionary.range().gt([2u8]).into_stream();
             assert_eq!(
-                value_list(range, false),
-                vec![3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
-            );
-        }
-        {
-            let range = term_dictionary.range().gt([2u8]).backward().into_stream();
-            assert_eq!(
-                value_list(range, true),
+                value_list(range),
                 vec![3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
             );
         }
         {
             let range = term_dictionary.range().lt([6u8]).into_stream();
-            assert_eq!(
-                value_list(range, false),
-                vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32]
-            );
-        }
-        {
-            let range = term_dictionary.range().lt([6u8]).backward().into_stream();
-            assert_eq!(
-                value_list(range, true),
-                vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32]
-            );
+            assert_eq!(value_list(range), vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32]);
         }
         {
             let range = term_dictionary.range().le([6u8]).into_stream();
             assert_eq!(
-                value_list(range, false),
-                vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32]
-            );
-        }
-        {
-            let range = term_dictionary.range().le([6u8]).backward().into_stream();
-            assert_eq!(
-                value_list(range, true),
+                value_list(range),
                 vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32]
             );
         }
         {
             let range = term_dictionary.range().ge([0u8]).lt([5u8]).into_stream();
-            assert_eq!(value_list(range, false), vec![0u32, 1u32, 2u32, 3u32, 4u32]);
-        }
-        {
-            let range = term_dictionary
-                .range()
-                .ge([0u8])
-                .lt([5u8])
-                .backward()
-                .into_stream();
-            assert_eq!(value_list(range, true), vec![0u32, 1u32, 2u32, 3u32, 4u32]);
+            assert_eq!(value_list(range), vec![0u32, 1u32, 2u32, 3u32, 4u32]);
         }
     }
 
@@ -450,7 +408,9 @@ mod tests {
         let path = PathBuf::from("TermDictionary");
         {
             let write = directory.open_write(&path).unwrap();
-            let mut term_dictionary_builder = TermDictionaryBuilder::create(write).unwrap();
+            let field_type = FieldType::Str(TEXT);
+            let mut term_dictionary_builder =
+                TermDictionaryBuilder::create(write, &field_type).unwrap();
             for term in COUNTRIES.iter() {
                 term_dictionary_builder
                     .insert(term.as_bytes(), &make_term_info(0u64))

src/termdict/streamer.rs

@@ -51,12 +51,6 @@ where
         self
     }
 
-    /// Iterate over the range backwards.
-    pub fn backward(mut self) -> Self {
-        self.stream_builder = self.stream_builder.backward();
-        self
-    }
-
     /// Creates the stream corresponding to the range
     /// of terms defined using the `TermStreamerBuilder`.
     pub fn into_stream(self) -> TermStreamer<'a, A> {

src/termdict/termdict.rs

@@ -4,8 +4,8 @@ use crate::common::BinarySerializable;
 use crate::common::CountingWriter;
 use crate::directory::ReadOnlySource;
 use crate::postings::TermInfo;
+use crate::schema::FieldType;
 use crate::termdict::TermOrdinal;
-use once_cell::sync::Lazy;
 use std::io::{self, Write};
 use tantivy_fst;
 use tantivy_fst::raw::Fst;
@@ -29,7 +29,7 @@ where
     W: Write,
 {
     /// Creates a new `TermDictionaryBuilder`
-    pub fn create(w: W) -> io::Result<Self> {
+    pub fn create(w: W, _field_type: &FieldType) -> io::Result<Self> {
         let fst_builder = tantivy_fst::MapBuilder::new(w).map_err(convert_fst_error)?;
         Ok(TermDictionaryBuilder {
             fst_builder,
@@ -92,14 +92,6 @@ fn open_fst_index(source: ReadOnlySource) -> tantivy_fst::Map<ReadOnlySource> {
     tantivy_fst::Map::from(fst)
 }
 
-static EMPTY_DATA_SOURCE: Lazy<ReadOnlySource> = Lazy::new(|| {
-    let term_dictionary_data: Vec<u8> = TermDictionaryBuilder::create(Vec::<u8>::new())
-        .expect("Creating a TermDictionaryBuilder in a Vec<u8> should never fail")
-        .finish()
-        .expect("Writing in a Vec<u8> should never fail");
-    ReadOnlySource::from(term_dictionary_data)
-});
-
 /// The term dictionary contains all of the terms in
 /// `tantivy index` in a sorted manner.
 ///
@@ -130,8 +122,14 @@ impl TermDictionary {
     }
 
     /// Creates an empty term dictionary which contains no terms.
-    pub fn empty() -> Self {
-        TermDictionary::from_source(&*EMPTY_DATA_SOURCE)
+    pub fn empty(field_type: &FieldType) -> Self {
+        let term_dictionary_data: Vec<u8> =
+            TermDictionaryBuilder::create(Vec::<u8>::new(), &field_type)
+                .expect("Creating a TermDictionaryBuilder in a Vec<u8> should never fail")
+                .finish()
+                .expect("Writing in a Vec<u8> should never fail");
+        let source = ReadOnlySource::from(term_dictionary_data);
+        Self::from_source(&source)
     }
 
     /// Returns the number of terms in the dictionary.

src/tokenizer/alphanum_only.rs

@@ -2,7 +2,7 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let tokenizer = TextAnalyzer::from(RawTokenizer)
+//! let tokenizer = RawTokenizer
 //!   .filter(AlphaNumOnlyFilter);
 //!
 //! let mut stream = tokenizer.token_stream("hello there");
@@ -10,7 +10,7 @@
 //! // contains a space
 //! assert!(stream.next().is_none());
 //!
-//! let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+//! let tokenizer = SimpleTokenizer
 //!   .filter(AlphaNumOnlyFilter);
 //!
 //! let mut stream = tokenizer.token_stream("hello there 💣");
@@ -19,30 +19,56 @@
 //! // the "emoji" is dropped because its not an alphanum
 //! assert!(stream.next().is_none());
 //! ```
-use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
+use super::{Token, TokenFilter, TokenStream};
 
 /// `TokenFilter` that removes all tokens that contain non
 /// ascii alphanumeric characters.
 #[derive(Clone)]
 pub struct AlphaNumOnlyFilter;
 
-pub struct AlphaNumOnlyFilterStream<'a> {
-    tail: BoxTokenStream<'a>,
+pub struct AlphaNumOnlyFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    tail: TailTokenStream,
 }
 
-impl<'a> AlphaNumOnlyFilterStream<'a> {
+impl<TailTokenStream> AlphaNumOnlyFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
     fn predicate(&self, token: &Token) -> bool {
         token.text.chars().all(|c| c.is_ascii_alphanumeric())
     }
-}
 
-impl TokenFilter for AlphaNumOnlyFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(AlphaNumOnlyFilterStream { tail: token_stream })
+    fn wrap(tail: TailTokenStream) -> AlphaNumOnlyFilterStream<TailTokenStream> {
+        AlphaNumOnlyFilterStream { tail }
     }
 }
 
-impl<'a> TokenStream for AlphaNumOnlyFilterStream<'a> {
+impl<TailTokenStream> TokenFilter<TailTokenStream> for AlphaNumOnlyFilter
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = AlphaNumOnlyFilterStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+        AlphaNumOnlyFilterStream::wrap(token_stream)
+    }
+}
+
+impl<TailTokenStream> TokenStream for AlphaNumOnlyFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+
     fn advance(&mut self) -> bool {
         while self.tail.advance() {
             if self.predicate(self.tail.token()) {
@@ -52,12 +78,4 @@ impl<'a> TokenStream for AlphaNumOnlyFilterStream<'a> {
 
         false
     }
-
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
-    }
 }

src/tokenizer/ascii_folding_filter.rs

@@ -1,4 +1,4 @@
-use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
+use super::{Token, TokenFilter, TokenStream};
 use std::mem;
 
 /// This class converts alphabetic, numeric, and symbolic Unicode characters
@@ -7,21 +7,26 @@ use std::mem;
 #[derive(Clone)]
 pub struct AsciiFoldingFilter;
 
-impl TokenFilter for AsciiFoldingFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        From::from(AsciiFoldingFilterTokenStream {
-            tail: token_stream,
-            buffer: String::with_capacity(100),
-        })
+impl<TailTokenStream> TokenFilter<TailTokenStream> for AsciiFoldingFilter
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = AsciiFoldingFilterTokenStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+        AsciiFoldingFilterTokenStream::wrap(token_stream)
     }
 }
 
-pub struct AsciiFoldingFilterTokenStream<'a> {
+pub struct AsciiFoldingFilterTokenStream<TailTokenStream> {
     buffer: String,
-    tail: BoxTokenStream<'a>,
+    tail: TailTokenStream,
 }
 
-impl<'a> TokenStream for AsciiFoldingFilterTokenStream<'a> {
+impl<TailTokenStream> TokenStream for AsciiFoldingFilterTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
     fn advance(&mut self) -> bool {
         if !self.tail.advance() {
             return false;
@@ -43,6 +48,18 @@ impl<'a> TokenStream for AsciiFoldingFilterTokenStream<'a> {
     }
 }
 
+impl<TailTokenStream> AsciiFoldingFilterTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn wrap(tail: TailTokenStream) -> AsciiFoldingFilterTokenStream<TailTokenStream> {
+        AsciiFoldingFilterTokenStream {
+            tail,
+            buffer: String::with_capacity(100),
+        }
+    }
+}
+
 // Returns a string that represents the ascii folded version of
 // the character. If the `char` does not require ascii folding
 // (e.g. simple ASCII chars like `A`) or if the `char`
@@ -1544,7 +1561,8 @@ mod tests {
     use crate::tokenizer::AsciiFoldingFilter;
     use crate::tokenizer::RawTokenizer;
     use crate::tokenizer::SimpleTokenizer;
-    use crate::tokenizer::TextAnalyzer;
+    use crate::tokenizer::TokenStream;
+    use crate::tokenizer::Tokenizer;
     use std::iter;
 
     #[test]
@@ -1561,7 +1579,7 @@ mod tests {
 
     fn folding_helper(text: &str) -> Vec<String> {
         let mut tokens = Vec::new();
-        TextAnalyzer::from(SimpleTokenizer)
+        SimpleTokenizer
             .filter(AsciiFoldingFilter)
             .token_stream(text)
             .process(&mut |token| {
@@ -1571,9 +1589,7 @@ mod tests {
     }
 
     fn folding_using_raw_tokenizer_helper(text: &str) -> String {
-        let mut token_stream = TextAnalyzer::from(RawTokenizer)
-            .filter(AsciiFoldingFilter)
-            .token_stream(text);
+        let mut token_stream = RawTokenizer.filter(AsciiFoldingFilter).token_stream(text);
         token_stream.advance();
         token_stream.token().text.clone()
     }

src/tokenizer/facet_tokenizer.rs

@@ -1,4 +1,4 @@
-use super::{BoxTokenStream, Token, TokenStream, Tokenizer};
+use super::{Token, TokenStream, Tokenizer};
 use crate::schema::FACET_SEP_BYTE;
 
 /// The `FacetTokenizer` process a `Facet` binary representation
@@ -25,14 +25,15 @@ pub struct FacetTokenStream<'a> {
     token: Token,
 }
 
-impl Tokenizer for FacetTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+impl<'a> Tokenizer<'a> for FacetTokenizer {
+    type TokenStreamImpl = FacetTokenStream<'a>;
+
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
         FacetTokenStream {
             text,
             state: State::RootFacetNotEmitted, //< pos is the first char that has not been processed yet.
             token: Token::default(),
         }
-        .into()
     }
 }
 
@@ -83,7 +84,7 @@ mod tests {
 
     use super::FacetTokenizer;
     use crate::schema::Facet;
-    use crate::tokenizer::{Token, Tokenizer};
+    use crate::tokenizer::{Token, TokenStream, Tokenizer};
 
     #[test]
     fn test_facet_tokenizer() {

src/tokenizer/lower_caser.rs

@@ -1,23 +1,24 @@
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;
 use std::mem;
 
-impl TokenFilter for LowerCaser {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(LowerCaserTokenStream {
-            tail: token_stream,
-            buffer: String::with_capacity(100),
-        })
-    }
-}
-
 /// Token filter that lowercase terms.
 #[derive(Clone)]
 pub struct LowerCaser;
 
-pub struct LowerCaserTokenStream<'a> {
+impl<TailTokenStream> TokenFilter<TailTokenStream> for LowerCaser
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = LowerCaserTokenStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+        LowerCaserTokenStream::wrap(token_stream)
+    }
+}
+
+pub struct LowerCaserTokenStream<TailTokenStream> {
     buffer: String,
-    tail: BoxTokenStream<'a>,
+    tail: TailTokenStream,
 }
 
 // writes a lowercased version of text into output.
@@ -30,7 +31,18 @@ fn to_lowercase_unicode(text: &mut String, output: &mut String) {
     }
 }
 
-impl<'a> TokenStream for LowerCaserTokenStream<'a> {
+impl<TailTokenStream> TokenStream for LowerCaserTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+
     fn advance(&mut self) -> bool {
         if !self.tail.advance() {
             return false;
@@ -44,19 +56,26 @@ impl<'a> TokenStream for LowerCaserTokenStream<'a> {
         }
         true
     }
+}
 
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
+impl<TailTokenStream> LowerCaserTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn wrap(tail: TailTokenStream) -> LowerCaserTokenStream<TailTokenStream> {
+        LowerCaserTokenStream {
+            tail,
+            buffer: String::with_capacity(100),
+        }
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use crate::tokenizer::{LowerCaser, SimpleTokenizer, TextAnalyzer};
+    use crate::tokenizer::LowerCaser;
+    use crate::tokenizer::SimpleTokenizer;
+    use crate::tokenizer::TokenStream;
+    use crate::tokenizer::Tokenizer;
 
     #[test]
     fn test_to_lower_case() {
@@ -68,9 +87,7 @@ mod tests {
 
     fn lowercase_helper(text: &str) -> Vec<String> {
         let mut tokens = vec![];
-        let mut token_stream = TextAnalyzer::from(SimpleTokenizer)
-            .filter(LowerCaser)
-            .token_stream(text);
+        let mut token_stream = SimpleTokenizer.filter(LowerCaser).token_stream(text);
         while token_stream.advance() {
             let token_text = token_stream.token().text.clone();
             tokens.push(token_text);

src/tokenizer/mod.rs

@@ -64,7 +64,7 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let en_stem = TextAnalyzer::from(SimpleTokenizer)
+//! let en_stem = SimpleTokenizer
 //!     .filter(RemoveLongFilter::limit(40))
 //!     .filter(LowerCaser)
 //!     .filter(Stemmer::new(Language::English));
@@ -109,7 +109,7 @@
 //! let index = Index::create_in_ram(schema);
 //!
 //! // We need to register our tokenizer :
-//! let custom_en_tokenizer = TextAnalyzer::from(SimpleTokenizer)
+//! let custom_en_tokenizer = SimpleTokenizer
 //!     .filter(RemoveLongFilter::limit(40))
 //!     .filter(LowerCaser);
 //! index
@@ -143,11 +143,10 @@ pub use self::simple_tokenizer::SimpleTokenizer;
 pub use self::stemmer::{Language, Stemmer};
 pub use self::stop_word_filter::StopWordFilter;
 pub(crate) use self::token_stream_chain::TokenStreamChain;
+pub use self::tokenizer::BoxedTokenizer;
 
 pub use self::tokenized_string::{PreTokenizedStream, PreTokenizedString};
-pub use self::tokenizer::{
-    BoxTokenFilter, BoxTokenStream, TextAnalyzer, Token, TokenFilter, TokenStream, Tokenizer,
-};
+pub use self::tokenizer::{Token, TokenFilter, TokenStream, Tokenizer};
 
 pub use self::tokenizer_manager::TokenizerManager;
 
@@ -161,9 +160,9 @@ pub const MAX_TOKEN_LEN: usize = u16::max_value() as usize - 4;
 #[cfg(test)]
 pub mod tests {
     use super::{
-        Language, LowerCaser, RemoveLongFilter, SimpleTokenizer, Stemmer, Token, TokenizerManager,
+        Language, LowerCaser, RemoveLongFilter, SimpleTokenizer, Stemmer, Token, Tokenizer,
+        TokenizerManager,
     };
-    use crate::tokenizer::TextAnalyzer;
 
     /// This is a function that can be used in tests and doc tests
     /// to assert a token's correctness.
@@ -230,7 +229,7 @@ pub mod tests {
         let tokenizer_manager = TokenizerManager::default();
         tokenizer_manager.register(
             "el_stem",
-            TextAnalyzer::from(SimpleTokenizer)
+            SimpleTokenizer
                 .filter(RemoveLongFilter::limit(40))
                 .filter(LowerCaser)
                 .filter(Stemmer::new(Language::Greek)),

src/tokenizer/ngram_tokenizer.rs

@@ -1,5 +1,4 @@
 use super::{Token, TokenStream, Tokenizer};
-use crate::tokenizer::BoxTokenStream;
 
 /// Tokenize the text by splitting words into n-grams of the given size(s)
 ///
@@ -130,9 +129,11 @@ pub struct NgramTokenStream<'a> {
     token: Token,
 }
 
-impl Tokenizer for NgramTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
-        From::from(NgramTokenStream {
+impl<'a> Tokenizer<'a> for NgramTokenizer {
+    type TokenStreamImpl = NgramTokenStream<'a>;
+
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
+        NgramTokenStream {
             ngram_charidx_iterator: StutteringIterator::new(
                 CodepointFrontiers::for_str(text),
                 self.min_gram,
@@ -141,7 +142,7 @@ impl Tokenizer for NgramTokenizer {
             prefix_only: self.prefix_only,
             text,
             token: Token::default(),
-        })
+        }
     }
 }
 
@@ -307,10 +308,10 @@ mod tests {
     use super::NgramTokenizer;
     use super::StutteringIterator;
     use crate::tokenizer::tests::assert_token;
-    use crate::tokenizer::tokenizer::Tokenizer;
-    use crate::tokenizer::{BoxTokenStream, Token};
+    use crate::tokenizer::tokenizer::{TokenStream, Tokenizer};
+    use crate::tokenizer::Token;
 
-    fn test_helper(mut tokenizer: BoxTokenStream) -> Vec<Token> {
+    fn test_helper<T: TokenStream>(mut tokenizer: T) -> Vec<Token> {
         let mut tokens: Vec<Token> = vec![];
         tokenizer.process(&mut |token: &Token| tokens.push(token.clone()));
         tokens

src/tokenizer/raw_tokenizer.rs

@@ -1,5 +1,4 @@
 use super::{Token, TokenStream, Tokenizer};
-use crate::tokenizer::BoxTokenStream;
 
 /// For each value of the field, emit a single unprocessed token.
 #[derive(Clone)]
@@ -10,8 +9,10 @@ pub struct RawTokenStream {
     has_token: bool,
 }
 
-impl Tokenizer for RawTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+impl<'a> Tokenizer<'a> for RawTokenizer {
+    type TokenStreamImpl = RawTokenStream;
+
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
         let token = Token {
             offset_from: 0,
             offset_to: text.len(),
@@ -23,7 +24,6 @@ impl Tokenizer for RawTokenizer {
             token,
             has_token: true,
         }
-        .into()
     }
 }
 

src/tokenizer/remove_long.rs

@@ -2,7 +2,7 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+//! let tokenizer = SimpleTokenizer
 //!   .filter(RemoveLongFilter::limit(5));
 //!
 //! let mut stream = tokenizer.token_stream("toolong nice");
@@ -13,7 +13,6 @@
 //! ```
 //!
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;
 
 /// `RemoveLongFilter` removes tokens that are longer
 /// than a given number of bytes (in UTF-8 representation).
@@ -32,27 +31,56 @@ impl RemoveLongFilter {
     }
 }
 
-impl<'a> RemoveLongFilterStream<'a> {
+impl<TailTokenStream> RemoveLongFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
     fn predicate(&self, token: &Token) -> bool {
         token.text.len() < self.token_length_limit
     }
-}
 
-impl TokenFilter for RemoveLongFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(RemoveLongFilterStream {
-            token_length_limit: self.length_limit,
-            tail: token_stream,
-        })
+    fn wrap(
+        token_length_limit: usize,
+        tail: TailTokenStream,
+    ) -> RemoveLongFilterStream<TailTokenStream> {
+        RemoveLongFilterStream {
+            token_length_limit,
+            tail,
+        }
     }
 }
 
-pub struct RemoveLongFilterStream<'a> {
-    token_length_limit: usize,
-    tail: BoxTokenStream<'a>,
+impl<TailTokenStream> TokenFilter<TailTokenStream> for RemoveLongFilter
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = RemoveLongFilterStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+        RemoveLongFilterStream::wrap(self.length_limit, token_stream)
+    }
 }
 
-impl<'a> TokenStream for RemoveLongFilterStream<'a> {
+pub struct RemoveLongFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    token_length_limit: usize,
+    tail: TailTokenStream,
+}
+
+impl<TailTokenStream> TokenStream for RemoveLongFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+
     fn advance(&mut self) -> bool {
         while self.tail.advance() {
             if self.predicate(self.tail.token()) {
@@ -61,12 +89,4 @@ impl<'a> TokenStream for RemoveLongFilterStream<'a> {
         }
         false
     }
-
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
-    }
 }

src/tokenizer/simple_tokenizer.rs

@@ -1,4 +1,3 @@
-use super::BoxTokenStream;
 use super::{Token, TokenStream, Tokenizer};
 use std::str::CharIndices;
 
@@ -12,13 +11,15 @@ pub struct SimpleTokenStream<'a> {
     token: Token,
 }
 
-impl Tokenizer for SimpleTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(SimpleTokenStream {
+impl<'a> Tokenizer<'a> for SimpleTokenizer {
+    type TokenStreamImpl = SimpleTokenStream<'a>;
+
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
+        SimpleTokenStream {
             text,
             chars: text.char_indices(),
             token: Token::default(),
-        })
+        }
     }
 }
 

src/tokenizer/stemmer.rs

@@ -1,5 +1,4 @@
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;
 use rust_stemmers::{self, Algorithm};
 
 /// Available stemmer languages.
@@ -76,22 +75,38 @@ impl Default for Stemmer {
     }
 }
 
-impl TokenFilter for Stemmer {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+impl<TailTokenStream> TokenFilter<TailTokenStream> for Stemmer
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = StemmerTokenStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
         let inner_stemmer = rust_stemmers::Stemmer::create(self.stemmer_algorithm);
-        BoxTokenStream::from(StemmerTokenStream {
-            tail: token_stream,
-            stemmer: inner_stemmer,
-        })
+        StemmerTokenStream::wrap(inner_stemmer, token_stream)
     }
 }
 
-pub struct StemmerTokenStream<'a> {
-    tail: BoxTokenStream<'a>,
+pub struct StemmerTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    tail: TailTokenStream,
     stemmer: rust_stemmers::Stemmer,
 }
 
-impl<'a> TokenStream for StemmerTokenStream<'a> {
+impl<TailTokenStream> TokenStream for StemmerTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+
     fn advance(&mut self) -> bool {
         if !self.tail.advance() {
             return false;
@@ -102,12 +117,16 @@ impl<'a> TokenStream for StemmerTokenStream<'a> {
         self.token_mut().text.push_str(&stemmed_str);
         true
     }
+}
 
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
+impl<TailTokenStream> StemmerTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn wrap(
+        stemmer: rust_stemmers::Stemmer,
+        tail: TailTokenStream,
+    ) -> StemmerTokenStream<TailTokenStream> {
+        StemmerTokenStream { tail, stemmer }
     }
 }

src/tokenizer/stop_word_filter.rs

@@ -2,7 +2,7 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+//! let tokenizer = SimpleTokenizer
 //!   .filter(StopWordFilter::remove(vec!["the".to_string(), "is".to_string()]));
 //!
 //! let mut stream = tokenizer.token_stream("the fox is crafty");
@@ -11,7 +11,6 @@
 //! assert!(stream.next().is_none());
 //! ```
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;
 use fnv::FnvHasher;
 use std::collections::HashSet;
 use std::hash::BuildHasherDefault;
@@ -49,27 +48,53 @@ impl StopWordFilter {
     }
 }
 
-pub struct StopWordFilterStream<'a> {
+pub struct StopWordFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
     words: StopWordHashSet,
-    tail: BoxTokenStream<'a>,
+    tail: TailTokenStream,
 }
 
-impl TokenFilter for StopWordFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(StopWordFilterStream {
-            words: self.words.clone(),
-            tail: token_stream,
-        })
+impl<TailTokenStream> TokenFilter<TailTokenStream> for StopWordFilter
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = StopWordFilterStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+        StopWordFilterStream::wrap(self.words.clone(), token_stream)
     }
 }
 
-impl<'a> StopWordFilterStream<'a> {
+impl<TailTokenStream> StopWordFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
     fn predicate(&self, token: &Token) -> bool {
         !self.words.contains(&token.text)
     }
+
+    fn wrap(
+        words: StopWordHashSet,
+        tail: TailTokenStream,
+    ) -> StopWordFilterStream<TailTokenStream> {
+        StopWordFilterStream { words, tail }
+    }
 }
 
-impl<'a> TokenStream for StopWordFilterStream<'a> {
+impl<TailTokenStream> TokenStream for StopWordFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+
     fn advance(&mut self) -> bool {
         while self.tail.advance() {
             if self.predicate(self.tail.token()) {
@@ -78,14 +103,6 @@ impl<'a> TokenStream for StopWordFilterStream<'a> {
         }
         false
     }
-
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
-    }
 }
 
 impl Default for StopWordFilter {

src/tokenizer/token_stream_chain.rs

@@ -1,21 +1,23 @@
-use crate::tokenizer::{BoxTokenStream, Token, TokenStream};
-use std::ops::DerefMut;
+use crate::tokenizer::{Token, TokenStream};
 
 const POSITION_GAP: usize = 2;
 
-pub(crate) struct TokenStreamChain<'a> {
+pub(crate) struct TokenStreamChain<TTokenStream: TokenStream> {
     offsets: Vec<usize>,
-    token_streams: Vec<BoxTokenStream<'a>>,
+    token_streams: Vec<TTokenStream>,
     position_shift: usize,
     stream_idx: usize,
     token: Token,
 }
 
-impl<'a> TokenStreamChain<'a> {
+impl<'a, TTokenStream> TokenStreamChain<TTokenStream>
+where
+    TTokenStream: TokenStream,
+{
     pub fn new(
         offsets: Vec<usize>,
-        token_streams: Vec<BoxTokenStream<'a>>,
-    ) -> TokenStreamChain<'a> {
+        token_streams: Vec<TTokenStream>,
+    ) -> TokenStreamChain<TTokenStream> {
         TokenStreamChain {
             offsets,
             stream_idx: 0,
@@ -26,10 +28,13 @@ impl<'a> TokenStreamChain<'a> {
     }
 }
 
-impl<'a> TokenStream for TokenStreamChain<'a> {
+impl<'a, TTokenStream> TokenStream for TokenStreamChain<TTokenStream>
+where
+    TTokenStream: TokenStream,
+{
     fn advance(&mut self) -> bool {
         while self.stream_idx < self.token_streams.len() {
-            let token_stream = self.token_streams[self.stream_idx].deref_mut();
+            let token_stream = &mut self.token_streams[self.stream_idx];
             if token_stream.advance() {
                 let token = token_stream.token();
                 let offset_offset = self.offsets[self.stream_idx];

src/tokenizer/tokenized_string.rs

@@ -1,4 +1,4 @@
-use crate::tokenizer::{BoxTokenStream, Token, TokenStream, TokenStreamChain};
+use crate::tokenizer::{Token, TokenStream, TokenStreamChain};
 use std::cmp::Ordering;
 
 /// Struct representing pre-tokenized text
@@ -41,9 +41,9 @@ impl PreTokenizedStream {
     /// Creates a TokenStream from PreTokenizedString array
     pub fn chain_tokenized_strings<'a>(
         tok_strings: &'a [&'a PreTokenizedString],
-    ) -> BoxTokenStream {
+    ) -> Box<dyn TokenStream + 'a> {
         if tok_strings.len() == 1 {
-            PreTokenizedStream::from((*tok_strings[0]).clone()).into()
+            Box::new(PreTokenizedStream::from((*tok_strings[0]).clone()))
         } else {
             let mut offsets = vec![];
             let mut total_offset = 0;
@@ -53,12 +53,11 @@ impl PreTokenizedStream {
                     total_offset += last_token.offset_to;
                 }
             }
-            // TODO remove the string cloning.
-            let token_streams: Vec<BoxTokenStream<'static>> = tok_strings
+            let token_streams: Vec<_> = tok_strings
                 .iter()
-                .map(|&tok_string| PreTokenizedStream::from((*tok_string).clone()).into())
+                .map(|tok_string| PreTokenizedStream::from((*tok_string).clone()))
                 .collect();
-            TokenStreamChain::new(offsets, token_streams).into()
+            Box::new(TokenStreamChain::new(offsets, token_streams))
         }
     }
 }

src/tokenizer/tokenizer.rs

@@ -2,7 +2,6 @@ use crate::tokenizer::TokenStreamChain;
 /// The tokenizer module contains all of the tools used to process
 /// text in `tantivy`.
 use std::borrow::{Borrow, BorrowMut};
-use std::ops::{Deref, DerefMut};
 
 /// Token
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
@@ -34,31 +33,20 @@ impl Default for Token {
     }
 }
 
-/// `TextAnalyzer` tokenizes an input text into tokens and modifies the resulting `TokenStream`.
+/// `Tokenizer` are in charge of splitting text into a stream of token
+/// before indexing.
 ///
-/// It simply wraps a `Tokenizer` and a list of `TokenFilter` that are applied sequentially.
-pub struct TextAnalyzer {
-    tokenizer: Box<dyn Tokenizer>,
-    token_filters: Vec<BoxTokenFilter>,
-}
+/// See the [module documentation](./index.html) for more detail.
+///
+/// # Warning
+///
+/// This API may change to use associated types.
+pub trait Tokenizer<'a>: Sized + Clone {
+    /// Type associated to the resulting tokenstream tokenstream.
+    type TokenStreamImpl: TokenStream;
 
-impl<T: Tokenizer> From<T> for TextAnalyzer {
-    fn from(tokenizer: T) -> Self {
-        TextAnalyzer::new(tokenizer, Vec::new())
-    }
-}
-
-impl TextAnalyzer {
-    /// Creates a new `TextAnalyzer` given a tokenizer and a vector of `BoxTokenFilter`.
-    ///
-    /// When creating a `TextAnalyzer` from a `Tokenizer` alone, prefer using
-    /// `TextAnalyzer::from(tokenizer)`.
-    pub fn new<T: Tokenizer>(tokenizer: T, token_filters: Vec<BoxTokenFilter>) -> TextAnalyzer {
-        TextAnalyzer {
-            tokenizer: Box::new(tokenizer),
-            token_filters,
-        }
-    }
+    /// Creates a token stream for a given `str`.
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl;
 
     /// Appends a token filter to the current tokenizer.
     ///
@@ -70,26 +58,90 @@ impl TextAnalyzer {
     /// ```rust
     /// use tantivy::tokenizer::*;
     ///
-    /// let en_stem = TextAnalyzer::from(SimpleTokenizer)
+    /// let en_stem = SimpleTokenizer
     ///     .filter(RemoveLongFilter::limit(40))
     ///     .filter(LowerCaser)
     ///     .filter(Stemmer::default());
     /// ```
     ///
-    pub fn filter<F: Into<BoxTokenFilter>>(mut self, token_filter: F) -> Self {
-        self.token_filters.push(token_filter.into());
-        self
+    fn filter<NewFilter>(self, new_filter: NewFilter) -> ChainTokenizer<NewFilter, Self>
+    where
+        NewFilter: TokenFilter<<Self as Tokenizer<'a>>::TokenStreamImpl>,
+    {
+        ChainTokenizer {
+            head: new_filter,
+            tail: self,
+        }
+    }
+}
+
+/// A boxed tokenizer
+trait BoxedTokenizerTrait: Send + Sync {
+    /// Tokenize a `&str`
+    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a>;
+
+    /// Tokenize an array`&str`
+    ///
+    /// The resulting `TokenStream` is equivalent to what would be obtained if the &str were
+    /// one concatenated `&str`, with an artificial position gap of `2` between the different fields
+    /// to prevent accidental `PhraseQuery` to match accross two terms.
+    fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<dyn TokenStream + 'b>;
+
+    /// Return a boxed clone of the tokenizer
+    fn boxed_clone(&self) -> BoxedTokenizer;
+}
+
+/// A boxed tokenizer
+pub struct BoxedTokenizer(Box<dyn BoxedTokenizerTrait>);
+
+impl<T> From<T> for BoxedTokenizer
+where
+    T: 'static + Send + Sync + for<'a> Tokenizer<'a>,
+{
+    fn from(tokenizer: T) -> BoxedTokenizer {
+        BoxedTokenizer(Box::new(BoxableTokenizer(tokenizer)))
+    }
+}
+
+impl BoxedTokenizer {
+    /// Tokenize a `&str`
+    pub fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
+        self.0.token_stream(text)
     }
 
     /// Tokenize an array`&str`
     ///
-    /// The resulting `BoxTokenStream` is equivalent to what would be obtained if the &str were
+    /// The resulting `TokenStream` is equivalent to what would be obtained if the &str were
     /// one concatenated `&str`, with an artificial position gap of `2` between the different fields
     /// to prevent accidental `PhraseQuery` to match accross two terms.
-    pub fn token_stream_texts<'a>(&self, texts: &'a [&'a str]) -> BoxTokenStream<'a> {
+    pub fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<dyn TokenStream + 'b> {
+        self.0.token_stream_texts(texts)
+    }
+}
+
+impl Clone for BoxedTokenizer {
+    fn clone(&self) -> BoxedTokenizer {
+        self.0.boxed_clone()
+    }
+}
+
+#[derive(Clone)]
+struct BoxableTokenizer<A>(A)
+where
+    A: for<'a> Tokenizer<'a> + Send + Sync;
+
+impl<A> BoxedTokenizerTrait for BoxableTokenizer<A>
+where
+    A: 'static + Send + Sync + for<'a> Tokenizer<'a>,
+{
+    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
+        Box::new(self.0.token_stream(text))
+    }
+
+    fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<dyn TokenStream + 'b> {
         assert!(!texts.is_empty());
         if texts.len() == 1 {
-            self.token_stream(texts[0])
+            Box::new(self.0.token_stream(texts[0]))
         } else {
             let mut offsets = vec![];
             let mut total_offset = 0;
@@ -97,124 +149,34 @@ impl TextAnalyzer {
                 offsets.push(total_offset);
                 total_offset += text.len();
             }
-            let token_streams: Vec<BoxTokenStream<'a>> = texts
-                .iter()
-                .cloned()
-                .map(|text| self.token_stream(text))
-                .collect();
-            From::from(TokenStreamChain::new(offsets, token_streams))
+            let token_streams: Vec<_> =
+                texts.iter().map(|text| self.0.token_stream(text)).collect();
+            Box::new(TokenStreamChain::new(offsets, token_streams))
         }
     }
 
-    /// Creates a token stream for a given `str`.
-    pub fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
-        let mut token_stream = self.tokenizer.token_stream(text);
-        for token_filter in &self.token_filters {
-            token_stream = token_filter.transform(token_stream);
-        }
-        token_stream
+    fn boxed_clone(&self) -> BoxedTokenizer {
+        self.0.clone().into()
     }
 }
 
-impl Clone for TextAnalyzer {
-    fn clone(&self) -> Self {
-        TextAnalyzer {
-            tokenizer: self.tokenizer.box_clone(),
-            token_filters: self
-                .token_filters
-                .iter()
-                .map(|token_filter| token_filter.box_clone())
-                .collect(),
-        }
-    }
-}
-
-/// `Tokenizer` are in charge of splitting text into a stream of token
-/// before indexing.
-///
-/// See the [module documentation](./index.html) for more detail.
-///
-/// # Warning
-///
-/// This API may change to use associated types.
-pub trait Tokenizer: 'static + Send + Sync + TokenizerClone {
-    /// Creates a token stream for a given `str`.
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a>;
-}
-
-pub trait TokenizerClone {
-    fn box_clone(&self) -> Box<dyn Tokenizer>;
-}
-
-impl<T: Tokenizer + Clone> TokenizerClone for T {
-    fn box_clone(&self) -> Box<dyn Tokenizer> {
-        Box::new(self.clone())
-    }
-}
-
-impl<'a> TokenStream for Box<dyn TokenStream + 'a> {
+impl<'b> TokenStream for Box<dyn TokenStream + 'b> {
     fn advance(&mut self) -> bool {
         let token_stream: &mut dyn TokenStream = self.borrow_mut();
         token_stream.advance()
     }
 
-    fn token<'b>(&'b self) -> &'b Token {
-        let token_stream: &'b (dyn TokenStream + 'a) = self.borrow();
+    fn token(&self) -> &Token {
+        let token_stream: &dyn TokenStream = self.borrow();
         token_stream.token()
     }
 
-    fn token_mut<'b>(&'b mut self) -> &'b mut Token {
-        let token_stream: &'b mut (dyn TokenStream + 'a) = self.borrow_mut();
+    fn token_mut(&mut self) -> &mut Token {
+        let token_stream: &mut dyn TokenStream = self.borrow_mut();
         token_stream.token_mut()
     }
 }
 
-/// Simple wrapper of `Box<dyn TokenStream + 'a>`.
-///
-/// See `TokenStream` for more information.
-pub struct BoxTokenStream<'a>(Box<dyn TokenStream + 'a>);
-
-impl<'a, T> From<T> for BoxTokenStream<'a>
-where
-    T: TokenStream + 'a,
-{
-    fn from(token_stream: T) -> BoxTokenStream<'a> {
-        BoxTokenStream(Box::new(token_stream))
-    }
-}
-
-impl<'a> Deref for BoxTokenStream<'a> {
-    type Target = dyn TokenStream + 'a;
-
-    fn deref(&self) -> &Self::Target {
-        &*self.0
-    }
-}
-impl<'a> DerefMut for BoxTokenStream<'a> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut *self.0
-    }
-}
-
-/// Simple wrapper of `Box<dyn TokenFilter + 'a>`.
-///
-/// See `TokenStream` for more information.
-pub struct BoxTokenFilter(Box<dyn TokenFilter>);
-
-impl Deref for BoxTokenFilter {
-    type Target = dyn TokenFilter;
-
-    fn deref(&self) -> &dyn TokenFilter {
-        &*self.0
-    }
-}
-
-impl<T: TokenFilter> From<T> for BoxTokenFilter {
-    fn from(tokenizer: T) -> BoxTokenFilter {
-        BoxTokenFilter(Box::new(tokenizer))
-    }
-}
-
 /// `TokenStream` is the result of the tokenization.
 ///
 /// It consists consumable stream of `Token`s.
@@ -224,7 +186,7 @@ impl<T: TokenFilter> From<T> for BoxTokenFilter {
 /// ```
 /// use tantivy::tokenizer::*;
 ///
-/// let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+/// let tokenizer = SimpleTokenizer
 ///        .filter(RemoveLongFilter::limit(40))
 ///        .filter(LowerCaser);
 /// let mut token_stream = tokenizer.token_stream("Hello, happy tax payer");
@@ -263,7 +225,7 @@ pub trait TokenStream {
     /// ```
     /// use tantivy::tokenizer::*;
     ///
-    /// let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+    /// let tokenizer = SimpleTokenizer
     ///       .filter(RemoveLongFilter::limit(40))
     ///       .filter(LowerCaser);
     /// let mut token_stream = tokenizer.token_stream("Hello, happy tax payer");
@@ -281,8 +243,6 @@ pub trait TokenStream {
 
     /// Helper function to consume the entire `TokenStream`
     /// and push the tokens to a sink function.
-    ///
-    /// Remove this.
     fn process(&mut self, sink: &mut dyn FnMut(&Token)) -> u32 {
         let mut num_tokens_pushed = 0u32;
         while self.advance() {
@@ -293,20 +253,33 @@ pub trait TokenStream {
     }
 }
 
-pub trait TokenFilterClone {
-    fn box_clone(&self) -> BoxTokenFilter;
+#[derive(Clone)]
+pub struct ChainTokenizer<HeadTokenFilterFactory, TailTokenizer> {
+    head: HeadTokenFilterFactory,
+    tail: TailTokenizer,
+}
+
+impl<'a, HeadTokenFilterFactory, TailTokenizer> Tokenizer<'a>
+    for ChainTokenizer<HeadTokenFilterFactory, TailTokenizer>
+where
+    HeadTokenFilterFactory: TokenFilter<TailTokenizer::TokenStreamImpl>,
+    TailTokenizer: Tokenizer<'a>,
+{
+    type TokenStreamImpl = HeadTokenFilterFactory::ResultTokenStream;
+
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
+        let tail_token_stream = self.tail.token_stream(text);
+        self.head.transform(tail_token_stream)
+    }
 }
 
 /// Trait for the pluggable components of `Tokenizer`s.
-pub trait TokenFilter: 'static + Send + Sync + TokenFilterClone {
-    /// Wraps a token stream and returns the modified one.
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a>;
-}
+pub trait TokenFilter<TailTokenStream: TokenStream>: Clone {
+    /// The resulting `TokenStream` type.
+    type ResultTokenStream: TokenStream;
 
-impl<T: TokenFilter + Clone> TokenFilterClone for T {
-    fn box_clone(&self) -> BoxTokenFilter {
-        BoxTokenFilter::from(self.clone())
-    }
+    /// Wraps a token stream and returns the modified one.
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream;
 }
 
 #[cfg(test)]

src/tokenizer/tokenizer_manager.rs

@@ -1,10 +1,11 @@
 use crate::tokenizer::stemmer::Language;
-use crate::tokenizer::tokenizer::TextAnalyzer;
+use crate::tokenizer::BoxedTokenizer;
 use crate::tokenizer::LowerCaser;
 use crate::tokenizer::RawTokenizer;
 use crate::tokenizer::RemoveLongFilter;
 use crate::tokenizer::SimpleTokenizer;
 use crate::tokenizer::Stemmer;
+use crate::tokenizer::Tokenizer;
 use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
 
@@ -22,16 +23,16 @@ use std::sync::{Arc, RwLock};
 ///  search engine.
 #[derive(Clone)]
 pub struct TokenizerManager {
-    tokenizers: Arc<RwLock<HashMap<String, TextAnalyzer>>>,
+    tokenizers: Arc<RwLock<HashMap<String, BoxedTokenizer>>>,
 }
 
 impl TokenizerManager {
     /// Registers a new tokenizer associated with a given name.
-    pub fn register<T>(&self, tokenizer_name: &str, tokenizer: T)
+    pub fn register<A>(&self, tokenizer_name: &str, tokenizer: A)
     where
-        TextAnalyzer: From<T>,
+        A: Into<BoxedTokenizer>,
     {
-        let boxed_tokenizer: TextAnalyzer = TextAnalyzer::from(tokenizer);
+        let boxed_tokenizer = tokenizer.into();
         self.tokenizers
             .write()
             .expect("Acquiring the lock should never fail")
@@ -39,7 +40,7 @@ impl TokenizerManager {
     }
 
     /// Accessing a tokenizer given its name.
-    pub fn get(&self, tokenizer_name: &str) -> Option<TextAnalyzer> {
+    pub fn get(&self, tokenizer_name: &str) -> Option<BoxedTokenizer> {
         self.tokenizers
             .read()
             .expect("Acquiring the lock should never fail")
@@ -61,13 +62,13 @@ impl Default for TokenizerManager {
         manager.register("raw", RawTokenizer);
         manager.register(
             "default",
-            TextAnalyzer::from(SimpleTokenizer)
+            SimpleTokenizer
                 .filter(RemoveLongFilter::limit(40))
                 .filter(LowerCaser),
         );
         manager.register(
             "en_stem",
-            TextAnalyzer::from(SimpleTokenizer)
+            SimpleTokenizer
                 .filter(RemoveLongFilter::limit(40))
                 .filter(LowerCaser)
                 .filter(Stemmer::new(Language::English)),