Removed the SegmentCollector type from the Generics of the

FilterCollector

CHANGELOG.md

@@ -8,11 +8,6 @@ Tantivy 0.14.0
 - Added helper for building intersections and unions in BooleanQuery (@guilload)
 - Bugfix in `Query::explain`
 - Removed dependency on `notify` #924. Replaced with `FileWatcher` struct that polls meta file every 500ms in background thread. (@halvorboe @guilload)
-- Added `FilterCollector`, which wraps another collector and filters docs using a predicate over a fast field (@barrotsteindev)
-- Simplified the encoding of the skip reader struct. BlockWAND max tf is now encoded over a single byte. (@pmasurel)
-- `FilterCollector` now supports all Fast Field value types (@barrotsteindev)
-
-This version breaks compatibility and requires users to reindex everything.
 
 Tantivy 0.13.2
 ===================

Cargo.toml

@@ -53,11 +53,10 @@ lru = "0.6"
 winapi = "0.3"
 
 [dev-dependencies]
-rand = "0.8"
+rand = "0.7"
 maplit = "1"
 matches = "0.1.8"
 proptest = "0.10"
-criterion = "0.3"
 
 [dev-dependencies.fail]
 version = "0.4"
@@ -98,7 +97,3 @@ travis-ci = { repository = "tantivy-search/tantivy" }
 name = "failpoints"
 path = "tests/failpoints/mod.rs"
 required-features = ["fail/failpoints"]
-
-[[bench]]
-name = "analyzer"
-harness = false

benches/analyzer.rs

@@ -1,22 +0,0 @@
-use criterion::{criterion_group, criterion_main, Criterion};
-use tantivy::tokenizer::TokenizerManager;
-
-const ALICE_TXT: &'static str = include_str!("alice.txt");
-
-pub fn criterion_benchmark(c: &mut Criterion) {
-    let tokenizer_manager = TokenizerManager::default();
-    let tokenizer = tokenizer_manager.get("default").unwrap();
-    c.bench_function("default-tokenize-alice", |b| {
-        b.iter(|| {
-            let mut word_count = 0;
-            let mut token_stream = tokenizer.token_stream(ALICE_TXT);
-            for token in token_stream {
-                word_count += 1;
-            }
-            assert_eq!(word_count, 30_731);
-        })
-    });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

examples/faceted_search_with_tweaked_score.rs

@@ -61,7 +61,7 @@ fn main() -> tantivy::Result<()> {
 
                 let query_ords: HashSet<u64> = facets
                     .iter()
-                    .filter_map(|key| facet_dict.term_ord(key.encoded_str()).unwrap())
+                    .filter_map(|key| facet_dict.term_ord(key.encoded_str()))
                     .collect();
 
                 let mut facet_ords_buffer: Vec<u64> = Vec::with_capacity(20);

examples/pre_tokenized_text.rs

@@ -17,7 +17,12 @@ use tantivy::{doc, Index, ReloadPolicy};
 use tempfile::TempDir;
 
 fn pre_tokenize_text(text: &str) -> Vec<Token> {
-    SimpleTokenizer.token_stream(text).collect()
+    let mut token_stream = SimpleTokenizer.token_stream(text);
+    let mut tokens = vec![];
+    while token_stream.advance() {
+        tokens.push(token_stream.token().clone());
+    }
+    tokens
 }
 
 fn main() -> tantivy::Result<()> {

examples/snippet.rs

@@ -51,7 +51,7 @@ fn main() -> tantivy::Result<()> {
 
     let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;
 
-    let mut snippet_generator = SnippetGenerator::create(&searcher, &*query, body)?;
+    let snippet_generator = SnippetGenerator::create(&searcher, &*query, body)?;
 
     for (score, doc_address) in top_docs {
         let doc = searcher.doc(doc_address)?;

examples/stop_words.rs

@@ -50,13 +50,12 @@ fn main() -> tantivy::Result<()> {
 
     // This tokenizer lowers all of the text (to help with stop word matching)
     // then removes all instances of `the` and `and` from the corpus
-    let tokenizer = analyzer_builder(SimpleTokenizer)
-        .filter(LowerCaser::new())
+    let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+        .filter(LowerCaser)
         .filter(StopWordFilter::remove(vec![
             "the".to_string(),
             "and".to_string(),
-        ]))
-        .build();
+        ]));
 
     index.tokenizers().register("stoppy", tokenizer);
 

src/collector/facet_collector.rs

@@ -274,7 +274,7 @@ impl Collector for FacetCollector {
         let mut collapse_facet_it = self.facets.iter().peekable();
         collapse_facet_ords.push(0);
         {
-            let mut facet_streamer = facet_reader.facet_dict().range().into_stream()?;
+            let mut facet_streamer = facet_reader.facet_dict().range().into_stream();
             if facet_streamer.advance() {
                 'outer: loop {
                     // at the begining of this loop, facet_streamer
@@ -368,12 +368,9 @@ impl SegmentCollector for FacetSegmentCollector {
             }
             let mut facet = vec![];
             let facet_ord = self.collapse_facet_ords[collapsed_facet_ord];
-            // TODO handle errors.
-            if facet_dict.ord_to_term(facet_ord as u64, &mut facet).is_ok() {
-                if let Ok(facet) = Facet::from_encoded(facet) {
-                    facet_counts.insert(facet, count);
-                }
-            }
+            facet_dict.ord_to_term(facet_ord as u64, &mut facet);
+            // TODO
+            facet_counts.insert(Facet::from_encoded(facet).unwrap(), count);
         }
         FacetCounts { facet_counts }
     }

src/collector/filter_collector_wrapper.rs

@@ -9,10 +9,8 @@
 
 // ---
 // Importing tantivy...
-use std::marker::PhantomData;
-
 use crate::collector::{Collector, SegmentCollector};
-use crate::fastfield::{FastFieldReader, FastValue};
+use crate::fastfield::FastFieldReader;
 use crate::schema::Field;
 use crate::{Score, SegmentReader, TantivyError};
 
@@ -43,104 +41,73 @@ use crate::{Score, SegmentReader, TantivyError};
 ///
 /// let query_parser = QueryParser::for_index(&index, vec![title]);
 /// let query = query_parser.parse_query("diary").unwrap();
-/// let no_filter_collector = FilterCollector::new(price, &|value: u64| value > 20_120u64, TopDocs::with_limit(2));
+/// let no_filter_collector = FilterCollector::new(price, &|value| true, TopDocs::with_limit(2));
 /// let top_docs = searcher.search(&query, &no_filter_collector).unwrap();
 ///
-/// assert_eq!(top_docs.len(), 1);
+/// assert_eq!(top_docs.len(), 2);
 /// assert_eq!(top_docs[0].1, DocAddress(0, 1));
+/// assert_eq!(top_docs[1].1, DocAddress(0, 3));
 ///
-/// let filter_all_collector: FilterCollector<_, _, u64> = FilterCollector::new(price, &|value| value < 5u64, TopDocs::with_limit(2));
+/// let filter_all_collector = FilterCollector::new(price, &|value| false, TopDocs::with_limit(2));
 /// let filtered_top_docs = searcher.search(&query, &filter_all_collector).unwrap();
 ///
 /// assert_eq!(filtered_top_docs.len(), 0);
 /// ```
-pub struct FilterCollector<TCollector, TPredicate, TPredicateValue: FastValue>
-where
-    TPredicate: 'static,
-{
+pub struct FilterCollector<TCollector> {
     field: Field,
     collector: TCollector,
-    predicate: &'static TPredicate,
-    t_predicate_value: PhantomData<TPredicateValue>,
+    predicate: &'static (dyn Fn(u64) -> bool + Send + Sync),
 }
 
-impl<TCollector, TPredicate, TPredicateValue: FastValue>
-    FilterCollector<TCollector, TPredicate, TPredicateValue>
+impl<TCollector> FilterCollector<TCollector>
 where
     TCollector: Collector + Send + Sync,
-    TPredicate: Fn(TPredicateValue) -> bool + Send + Sync,
 {
-    /// Create a new FilterCollector.
     pub fn new(
         field: Field,
-        predicate: &'static TPredicate,
+        predicate: &'static (dyn Fn(u64) -> bool + Send + Sync),
         collector: TCollector,
-    ) -> FilterCollector<TCollector, TPredicate, TPredicateValue> {
+    ) -> FilterCollector<TCollector> {
         FilterCollector {
             field,
             predicate,
             collector,
-            t_predicate_value: PhantomData,
         }
     }
 }
 
-impl<TCollector, TPredicate, TPredicateValue: FastValue> Collector
-    for FilterCollector<TCollector, TPredicate, TPredicateValue>
+impl<TCollector> Collector for FilterCollector<TCollector>
 where
     TCollector: Collector + Send + Sync,
-    TPredicate: 'static + Fn(TPredicateValue) -> bool + Send + Sync,
-    TPredicateValue: 'static + FastValue,
 {
     // That's the type of our result.
     // Our standard deviation will be a float.
     type Fruit = TCollector::Fruit;
 
-    type Child = FilterSegmentCollector<TCollector::Child, TPredicate, TPredicateValue>;
+    type Child = FilterSegmentCollector<TCollector::Child>;
 
     fn for_segment(
         &self,
         segment_local_id: u32,
         segment_reader: &SegmentReader,
-    ) -> crate::Result<FilterSegmentCollector<TCollector::Child, TPredicate, TPredicateValue>> {
-        let schema = segment_reader.schema();
-        let field_entry = schema.get_field_entry(self.field);
-        if !field_entry.is_fast() {
-            return Err(TantivyError::SchemaError(format!(
-                "Field {:?} is not a fast field.",
-                field_entry.name()
-            )));
-        }
-        let requested_type = TPredicateValue::to_type();
-        let field_schema_type = field_entry.field_type().value_type();
-        if requested_type != field_schema_type {
-            return Err(TantivyError::SchemaError(format!(
-                "Field {:?} is of type {:?}!={:?}",
-                field_entry.name(),
-                requested_type,
-                field_schema_type
-            )));
-        }
-
+    ) -> crate::Result<FilterSegmentCollector<TCollector::Child>> {
         let fast_field_reader = segment_reader
             .fast_fields()
-            .typed_fast_field_reader(self.field)
+            .u64(self.field)
             .ok_or_else(|| {
+                let field_name = segment_reader.schema().get_field_name(self.field);
                 TantivyError::SchemaError(format!(
-                    "{:?} is not declared as a fast field in the schema.",
-                    self.field
+                    "Field {:?} is not a u64 fast field.",
+                    field_name
                 ))
             })?;
-
         let segment_collector = self
             .collector
             .for_segment(segment_local_id, segment_reader)?;
-
         Ok(FilterSegmentCollector {
             fast_field_reader,
-            segment_collector,
+            segment_collector: segment_collector,
             predicate: self.predicate,
-            t_predicate_value: PhantomData,
         })
     }
 
@@ -156,23 +123,15 @@ where
     }
 }
 
-pub struct FilterSegmentCollector<TSegmentCollector, TPredicate, TPredicateValue>
-where
-    TPredicate: 'static,
-    TPredicateValue: 'static + FastValue,
-{
-    fast_field_reader: FastFieldReader<TPredicateValue>,
+pub struct FilterSegmentCollector<TSegmentCollector> {
+    fast_field_reader: FastFieldReader<u64>,
     segment_collector: TSegmentCollector,
-    predicate: &'static TPredicate,
-    t_predicate_value: PhantomData<TPredicateValue>,
+    predicate: &'static (dyn Fn(u64) -> bool + Send + Sync),
 }
 
-impl<TSegmentCollector, TPredicate, TPredicateValue> SegmentCollector
-    for FilterSegmentCollector<TSegmentCollector, TPredicate, TPredicateValue>
+impl<TSegmentCollector> SegmentCollector for FilterSegmentCollector<TSegmentCollector>
 where
     TSegmentCollector: SegmentCollector,
-    TPredicate: 'static + Fn(TPredicateValue) -> bool + Send + Sync,
-    TPredicateValue: 'static + FastValue,
 {
     type Fruit = TSegmentCollector::Fruit;
 

src/collector/tests.rs

@@ -8,13 +8,6 @@ use crate::DocId;
 use crate::Score;
 use crate::SegmentLocalId;
 
-use crate::collector::{FilterCollector, TopDocs};
-use crate::query::QueryParser;
-use crate::schema::{Schema, FAST, TEXT};
-use crate::DateTime;
-use crate::{doc, Index};
-use std::str::FromStr;
-
 pub const TEST_COLLECTOR_WITH_SCORE: TestCollector = TestCollector {
     compute_score: true,
 };
@@ -23,54 +16,6 @@ pub const TEST_COLLECTOR_WITHOUT_SCORE: TestCollector = TestCollector {
     compute_score: true,
 };
 
-#[test]
-pub fn test_filter_collector() {
-    let mut schema_builder = Schema::builder();
-    let title = schema_builder.add_text_field("title", TEXT);
-    let price = schema_builder.add_u64_field("price", FAST);
-    let date = schema_builder.add_date_field("date", FAST);
-    let schema = schema_builder.build();
-    let index = Index::create_in_ram(schema);
-
-    let mut index_writer = index.writer_with_num_threads(1, 10_000_000).unwrap();
-    index_writer.add_document(doc!(title => "The Name of the Wind", price => 30_200u64, date => DateTime::from_str("1898-04-09T00:00:00+00:00").unwrap()));
-    index_writer.add_document(doc!(title => "The Diary of Muadib", price => 29_240u64, date => DateTime::from_str("2020-04-09T00:00:00+00:00").unwrap()));
-    index_writer.add_document(doc!(title => "The Diary of Anne Frank", price => 18_240u64, date => DateTime::from_str("2019-04-20T00:00:00+00:00").unwrap()));
-    index_writer.add_document(doc!(title => "A Dairy Cow", price => 21_240u64, date => DateTime::from_str("2019-04-09T00:00:00+00:00").unwrap()));
-    index_writer.add_document(doc!(title => "The Diary of a Young Girl", price => 20_120u64, date => DateTime::from_str("2018-04-09T00:00:00+00:00").unwrap()));
-    assert!(index_writer.commit().is_ok());
-
-    let reader = index.reader().unwrap();
-    let searcher = reader.searcher();
-
-    let query_parser = QueryParser::for_index(&index, vec![title]);
-    let query = query_parser.parse_query("diary").unwrap();
-    let filter_some_collector = FilterCollector::new(
-        price,
-        &|value: u64| value > 20_120u64,
-        TopDocs::with_limit(2),
-    );
-    let top_docs = searcher.search(&query, &filter_some_collector).unwrap();
-
-    assert_eq!(top_docs.len(), 1);
-    assert_eq!(top_docs[0].1, DocAddress(0, 1));
-
-    let filter_all_collector: FilterCollector<_, _, u64> =
-        FilterCollector::new(price, &|value| value < 5u64, TopDocs::with_limit(2));
-    let filtered_top_docs = searcher.search(&query, &filter_all_collector).unwrap();
-
-    assert_eq!(filtered_top_docs.len(), 0);
-
-    fn date_filter(value: DateTime) -> bool {
-        (value - DateTime::from_str("2019-04-09T00:00:00+00:00").unwrap()).num_weeks() > 0
-    }
-
-    let filter_dates_collector = FilterCollector::new(date, &date_filter, TopDocs::with_limit(5));
-    let filtered_date_docs = searcher.search(&query, &filter_dates_collector).unwrap();
-
-    assert_eq!(filtered_date_docs.len(), 2);
-}
-
 /// Stores all of the doc ids.
 /// This collector is only used for tests.
 /// It is unusable in pr

src/collector/top_score_collector.rs

@@ -728,7 +728,7 @@ mod tests {
     }
 
     #[test]
-    fn test_top_collector_not_at_capacity_without_offset() {
+    fn test_top_collector_not_at_capacity() {
         let index = make_index();
         let field = index.schema().get_field("text").unwrap();
         let query_parser = QueryParser::for_index(&index, vec![field]);

src/common/counting_writer.rs

@@ -20,10 +20,9 @@ impl<W: Write> CountingWriter<W> {
         self.written_bytes
     }
 
-    /// Returns the underlying write object.
-    /// Note that this method does not trigger any flushing.
-    pub fn finish(self) -> W {
-        self.underlying
+    pub fn finish(mut self) -> io::Result<(W, u64)> {
+        self.flush()?;
+        Ok((self.underlying, self.written_bytes))
     }
 }
 
@@ -47,6 +46,7 @@ impl<W: Write> Write for CountingWriter<W> {
 
 impl<W: TerminatingWrite> TerminatingWrite for CountingWriter<W> {
     fn terminate_ref(&mut self, token: AntiCallToken) -> io::Result<()> {
+        self.flush()?;
         self.underlying.terminate_ref(token)
     }
 }
@@ -63,9 +63,8 @@ mod test {
         let mut counting_writer = CountingWriter::wrap(buffer);
         let bytes = (0u8..10u8).collect::<Vec<u8>>();
         counting_writer.write_all(&bytes).unwrap();
-        let len = counting_writer.written_bytes();
-        let buffer_restituted: Vec<u8> = counting_writer.finish();
+        let (w, len): (Vec<u8>, u64) = counting_writer.finish().unwrap();
         assert_eq!(len, 10u64);
-        assert_eq!(buffer_restituted.len(), 10);
+        assert_eq!(w.len(), 10);
     }
 }

src/common/mod.rs

@@ -115,16 +115,11 @@ pub fn u64_to_i64(val: u64) -> i64 {
 /// For simplicity, tantivy internally handles `f64` as `u64`.
 /// The mapping is defined by this function.
 ///
-/// Maps `f64` to `u64` in a monotonic manner, so that bytes lexical order is preserved.
+/// Maps `f64` to `u64` so that lexical order is preserved.
 ///
 /// This is more suited than simply casting (`val as u64`)
 /// which would truncate the result
 ///
-/// # Reference
-///
-/// Daniel Lemire's [blog post](https://lemire.me/blog/2020/12/14/converting-floating-point-numbers-to-integers-while-preserving-order/)
-/// explains the mapping in a clear manner.
-///
 /// # See also
 /// The [reverse mapping is `u64_to_f64`](./fn.u64_to_f64.html).
 #[inline(always)]
@@ -153,7 +148,6 @@ pub(crate) mod test {
     pub use super::minmax;
     pub use super::serialize::test::fixed_size_test;
     use super::{compute_num_bits, f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
-    use proptest::prelude::*;
     use std::f64;
 
     fn test_i64_converter_helper(val: i64) {
@@ -164,15 +158,6 @@ pub(crate) mod test {
         assert_eq!(u64_to_f64(f64_to_u64(val)), val);
     }
 
-    proptest! {
-        #[test]
-        fn test_f64_converter_monotonicity_proptest((left, right) in (proptest::num::f64::NORMAL, proptest::num::f64::NORMAL)) {
-            let left_u64 = f64_to_u64(left);
-            let right_u64 = f64_to_u64(right);
-            assert_eq!(left_u64 < right_u64,  left < right);
-        }
-    }
-
     #[test]
     fn test_i64_converter() {
         assert_eq!(i64_to_u64(i64::min_value()), u64::min_value());

src/core/index.rs

@@ -20,7 +20,7 @@ use crate::reader::IndexReaderBuilder;
 use crate::schema::Field;
 use crate::schema::FieldType;
 use crate::schema::Schema;
-use crate::tokenizer::{TextAnalyzerT, TokenizerManager};
+use crate::tokenizer::{TextAnalyzer, TokenizerManager};
 use crate::IndexWriter;
 use std::collections::HashSet;
 use std::fmt;
@@ -119,12 +119,13 @@ impl Index {
             return Index::create(dir, schema);
         }
         let index = Index::open(dir)?;
-        if index.schema() != schema {
-            return Err(TantivyError::SchemaError(
+        if index.schema() == schema {
+            Ok(index)
+        } else {
+            Err(TantivyError::SchemaError(
                 "An index exists but the schema does not match.".to_string(),
-            ));
+            ))
         }
-        Ok(index)
     }
 
     /// Creates a new index in a temp directory.
@@ -180,11 +181,11 @@ impl Index {
     }
 
     /// Helper to access the tokenizer associated to a specific field.
-    pub fn tokenizer_for_field(&self, field: Field) -> crate::Result<Box<dyn TextAnalyzerT>> {
+    pub fn tokenizer_for_field(&self, field: Field) -> crate::Result<TextAnalyzer> {
         let field_entry = self.schema.get_field_entry(field);
         let field_type = field_entry.field_type();
         let tokenizer_manager: &TokenizerManager = self.tokenizers();
-        let tokenizer_name_opt: Option<Box<dyn TextAnalyzerT>> = match field_type {
+        let tokenizer_name_opt: Option<TextAnalyzer> = match field_type {
             FieldType::Str(text_options) => text_options
                 .get_indexing_options()
                 .map(|text_indexing_options| text_indexing_options.tokenizer().to_string())
@@ -510,28 +511,28 @@ mod tests {
         }
 
         #[test]
-        fn test_index_manual_policy_mmap() -> crate::Result<()> {
+        fn test_index_manual_policy_mmap() {
             let schema = throw_away_schema();
             let field = schema.get_field("num_likes").unwrap();
-            let mut index = Index::create_from_tempdir(schema)?;
-            let mut writer = index.writer_for_tests()?;
-            writer.commit()?;
+            let mut index = Index::create_from_tempdir(schema).unwrap();
+            let mut writer = index.writer_for_tests().unwrap();
+            writer.commit().unwrap();
             let reader = index
                 .reader_builder()
                 .reload_policy(ReloadPolicy::Manual)
-                .try_into()?;
+                .try_into()
+                .unwrap();
             assert_eq!(reader.searcher().num_docs(), 0);
             writer.add_document(doc!(field=>1u64));
             let (sender, receiver) = crossbeam::channel::unbounded();
             let _handle = index.directory_mut().watch(WatchCallback::new(move || {
                 let _ = sender.send(());
             }));
-            writer.commit()?;
+            writer.commit().unwrap();
             assert!(receiver.recv().is_ok());
             assert_eq!(reader.searcher().num_docs(), 0);
-            reader.reload()?;
+            reader.reload().unwrap();
             assert_eq!(reader.searcher().num_docs(), 1);
-            Ok(())
         }
 
         #[test]

src/core/inverted_index_reader.rs

@@ -66,7 +66,7 @@ impl InvertedIndexReader {
     }
 
     /// Returns the term info associated with the term.
-    pub fn get_term_info(&self, term: &Term) -> io::Result<Option<TermInfo>> {
+    pub fn get_term_info(&self, term: &Term) -> Option<TermInfo> {
         self.termdict.get(term.value_bytes())
     }
 
@@ -106,9 +106,10 @@ impl InvertedIndexReader {
         term: &Term,
         option: IndexRecordOption,
     ) -> io::Result<Option<BlockSegmentPostings>> {
-        self.get_term_info(term)?
+        Ok(self
+            .get_term_info(term)
             .map(move |term_info| self.read_block_postings_from_terminfo(&term_info, option))
-            .transpose()
+            .transpose()?)
     }
 
     /// Returns a block postings given a `term_info`.
@@ -180,7 +181,7 @@ impl InvertedIndexReader {
         term: &Term,
         option: IndexRecordOption,
     ) -> io::Result<Option<SegmentPostings>> {
-        self.get_term_info(term)?
+        self.get_term_info(term)
             .map(move |term_info| self.read_postings_from_terminfo(&term_info, option))
             .transpose()
     }
@@ -190,7 +191,7 @@ impl InvertedIndexReader {
         term: &Term,
         option: IndexRecordOption,
     ) -> io::Result<Option<SegmentPostings>> {
-        self.get_term_info(term)?
+        self.get_term_info(term)
             .map(|term_info| self.read_postings_from_terminfo(&term_info, option))
             .transpose()
     }
@@ -198,7 +199,7 @@ impl InvertedIndexReader {
     /// Returns the number of documents containing the term.
     pub fn doc_freq(&self, term: &Term) -> io::Result<u32> {
         Ok(self
-            .get_term_info(term)?
+            .get_term_info(term)
             .map(|term_info| term_info.doc_freq)
             .unwrap_or(0u32))
     }

src/core/mod.rs

@@ -12,7 +12,7 @@ pub use self::executor::Executor;
 pub use self::index::Index;
 pub use self::index_meta::{IndexMeta, SegmentMeta, SegmentMetaInventory};
 pub use self::inverted_index_reader::InvertedIndexReader;
-pub use self::searcher::Searcher;
+pub use self::searcher::{FieldSearcher, Searcher};
 pub use self::segment::Segment;
 pub use self::segment::SerializableSegment;
 pub use self::segment_component::SegmentComponent;

src/core/searcher.rs

@@ -1,16 +1,17 @@
 use crate::collector::Collector;
 use crate::core::Executor;
-
+use crate::core::InvertedIndexReader;
 use crate::core::SegmentReader;
 use crate::query::Query;
 use crate::schema::Document;
 use crate::schema::Schema;
-use crate::schema::Term;
+use crate::schema::{Field, Term};
 use crate::space_usage::SearcherSpaceUsage;
 use crate::store::StoreReader;
+use crate::termdict::TermMerger;
 use crate::DocAddress;
 use crate::Index;
-
+use std::sync::Arc;
 use std::{fmt, io};
 
 /// Holds a list of `SegmentReader`s ready for search.
@@ -147,6 +148,16 @@ impl Searcher {
         collector.merge_fruits(fruits)
     }
 
+    /// Return the field searcher associated to a `Field`.
+    pub fn field(&self, field: Field) -> crate::Result<FieldSearcher> {
+        let inv_index_readers: Vec<Arc<InvertedIndexReader>> = self
+            .segment_readers
+            .iter()
+            .map(|segment_reader| segment_reader.inverted_index(field))
+            .collect::<crate::Result<Vec<_>>>()?;
+        Ok(FieldSearcher::new(inv_index_readers))
+    }
+
     /// Summarize total space usage of this searcher.
     pub fn space_usage(&self) -> io::Result<SearcherSpaceUsage> {
         let mut space_usage = SearcherSpaceUsage::new();
@@ -157,6 +168,32 @@ impl Searcher {
     }
 }
 
+/// **Experimental API** `FieldSearcher` only gives access to a stream over the terms of a field.
+pub struct FieldSearcher {
+    inv_index_readers: Vec<Arc<InvertedIndexReader>>,
+}
+
+impl FieldSearcher {
+    fn new(inv_index_readers: Vec<Arc<InvertedIndexReader>>) -> FieldSearcher {
+        FieldSearcher { inv_index_readers }
+    }
+
+    /// Returns a Stream over all of the sorted unique terms of
+    /// for the given field.
+    ///
+    /// This method does not take into account which documents are deleted, so
+    /// in presence of deletes some terms may not actually exist in any document
+    /// anymore.
+    pub fn terms(&self) -> TermMerger {
+        let term_streamers: Vec<_> = self
+            .inv_index_readers
+            .iter()
+            .map(|inverted_index| inverted_index.terms().stream())
+            .collect();
+        TermMerger::new(term_streamers)
+    }
+}
+
 impl fmt::Debug for Searcher {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let segment_ids = self

src/core/segment_reader.rs

@@ -310,7 +310,7 @@ impl SegmentReader {
     }
 
     /// Returns an iterator that will iterate over the alive document ids
-    pub fn doc_ids_alive(&self) -> impl Iterator<Item = DocId> + '_ {
+    pub fn doc_ids_alive<'a>(&'a self) -> impl Iterator<Item = DocId> + 'a {
         (0u32..self.max_doc).filter(move |doc| !self.is_deleted(*doc))
     }
 

src/directory/directory.rs

@@ -1,8 +1,8 @@
 use crate::directory::directory_lock::Lock;
 use crate::directory::error::LockError;
 use crate::directory::error::{DeleteError, OpenReadError, OpenWriteError};
+use crate::directory::WatchCallback;
 use crate::directory::WatchHandle;
-use crate::directory::{FileHandle, WatchCallback};
 use crate::directory::{FileSlice, WritePtr};
 use std::fmt;
 use std::io;
@@ -108,13 +108,10 @@ fn retry_policy(is_blocking: bool) -> RetryPolicy {
 /// should be your default choice.
 /// - The [`RAMDirectory`](struct.RAMDirectory.html), which
 /// should be used mostly for tests.
+///
 pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
-    /// Opens a file and returns a boxed `FileHandle`.
+    /// Opens a virtual file for read.
     ///
-    /// Users of `Directory` should typically call `Directory::open_read(...)`,
-    /// while `Directory` implementor should implement `get_file_handle()`.
-    fn get_file_handle(&self, path: &Path) -> Result<Box<dyn FileHandle>, OpenReadError>;
-
     /// Once a virtual file is open, its data may not
     /// change.
     ///
@@ -122,10 +119,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     /// have no effect on the returned `FileSlice` object.
     ///
     /// You should only use this to read files create with [Directory::open_write].
-    fn open_read(&self, path: &Path) -> Result<FileSlice, OpenReadError> {
-        let file_handle = self.get_file_handle(path)?;
-        Ok(FileSlice::new(file_handle))
-    }
+    fn open_read(&self, path: &Path) -> Result<FileSlice, OpenReadError>;
 
     /// Removes a file
     ///

src/directory/error.rs

@@ -58,8 +58,7 @@ pub enum OpenWriteError {
 }
 
 impl OpenWriteError {
-    /// Wraps an io error.
-    pub fn wrap_io_error(io_error: io::Error, filepath: PathBuf) -> Self {
+    pub(crate) fn wrap_io_error(io_error: io::Error, filepath: PathBuf) -> Self {
         Self::IOError { io_error, filepath }
     }
 }
@@ -144,8 +143,7 @@ pub enum OpenReadError {
 }
 
 impl OpenReadError {
-    /// Wraps an io error.
-    pub fn wrap_io_error(io_error: io::Error, filepath: PathBuf) -> Self {
+    pub(crate) fn wrap_io_error(io_error: io::Error, filepath: PathBuf) -> Self {
         Self::IOError { io_error, filepath }
     }
 }

src/directory/file_slice.rs

@@ -2,11 +2,10 @@ use stable_deref_trait::StableDeref;
 
 use crate::common::HasLen;
 use crate::directory::OwnedBytes;
-use std::sync::{Arc, Weak};
+use std::sync::Arc;
 use std::{io, ops::Deref};
 
-pub type ArcBytes = Arc<dyn Deref<Target = [u8]> + Send + Sync + 'static>;
-pub type WeakArcBytes = Weak<dyn Deref<Target = [u8]> + Send + Sync + 'static>;
+pub type BoxedData = Box<dyn Deref<Target = [u8]> + Send + Sync + 'static>;
 
 /// Objects that represents files sections in tantivy.
 ///
@@ -41,7 +40,7 @@ where
     B: StableDeref + Deref<Target = [u8]> + 'static + Send + Sync,
 {
     fn from(bytes: B) -> FileSlice {
-        FileSlice::new(Box::new(OwnedBytes::new(bytes)))
+        FileSlice::new(OwnedBytes::new(bytes))
     }
 }
 
@@ -51,25 +50,22 @@ where
 ///
 #[derive(Clone)]
 pub struct FileSlice {
-    data: Arc<dyn FileHandle>,
+    data: Arc<Box<dyn FileHandle>>,
     start: usize,
     stop: usize,
 }
 
 impl FileSlice {
     /// Wraps a FileHandle.
-    pub fn new(file_handle: Box<dyn FileHandle>) -> Self {
-        let num_bytes = file_handle.len();
-        FileSlice::new_with_num_bytes(file_handle, num_bytes)
-    }
-
-    /// Wraps a FileHandle.
-    #[doc(hidden)]
-    pub fn new_with_num_bytes(file_handle: Box<dyn FileHandle>, num_bytes: usize) -> Self {
+    pub fn new<D>(data: D) -> Self
+    where
+        D: FileHandle,
+    {
+        let len = data.len();
         FileSlice {
-            data: Arc::from(file_handle),
+            data: Arc::new(Box::new(data)),
             start: 0,
-            stop: num_bytes,
+            stop: len,
         }
     }
 
@@ -150,12 +146,6 @@ impl FileSlice {
     }
 }
 
-impl FileHandle for FileSlice {
-    fn read_bytes(&self, from: usize, to: usize) -> io::Result<OwnedBytes> {
-        self.read_bytes_slice(from, to)
-    }
-}
-
 impl HasLen for FileSlice {
     fn len(&self) -> usize {
         self.stop - self.start
@@ -170,7 +160,7 @@ mod tests {
 
     #[test]
     fn test_file_slice() -> io::Result<()> {
-        let file_slice = FileSlice::new(Box::new(b"abcdef".as_ref()));
+        let file_slice = FileSlice::new(b"abcdef".as_ref());
         assert_eq!(file_slice.len(), 6);
         assert_eq!(file_slice.slice_from(2).read_bytes()?.as_slice(), b"cdef");
         assert_eq!(file_slice.slice_to(2).read_bytes()?.as_slice(), b"ab");
@@ -214,7 +204,7 @@ mod tests {
 
     #[test]
     fn test_slice_simple_read() -> io::Result<()> {
-        let slice = FileSlice::new(Box::new(&b"abcdef"[..]));
+        let slice = FileSlice::new(&b"abcdef"[..]);
         assert_eq!(slice.len(), 6);
         assert_eq!(slice.read_bytes()?.as_ref(), b"abcdef");
         assert_eq!(slice.slice(1, 4).read_bytes()?.as_ref(), b"bcd");
@@ -223,7 +213,7 @@ mod tests {
 
     #[test]
     fn test_slice_read_slice() -> io::Result<()> {
-        let slice_deref = FileSlice::new(Box::new(&b"abcdef"[..]));
+        let slice_deref = FileSlice::new(&b"abcdef"[..]);
         assert_eq!(slice_deref.read_bytes_slice(1, 4)?.as_ref(), b"bcd");
         Ok(())
     }
@@ -231,14 +221,14 @@ mod tests {
     #[test]
     #[should_panic(expected = "assertion failed: from <= to")]
     fn test_slice_read_slice_invalid_range() {
-        let slice_deref = FileSlice::new(Box::new(&b"abcdef"[..]));
+        let slice_deref = FileSlice::new(&b"abcdef"[..]);
         assert_eq!(slice_deref.read_bytes_slice(1, 0).unwrap().as_ref(), b"bcd");
     }
 
     #[test]
     #[should_panic(expected = "`to` exceeds the fileslice length")]
     fn test_slice_read_slice_invalid_range_exceeds() {
-        let slice_deref = FileSlice::new(Box::new(&b"abcdef"[..]));
+        let slice_deref = FileSlice::new(&b"abcdef"[..]);
         assert_eq!(
             slice_deref.read_bytes_slice(0, 10).unwrap().as_ref(),
             b"bcd"

src/directory/file_watcher.rs

@@ -3,7 +3,7 @@ use crc32fast::Hasher;
 use std::fs;
 use std::io;
 use std::io::BufRead;
-use std::path::Path;
+use std::path::PathBuf;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::thread;
@@ -13,15 +13,15 @@ pub const POLLING_INTERVAL: Duration = Duration::from_millis(if cfg!(test) { 1 }
 
 // Watches a file and executes registered callbacks when the file is modified.
 pub struct FileWatcher {
-    path: Arc<Path>,
+    path: Arc<PathBuf>,
     callbacks: Arc<WatchCallbackList>,
     state: Arc<AtomicUsize>, // 0: new, 1: runnable, 2: terminated
 }
 
 impl FileWatcher {
-    pub fn new(path: &Path) -> FileWatcher {
+    pub fn new(path: &PathBuf) -> FileWatcher {
         FileWatcher {
-            path: Arc::from(path),
+            path: Arc::new(path.clone()),
             callbacks: Default::default(),
             state: Default::default(),
         }
@@ -63,7 +63,7 @@ impl FileWatcher {
         handle
     }
 
-    fn compute_checksum(path: &Path) -> Result<u32, io::Error> {
+    fn compute_checksum(path: &PathBuf) -> Result<u32, io::Error> {
         let reader = match fs::File::open(path) {
             Ok(f) => io::BufReader::new(f),
             Err(e) => {

src/directory/footer.rs

@@ -115,18 +115,6 @@ impl Footer {
                 }
                 Ok(())
             }
-            VersionedFooter::V3 {
-                crc32: _crc,
-                store_compression,
-            } => {
-                if &library_version.store_compression != store_compression {
-                    return Err(Incompatibility::CompressionMismatch {
-                        library_compression_format: library_version.store_compression.to_string(),
-                        index_compression_format: store_compression.to_string(),
-                    });
-                }
-                Ok(())
-            }
             VersionedFooter::UnknownVersion => Err(Incompatibility::IndexMismatch {
                 library_version: library_version.clone(),
                 index_version: self.version.clone(),
@@ -148,31 +136,24 @@ pub enum VersionedFooter {
         crc32: CrcHashU32,
         store_compression: String,
     },
-    // Block wand max termfred on 1 byte
-    V3 {
-        crc32: CrcHashU32,
-        store_compression: String,
-    },
 }
 
 impl BinarySerializable for VersionedFooter {
     fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
         let mut buf = Vec::new();
         match self {
-            VersionedFooter::V3 {
+            VersionedFooter::V2 {
                 crc32,
                 store_compression: compression,
             } => {
                 // Serializes a valid `VersionedFooter` or panics if the version is unknown
                 // [   version    |   crc_hash  | compression_mode ]
                 // [    0..4      |     4..8    |     variable     ]
-                BinarySerializable::serialize(&3u32, &mut buf)?;
+                BinarySerializable::serialize(&2u32, &mut buf)?;
                 BinarySerializable::serialize(crc32, &mut buf)?;
                 BinarySerializable::serialize(compression, &mut buf)?;
             }
-            VersionedFooter::V2 { .. }
-            | VersionedFooter::V1 { .. }
-            | VersionedFooter::UnknownVersion => {
+            VersionedFooter::V1 { .. } | VersionedFooter::UnknownVersion => {
                 return Err(io::Error::new(
                     io::ErrorKind::InvalidInput,
                     "Cannot serialize an unknown versioned footer ",
@@ -201,7 +182,7 @@ impl BinarySerializable for VersionedFooter {
         reader.read_exact(&mut buf[..])?;
         let mut cursor = &buf[..];
         let version = u32::deserialize(&mut cursor)?;
-        if version > 3 {
+        if version != 1 && version != 2 {
             return Ok(VersionedFooter::UnknownVersion);
         }
         let crc32 = u32::deserialize(&mut cursor)?;
@@ -211,14 +192,9 @@ impl BinarySerializable for VersionedFooter {
                 crc32,
                 store_compression,
             }
-        } else if version == 2 {
-            VersionedFooter::V2 {
-                crc32,
-                store_compression,
-            }
         } else {
-            assert_eq!(version, 3);
-            VersionedFooter::V3 {
+            assert_eq!(version, 2);
+            VersionedFooter::V2 {
                 crc32,
                 store_compression,
             }
@@ -229,7 +205,6 @@ impl BinarySerializable for VersionedFooter {
 impl VersionedFooter {
     pub fn crc(&self) -> Option<CrcHashU32> {
         match self {
-            VersionedFooter::V3 { crc32, .. } => Some(*crc32),
             VersionedFooter::V2 { crc32, .. } => Some(*crc32),
             VersionedFooter::V1 { crc32, .. } => Some(*crc32),
             VersionedFooter::UnknownVersion { .. } => None,
@@ -268,7 +243,7 @@ impl<W: TerminatingWrite> Write for FooterProxy<W> {
 impl<W: TerminatingWrite> TerminatingWrite for FooterProxy<W> {
     fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()> {
         let crc32 = self.hasher.take().unwrap().finalize();
-        let footer = Footer::new(VersionedFooter::V3 {
+        let footer = Footer::new(VersionedFooter::V2 {
             crc32,
             store_compression: crate::store::COMPRESSION.to_string(),
         });
@@ -303,7 +278,7 @@ mod tests {
         let footer = Footer::deserialize(&mut &vec[..]).unwrap();
         assert!(matches!(
            footer.versioned_footer,
-           VersionedFooter::V3 { store_compression, .. }
+           VersionedFooter::V2 { store_compression, .. }
            if store_compression == crate::store::COMPRESSION
         ));
         assert_eq!(&footer.version, crate::version());
@@ -313,7 +288,7 @@ mod tests {
     fn test_serialize_deserialize_footer() {
         let mut buffer = Vec::new();
         let crc32 = 123456u32;
-        let footer: Footer = Footer::new(VersionedFooter::V3 {
+        let footer: Footer = Footer::new(VersionedFooter::V2 {
             crc32,
             store_compression: "lz4".to_string(),
         });
@@ -325,7 +300,7 @@ mod tests {
     #[test]
     fn footer_length() {
         let crc32 = 1111111u32;
-        let versioned_footer = VersionedFooter::V3 {
+        let versioned_footer = VersionedFooter::V2 {
             crc32,
             store_compression: "lz4".to_string(),
         };
@@ -346,7 +321,7 @@ mod tests {
             // versionned footer length
             12 | 128,
             // index format version
-            3,
+            2,
             0,
             0,
             0,
@@ -365,7 +340,7 @@ mod tests {
         let versioned_footer = VersionedFooter::deserialize(&mut cursor).unwrap();
         assert!(cursor.is_empty());
         let expected_crc: u32 = LittleEndian::read_u32(&v_footer_bytes[5..9]) as CrcHashU32;
-        let expected_versioned_footer: VersionedFooter = VersionedFooter::V3 {
+        let expected_versioned_footer: VersionedFooter = VersionedFooter::V2 {
             crc32: expected_crc,
             store_compression: "lz4".to_string(),
         };

src/directory/managed_directory.rs

@@ -1,10 +1,10 @@
 use crate::core::{MANAGED_FILEPATH, META_FILEPATH};
 use crate::directory::error::{DeleteError, LockError, OpenReadError, OpenWriteError};
 use crate::directory::footer::{Footer, FooterProxy};
+use crate::directory::DirectoryLock;
 use crate::directory::GarbageCollectionResult;
 use crate::directory::Lock;
 use crate::directory::META_LOCK;
-use crate::directory::{DirectoryLock, FileHandle};
 use crate::directory::{FileSlice, WritePtr};
 use crate::directory::{WatchCallback, WatchHandle};
 use crate::error::DataCorruption;
@@ -274,11 +274,6 @@ impl ManagedDirectory {
 }
 
 impl Directory for ManagedDirectory {
-    fn get_file_handle(&self, path: &Path) -> Result<Box<dyn FileHandle>, OpenReadError> {
-        let file_slice = self.open_read(path)?;
-        Ok(Box::new(file_slice))
-    }
-
     fn open_read(&self, path: &Path) -> result::Result<FileSlice, OpenReadError> {
         let file_slice = self.directory.open_read(path)?;
         let (footer, reader) = Footer::extract_footer(file_slice)

src/directory/mmap_directory.rs

@@ -2,13 +2,14 @@ use crate::core::META_FILEPATH;
 use crate::directory::error::LockError;
 use crate::directory::error::{DeleteError, OpenDirectoryError, OpenReadError, OpenWriteError};
 use crate::directory::file_watcher::FileWatcher;
+use crate::directory::AntiCallToken;
+use crate::directory::BoxedData;
 use crate::directory::Directory;
 use crate::directory::DirectoryLock;
+use crate::directory::FileSlice;
 use crate::directory::Lock;
 use crate::directory::WatchCallback;
 use crate::directory::WatchHandle;
-use crate::directory::{AntiCallToken, FileHandle, OwnedBytes};
-use crate::directory::{ArcBytes, WeakArcBytes};
 use crate::directory::{TerminatingWrite, WritePtr};
 use fs2::FileExt;
 use memmap::Mmap;
@@ -24,6 +25,7 @@ use std::path::{Path, PathBuf};
 use std::result;
 use std::sync::Arc;
 use std::sync::RwLock;
+use std::sync::Weak;
 use std::{collections::HashMap, ops::Deref};
 use tempfile::TempDir;
 
@@ -76,7 +78,7 @@ pub struct CacheInfo {
 
 struct MmapCache {
     counters: CacheCounters,
-    cache: HashMap<PathBuf, WeakArcBytes>,
+    cache: HashMap<PathBuf, Weak<BoxedData>>,
 }
 
 impl Default for MmapCache {
@@ -110,7 +112,7 @@ impl MmapCache {
     }
 
     // Returns None if the file exists but as a len of 0 (and hence is not mmappable).
-    fn get_mmap(&mut self, full_path: &Path) -> Result<Option<ArcBytes>, OpenReadError> {
+    fn get_mmap(&mut self, full_path: &Path) -> Result<Option<Arc<BoxedData>>, OpenReadError> {
         if let Some(mmap_weak) = self.cache.get(full_path) {
             if let Some(mmap_arc) = mmap_weak.upgrade() {
                 self.counters.hit += 1;
@@ -121,7 +123,7 @@ impl MmapCache {
         self.counters.miss += 1;
         let mmap_opt = open_mmap(full_path)?;
         Ok(mmap_opt.map(|mmap| {
-            let mmap_arc: ArcBytes = Arc::new(mmap);
+            let mmap_arc: Arc<BoxedData> = Arc::new(Box::new(mmap));
             let mmap_weak = Arc::downgrade(&mmap_arc);
             self.cache.insert(full_path.to_owned(), mmap_weak);
             mmap_arc
@@ -159,7 +161,7 @@ impl MmapDirectoryInner {
             mmap_cache: Default::default(),
             _temp_directory: temp_directory,
             watcher: FileWatcher::new(&root_path.join(*META_FILEPATH)),
-            root_path,
+            root_path: root_path,
         }
     }
 
@@ -314,7 +316,7 @@ impl TerminatingWrite for SafeFileWriter {
 }
 
 #[derive(Clone)]
-struct MmapArc(Arc<dyn Deref<Target = [u8]> + Send + Sync>);
+struct MmapArc(Arc<Box<dyn Deref<Target = [u8]> + Send + Sync>>);
 
 impl Deref for MmapArc {
     type Target = [u8];
@@ -344,7 +346,7 @@ pub(crate) fn atomic_write(path: &Path, content: &[u8]) -> io::Result<()> {
 }
 
 impl Directory for MmapDirectory {
-    fn get_file_handle(&self, path: &Path) -> result::Result<Box<dyn FileHandle>, OpenReadError> {
+    fn open_read(&self, path: &Path) -> result::Result<FileSlice, OpenReadError> {
         debug!("Open Read {:?}", path);
         let full_path = self.resolve_path(path);
 
@@ -357,16 +359,11 @@ impl Directory for MmapDirectory {
             let io_err = make_io_err(msg);
             OpenReadError::wrap_io_error(io_err, path.to_path_buf())
         })?;
-
-        let owned_bytes = mmap_cache
-            .get_mmap(&full_path)?
-            .map(|mmap_arc| {
-                let mmap_arc_obj = MmapArc(mmap_arc);
-                OwnedBytes::new(mmap_arc_obj)
-            })
-            .unwrap_or_else(OwnedBytes::empty);
-
-        Ok(Box::new(owned_bytes))
+        if let Some(mmap_arc) = mmap_cache.get_mmap(&full_path)? {
+            Ok(FileSlice::from(MmapArc(mmap_arc)))
+        } else {
+            Ok(FileSlice::empty())
+        }
     }
 
     /// Any entry associated to the path in the mmap will be
@@ -449,8 +446,7 @@ impl Directory for MmapDirectory {
     fn atomic_write(&self, path: &Path, content: &[u8]) -> io::Result<()> {
         debug!("Atomic Write {:?}", path);
         let full_path = self.resolve_path(path);
-        atomic_write(&full_path, content)?;
-        self.sync_directory()
+        atomic_write(&full_path, content)
     }
 
     fn acquire_lock(&self, lock: &Lock) -> Result<DirectoryLock, LockError> {

src/directory/mod.rs

@@ -23,7 +23,7 @@ pub mod error;
 pub use self::directory::DirectoryLock;
 pub use self::directory::{Directory, DirectoryClone};
 pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};
-pub(crate) use self::file_slice::{ArcBytes, WeakArcBytes};
+pub(crate) use self::file_slice::BoxedData;
 pub use self::file_slice::{FileHandle, FileSlice};
 pub use self::owned_bytes::OwnedBytes;
 pub use self::ram_directory::RAMDirectory;

src/directory/owned_bytes.rs

@@ -1,6 +1,5 @@
 use crate::directory::FileHandle;
 use stable_deref_trait::StableDeref;
-use std::convert::TryInto;
 use std::mem;
 use std::ops::Deref;
 use std::sync::Arc;
@@ -96,24 +95,6 @@ impl OwnedBytes {
     pub fn advance(&mut self, advance_len: usize) {
         self.data = &self.data[advance_len..]
     }
-
-    /// Reads an `u8` from the `OwnedBytes` and advance by one byte.
-    pub fn read_u8(&mut self) -> u8 {
-        assert!(!self.is_empty());
-
-        let byte = self.as_slice()[0];
-        self.advance(1);
-        byte
-    }
-
-    /// Reads an `u64` encoded as little-endian from the `OwnedBytes` and advance by 8 bytes.
-    pub fn read_u64(&mut self) -> u64 {
-        assert!(self.len() > 7);
-
-        let octlet: [u8; 8] = self.as_slice()[..8].try_into().unwrap();
-        self.advance(8);
-        u64::from_le_bytes(octlet)
-    }
 }
 
 impl fmt::Debug for OwnedBytes {
@@ -249,22 +230,6 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn test_owned_bytes_read_u8() -> io::Result<()> {
-        let mut bytes = OwnedBytes::new(b"\xFF".as_ref());
-        assert_eq!(bytes.read_u8(), 255);
-        assert_eq!(bytes.len(), 0);
-        Ok(())
-    }
-
-    #[test]
-    fn test_owned_bytes_read_u64() -> io::Result<()> {
-        let mut bytes = OwnedBytes::new(b"\0\xFF\xFF\xFF\xFF\xFF\xFF\xFF".as_ref());
-        assert_eq!(bytes.read_u64(), u64::MAX - 255);
-        assert_eq!(bytes.len(), 0);
-        Ok(())
-    }
-
     #[test]
     fn test_owned_bytes_split() {
         let bytes = OwnedBytes::new(b"abcdefghi".as_ref());

src/directory/ram_directory.rs

@@ -12,8 +12,6 @@ use std::path::{Path, PathBuf};
 use std::result;
 use std::sync::{Arc, RwLock};
 
-use super::FileHandle;
-
 /// Writer associated with the `RAMDirectory`
 ///
 /// The Writer just writes a buffer.
@@ -165,11 +163,6 @@ impl RAMDirectory {
 }
 
 impl Directory for RAMDirectory {
-    fn get_file_handle(&self, path: &Path) -> Result<Box<dyn FileHandle>, OpenReadError> {
-        let file_slice = self.open_read(path)?;
-        Ok(Box::new(file_slice))
-    }
-
     fn open_read(&self, path: &Path) -> result::Result<FileSlice, OpenReadError> {
         self.fs.read().unwrap().open_read(path)
     }

src/directory/watch_event_router.rs

@@ -6,12 +6,12 @@ use std::sync::Weak;
 
 /// Cloneable wrapper for callbacks registered when watching files of a `Directory`.
 #[derive(Clone)]
-pub struct WatchCallback(Arc<dyn Fn() + Sync + Send>);
+pub struct WatchCallback(Arc<Box<dyn Fn() + Sync + Send>>);
 
 impl WatchCallback {
     /// Wraps a `Fn()` to create a WatchCallback.
     pub fn new<F: Fn() + Sync + Send + 'static>(op: F) -> Self {
-        WatchCallback(Arc::new(op))
+        WatchCallback(Arc::new(Box::new(op)))
     }
 
     fn call(&self) {

src/docset.rs

@@ -10,7 +10,7 @@ use std::borrow::BorrowMut;
 pub const TERMINATED: DocId = std::i32::MAX as u32;
 
 /// Represents an iterable set of sorted doc ids.
-pub trait DocSet: Send {
+pub trait DocSet {
     /// Goes to the next element.
     ///
     /// The DocId of the next element is returned.
@@ -129,14 +129,6 @@ impl<'a> DocSet for &'a mut dyn DocSet {
     fn size_hint(&self) -> u32 {
         (**self).size_hint()
     }
-
-    fn count(&mut self, delete_bitset: &DeleteBitSet) -> u32 {
-        (**self).count(delete_bitset)
-    }
-
-    fn count_including_deleted(&mut self) -> u32 {
-        (**self).count_including_deleted()
-    }
 }
 
 impl<TDocSet: DocSet + ?Sized> DocSet for Box<TDocSet> {

src/fastfield/facet_reader.rs

@@ -1,5 +1,4 @@
 use super::MultiValueIntFastFieldReader;
-use crate::error::DataCorruption;
 use crate::schema::Facet;
 use crate::termdict::TermDictionary;
 use crate::termdict::TermOrdinal;
@@ -63,13 +62,12 @@ impl FacetReader {
         &mut self,
         facet_ord: TermOrdinal,
         output: &mut Facet,
-    ) -> crate::Result<()> {
+    ) -> Result<(), str::Utf8Error> {
         let found_term = self
             .term_dict
-            .ord_to_term(facet_ord as u64, &mut self.buffer)?;
+            .ord_to_term(facet_ord as u64, &mut self.buffer);
         assert!(found_term, "Term ordinal {} no found.", facet_ord);
-        let facet_str = str::from_utf8(&self.buffer[..])
-            .map_err(|utf8_err| DataCorruption::comment_only(utf8_err.to_string()))?;
+        let facet_str = str::from_utf8(&self.buffer[..])?;
         output.set_facet_str(facet_str);
         Ok(())
     }

src/fastfield/reader.rs

@@ -51,15 +51,6 @@ impl<Item: FastValue> FastFieldReader<Item> {
         }
     }
 
-    pub(crate) fn cast<TFastValue: FastValue>(self) -> FastFieldReader<TFastValue> {
-        FastFieldReader {
-            bit_unpacker: self.bit_unpacker,
-            min_value_u64: self.min_value_u64,
-            max_value_u64: self.max_value_u64,
-            _phantom: PhantomData,
-        }
-    }
-
     /// Return the value associated to the given document.
     ///
     /// This accessor should return as fast as possible.

src/fastfield/readers.rs

@@ -1,6 +1,6 @@
 use crate::common::CompositeFile;
+use crate::fastfield::BytesFastFieldReader;
 use crate::fastfield::MultiValueIntFastFieldReader;
-use crate::fastfield::{BytesFastFieldReader, FastValue};
 use crate::fastfield::{FastFieldNotAvailableError, FastFieldReader};
 use crate::schema::{Cardinality, Field, FieldType, Schema};
 use crate::space_usage::PerFieldSpaceUsage;
@@ -201,14 +201,6 @@ impl FastFieldReaders {
         None
     }
 
-    pub(crate) fn typed_fast_field_reader<TFastValue: FastValue>(
-        &self,
-        field: Field,
-    ) -> Option<FastFieldReader<TFastValue>> {
-        self.u64_lenient(field)
-            .map(|fast_field_reader| fast_field_reader.cast())
-    }
-
     /// Returns the `i64` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a i64 fast field, this method returns `None`.

src/fieldnorm/reader.rs

@@ -61,38 +61,16 @@ impl FieldNormReaders {
 /// precompute computationally expensive functions of the fieldnorm
 /// in a very short array.
 #[derive(Clone)]
-pub struct FieldNormReader(ReaderImplEnum);
-
-impl From<ReaderImplEnum> for FieldNormReader {
-    fn from(reader_enum: ReaderImplEnum) -> FieldNormReader {
-        FieldNormReader(reader_enum)
-    }
-}
-
-#[derive(Clone)]
-enum ReaderImplEnum {
-    FromData(OwnedBytes),
-    Const {
-        num_docs: u32,
-        fieldnorm_id: u8,
-        fieldnorm: u32,
-    },
+pub struct FieldNormReader {
+    data: OwnedBytes,
 }
 
 impl FieldNormReader {
     /// Creates a `FieldNormReader` with a constant fieldnorm.
-    ///
-    /// The fieldnorm will be subjected to compression as if it was coming
-    /// from an array-backed fieldnorm reader.
     pub fn constant(num_docs: u32, fieldnorm: u32) -> FieldNormReader {
         let fieldnorm_id = fieldnorm_to_id(fieldnorm);
-        let fieldnorm = id_to_fieldnorm(fieldnorm_id);
-        ReaderImplEnum::Const {
-            num_docs,
-            fieldnorm_id,
-            fieldnorm,
-        }
-        .into()
+        let field_norms_data = OwnedBytes::new(vec![fieldnorm_id; num_docs as usize]);
+        FieldNormReader::new(field_norms_data)
     }
 
     /// Opens a field norm reader given its file.
@@ -102,15 +80,12 @@ impl FieldNormReader {
     }
 
     fn new(data: OwnedBytes) -> Self {
-        ReaderImplEnum::FromData(data).into()
+        FieldNormReader { data }
     }
 
     /// Returns the number of documents in this segment.
     pub fn num_docs(&self) -> u32 {
-        match &self.0 {
-            ReaderImplEnum::FromData(data) => data.len() as u32,
-            ReaderImplEnum::Const { num_docs, .. } => *num_docs,
-        }
+        self.data.len() as u32
     }
 
     /// Returns the `fieldnorm` associated to a doc id.
@@ -123,25 +98,14 @@ impl FieldNormReader {
     /// The fieldnorm is effectively decoded from the
     /// `fieldnorm_id` by doing a simple table lookup.
     pub fn fieldnorm(&self, doc_id: DocId) -> u32 {
-        match &self.0 {
-            ReaderImplEnum::FromData(data) => {
-                let fieldnorm_id = data.as_slice()[doc_id as usize];
-                id_to_fieldnorm(fieldnorm_id)
-            }
-            ReaderImplEnum::Const { fieldnorm, .. } => *fieldnorm,
-        }
+        let fieldnorm_id = self.fieldnorm_id(doc_id);
+        id_to_fieldnorm(fieldnorm_id)
     }
 
     /// Returns the `fieldnorm_id` associated to a document.
     #[inline(always)]
     pub fn fieldnorm_id(&self, doc_id: DocId) -> u8 {
-        match &self.0 {
-            ReaderImplEnum::FromData(data) => {
-                let fieldnorm_id = data.as_slice()[doc_id as usize];
-                fieldnorm_id
-            }
-            ReaderImplEnum::Const { fieldnorm_id, .. } => *fieldnorm_id,
-        }
+        self.data.as_slice()[doc_id as usize]
     }
 
     /// Converts a `fieldnorm_id` into a fieldnorm.
@@ -165,7 +129,9 @@ impl FieldNormReader {
             .map(FieldNormReader::fieldnorm_to_id)
             .collect::<Vec<u8>>();
         let field_norms_data = OwnedBytes::new(field_norms_id);
-        FieldNormReader::new(field_norms_data)
+        FieldNormReader {
+            data: field_norms_data,
+        }
     }
 }
 
@@ -184,20 +150,4 @@ mod tests {
         assert_eq!(fieldnorm_reader.fieldnorm(3), 4);
         assert_eq!(fieldnorm_reader.fieldnorm(4), 983_064);
     }
-
-    #[test]
-    fn test_const_fieldnorm_reader_small_fieldnorm_id() {
-        let fieldnorm_reader = FieldNormReader::constant(1_000_000u32, 10u32);
-        assert_eq!(fieldnorm_reader.num_docs(), 1_000_000u32);
-        assert_eq!(fieldnorm_reader.fieldnorm(0u32), 10u32);
-        assert_eq!(fieldnorm_reader.fieldnorm_id(0u32), 10u8);
-    }
-
-    #[test]
-    fn test_const_fieldnorm_reader_large_fieldnorm_id() {
-        let fieldnorm_reader = FieldNormReader::constant(1_000_000u32, 300u32);
-        assert_eq!(fieldnorm_reader.num_docs(), 1_000_000u32);
-        assert_eq!(fieldnorm_reader.fieldnorm(0u32), 280u32);
-        assert_eq!(fieldnorm_reader.fieldnorm_id(0u32), 72u8);
-    }
 }

src/functional_test.rs

@@ -31,7 +31,7 @@ fn test_indexing() {
     let mut uncommitted_docs: HashSet<u64> = HashSet::new();
 
     for _ in 0..200 {
-        let random_val = rng.gen_range(0..20);
+        let random_val = rng.gen_range(0, 20);
         if random_val == 0 {
             index_writer.commit().expect("Commit failed");
             committed_docs.extend(&uncommitted_docs);

src/indexer/delete_queue.rs

@@ -53,7 +53,7 @@ impl DeleteQueue {
             return block;
         }
         let block = Arc::new(Block {
-            operations: Arc::new([]),
+            operations: Arc::default(),
             next: NextBlock::from(self.clone()),
         });
         wlock.last_block = Arc::downgrade(&block);
@@ -108,7 +108,7 @@ impl DeleteQueue {
         let delete_operations = mem::replace(&mut self_wlock.writer, vec![]);
 
         let new_block = Arc::new(Block {
-            operations: Arc::from(delete_operations.into_boxed_slice()),
+            operations: Arc::new(delete_operations.into_boxed_slice()),
             next: NextBlock::from(self.clone()),
         });
 
@@ -167,7 +167,7 @@ impl NextBlock {
 }
 
 struct Block {
-    operations: Arc<[DeleteOperation]>,
+    operations: Arc<Box<[DeleteOperation]>>,
     next: NextBlock,
 }
 

src/indexer/index_writer.rs

@@ -449,7 +449,7 @@ impl IndexWriter {
     }
 
     /// Accessor to the merge policy.
-    pub fn get_merge_policy(&self) -> Arc<dyn MergePolicy> {
+    pub fn get_merge_policy(&self) -> Arc<Box<dyn MergePolicy>> {
         self.segment_updater.get_merge_policy()
     }
 

src/indexer/log_merge_policy.rs

@@ -8,7 +8,7 @@ const DEFAULT_MIN_LAYER_SIZE: u32 = 10_000;
 const DEFAULT_MIN_MERGE_SIZE: usize = 8;
 const DEFAULT_MAX_MERGE_SIZE: usize = 10_000_000;
 
-/// `LogMergePolicy` tries to merge segments that have a similar number of
+/// `LogMergePolicy` tries tries to merge segments that have a similar number of
 /// documents.
 #[derive(Debug, Clone)]
 pub struct LogMergePolicy {

src/indexer/merger.rs

@@ -503,6 +503,7 @@ impl IndexMerger {
         let mut positions_buffer: Vec<u32> = Vec::with_capacity(1_000);
         let mut delta_computer = DeltaComputer::new();
 
+        let mut field_term_streams = Vec::new();
         let mut max_term_ords: Vec<TermOrdinal> = Vec::new();
 
         let field_readers: Vec<Arc<InvertedIndexReader>> = self
@@ -511,10 +512,9 @@ impl IndexMerger {
             .map(|reader| reader.inverted_index(indexed_field))
             .collect::<crate::Result<Vec<_>>>()?;
 
-        let mut field_term_streams = Vec::new();
         for field_reader in &field_readers {
             let terms = field_reader.terms();
-            field_term_streams.push(terms.stream()?);
+            field_term_streams.push(terms.stream());
             max_term_ords.push(terms.num_terms() as u64);
         }
 

src/indexer/operation.rs

@@ -9,15 +9,6 @@ pub struct DeleteOperation {
     pub term: Term,
 }
 
-impl Default for DeleteOperation {
-    fn default() -> Self {
-        DeleteOperation {
-            opstamp: 0u64,
-            term: Term::new(),
-        }
-    }
-}
-
 /// Timestamped Add operation.
 #[derive(Eq, PartialEq, Debug)]
 pub struct AddOperation {

src/indexer/segment_updater.rs

@@ -154,7 +154,7 @@ pub(crate) struct InnerSegmentUpdater {
 
     index: Index,
     segment_manager: SegmentManager,
-    merge_policy: RwLock<Arc<dyn MergePolicy>>,
+    merge_policy: RwLock<Arc<Box<dyn MergePolicy>>>,
     killed: AtomicBool,
     stamper: Stamper,
     merge_operations: MergeOperationInventory,
@@ -193,19 +193,19 @@ impl SegmentUpdater {
             merge_thread_pool,
             index,
             segment_manager,
-            merge_policy: RwLock::new(Arc::new(DefaultMergePolicy::default())),
+            merge_policy: RwLock::new(Arc::new(Box::new(DefaultMergePolicy::default()))),
             killed: AtomicBool::new(false),
             stamper,
             merge_operations: Default::default(),
         })))
     }
 
-    pub fn get_merge_policy(&self) -> Arc<dyn MergePolicy> {
+    pub fn get_merge_policy(&self) -> Arc<Box<dyn MergePolicy>> {
         self.merge_policy.read().unwrap().clone()
     }
 
     pub fn set_merge_policy(&self, merge_policy: Box<dyn MergePolicy>) {
-        let arc_merge_policy = Arc::from(merge_policy);
+        let arc_merge_policy = Arc::new(merge_policy);
         *self.merge_policy.write().unwrap() = arc_merge_policy;
     }
 

src/indexer/segment_writer.rs

@@ -10,9 +10,10 @@ use crate::schema::FieldType;
 use crate::schema::Schema;
 use crate::schema::Term;
 use crate::schema::Value;
-use crate::tokenizer::PreTokenizedStream;
-use crate::tokenizer::{DynTokenStreamChain, Tokenizer};
-use crate::tokenizer::{FacetTokenizer, TextAnalyzerT, Token};
+use crate::schema::{Field, FieldEntry};
+use crate::tokenizer::{BoxTokenStream, PreTokenizedStream};
+use crate::tokenizer::{FacetTokenizer, TextAnalyzer};
+use crate::tokenizer::{TokenStreamChain, Tokenizer};
 use crate::Opstamp;
 use crate::{DocId, SegmentComponent};
 
@@ -22,7 +23,7 @@ use crate::{DocId, SegmentComponent};
 fn initial_table_size(per_thread_memory_budget: usize) -> crate::Result<usize> {
     let table_memory_upper_bound = per_thread_memory_budget / 3;
     if let Some(limit) = (10..)
-        .take_while(|&num_bits| compute_table_size(num_bits) < table_memory_upper_bound)
+        .take_while(|num_bits: &usize| compute_table_size(*num_bits) < table_memory_upper_bound)
         .last()
     {
         Ok(limit.min(19)) // we cap it at 2^19 = 512K.
@@ -44,8 +45,7 @@ pub struct SegmentWriter {
     fast_field_writers: FastFieldsWriter,
     fieldnorms_writer: FieldNormsWriter,
     doc_opstamps: Vec<Opstamp>,
-    // TODO: change type
-    tokenizers: Vec<Option<Box<dyn TextAnalyzerT>>>,
+    tokenizers: Vec<Option<TextAnalyzer>>,
     term_buffer: Term,
 }
 
@@ -70,17 +70,17 @@ impl SegmentWriter {
         let multifield_postings = MultiFieldPostingsWriter::new(schema, table_num_bits);
         let tokenizers = schema
             .fields()
-            .map(|(_, field_entry)| match field_entry.field_type() {
-                FieldType::Str(text_options) => {
-                    text_options
+            .map(
+                |(_, field_entry): (Field, &FieldEntry)| match field_entry.field_type() {
+                    FieldType::Str(ref text_options) => text_options
                         .get_indexing_options()
                         .and_then(|text_index_option| {
                             let tokenizer_name = &text_index_option.tokenizer();
                             tokenizer_manager.get(tokenizer_name)
-                        })
-                }
-                _ => None,
-            })
+                        }),
+                    _ => None,
+                },
+            )
             .collect();
         Ok(SegmentWriter {
             max_doc: 0,
@@ -141,13 +141,13 @@ impl SegmentWriter {
             }
             let (term_buffer, multifield_postings) =
                 (&mut self.term_buffer, &mut self.multifield_postings);
-            match field_entry.field_type() {
+            match *field_entry.field_type() {
                 FieldType::HierarchicalFacet => {
                     term_buffer.set_field(field);
                     let facets =
                         field_values
                             .iter()
-                            .flat_map(|field_value| match field_value.value() {
+                            .flat_map(|field_value| match *field_value.value() {
                                 Value::Facet(ref facet) => Some(facet.encoded_str()),
                                 _ => {
                                     panic!("Expected hierarchical facet");
@@ -157,13 +157,12 @@ impl SegmentWriter {
                         let mut unordered_term_id_opt = None;
                         FacetTokenizer
                             .token_stream(facet_str)
-                            .map(|token| {
+                            .process(&mut |token| {
                                 term_buffer.set_text(&token.text);
                                 let unordered_term_id =
                                     multifield_postings.subscribe(doc_id, &term_buffer);
                                 unordered_term_id_opt = Some(unordered_term_id);
-                            })
-                            .count();
+                            });
                         if let Some(unordered_term_id) = unordered_term_id_opt {
                             self.fast_field_writers
                                 .get_multivalue_writer(field)
@@ -173,38 +172,37 @@ impl SegmentWriter {
                     }
                 }
                 FieldType::Str(_) => {
-                    let mut streams_with_offsets = vec![];
+                    let mut token_streams: Vec<BoxTokenStream> = vec![];
+                    let mut offsets = vec![];
                     let mut total_offset = 0;
 
                     for field_value in field_values {
                         match field_value.value() {
                             Value::PreTokStr(tok_str) => {
-                                streams_with_offsets.push((
-                                    Box::new(PreTokenizedStream::from(tok_str.clone()))
-                                        as Box<dyn Iterator<Item = Token>>,
-                                    total_offset,
-                                ));
+                                offsets.push(total_offset);
                                 if let Some(last_token) = tok_str.tokens.last() {
                                     total_offset += last_token.offset_to;
                                 }
+                                token_streams
+                                    .push(PreTokenizedStream::from(tok_str.clone()).into());
                             }
-                            Value::Str(text) => {
+                            Value::Str(ref text) => {
                                 if let Some(ref mut tokenizer) =
                                     self.tokenizers[field.field_id() as usize]
                                 {
-                                    streams_with_offsets
-                                        .push((tokenizer.token_stream(text), total_offset));
+                                    offsets.push(total_offset);
                                     total_offset += text.len();
+                                    token_streams.push(tokenizer.token_stream(text));
                                 }
                             }
                             _ => (),
                         }
                     }
 
-                    let num_tokens = if streams_with_offsets.is_empty() {
+                    let num_tokens = if token_streams.is_empty() {
                         0
                     } else {
-                        let mut token_stream = DynTokenStreamChain::from_vec(streams_with_offsets);
+                        let mut token_stream = TokenStreamChain::new(offsets, token_streams);
                         multifield_postings.index_text(
                             doc_id,
                             field,
@@ -215,62 +213,71 @@ impl SegmentWriter {
 
                     self.fieldnorms_writer.record(doc_id, field, num_tokens);
                 }
-                FieldType::U64(int_option) if int_option.is_indexed() => {
-                    for field_value in field_values {
-                        term_buffer.set_field(field_value.field());
-                        let u64_val = field_value
-                            .value()
-                            .u64_value()
-                            .ok_or_else(make_schema_error)?;
-                        term_buffer.set_u64(u64_val);
-                        multifield_postings.subscribe(doc_id, &term_buffer);
+                FieldType::U64(ref int_option) => {
+                    if int_option.is_indexed() {
+                        for field_value in field_values {
+                            term_buffer.set_field(field_value.field());
+                            let u64_val = field_value
+                                .value()
+                                .u64_value()
+                                .ok_or_else(make_schema_error)?;
+                            term_buffer.set_u64(u64_val);
+                            multifield_postings.subscribe(doc_id, &term_buffer);
+                        }
                     }
                 }
-                FieldType::Date(int_option) if int_option.is_indexed() => {
-                    for field_value in field_values {
-                        term_buffer.set_field(field_value.field());
-                        let date_val = field_value
-                            .value()
-                            .date_value()
-                            .ok_or_else(make_schema_error)?;
-                        term_buffer.set_i64(date_val.timestamp());
-                        multifield_postings.subscribe(doc_id, &term_buffer);
+                FieldType::Date(ref int_option) => {
+                    if int_option.is_indexed() {
+                        for field_value in field_values {
+                            term_buffer.set_field(field_value.field());
+                            let date_val = field_value
+                                .value()
+                                .date_value()
+                                .ok_or_else(make_schema_error)?;
+                            term_buffer.set_i64(date_val.timestamp());
+                            multifield_postings.subscribe(doc_id, &term_buffer);
+                        }
                     }
                 }
-                FieldType::I64(int_option) if int_option.is_indexed() => {
-                    for field_value in field_values {
-                        term_buffer.set_field(field_value.field());
-                        let i64_val = field_value
-                            .value()
-                            .i64_value()
-                            .ok_or_else(make_schema_error)?;
-                        term_buffer.set_i64(i64_val);
-                        multifield_postings.subscribe(doc_id, &term_buffer);
+                FieldType::I64(ref int_option) => {
+                    if int_option.is_indexed() {
+                        for field_value in field_values {
+                            term_buffer.set_field(field_value.field());
+                            let i64_val = field_value
+                                .value()
+                                .i64_value()
+                                .ok_or_else(make_schema_error)?;
+                            term_buffer.set_i64(i64_val);
+                            multifield_postings.subscribe(doc_id, &term_buffer);
+                        }
                     }
                 }
-                FieldType::F64(int_option) if int_option.is_indexed() => {
-                    for field_value in field_values {
-                        term_buffer.set_field(field_value.field());
-                        let f64_val = field_value
-                            .value()
-                            .f64_value()
-                            .ok_or_else(make_schema_error)?;
-                        term_buffer.set_f64(f64_val);
-                        multifield_postings.subscribe(doc_id, &term_buffer);
+                FieldType::F64(ref int_option) => {
+                    if int_option.is_indexed() {
+                        for field_value in field_values {
+                            term_buffer.set_field(field_value.field());
+                            let f64_val = field_value
+                                .value()
+                                .f64_value()
+                                .ok_or_else(make_schema_error)?;
+                            term_buffer.set_f64(f64_val);
+                            multifield_postings.subscribe(doc_id, &term_buffer);
+                        }
                     }
                 }
-                FieldType::Bytes(option) if option.is_indexed() => {
-                    for field_value in field_values {
-                        term_buffer.set_field(field_value.field());
-                        let bytes = field_value
-                            .value()
-                            .bytes_value()
-                            .ok_or_else(make_schema_error)?;
-                        term_buffer.set_bytes(bytes);
-                        self.multifield_postings.subscribe(doc_id, &term_buffer);
+                FieldType::Bytes(ref option) => {
+                    if option.is_indexed() {
+                        for field_value in field_values {
+                            term_buffer.set_field(field_value.field());
+                            let bytes = field_value
+                                .value()
+                                .bytes_value()
+                                .ok_or_else(make_schema_error)?;
+                            term_buffer.set_bytes(bytes);
+                            self.multifield_postings.subscribe(doc_id, &term_buffer);
+                        }
                     }
                 }
-                _ => {}
             }
         }
         doc.filter_fields(|field| schema.get_field_entry(field).is_stored());

src/lib.rs

@@ -160,7 +160,7 @@ pub use self::docset::{DocSet, TERMINATED};
 pub use crate::common::HasLen;
 pub use crate::common::{f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
 pub use crate::core::{Executor, SegmentComponent};
-pub use crate::core::{Index, IndexMeta, Searcher, Segment, SegmentId, SegmentMeta};
+pub use crate::core::{FieldSearcher, Index, IndexMeta, Searcher, Segment, SegmentId, SegmentMeta};
 pub use crate::core::{InvertedIndexReader, SegmentReader};
 pub use crate::directory::Directory;
 pub use crate::indexer::operation::UserOperation;
@@ -174,7 +174,7 @@ use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
 /// Index format version.
-const INDEX_FORMAT_VERSION: u32 = 3;
+const INDEX_FORMAT_VERSION: u32 = 2;
 
 /// Structure version for the index.
 #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]

src/positions/reader.rs

@@ -132,7 +132,7 @@ impl PositionReader {
             "offset arguments should be increasing."
         );
         let delta_to_block_offset = offset as i64 - self.block_offset as i64;
-        if !(0..128).contains(&delta_to_block_offset) {
+        if delta_to_block_offset < 0 || delta_to_block_offset >= 128 {
             // The first position is not within the first block.
             // We need to decompress the first block.
             let delta_to_anchor_offset = offset - self.anchor_offset;

src/positions/serializer.rs

@@ -8,7 +8,7 @@ use std::io::{self, Write};
 pub struct PositionSerializer<W: io::Write> {
     bit_packer: BitPacker4x,
     write_stream: CountingWriter<W>,
-    write_skip_index: W,
+    write_skiplist: W,
     block: Vec<u32>,
     buffer: Vec<u8>,
     num_ints: u64,
@@ -16,11 +16,11 @@ pub struct PositionSerializer<W: io::Write> {
 }
 
 impl<W: io::Write> PositionSerializer<W> {
-    pub fn new(write_stream: W, write_skip_index: W) -> PositionSerializer<W> {
+    pub fn new(write_stream: W, write_skiplist: W) -> PositionSerializer<W> {
         PositionSerializer {
             bit_packer: BitPacker4x::new(),
             write_stream: CountingWriter::wrap(write_stream),
-            write_skip_index,
+            write_skiplist,
             block: Vec::with_capacity(128),
             buffer: vec![0u8; 128 * 4],
             num_ints: 0u64,
@@ -52,7 +52,7 @@ impl<W: io::Write> PositionSerializer<W> {
 
     fn flush_block(&mut self) -> io::Result<()> {
         let num_bits = self.bit_packer.num_bits(&self.block[..]);
-        self.write_skip_index.write_all(&[num_bits])?;
+        self.write_skiplist.write_all(&[num_bits])?;
         let written_len = self
             .bit_packer
             .compress(&self.block[..], &mut self.buffer, num_bits);
@@ -70,10 +70,10 @@ impl<W: io::Write> PositionSerializer<W> {
             self.flush_block()?;
         }
         for &long_skip in &self.long_skips {
-            long_skip.serialize(&mut self.write_skip_index)?;
+            long_skip.serialize(&mut self.write_skiplist)?;
         }
-        (self.long_skips.len() as u32).serialize(&mut self.write_skip_index)?;
-        self.write_skip_index.flush()?;
+        (self.long_skips.len() as u32).serialize(&mut self.write_skiplist)?;
+        self.write_skiplist.flush()?;
         self.write_stream.flush()?;
         Ok(())
     }

src/postings/block_search.rs

@@ -109,9 +109,9 @@ impl BlockSearcher {
     /// The results should be equivalent to
     /// ```compile_fail
     /// block[..]
-    ///       .iter()
-    ///       .take_while(|&&val| val < target)
-    ///       .count()
+    //       .iter()
+    //       .take_while(|&&val| val < target)
+    //       .count()
     /// ```
     ///
     /// The `start` argument is just used to hint that the response is

src/postings/block_segment_postings.rs

@@ -469,7 +469,7 @@ mod tests {
         let segment_reader = searcher.segment_reader(0);
         let inverted_index = segment_reader.inverted_index(int_field).unwrap();
         let term = Term::from_field_u64(int_field, 0u64);
-        let term_info = inverted_index.get_term_info(&term).unwrap().unwrap();
+        let term_info = inverted_index.get_term_info(&term).unwrap();
         inverted_index
             .read_block_postings_from_terminfo(&term_info, IndexRecordOption::Basic)
             .unwrap()
@@ -513,7 +513,7 @@ mod tests {
         {
             let term = Term::from_field_u64(int_field, 0u64);
             let inverted_index = segment_reader.inverted_index(int_field)?;
-            let term_info = inverted_index.get_term_info(&term)?.unwrap();
+            let term_info = inverted_index.get_term_info(&term).unwrap();
             block_segments = inverted_index
                 .read_block_postings_from_terminfo(&term_info, IndexRecordOption::Basic)?;
         }
@@ -521,7 +521,7 @@ mod tests {
         {
             let term = Term::from_field_u64(int_field, 1u64);
             let inverted_index = segment_reader.inverted_index(int_field)?;
-            let term_info = inverted_index.get_term_info(&term)?.unwrap();
+            let term_info = inverted_index.get_term_info(&term).unwrap();
             inverted_index.reset_block_postings_from_terminfo(&term_info, &mut block_segments)?;
         }
         assert_eq!(block_segments.docs(), &[1, 3, 5]);

src/postings/mod.rs

@@ -54,7 +54,7 @@ pub mod tests {
     use crate::DocId;
     use crate::HasLen;
     use crate::Score;
-    use std::{iter, mem};
+    use std::iter;
 
     #[test]
     pub fn test_position_write() -> crate::Result<()> {
@@ -71,7 +71,6 @@ pub mod tests {
             field_serializer.write_doc(doc_id, 4, &delta_positions)?;
         }
         field_serializer.close_term()?;
-        mem::drop(field_serializer);
         posting_serializer.close()?;
         let read = segment.open_read(SegmentComponent::POSITIONS)?;
         assert!(read.len() <= 140);
@@ -180,7 +179,7 @@ pub mod tests {
             let inverted_index = segment_reader.inverted_index(text_field)?;
             assert_eq!(inverted_index.terms().num_terms(), 1);
             let mut bytes = vec![];
-            assert!(inverted_index.terms().ord_to_term(0, &mut bytes)?);
+            assert!(inverted_index.terms().ord_to_term(0, &mut bytes));
             assert_eq!(&bytes, b"hello");
         }
         {
@@ -192,7 +191,7 @@ pub mod tests {
             let inverted_index = segment_reader.inverted_index(text_field)?;
             assert_eq!(inverted_index.terms().num_terms(), 1);
             let mut bytes = vec![];
-            assert!(inverted_index.terms().ord_to_term(0, &mut bytes)?);
+            assert!(inverted_index.terms().ord_to_term(0, &mut bytes));
             assert_eq!(&bytes[..], ok_token_text.as_bytes());
         }
         Ok(())

src/postings/postings_writer.rs

@@ -9,6 +9,7 @@ use crate::postings::{FieldSerializer, InvertedIndexSerializer};
 use crate::schema::IndexRecordOption;
 use crate::schema::{Field, FieldEntry, FieldType, Schema, Term};
 use crate::termdict::TermOrdinal;
+use crate::tokenizer::TokenStream;
 use crate::tokenizer::{Token, MAX_TOKEN_LEN};
 use crate::DocId;
 use fnv::FnvHashMap;
@@ -99,10 +100,12 @@ impl MultiFieldPostingsWriter {
         &mut self,
         doc: DocId,
         field: Field,
-        token_stream: &mut dyn Iterator<Item = Token>,
+        token_stream: &mut dyn TokenStream,
         term_buffer: &mut Term,
     ) -> u32 {
-        self.per_field_postings_writers[field.field_id() as usize].index_text(
+        let postings_writer =
+            self.per_field_postings_writers[field.field_id() as usize].deref_mut();
+        postings_writer.index_text(
             &mut self.term_index,
             doc,
             field,
@@ -214,7 +217,7 @@ pub trait PostingsWriter {
         term_index: &mut TermHashMap,
         doc_id: DocId,
         field: Field,
-        token_stream: &mut dyn Iterator<Item = Token>,
+        token_stream: &mut dyn TokenStream,
         heap: &mut MemoryArena,
         term_buffer: &mut Term,
     ) -> u32 {
@@ -239,7 +242,7 @@ pub trait PostingsWriter {
                 );
             }
         };
-        token_stream.map(|tok| sink(&tok)).count() as u32
+        token_stream.process(&mut sink)
     }
 
     fn total_num_tokens(&self) -> u64;

src/postings/skip.rs

@@ -1,46 +1,32 @@
-use std::convert::TryInto;
-
+use crate::common::{read_u32_vint_no_advance, serialize_vint_u32, BinarySerializable};
 use crate::directory::OwnedBytes;
 use crate::postings::compression::{compressed_block_size, COMPRESSION_BLOCK_SIZE};
 use crate::query::BM25Weight;
 use crate::schema::IndexRecordOption;
 use crate::{DocId, Score, TERMINATED};
 
-#[inline(always)]
-fn encode_block_wand_max_tf(max_tf: u32) -> u8 {
-    max_tf.min(u8::MAX as u32) as u8
-}
-
-#[inline(always)]
-fn decode_block_wand_max_tf(max_tf_code: u8) -> u32 {
-    if max_tf_code == u8::MAX {
-        u32::MAX
-    } else {
-        max_tf_code as u32
-    }
-}
-
-#[inline(always)]
-fn read_u32(data: &[u8]) -> u32 {
-    u32::from_le_bytes(data[..4].try_into().unwrap())
-}
-
-#[inline(always)]
-fn write_u32(val: u32, buf: &mut Vec<u8>) {
-    buf.extend_from_slice(&val.to_le_bytes());
-}
-
 pub struct SkipSerializer {
     buffer: Vec<u8>,
+    prev_doc: DocId,
 }
 
 impl SkipSerializer {
     pub fn new() -> SkipSerializer {
-        SkipSerializer { buffer: Vec::new() }
+        SkipSerializer {
+            buffer: Vec::new(),
+            prev_doc: 0u32,
+        }
     }
 
     pub fn write_doc(&mut self, last_doc: DocId, doc_num_bits: u8) {
-        write_u32(last_doc, &mut self.buffer);
+        assert!(
+            last_doc > self.prev_doc,
+            "write_doc(...) called with non-increasing doc ids. \
+             Did you forget to call clear maybe?"
+        );
+        let delta_doc = last_doc - self.prev_doc;
+        self.prev_doc = last_doc;
+        delta_doc.serialize(&mut self.buffer).unwrap();
         self.buffer.push(doc_num_bits);
     }
 
@@ -49,13 +35,16 @@ impl SkipSerializer {
     }
 
     pub fn write_total_term_freq(&mut self, tf_sum: u32) {
-        write_u32(tf_sum, &mut self.buffer);
+        tf_sum
+            .serialize(&mut self.buffer)
+            .expect("Should never fail");
     }
 
     pub fn write_blockwand_max(&mut self, fieldnorm_id: u8, term_freq: u32) {
-        let block_wand_tf = encode_block_wand_max_tf(term_freq);
-        self.buffer
-            .extend_from_slice(&[fieldnorm_id, block_wand_tf]);
+        self.buffer.push(fieldnorm_id);
+        let mut buf = [0u8; 8];
+        let bytes = serialize_vint_u32(term_freq, &mut buf);
+        self.buffer.extend_from_slice(bytes);
     }
 
     pub fn data(&self) -> &[u8] {
@@ -63,6 +52,7 @@ impl SkipSerializer {
     }
 
     pub fn clear(&mut self) {
+        self.prev_doc = 0u32;
         self.buffer.clear();
     }
 }
@@ -169,13 +159,18 @@ impl SkipReader {
     }
 
     fn read_block_info(&mut self) {
-        let bytes = self.owned_read.as_slice();
-        let advance_len: usize;
-        self.last_doc_in_block = read_u32(bytes);
-        let doc_num_bits = bytes[4];
+        let doc_delta = {
+            let bytes = self.owned_read.as_slice();
+            let mut buf = [0; 4];
+            buf.copy_from_slice(&bytes[..4]);
+            u32::from_le_bytes(buf)
+        };
+        self.last_doc_in_block += doc_delta as DocId;
+        let doc_num_bits = self.owned_read.as_slice()[4];
+
         match self.skip_info {
             IndexRecordOption::Basic => {
-                advance_len = 5;
+                self.owned_read.advance(5);
                 self.block_info = BlockInfo::BitPacked {
                     doc_num_bits,
                     tf_num_bits: 0,
@@ -185,10 +180,11 @@ impl SkipReader {
                 };
             }
             IndexRecordOption::WithFreqs => {
+                let bytes = self.owned_read.as_slice();
                 let tf_num_bits = bytes[5];
                 let block_wand_fieldnorm_id = bytes[6];
-                let block_wand_term_freq = decode_block_wand_max_tf(bytes[7]);
-                advance_len = 8;
+                let (block_wand_term_freq, num_bytes) = read_u32_vint_no_advance(&bytes[7..]);
+                self.owned_read.advance(7 + num_bytes);
                 self.block_info = BlockInfo::BitPacked {
                     doc_num_bits,
                     tf_num_bits,
@@ -198,11 +194,16 @@ impl SkipReader {
                 };
             }
             IndexRecordOption::WithFreqsAndPositions => {
+                let bytes = self.owned_read.as_slice();
                 let tf_num_bits = bytes[5];
-                let tf_sum = read_u32(&bytes[6..10]);
+                let tf_sum = {
+                    let mut buf = [0; 4];
+                    buf.copy_from_slice(&bytes[6..10]);
+                    u32::from_le_bytes(buf)
+                };
                 let block_wand_fieldnorm_id = bytes[10];
-                let block_wand_term_freq = decode_block_wand_max_tf(bytes[11]);
-                advance_len = 12;
+                let (block_wand_term_freq, num_bytes) = read_u32_vint_no_advance(&bytes[11..]);
+                self.owned_read.advance(11 + num_bytes);
                 self.block_info = BlockInfo::BitPacked {
                     doc_num_bits,
                     tf_num_bits,
@@ -212,7 +213,6 @@ impl SkipReader {
                 };
             }
         }
-        self.owned_read.advance(advance_len);
     }
 
     pub fn block_info(&self) -> BlockInfo {
@@ -274,24 +274,6 @@ mod tests {
     use crate::directory::OwnedBytes;
     use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
 
-    #[test]
-    fn test_encode_block_wand_max_tf() {
-        for tf in 0..255 {
-            assert_eq!(super::encode_block_wand_max_tf(tf), tf as u8);
-        }
-        for &tf in &[255, 256, 1_000_000, u32::MAX] {
-            assert_eq!(super::encode_block_wand_max_tf(tf), 255);
-        }
-    }
-
-    #[test]
-    fn test_decode_block_wand_max_tf() {
-        for tf in 0..255 {
-            assert_eq!(super::decode_block_wand_max_tf(tf), tf as u32);
-        }
-        assert_eq!(super::decode_block_wand_max_tf(255), u32::MAX);
-    }
-
     #[test]
     fn test_skip_with_freq() {
         let buf = {

src/query/automaton_weight.rs

@@ -7,7 +7,6 @@ use crate::schema::{Field, IndexRecordOption};
 use crate::termdict::{TermDictionary, TermStreamer};
 use crate::TantivyError;
 use crate::{DocId, Score};
-use std::io;
 use std::sync::Arc;
 use tantivy_fst::Automaton;
 
@@ -20,7 +19,6 @@ pub struct AutomatonWeight<A> {
 impl<A> AutomatonWeight<A>
 where
     A: Automaton + Send + Sync + 'static,
-    A::State: Clone,
 {
     /// Create a new AutomationWeight
     pub fn new<IntoArcA: Into<Arc<A>>>(field: Field, automaton: IntoArcA) -> AutomatonWeight<A> {
@@ -30,10 +28,7 @@ where
         }
     }
 
-    fn automaton_stream<'a>(
-        &'a self,
-        term_dict: &'a TermDictionary,
-    ) -> io::Result<TermStreamer<'a, &'a A>> {
+    fn automaton_stream<'a>(&'a self, term_dict: &'a TermDictionary) -> TermStreamer<'a, &'a A> {
         let automaton: &A = &*self.automaton;
         let term_stream_builder = term_dict.search(automaton);
         term_stream_builder.into_stream()
@@ -43,14 +38,13 @@ where
 impl<A> Weight for AutomatonWeight<A>
 where
     A: Automaton + Send + Sync + 'static,
-    A::State: Clone,
 {
     fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
         let max_doc = reader.max_doc();
         let mut doc_bitset = BitSet::with_max_value(max_doc);
         let inverted_index = reader.inverted_index(self.field)?;
         let term_dict = inverted_index.terms();
-        let mut term_stream = self.automaton_stream(term_dict)?;
+        let mut term_stream = self.automaton_stream(term_dict);
         while term_stream.advance() {
             let term_info = term_stream.value();
             let mut block_segment_postings = inverted_index
@@ -104,7 +98,6 @@ mod tests {
         index
     }
 
-    #[derive(Clone, Copy)]
     enum State {
         Start,
         NotMatching,

src/query/bm25.rs

@@ -106,7 +106,7 @@ impl BM25Weight {
         BM25Weight::new(idf_explain, avg_fieldnorm)
     }
 
-    pub(crate) fn new(idf_explain: Explanation, average_fieldnorm: Score) -> BM25Weight {
+    fn new(idf_explain: Explanation, average_fieldnorm: Score) -> BM25Weight {
         let weight = idf_explain.value() * (1.0 + K1);
         BM25Weight {
             idf_explain,

src/query/query_parser/query_parser.rs

@@ -289,7 +289,7 @@ impl QueryParser {
             let field_name = field_entry.name().to_string();
             return Err(QueryParserError::FieldNotIndexed(field_name));
         }
-        match field_type {
+        match *field_type {
             FieldType::I64(_) => {
                 let val: i64 = i64::from_str(phrase)?;
                 let term = Term::from_field_i64(field, val);
@@ -312,7 +312,7 @@ impl QueryParser {
                 let term = Term::from_field_u64(field, val);
                 Ok(vec![(0, term)])
             }
-            FieldType::Str(str_options) => {
+            FieldType::Str(ref str_options) => {
                 if let Some(option) = str_options.get_indexing_options() {
                     let tokenizer =
                         self.tokenizer_manager
@@ -323,14 +323,15 @@ impl QueryParser {
                                     option.tokenizer().to_string(),
                                 )
                             })?;
-                    let token_stream = tokenizer.token_stream(phrase);
-                    let terms: Vec<_> = token_stream
-                        .map(|token| {
-                            let term = Term::from_field_text(field, &token.text);
-                            (token.position, term)
-                        })
-                        .collect();
-                    if terms.len() <= 1 {
+                    let mut terms: Vec<(usize, Term)> = Vec::new();
+                    let mut token_stream = tokenizer.token_stream(phrase);
+                    token_stream.process(&mut |token| {
+                        let term = Term::from_field_text(field, &token.text);
+                        terms.push((token.position, term));
+                    });
+                    if terms.is_empty() {
+                        Ok(vec![])
+                    } else if terms.len() == 1 {
                         Ok(terms)
                     } else {
                         let field_entry = self.schema.get_field_entry(field);
@@ -413,7 +414,7 @@ impl QueryParser {
         &self,
         given_field: &Option<String>,
     ) -> Result<Cow<'_, [Field]>, QueryParserError> {
-        match given_field {
+        match *given_field {
             None => {
                 if self.default_fields.is_empty() {
                     Err(QueryParserError::NoDefaultFieldDeclared)
@@ -421,7 +422,7 @@ impl QueryParser {
                     Ok(Cow::from(&self.default_fields[..]))
                 }
             }
-            Some(field) => Ok(Cow::from(vec![self.resolve_field_name(&*field)?])),
+            Some(ref field) => Ok(Cow::from(vec![self.resolve_field_name(&*field)?])),
         }
     }
 
@@ -573,12 +574,15 @@ fn convert_to_query(logical_ast: LogicalAST) -> Box<dyn Query> {
 #[cfg(test)]
 mod test {
     use super::super::logical_ast::*;
-    use super::*;
+    use super::QueryParser;
+    use super::QueryParserError;
     use crate::query::Query;
     use crate::schema::Field;
     use crate::schema::{IndexRecordOption, TextFieldIndexing, TextOptions};
     use crate::schema::{Schema, Term, INDEXED, STORED, STRING, TEXT};
-    use crate::tokenizer::{analyzer_builder, LowerCaser, SimpleTokenizer, StopWordFilter};
+    use crate::tokenizer::{
+        LowerCaser, SimpleTokenizer, StopWordFilter, TextAnalyzer, TokenizerManager,
+    };
     use crate::Index;
     use matches::assert_matches;
 
@@ -616,10 +620,9 @@ mod test {
         let tokenizer_manager = TokenizerManager::default();
         tokenizer_manager.register(
             "en_with_stop_words",
-            analyzer_builder(SimpleTokenizer)
-                .filter(LowerCaser::new())
-                .filter(StopWordFilter::remove(vec!["the".to_string()]))
-                .build(),
+            TextAnalyzer::from(SimpleTokenizer)
+                .filter(LowerCaser)
+                .filter(StopWordFilter::remove(vec!["the".to_string()])),
         );
         QueryParser::new(schema, default_fields, tokenizer_manager)
     }

src/query/range_query.rs

@@ -11,7 +11,6 @@ use crate::schema::{Field, IndexRecordOption, Term};
 use crate::termdict::{TermDictionary, TermStreamer};
 use crate::{DocId, Score};
 use std::collections::Bound;
-use std::io;
 use std::ops::Range;
 
 fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
@@ -275,7 +274,7 @@ pub struct RangeWeight {
 }
 
 impl RangeWeight {
-    fn term_range<'a>(&self, term_dict: &'a TermDictionary) -> io::Result<TermStreamer<'a>> {
+    fn term_range<'a>(&self, term_dict: &'a TermDictionary) -> TermStreamer<'a> {
         use std::collections::Bound::*;
         let mut term_stream_builder = term_dict.range();
         term_stream_builder = match self.left_bound {
@@ -299,7 +298,7 @@ impl Weight for RangeWeight {
 
         let inverted_index = reader.inverted_index(self.field)?;
         let term_dict = inverted_index.terms();
-        let mut term_range = self.term_range(term_dict)?;
+        let mut term_range = self.term_range(term_dict);
         while term_range.advance() {
             let term_info = term_range.value();
             let mut block_segment_postings = inverted_index

src/query/reqopt_scorer.rs

@@ -12,7 +12,7 @@ use std::marker::PhantomData;
 /// This is useful for queries like `+somethingrequired somethingoptional`.
 ///
 /// Note that `somethingoptional` has no impact on the `DocSet`.
-pub struct RequiredOptionalScorer<TReqScorer, TOptScorer, TScoreCombiner: ScoreCombiner> {
+pub struct RequiredOptionalScorer<TReqScorer, TOptScorer, TScoreCombiner> {
     req_scorer: TReqScorer,
     opt_scorer: TOptScorer,
     score_cache: Option<Score>,
@@ -23,7 +23,6 @@ impl<TReqScorer, TOptScorer, TScoreCombiner>
     RequiredOptionalScorer<TReqScorer, TOptScorer, TScoreCombiner>
 where
     TOptScorer: DocSet,
-    TScoreCombiner: ScoreCombiner,
 {
     /// Creates a new `RequiredOptionalScorer`.
     pub fn new(
@@ -44,7 +43,6 @@ impl<TReqScorer, TOptScorer, TScoreCombiner> DocSet
 where
     TReqScorer: DocSet,
     TOptScorer: DocSet,
-    TScoreCombiner: ScoreCombiner,
 {
     fn advance(&mut self) -> DocId {
         self.score_cache = None;

src/query/score_combiner.rs

@@ -3,7 +3,7 @@ use crate::Score;
 
 /// The `ScoreCombiner` trait defines how to compute
 /// an overall score given a list of scores.
-pub trait ScoreCombiner: Default + Clone + Send + Copy + 'static {
+pub trait ScoreCombiner: Default + Clone + Copy + 'static {
     /// Aggregates the score combiner with the given scorer.
     ///
     /// The `ScoreCombiner` may decide to call `.scorer.score()`

src/query/term_query/term_query.rs

@@ -1,7 +1,7 @@
 use super::term_weight::TermWeight;
 use crate::query::bm25::BM25Weight;
+use crate::query::Query;
 use crate::query::Weight;
-use crate::query::{Explanation, Query};
 use crate::schema::IndexRecordOption;
 use crate::Searcher;
 use crate::Term;
@@ -100,13 +100,7 @@ impl TermQuery {
                 field_entry.name()
             )));
         }
-        let bm25_weight;
-        if scoring_enabled {
-            bm25_weight = BM25Weight::for_terms(searcher, &[term])?;
-        } else {
-            bm25_weight =
-                BM25Weight::new(Explanation::new("<no score>".to_string(), 1.0f32), 1.0f32);
-        }
+        let bm25_weight = BM25Weight::for_terms(searcher, &[term])?;
         let index_record_option = if scoring_enabled {
             self.index_record_option
         } else {

src/query/term_query/term_scorer.rs

@@ -302,7 +302,7 @@ mod tests {
         let mut rng = rand::thread_rng();
         writer.set_merge_policy(Box::new(NoMergePolicy));
         for _ in 0..3_000 {
-            let term_freq = rng.gen_range(1..10000);
+            let term_freq = rng.gen_range(1, 10000);
             let words: Vec<&str> = std::iter::repeat("bbbb").take(term_freq).collect();
             let text = words.join(" ");
             writer.add_document(doc!(text_field=>text));

src/query/term_query/term_weight.rs

@@ -45,7 +45,7 @@ impl Weight for TermWeight {
         } else {
             let field = self.term.field();
             let inv_index = reader.inverted_index(field)?;
-            let term_info = inv_index.get_term_info(&self.term)?;
+            let term_info = inv_index.get_term_info(&self.term);
             Ok(term_info.map(|term_info| term_info.doc_freq).unwrap_or(0))
         }
     }

src/schema/facet.rs

@@ -233,7 +233,6 @@ mod tests {
         assert_eq!(Facet::root(), Facet::from("/"));
         assert_eq!(format!("{}", Facet::root()), "/");
         assert!(Facet::root().is_root());
-        assert_eq!(Facet::root().encoded_str(), "");
     }
 
     #[test]

src/schema/named_field_document.rs

@@ -1,5 +1,5 @@
 use crate::schema::Value;
-use serde::{Deserialize, Serialize};
+use serde::Serialize;
 use std::collections::BTreeMap;
 
 /// Internal representation of a document used for JSON
@@ -8,5 +8,5 @@ use std::collections::BTreeMap;
 /// A `NamedFieldDocument` is a simple representation of a document
 /// as a `BTreeMap<String, Vec<Value>>`.
 ///
-#[derive(Debug, Deserialize, Serialize)]
+#[derive(Serialize)]
 pub struct NamedFieldDocument(pub BTreeMap<String, Vec<Value>>);

src/snippet/mod.rs

@@ -1,7 +1,7 @@
 use crate::query::Query;
 use crate::schema::Field;
 use crate::schema::Value;
-use crate::tokenizer::{TextAnalyzerT, Token};
+use crate::tokenizer::{TextAnalyzer, Token};
 use crate::Searcher;
 use crate::{Document, Score};
 use htmlescape::encode_minimal;
@@ -139,9 +139,9 @@ impl Snippet {
 ///
 /// Fragments must be valid in the sense that `&text[fragment.start..fragment.stop]`\
 /// has to be a valid string.
-fn search_fragments(
-    tokenizer: &dyn TextAnalyzerT,
-    text: &str,
+fn search_fragments<'a>(
+    tokenizer: &TextAnalyzer,
+    text: &'a str,
     terms: &BTreeMap<String, Score>,
     max_num_chars: usize,
 ) -> Vec<FragmentCandidate> {
@@ -155,7 +155,7 @@ fn search_fragments(
             };
             fragment = FragmentCandidate::new(next.offset_from);
         }
-        fragment.try_add_token(&next, &terms);
+        fragment.try_add_token(next, &terms);
     }
     if fragment.score > 0.0 {
         fragments.push(fragment)
@@ -249,7 +249,7 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str)
 /// ```
 pub struct SnippetGenerator {
     terms_text: BTreeMap<String, Score>,
-    tokenizer: Box<dyn TextAnalyzerT>,
+    tokenizer: TextAnalyzer,
     field: Field,
     max_num_chars: usize,
 }
@@ -297,37 +297,33 @@ impl SnippetGenerator {
     ///
     /// This method extract the text associated to the `SnippetGenerator`'s field
     /// and computes a snippet.
-    pub fn snippet_from_doc(&mut self, doc: &Document) -> Snippet {
+    pub fn snippet_from_doc(&self, doc: &Document) -> Snippet {
         let text: String = doc
             .get_all(self.field)
             .flat_map(Value::text)
             .collect::<Vec<&str>>()
             .join(" ");
-        self.snippet(text.as_ref())
+        self.snippet(&text)
     }
 
     /// Generates a snippet for the given text.
-    pub fn snippet(&mut self, text: &str) -> Snippet {
-        let fragment_candidates = search_fragments(
-            &mut *self.tokenizer,
-            text,
-            &self.terms_text,
-            self.max_num_chars,
-        );
-        select_best_fragment_combination(&fragment_candidates[..], text)
+    pub fn snippet(&self, text: &str) -> Snippet {
+        let fragment_candidates =
+            search_fragments(&self.tokenizer, &text, &self.terms_text, self.max_num_chars);
+        select_best_fragment_combination(&fragment_candidates[..], &text)
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use super::*;
+    use super::{search_fragments, select_best_fragment_combination};
     use crate::query::QueryParser;
     use crate::schema::{IndexRecordOption, Schema, TextFieldIndexing, TextOptions, TEXT};
     use crate::tokenizer::SimpleTokenizer;
-    use crate::tokenizer::TextAnalyzer;
     use crate::Index;
     use crate::SnippetGenerator;
     use maplit::btreemap;
+    use std::collections::BTreeMap;
     use std::iter::Iterator;
 
     const TEST_TEXT: &'static str = r#"Rust is a systems programming language sponsored by
@@ -350,13 +346,7 @@ Survey in 2016, 2017, and 2018."#;
             String::from("rust") => 1.0,
             String::from("language") => 0.9
         };
-
-        let fragments = search_fragments(
-            &Into::<TextAnalyzer<_>>::into(SimpleTokenizer),
-            TEST_TEXT,
-            &terms,
-            100,
-        );
+        let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 100);
         assert_eq!(fragments.len(), 7);
         {
             let first = &fragments[0];
@@ -383,12 +373,7 @@ Survey in 2016, 2017, and 2018."#;
                 String::from("rust") =>1.0,
                 String::from("language") => 0.9
             };
-            let fragments = search_fragments(
-                &Into::<TextAnalyzer<_>>::into(SimpleTokenizer),
-                TEST_TEXT,
-                &terms,
-                20,
-            );
+            let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 20);
             {
                 let first = &fragments[0];
                 assert_eq!(first.score, 1.0);
@@ -402,12 +387,7 @@ Survey in 2016, 2017, and 2018."#;
                 String::from("rust") =>0.9,
                 String::from("language") => 1.0
             };
-            let fragments = search_fragments(
-                &Into::<TextAnalyzer<_>>::into(SimpleTokenizer),
-                TEST_TEXT,
-                &terms,
-                20,
-            );
+            let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 20);
             //assert_eq!(fragments.len(), 7);
             {
                 let first = &fragments[0];
@@ -426,12 +406,7 @@ Survey in 2016, 2017, and 2018."#;
         let mut terms = BTreeMap::new();
         terms.insert(String::from("c"), 1.0);
 
-        let fragments = search_fragments(
-            &Into::<TextAnalyzer<_>>::into(SimpleTokenizer),
-            &text,
-            &terms,
-            3,
-        );
+        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
 
         assert_eq!(fragments.len(), 1);
         {
@@ -453,12 +428,7 @@ Survey in 2016, 2017, and 2018."#;
         let mut terms = BTreeMap::new();
         terms.insert(String::from("f"), 1.0);
 
-        let fragments = search_fragments(
-            &Into::<TextAnalyzer<_>>::into(SimpleTokenizer),
-            &text,
-            &terms,
-            3,
-        );
+        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
 
         assert_eq!(fragments.len(), 2);
         {
@@ -481,12 +451,7 @@ Survey in 2016, 2017, and 2018."#;
         terms.insert(String::from("f"), 1.0);
         terms.insert(String::from("a"), 0.9);
 
-        let fragments = search_fragments(
-            &Into::<TextAnalyzer<_>>::into(SimpleTokenizer),
-            &text,
-            &terms,
-            7,
-        );
+        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 7);
 
         assert_eq!(fragments.len(), 2);
         {
@@ -508,12 +473,7 @@ Survey in 2016, 2017, and 2018."#;
         let mut terms = BTreeMap::new();
         terms.insert(String::from("z"), 1.0);
 
-        let fragments = search_fragments(
-            &Into::<TextAnalyzer<_>>::into(SimpleTokenizer),
-            &text,
-            &terms,
-            3,
-        );
+        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
 
         assert_eq!(fragments.len(), 0);
 
@@ -527,12 +487,7 @@ Survey in 2016, 2017, and 2018."#;
         let text = "a b c d";
 
         let terms = BTreeMap::new();
-        let fragments = search_fragments(
-            &Into::<TextAnalyzer<_>>::into(SimpleTokenizer),
-            &text,
-            &terms,
-            3,
-        );
+        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
         assert_eq!(fragments.len(), 0);
 
         let snippet = select_best_fragment_combination(&fragments[..], &text);
@@ -617,12 +572,12 @@ Survey in 2016, 2017, and 2018."#;
         let mut snippet_generator =
             SnippetGenerator::create(&searcher, &*query, text_field).unwrap();
         {
-            let snippet = snippet_generator.snippet(TEST_TEXT.into());
+            let snippet = snippet_generator.snippet(TEST_TEXT);
             assert_eq!(snippet.to_html(), "imperative-procedural paradigms. <b>Rust</b> is syntactically similar to C++[according to whom?],\nbut its <b>designers</b> intend it to provide better memory safety");
         }
         {
             snippet_generator.set_max_num_chars(90);
-            let snippet = snippet_generator.snippet(TEST_TEXT.into());
+            let snippet = snippet_generator.snippet(TEST_TEXT);
             assert_eq!(snippet.to_html(), "<b>Rust</b> is syntactically similar to C++[according to whom?],\nbut its <b>designers</b> intend it to");
         }
     }

src/store/compression_lz4.rs

@@ -3,7 +3,7 @@ use std::io::{self, Read, Write};
 /// Name of the compression scheme used in the doc store.
 ///
 /// This name is appended to the version string of tantivy.
-pub const COMPRESSION: &str = "lz4";
+pub const COMPRESSION: &'static str = "lz4";
 
 pub fn compress(uncompressed: &[u8], compressed: &mut Vec<u8>) -> io::Result<()> {
     compressed.clear();

src/store/index/skip_index.rs

@@ -19,7 +19,7 @@ impl<'a> Iterator for LayerCursor<'a> {
                 return None;
             }
             let (block_mut, remaining_mut) = (&mut self.block, &mut self.remaining);
-            if block_mut.deserialize(remaining_mut).is_err() {
+            if let Err(_) = block_mut.deserialize(remaining_mut) {
                 return None;
             }
             self.cursor = 0;
@@ -35,11 +35,11 @@ struct Layer {
 }
 
 impl Layer {
-    fn cursor(&self) -> impl Iterator<Item = Checkpoint> + '_ {
+    fn cursor<'a>(&'a self) -> impl Iterator<Item = Checkpoint> + 'a {
         self.cursor_at_offset(0u64)
     }
 
-    fn cursor_at_offset(&self, start_offset: u64) -> impl Iterator<Item = Checkpoint> + '_ {
+    fn cursor_at_offset<'a>(&'a self, start_offset: u64) -> impl Iterator<Item = Checkpoint> + 'a {
         let data = &self.data.as_slice();
         LayerCursor {
             remaining: &data[start_offset as usize..],
@@ -50,7 +50,8 @@ impl Layer {
 
     fn seek_start_at_offset(&self, target: DocId, offset: u64) -> Option<Checkpoint> {
         self.cursor_at_offset(offset)
-            .find(|checkpoint| checkpoint.end_doc > target)
+            .filter(|checkpoint| checkpoint.end_doc > target)
+            .next()
     }
 }
 
@@ -59,7 +60,7 @@ pub struct SkipIndex {
 }
 
 impl SkipIndex {
-    pub(crate) fn checkpoints(&self) -> impl Iterator<Item = Checkpoint> + '_ {
+    pub(crate) fn checkpoints<'a>(&'a self) -> impl Iterator<Item = Checkpoint> + 'a {
         self.layers
             .last()
             .into_iter()

src/store/reader.rs

@@ -46,7 +46,7 @@ impl StoreReader {
         })
     }
 
-    pub(crate) fn block_checkpoints(&self) -> impl Iterator<Item = Checkpoint> + '_ {
+    pub(crate) fn block_checkpoints<'a>(&'a self) -> impl Iterator<Item = Checkpoint> + 'a {
         self.skip_index.checkpoints()
     }
 

src/termdict/fst_termdict/mod.rs

@@ -1,27 +0,0 @@
-/*!
-The term dictionary main role is to associate the sorted [`Term`s](../struct.Term.html) to
-a [`TermInfo`](../postings/struct.TermInfo.html) struct that contains some meta-information
-about the term.
-
-Internally, the term dictionary relies on the `fst` crate to store
-a sorted mapping that associate each term to its rank in the lexicographical order.
-For instance, in a dictionary containing the sorted terms "abba", "bjork", "blur" and "donovan",
-the `TermOrdinal` are respectively `0`, `1`, `2`, and `3`.
-
-For `u64`-terms, tantivy explicitely uses a `BigEndian` representation to ensure that the
-lexicographical order matches the natural order of integers.
-
-`i64`-terms are transformed to `u64` using a continuous mapping `val ⟶ val - i64::min_value()`
-and then treated as a `u64`.
-
-`f64`-terms are transformed to `u64` using a mapping that preserve order, and are then treated
-as `u64`.
-
-A second datastructure makes it possible to access a [`TermInfo`](../postings/struct.TermInfo.html).
-*/
-mod streamer;
-mod term_info_store;
-mod termdict;
-
-pub use self::streamer::{TermStreamer, TermStreamerBuilder};
-pub use self::termdict::{TermDictionary, TermDictionaryBuilder};

src/termdict/mod.rs

@@ -20,37 +20,438 @@ as `u64`.
 A second datastructure makes it possible to access a [`TermInfo`](../postings/struct.TermInfo.html).
 */
 
-use tantivy_fst::automaton::AlwaysMatch;
-
-mod fst_termdict;
-use fst_termdict as termdict;
-
-mod merger;
-
-#[cfg(test)]
-mod tests;
-
 /// Position of the term in the sorted list of terms.
 pub type TermOrdinal = u64;
 
-/// The term dictionary contains all of the terms in
-/// `tantivy index` in a sorted manner.
-pub type TermDictionary = self::termdict::TermDictionary;
+mod merger;
+mod streamer;
+mod term_info_store;
+mod termdict;
 
-/// Builder for the new term dictionary.
-///
-/// Inserting must be done in the order of the `keys`.
-pub type TermDictionaryBuilder<W> = self::termdict::TermDictionaryBuilder<W>;
+pub use self::merger::TermMerger;
+pub use self::streamer::{TermStreamer, TermStreamerBuilder};
+pub use self::termdict::{TermDictionary, TermDictionaryBuilder};
 
-/// Given a list of sorted term streams,
-/// returns an iterator over sorted unique terms.
-///
-/// The item yield is actually a pair with
-/// - the term
-/// - a slice with the ordinal of the segments containing
-/// the terms.
-pub type TermMerger<'a> = self::merger::TermMerger<'a>;
+#[cfg(test)]
+mod tests {
+    use super::{TermDictionary, TermDictionaryBuilder, TermStreamer};
+    use crate::core::Index;
+    use crate::directory::{Directory, FileSlice, RAMDirectory};
+    use crate::postings::TermInfo;
+    use crate::schema::{Schema, TEXT};
+    use std::path::PathBuf;
+    use std::str;
 
-/// `TermStreamer` acts as a cursor over a range of terms of a segment.
-/// Terms are guaranteed to be sorted.
-pub type TermStreamer<'a, A = AlwaysMatch> = self::termdict::TermStreamer<'a, A>;
+    const BLOCK_SIZE: usize = 1_500;
+
+    fn make_term_info(term_ord: u64) -> TermInfo {
+        let offset = |term_ord: u64| term_ord * 100 + term_ord * term_ord;
+        TermInfo {
+            doc_freq: term_ord as u32,
+            postings_start_offset: offset(term_ord),
+            postings_stop_offset: offset(term_ord + 1),
+            positions_idx: offset(term_ord) * 2u64,
+        }
+    }
+
+    #[test]
+    fn test_empty_term_dictionary() {
+        let empty = TermDictionary::empty();
+        assert!(empty.stream().next().is_none());
+    }
+
+    #[test]
+    fn test_term_ordinals() -> crate::Result<()> {
+        const COUNTRIES: [&'static str; 7] = [
+            "San Marino",
+            "Serbia",
+            "Slovakia",
+            "Slovenia",
+            "Spain",
+            "Sweden",
+            "Switzerland",
+        ];
+        let directory = RAMDirectory::create();
+        let path = PathBuf::from("TermDictionary");
+        {
+            let write = directory.open_write(&path)?;
+            let mut term_dictionary_builder = TermDictionaryBuilder::create(write)?;
+            for term in COUNTRIES.iter() {
+                term_dictionary_builder.insert(term.as_bytes(), &make_term_info(0u64))?;
+            }
+            term_dictionary_builder.finish()?;
+        }
+        let term_file = directory.open_read(&path)?;
+        let term_dict: TermDictionary = TermDictionary::open(term_file)?;
+        for (term_ord, term) in COUNTRIES.iter().enumerate() {
+            assert_eq!(term_dict.term_ord(term).unwrap(), term_ord as u64);
+            let mut bytes = vec![];
+            assert!(term_dict.ord_to_term(term_ord as u64, &mut bytes));
+            assert_eq!(bytes, term.as_bytes());
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn test_term_dictionary_simple() -> crate::Result<()> {
+        let directory = RAMDirectory::create();
+        let path = PathBuf::from("TermDictionary");
+        {
+            let write = directory.open_write(&path)?;
+            let mut term_dictionary_builder = TermDictionaryBuilder::create(write)?;
+            term_dictionary_builder.insert("abc".as_bytes(), &make_term_info(34u64))?;
+            term_dictionary_builder.insert("abcd".as_bytes(), &make_term_info(346u64))?;
+            term_dictionary_builder.finish()?;
+        }
+        let file = directory.open_read(&path)?;
+        let term_dict: TermDictionary = TermDictionary::open(file)?;
+        assert_eq!(term_dict.get("abc").unwrap().doc_freq, 34u32);
+        assert_eq!(term_dict.get("abcd").unwrap().doc_freq, 346u32);
+        let mut stream = term_dict.stream();
+        {
+            {
+                let (k, v) = stream.next().unwrap();
+                assert_eq!(k.as_ref(), "abc".as_bytes());
+                assert_eq!(v.doc_freq, 34u32);
+            }
+            assert_eq!(stream.key(), "abc".as_bytes());
+            assert_eq!(stream.value().doc_freq, 34u32);
+        }
+        {
+            {
+                let (k, v) = stream.next().unwrap();
+                assert_eq!(k, "abcd".as_bytes());
+                assert_eq!(v.doc_freq, 346u32);
+            }
+            assert_eq!(stream.key(), "abcd".as_bytes());
+            assert_eq!(stream.value().doc_freq, 346u32);
+        }
+        assert!(!stream.advance());
+        Ok(())
+    }
+
+    #[test]
+    fn test_term_iterator() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let text_field = schema_builder.add_text_field("text", TEXT);
+        let index = Index::create_in_ram(schema_builder.build());
+        {
+            let mut index_writer = index.writer_for_tests()?;
+            index_writer.add_document(doc!(text_field=>"a b d f"));
+            index_writer.commit()?;
+            index_writer.add_document(doc!(text_field=>"a b c d f"));
+            index_writer.commit()?;
+            index_writer.add_document(doc!(text_field => "e f"));
+            index_writer.commit()?;
+        }
+        let searcher = index.reader()?.searcher();
+
+        let field_searcher = searcher.field(text_field)?;
+        let mut term_it = field_searcher.terms();
+        let mut term_string = String::new();
+        while term_it.advance() {
+            //let term = Term::from_bytes(term_it.key());
+            term_string.push_str(str::from_utf8(term_it.key()).expect("test"));
+        }
+        assert_eq!(&*term_string, "abcdef");
+        Ok(())
+    }
+
+    #[test]
+    fn test_term_dictionary_stream() -> crate::Result<()> {
+        let ids: Vec<_> = (0u32..10_000u32)
+            .map(|i| (format!("doc{:0>6}", i), i))
+            .collect();
+        let buffer: Vec<u8> = {
+            let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
+            for &(ref id, ref i) in &ids {
+                term_dictionary_builder
+                    .insert(id.as_bytes(), &make_term_info(*i as u64))
+                    .unwrap();
+            }
+            term_dictionary_builder.finish().unwrap()
+        };
+        let term_file = FileSlice::from(buffer);
+        let term_dictionary: TermDictionary = TermDictionary::open(term_file)?;
+        {
+            let mut streamer = term_dictionary.stream();
+            let mut i = 0;
+            while let Some((streamer_k, streamer_v)) = streamer.next() {
+                let &(ref key, ref v) = &ids[i];
+                assert_eq!(streamer_k.as_ref(), key.as_bytes());
+                assert_eq!(streamer_v, &make_term_info(*v as u64));
+                i += 1;
+            }
+        }
+
+        let &(ref key, ref val) = &ids[2047];
+        assert_eq!(
+            term_dictionary.get(key.as_bytes()),
+            Some(make_term_info(*val as u64))
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn test_stream_high_range_prefix_suffix() -> crate::Result<()> {
+        let buffer: Vec<u8> = {
+            let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
+            // term requires more than 16bits
+            term_dictionary_builder.insert("abcdefghijklmnopqrstuvwxy", &make_term_info(1))?;
+            term_dictionary_builder.insert("abcdefghijklmnopqrstuvwxyz", &make_term_info(2))?;
+            term_dictionary_builder.insert("abr", &make_term_info(3))?;
+            term_dictionary_builder.finish()?
+        };
+        let term_dict_file = FileSlice::from(buffer);
+        let term_dictionary: TermDictionary = TermDictionary::open(term_dict_file)?;
+        let mut kv_stream = term_dictionary.stream();
+        assert!(kv_stream.advance());
+        assert_eq!(kv_stream.key(), "abcdefghijklmnopqrstuvwxy".as_bytes());
+        assert_eq!(kv_stream.value(), &make_term_info(1));
+        assert!(kv_stream.advance());
+        assert_eq!(kv_stream.key(), "abcdefghijklmnopqrstuvwxyz".as_bytes());
+        assert_eq!(kv_stream.value(), &make_term_info(2));
+        assert!(kv_stream.advance());
+        assert_eq!(kv_stream.key(), "abr".as_bytes());
+        assert_eq!(kv_stream.value(), &make_term_info(3));
+        assert!(!kv_stream.advance());
+        Ok(())
+    }
+
+    #[test]
+    fn test_stream_range() -> crate::Result<()> {
+        let ids: Vec<_> = (0u32..10_000u32)
+            .map(|i| (format!("doc{:0>6}", i), i))
+            .collect();
+        let buffer: Vec<u8> = {
+            let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
+            for &(ref id, ref i) in &ids {
+                term_dictionary_builder
+                    .insert(id.as_bytes(), &make_term_info(*i as u64))
+                    .unwrap();
+            }
+            term_dictionary_builder.finish().unwrap()
+        };
+
+        let file = FileSlice::from(buffer);
+
+        let term_dictionary: TermDictionary = TermDictionary::open(file)?;
+        {
+            for i in (0..20).chain(6000..8_000) {
+                let &(ref target_key, _) = &ids[i];
+                let mut streamer = term_dictionary
+                    .range()
+                    .ge(target_key.as_bytes())
+                    .into_stream();
+                for j in 0..3 {
+                    let (streamer_k, streamer_v) = streamer.next().unwrap();
+                    let &(ref key, ref v) = &ids[i + j];
+                    assert_eq!(str::from_utf8(streamer_k.as_ref()).unwrap(), key);
+                    assert_eq!(streamer_v.doc_freq, *v);
+                    assert_eq!(streamer_v, &make_term_info(*v as u64));
+                }
+            }
+        }
+
+        {
+            for i in (0..20).chain(BLOCK_SIZE - 10..BLOCK_SIZE + 10) {
+                let &(ref target_key, _) = &ids[i];
+                let mut streamer = term_dictionary
+                    .range()
+                    .gt(target_key.as_bytes())
+                    .into_stream();
+                for j in 0..3 {
+                    let (streamer_k, streamer_v) = streamer.next().unwrap();
+                    let &(ref key, ref v) = &ids[i + j + 1];
+                    assert_eq!(streamer_k.as_ref(), key.as_bytes());
+                    assert_eq!(streamer_v.doc_freq, *v);
+                }
+            }
+        }
+
+        {
+            for i in (0..20).chain(BLOCK_SIZE - 10..BLOCK_SIZE + 10) {
+                for j in 0..3 {
+                    let &(ref fst_key, _) = &ids[i];
+                    let &(ref last_key, _) = &ids[i + j];
+                    let mut streamer = term_dictionary
+                        .range()
+                        .ge(fst_key.as_bytes())
+                        .lt(last_key.as_bytes())
+                        .into_stream();
+                    for _ in 0..j {
+                        assert!(streamer.next().is_some());
+                    }
+                    assert!(streamer.next().is_none());
+                }
+            }
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn test_empty_string() -> crate::Result<()> {
+        let buffer: Vec<u8> = {
+            let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
+            term_dictionary_builder
+                .insert(&[], &make_term_info(1 as u64))
+                .unwrap();
+            term_dictionary_builder
+                .insert(&[1u8], &make_term_info(2 as u64))
+                .unwrap();
+            term_dictionary_builder.finish().unwrap()
+        };
+        let file = FileSlice::from(buffer);
+        let term_dictionary: TermDictionary = TermDictionary::open(file)?;
+        let mut stream = term_dictionary.stream();
+        assert!(stream.advance());
+        assert!(stream.key().is_empty());
+        assert!(stream.advance());
+        assert_eq!(stream.key(), &[1u8]);
+        assert!(!stream.advance());
+        Ok(())
+    }
+
+    #[test]
+    fn test_stream_range_boundaries() -> crate::Result<()> {
+        let buffer: Vec<u8> = {
+            let mut term_dictionary_builder = TermDictionaryBuilder::create(Vec::new())?;
+            for i in 0u8..10u8 {
+                let number_arr = [i; 1];
+                term_dictionary_builder.insert(&number_arr, &make_term_info(i as u64))?;
+            }
+            term_dictionary_builder.finish()?
+        };
+        let file = FileSlice::from(buffer);
+        let term_dictionary: TermDictionary = TermDictionary::open(file)?;
+
+        let value_list = |mut streamer: TermStreamer<'_>, backwards: bool| {
+            let mut res: Vec<u32> = vec![];
+            while let Some((_, ref v)) = streamer.next() {
+                res.push(v.doc_freq);
+            }
+            if backwards {
+                res.reverse();
+            }
+            res
+        };
+        {
+            let range = term_dictionary.range().backward().into_stream();
+            assert_eq!(
+                value_list(range, true),
+                vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
+            );
+        }
+        {
+            let range = term_dictionary.range().ge([2u8]).into_stream();
+            assert_eq!(
+                value_list(range, false),
+                vec![2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
+            );
+        }
+        {
+            let range = term_dictionary.range().ge([2u8]).backward().into_stream();
+            assert_eq!(
+                value_list(range, true),
+                vec![2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
+            );
+        }
+        {
+            let range = term_dictionary.range().gt([2u8]).into_stream();
+            assert_eq!(
+                value_list(range, false),
+                vec![3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
+            );
+        }
+        {
+            let range = term_dictionary.range().gt([2u8]).backward().into_stream();
+            assert_eq!(
+                value_list(range, true),
+                vec![3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
+            );
+        }
+        {
+            let range = term_dictionary.range().lt([6u8]).into_stream();
+            assert_eq!(
+                value_list(range, false),
+                vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32]
+            );
+        }
+        {
+            let range = term_dictionary.range().lt([6u8]).backward().into_stream();
+            assert_eq!(
+                value_list(range, true),
+                vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32]
+            );
+        }
+        {
+            let range = term_dictionary.range().le([6u8]).into_stream();
+            assert_eq!(
+                value_list(range, false),
+                vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32]
+            );
+        }
+        {
+            let range = term_dictionary.range().le([6u8]).backward().into_stream();
+            assert_eq!(
+                value_list(range, true),
+                vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32]
+            );
+        }
+        {
+            let range = term_dictionary.range().ge([0u8]).lt([5u8]).into_stream();
+            assert_eq!(value_list(range, false), vec![0u32, 1u32, 2u32, 3u32, 4u32]);
+        }
+        {
+            let range = term_dictionary
+                .range()
+                .ge([0u8])
+                .lt([5u8])
+                .backward()
+                .into_stream();
+            assert_eq!(value_list(range, true), vec![0u32, 1u32, 2u32, 3u32, 4u32]);
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn test_automaton_search() -> crate::Result<()> {
+        use crate::query::DFAWrapper;
+        use levenshtein_automata::LevenshteinAutomatonBuilder;
+
+        const COUNTRIES: [&'static str; 7] = [
+            "San Marino",
+            "Serbia",
+            "Slovakia",
+            "Slovenia",
+            "Spain",
+            "Sweden",
+            "Switzerland",
+        ];
+
+        let directory = RAMDirectory::create();
+        let path = PathBuf::from("TermDictionary");
+        {
+            let write = directory.open_write(&path)?;
+            let mut term_dictionary_builder = TermDictionaryBuilder::create(write)?;
+            for term in COUNTRIES.iter() {
+                term_dictionary_builder.insert(term.as_bytes(), &make_term_info(0u64))?;
+            }
+            term_dictionary_builder.finish()?;
+        }
+        let file = directory.open_read(&path)?;
+        let term_dict: TermDictionary = TermDictionary::open(file)?;
+
+        // We can now build an entire dfa.
+        let lev_automaton_builder = LevenshteinAutomatonBuilder::new(2, true);
+        let automaton = DFAWrapper(lev_automaton_builder.build_dfa("Spaen"));
+
+        let mut range = term_dict.search(automaton).into_stream();
+
+        // get the first finding
+        assert!(range.advance());
+        assert_eq!("Spain".as_bytes(), range.key());
+        assert!(!range.advance());
+        Ok(())
+    }
+}

src/termdict/streamer.rs

@@ -1,5 +1,3 @@
-use std::io;
-
 use super::TermDictionary;
 use crate::postings::TermInfo;
 use crate::termdict::TermOrdinal;
@@ -61,14 +59,14 @@ where
 
     /// Creates the stream corresponding to the range
     /// of terms defined using the `TermStreamerBuilder`.
-    pub fn into_stream(self) -> io::Result<TermStreamer<'a, A>> {
-        Ok(TermStreamer {
+    pub fn into_stream(self) -> TermStreamer<'a, A> {
+        TermStreamer {
             fst_map: self.fst_map,
             stream: self.stream_builder.into_stream(),
             term_ord: 0u64,
             current_key: Vec::with_capacity(100),
             current_value: TermInfo::default(),
-        })
+        }
     }
 }
 

src/termdict/termdict.rs

@@ -80,6 +80,7 @@ where
                 .serialize(&mut counting_writer)?;
             let footer_size = counting_writer.written_bytes();
             (footer_size as u64).serialize(&mut counting_writer)?;
+            counting_writer.flush()?;
         }
         Ok(file)
     }
@@ -138,8 +139,8 @@ impl TermDictionary {
     }
 
     /// Returns the ordinal associated to a given term.
-    pub fn term_ord<K: AsRef<[u8]>>(&self, key: K) -> io::Result<Option<TermOrdinal>> {
-        Ok(self.fst_index.get(key))
+    pub fn term_ord<K: AsRef<[u8]>>(&self, key: K) -> Option<TermOrdinal> {
+        self.fst_index.get(key)
     }
 
     /// Returns the term associated to a given term ordinal.
@@ -151,7 +152,7 @@ impl TermDictionary {
     ///
     /// Regardless of whether the term is found or not,
     /// the buffer may be modified.
-    pub fn ord_to_term(&self, mut ord: TermOrdinal, bytes: &mut Vec<u8>) -> io::Result<bool> {
+    pub fn ord_to_term(&self, mut ord: TermOrdinal, bytes: &mut Vec<u8>) -> bool {
         bytes.clear();
         let fst = self.fst_index.as_fst();
         let mut node = fst.root();
@@ -166,10 +167,10 @@ impl TermDictionary {
                 let new_node_addr = transition.addr;
                 node = fst.node(new_node_addr);
             } else {
-                return Ok(false);
+                return false;
             }
         }
-        Ok(true)
+        true
     }
 
     /// Returns the number of terms in the dictionary.
@@ -178,10 +179,9 @@ impl TermDictionary {
     }
 
     /// Lookups the value corresponding to the key.
-    pub fn get<K: AsRef<[u8]>>(&self, key: K) -> io::Result<Option<TermInfo>> {
-        Ok(self
-            .term_ord(key)?
-            .map(|term_ord| self.term_info_from_ord(term_ord)))
+    pub fn get<K: AsRef<[u8]>>(&self, key: K) -> Option<TermInfo> {
+        self.term_ord(key)
+            .map(|term_ord| self.term_info_from_ord(term_ord))
     }
 
     /// Returns a range builder, to stream all of the terms
@@ -191,7 +191,7 @@ impl TermDictionary {
     }
 
     /// A stream of all the sorted terms. [See also `.stream_field()`](#method.stream_field)
-    pub fn stream(&self) -> io::Result<TermStreamer<'_>> {
+    pub fn stream(&self) -> TermStreamer<'_> {
         self.range().into_stream()
     }
 

src/termdict/tests.rs

@@ -1,431 +0,0 @@
-use super::{TermDictionary, TermDictionaryBuilder, TermStreamer};
-
-use crate::directory::{Directory, FileSlice, RAMDirectory, TerminatingWrite};
-use crate::postings::TermInfo;
-
-use std::path::PathBuf;
-use std::str;
-
-const BLOCK_SIZE: usize = 1_500;
-
-fn make_term_info(term_ord: u64) -> TermInfo {
-    let offset = |term_ord: u64| term_ord * 100 + term_ord * term_ord;
-    TermInfo {
-        doc_freq: term_ord as u32,
-        postings_start_offset: offset(term_ord),
-        postings_stop_offset: offset(term_ord + 1),
-        positions_idx: offset(term_ord) * 2u64,
-    }
-}
-
-#[test]
-fn test_empty_term_dictionary() {
-    let empty = TermDictionary::empty();
-    assert!(empty.stream().unwrap().next().is_none());
-}
-
-#[test]
-fn test_term_ordinals() -> crate::Result<()> {
-    const COUNTRIES: [&'static str; 7] = [
-        "San Marino",
-        "Serbia",
-        "Slovakia",
-        "Slovenia",
-        "Spain",
-        "Sweden",
-        "Switzerland",
-    ];
-    let directory = RAMDirectory::create();
-    let path = PathBuf::from("TermDictionary");
-    {
-        let write = directory.open_write(&path)?;
-        let mut term_dictionary_builder = TermDictionaryBuilder::create(write)?;
-        for term in COUNTRIES.iter() {
-            term_dictionary_builder.insert(term.as_bytes(), &make_term_info(0u64))?;
-        }
-        term_dictionary_builder.finish()?.terminate()?;
-    }
-    let term_file = directory.open_read(&path)?;
-    let term_dict: TermDictionary = TermDictionary::open(term_file)?;
-    for (term_ord, term) in COUNTRIES.iter().enumerate() {
-        assert_eq!(term_dict.term_ord(term)?, Some(term_ord as u64));
-        let mut bytes = vec![];
-        assert!(term_dict.ord_to_term(term_ord as u64, &mut bytes)?);
-        assert_eq!(bytes, term.as_bytes());
-    }
-    Ok(())
-}
-
-#[test]
-fn test_term_dictionary_simple() -> crate::Result<()> {
-    let directory = RAMDirectory::create();
-    let path = PathBuf::from("TermDictionary");
-    {
-        let write = directory.open_write(&path)?;
-        let mut term_dictionary_builder = TermDictionaryBuilder::create(write)?;
-        term_dictionary_builder.insert("abc".as_bytes(), &make_term_info(34u64))?;
-        term_dictionary_builder.insert("abcd".as_bytes(), &make_term_info(346u64))?;
-        term_dictionary_builder.finish()?.terminate()?;
-    }
-    let file = directory.open_read(&path)?;
-    let term_dict: TermDictionary = TermDictionary::open(file)?;
-    assert_eq!(term_dict.get("abc")?.unwrap().doc_freq, 34u32);
-    assert_eq!(term_dict.get("abcd")?.unwrap().doc_freq, 346u32);
-    let mut stream = term_dict.stream()?;
-    {
-        {
-            let (k, v) = stream.next().unwrap();
-            assert_eq!(k.as_ref(), "abc".as_bytes());
-            assert_eq!(v.doc_freq, 34u32);
-        }
-        assert_eq!(stream.key(), "abc".as_bytes());
-        assert_eq!(stream.value().doc_freq, 34u32);
-    }
-    {
-        {
-            let (k, v) = stream.next().unwrap();
-            assert_eq!(k, "abcd".as_bytes());
-            assert_eq!(v.doc_freq, 346u32);
-        }
-        assert_eq!(stream.key(), "abcd".as_bytes());
-        assert_eq!(stream.value().doc_freq, 346u32);
-    }
-    assert!(!stream.advance());
-    Ok(())
-}
-
-#[test]
-fn test_term_dictionary_stream() -> crate::Result<()> {
-    let ids: Vec<_> = (0u32..10_000u32)
-        .map(|i| (format!("doc{:0>6}", i), i))
-        .collect();
-    let buffer: Vec<u8> = {
-        let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
-        for &(ref id, ref i) in &ids {
-            term_dictionary_builder
-                .insert(id.as_bytes(), &make_term_info(*i as u64))
-                .unwrap();
-        }
-        term_dictionary_builder.finish()?
-    };
-    let term_file = FileSlice::from(buffer);
-    let term_dictionary: TermDictionary = TermDictionary::open(term_file)?;
-    {
-        let mut streamer = term_dictionary.stream()?;
-        let mut i = 0;
-        while let Some((streamer_k, streamer_v)) = streamer.next() {
-            let &(ref key, ref v) = &ids[i];
-            assert_eq!(streamer_k.as_ref(), key.as_bytes());
-            assert_eq!(streamer_v, &make_term_info(*v as u64));
-            i += 1;
-        }
-    }
-
-    let &(ref key, ref val) = &ids[2047];
-    assert_eq!(
-        term_dictionary.get(key.as_bytes())?,
-        Some(make_term_info(*val as u64))
-    );
-    Ok(())
-}
-
-#[test]
-fn test_stream_high_range_prefix_suffix() -> crate::Result<()> {
-    let buffer: Vec<u8> = {
-        let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
-        // term requires more than 16bits
-        term_dictionary_builder.insert("abcdefghijklmnopqrstuvwxy", &make_term_info(1))?;
-        term_dictionary_builder.insert("abcdefghijklmnopqrstuvwxyz", &make_term_info(2))?;
-        term_dictionary_builder.insert("abr", &make_term_info(3))?;
-        term_dictionary_builder.finish()?
-    };
-    let term_dict_file = FileSlice::from(buffer);
-    let term_dictionary: TermDictionary = TermDictionary::open(term_dict_file)?;
-    let mut kv_stream = term_dictionary.stream()?;
-    assert!(kv_stream.advance());
-    assert_eq!(kv_stream.key(), "abcdefghijklmnopqrstuvwxy".as_bytes());
-    assert_eq!(kv_stream.value(), &make_term_info(1));
-    assert!(kv_stream.advance());
-    assert_eq!(kv_stream.key(), "abcdefghijklmnopqrstuvwxyz".as_bytes());
-    assert_eq!(kv_stream.value(), &make_term_info(2));
-    assert!(kv_stream.advance());
-    assert_eq!(kv_stream.key(), "abr".as_bytes());
-    assert_eq!(kv_stream.value(), &make_term_info(3));
-    assert!(!kv_stream.advance());
-    Ok(())
-}
-
-#[test]
-fn test_stream_range() -> crate::Result<()> {
-    let ids: Vec<_> = (0u32..10_000u32)
-        .map(|i| (format!("doc{:0>6}", i), i))
-        .collect();
-    let buffer: Vec<u8> = {
-        let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
-        for &(ref id, ref i) in &ids {
-            term_dictionary_builder
-                .insert(id.as_bytes(), &make_term_info(*i as u64))
-                .unwrap();
-        }
-        term_dictionary_builder.finish()?
-    };
-
-    let file = FileSlice::from(buffer);
-
-    let term_dictionary: TermDictionary = TermDictionary::open(file)?;
-    {
-        for i in (0..20).chain(6000..8_000) {
-            let &(ref target_key, _) = &ids[i];
-            let mut streamer = term_dictionary
-                .range()
-                .ge(target_key.as_bytes())
-                .into_stream()?;
-            for j in 0..3 {
-                let (streamer_k, streamer_v) = streamer.next().unwrap();
-                let &(ref key, ref v) = &ids[i + j];
-                assert_eq!(str::from_utf8(streamer_k.as_ref()).unwrap(), key);
-                assert_eq!(streamer_v.doc_freq, *v);
-                assert_eq!(streamer_v, &make_term_info(*v as u64));
-            }
-        }
-    }
-
-    {
-        for i in (0..20).chain(BLOCK_SIZE - 10..BLOCK_SIZE + 10) {
-            let &(ref target_key, _) = &ids[i];
-            let mut streamer = term_dictionary
-                .range()
-                .gt(target_key.as_bytes())
-                .into_stream()?;
-            for j in 0..3 {
-                let (streamer_k, streamer_v) = streamer.next().unwrap();
-                let &(ref key, ref v) = &ids[i + j + 1];
-                assert_eq!(streamer_k.as_ref(), key.as_bytes());
-                assert_eq!(streamer_v.doc_freq, *v);
-            }
-        }
-    }
-
-    {
-        for i in (0..20).chain(BLOCK_SIZE - 10..BLOCK_SIZE + 10) {
-            for j in 0..3 {
-                let &(ref fst_key, _) = &ids[i];
-                let &(ref last_key, _) = &ids[i + j];
-                let mut streamer = term_dictionary
-                    .range()
-                    .ge(fst_key.as_bytes())
-                    .lt(last_key.as_bytes())
-                    .into_stream()?;
-                for _ in 0..j {
-                    assert!(streamer.next().is_some());
-                }
-                assert!(streamer.next().is_none());
-            }
-        }
-    }
-    Ok(())
-}
-
-#[test]
-fn test_empty_string() -> crate::Result<()> {
-    let buffer: Vec<u8> = {
-        let mut term_dictionary_builder = TermDictionaryBuilder::create(vec![]).unwrap();
-        term_dictionary_builder
-            .insert(&[], &make_term_info(1 as u64))
-            .unwrap();
-        term_dictionary_builder
-            .insert(&[1u8], &make_term_info(2 as u64))
-            .unwrap();
-        term_dictionary_builder.finish()?
-    };
-    let file = FileSlice::from(buffer);
-    let term_dictionary: TermDictionary = TermDictionary::open(file)?;
-    let mut stream = term_dictionary.stream()?;
-    assert!(stream.advance());
-    assert!(stream.key().is_empty());
-    assert!(stream.advance());
-    assert_eq!(stream.key(), &[1u8]);
-    assert!(!stream.advance());
-    Ok(())
-}
-
-fn stream_range_test_dict() -> crate::Result<TermDictionary> {
-    let buffer: Vec<u8> = {
-        let mut term_dictionary_builder = TermDictionaryBuilder::create(Vec::new())?;
-        for i in 0u8..10u8 {
-            let number_arr = [i; 1];
-            term_dictionary_builder.insert(&number_arr, &make_term_info(i as u64))?;
-        }
-        term_dictionary_builder.finish()?
-    };
-    let file = FileSlice::from(buffer);
-    TermDictionary::open(file)
-}
-
-#[test]
-fn test_stream_range_boundaries_forward() -> crate::Result<()> {
-    let term_dictionary = stream_range_test_dict()?;
-    let value_list = |mut streamer: TermStreamer<'_>| {
-        let mut res: Vec<u32> = vec![];
-        while let Some((_, ref v)) = streamer.next() {
-            res.push(v.doc_freq);
-        }
-        res
-    };
-    {
-        let range = term_dictionary.range().ge([2u8]).into_stream()?;
-        assert_eq!(
-            value_list(range),
-            vec![2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
-        );
-    }
-    {
-        let range = term_dictionary.range().gt([2u8]).into_stream()?;
-        assert_eq!(
-            value_list(range),
-            vec![3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
-        );
-    }
-    {
-        let range = term_dictionary.range().lt([6u8]).into_stream()?;
-        assert_eq!(value_list(range), vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32]);
-    }
-    {
-        let range = term_dictionary.range().le([6u8]).into_stream()?;
-        assert_eq!(
-            value_list(range),
-            vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32]
-        );
-    }
-    {
-        let range = term_dictionary.range().ge([0u8]).lt([5u8]).into_stream()?;
-        assert_eq!(value_list(range), vec![0u32, 1u32, 2u32, 3u32, 4u32]);
-    }
-    Ok(())
-}
-
-#[test]
-fn test_stream_range_boundaries_backward() -> crate::Result<()> {
-    let term_dictionary = stream_range_test_dict()?;
-    let value_list_backward = |mut streamer: TermStreamer<'_>| {
-        let mut res: Vec<u32> = vec![];
-        while let Some((_, ref v)) = streamer.next() {
-            res.push(v.doc_freq);
-        }
-        res.reverse();
-        res
-    };
-    {
-        let range = term_dictionary.range().backward().into_stream()?;
-        assert_eq!(
-            value_list_backward(range),
-            vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
-        );
-    }
-    {
-        let range = term_dictionary.range().ge([2u8]).backward().into_stream()?;
-        assert_eq!(
-            value_list_backward(range),
-            vec![2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
-        );
-    }
-    {
-        let range = term_dictionary.range().gt([2u8]).backward().into_stream()?;
-        assert_eq!(
-            value_list_backward(range),
-            vec![3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
-        );
-    }
-    {
-        let range = term_dictionary.range().lt([6u8]).backward().into_stream()?;
-        assert_eq!(
-            value_list_backward(range),
-            vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32]
-        );
-    }
-    {
-        let range = term_dictionary.range().le([6u8]).backward().into_stream()?;
-        assert_eq!(
-            value_list_backward(range),
-            vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32]
-        );
-    }
-    {
-        let range = term_dictionary
-            .range()
-            .ge([0u8])
-            .lt([5u8])
-            .backward()
-            .into_stream()?;
-        assert_eq!(
-            value_list_backward(range),
-            vec![0u32, 1u32, 2u32, 3u32, 4u32]
-        );
-    }
-    Ok(())
-}
-
-#[test]
-fn test_ord_to_term() -> crate::Result<()> {
-    let termdict = stream_range_test_dict()?;
-    let mut bytes = vec![];
-    for b in 0u8..10u8 {
-        termdict.ord_to_term(b as u64, &mut bytes)?;
-        assert_eq!(&bytes, &[b]);
-    }
-    Ok(())
-}
-
-#[test]
-fn test_stream_term_ord() -> crate::Result<()> {
-    let termdict = stream_range_test_dict()?;
-    let mut stream = termdict.stream()?;
-    for b in 0u8..10u8 {
-        assert!(stream.advance(), true);
-        assert_eq!(stream.term_ord(), b as u64);
-        assert_eq!(stream.key(), &[b]);
-    }
-    assert!(!stream.advance());
-    Ok(())
-}
-
-#[test]
-fn test_automaton_search() -> crate::Result<()> {
-    use crate::query::DFAWrapper;
-    use levenshtein_automata::LevenshteinAutomatonBuilder;
-
-    const COUNTRIES: [&'static str; 7] = [
-        "San Marino",
-        "Serbia",
-        "Slovakia",
-        "Slovenia",
-        "Spain",
-        "Sweden",
-        "Switzerland",
-    ];
-
-    let directory = RAMDirectory::create();
-    let path = PathBuf::from("TermDictionary");
-    {
-        let write = directory.open_write(&path)?;
-        let mut term_dictionary_builder = TermDictionaryBuilder::create(write)?;
-        for term in COUNTRIES.iter() {
-            term_dictionary_builder.insert(term.as_bytes(), &make_term_info(0u64))?;
-        }
-        term_dictionary_builder.finish()?.terminate()?;
-    }
-    let file = directory.open_read(&path)?;
-    let term_dict: TermDictionary = TermDictionary::open(file)?;
-
-    // We can now build an entire dfa.
-    let lev_automaton_builder = LevenshteinAutomatonBuilder::new(2, true);
-    let automaton = DFAWrapper(lev_automaton_builder.build_dfa("Spaen"));
-
-    let mut range = term_dict.search(automaton).into_stream()?;
-
-    // get the first finding
-    assert!(range.advance());
-    assert_eq!("Spain".as_bytes(), range.key());
-    assert!(!range.advance());
-    Ok(())
-}

src/tokenizer/alphanum_only.rs

@@ -2,16 +2,16 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let tokenizer = analyzer_builder(RawTokenizer)
-//!   .filter(AlphaNumOnlyFilter).build();
+//! let tokenizer = TextAnalyzer::from(RawTokenizer)
+//!   .filter(AlphaNumOnlyFilter);
 //!
 //! let mut stream = tokenizer.token_stream("hello there");
 //! // is none because the raw filter emits one token that
 //! // contains a space
 //! assert!(stream.next().is_none());
 //!
-//! let tokenizer = analyzer_builder(SimpleTokenizer)
-//!   .filter(AlphaNumOnlyFilter).build();
+//! let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+//!   .filter(AlphaNumOnlyFilter);
 //!
 //! let mut stream = tokenizer.token_stream("hello there 💣");
 //! assert!(stream.next().is_some());
@@ -19,18 +19,45 @@
 //! // the "emoji" is dropped because its not an alphanum
 //! assert!(stream.next().is_none());
 //! ```
-use super::{Token, TokenFilter};
+use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
 
 /// `TokenFilter` that removes all tokens that contain non
 /// ascii alphanumeric characters.
-#[derive(Clone, Debug, Default)]
+#[derive(Clone)]
 pub struct AlphaNumOnlyFilter;
 
-impl TokenFilter for AlphaNumOnlyFilter {
-    fn transform(&mut self, token: Token) -> Option<Token> {
-        if token.text.chars().all(|c| c.is_ascii_alphanumeric()) {
-            return Some(token);
-        }
-        None
+pub struct AlphaNumOnlyFilterStream<'a> {
+    tail: BoxTokenStream<'a>,
+}
+
+impl<'a> AlphaNumOnlyFilterStream<'a> {
+    fn predicate(&self, token: &Token) -> bool {
+        token.text.chars().all(|c| c.is_ascii_alphanumeric())
+    }
+}
+
+impl TokenFilter for AlphaNumOnlyFilter {
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(AlphaNumOnlyFilterStream { tail: token_stream })
+    }
+}
+
+impl<'a> TokenStream for AlphaNumOnlyFilterStream<'a> {
+    fn advance(&mut self) -> bool {
+        while self.tail.advance() {
+            if self.predicate(self.tail.token()) {
+                return true;
+            }
+        }
+
+        false
+    }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
     }
 }

src/tokenizer/ascii_folding_filter.rs

@@ -1,31 +1,45 @@
-use super::{Token, TokenFilter};
+use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
 use std::mem;
 
 /// This class converts alphabetic, numeric, and symbolic Unicode characters
 /// which are not in the first 127 ASCII characters (the "Basic Latin" Unicode
 /// block) into their ASCII equivalents, if one exists.
-#[derive(Clone, Debug, Default)]
-pub struct AsciiFolding {
-    buffer: String,
-}
+#[derive(Clone)]
+pub struct AsciiFoldingFilter;
 
-impl AsciiFolding {
-    /// Construct a new `AsciiFolding` filter.
-    pub fn new() -> Self {
-        Self {
+impl TokenFilter for AsciiFoldingFilter {
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        From::from(AsciiFoldingFilterTokenStream {
+            tail: token_stream,
             buffer: String::with_capacity(100),
-        }
+        })
     }
 }
 
-impl TokenFilter for AsciiFolding {
-    fn transform(&mut self, mut token: Token) -> Option<Token> {
-        if !token.text.is_ascii() {
-            // ignore its already ascii
-            to_ascii(&token.text, &mut self.buffer);
-            mem::swap(&mut token.text, &mut self.buffer);
+pub struct AsciiFoldingFilterTokenStream<'a> {
+    buffer: String,
+    tail: BoxTokenStream<'a>,
+}
+
+impl<'a> TokenStream for AsciiFoldingFilterTokenStream<'a> {
+    fn advance(&mut self) -> bool {
+        if !self.tail.advance() {
+            return false;
         }
-        Some(token)
+        if !self.token_mut().text.is_ascii() {
+            // ignore its already ascii
+            to_ascii(&mut self.tail.token_mut().text, &mut self.buffer);
+            mem::swap(&mut self.tail.token_mut().text, &mut self.buffer);
+        }
+        true
+    }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
     }
 }
 
@@ -1512,7 +1526,7 @@ fn fold_non_ascii_char(c: char) -> Option<&'static str> {
 }
 
 // https://github.com/apache/lucene-solr/blob/master/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java#L187
-fn to_ascii(text: &String, output: &mut String) {
+fn to_ascii(text: &mut String, output: &mut String) {
     output.clear();
 
     for c in text.chars() {
@@ -1526,8 +1540,11 @@ fn to_ascii(text: &String, output: &mut String) {
 
 #[cfg(test)]
 mod tests {
-    use super::super::*;
-    use super::*;
+    use super::to_ascii;
+    use crate::tokenizer::AsciiFoldingFilter;
+    use crate::tokenizer::RawTokenizer;
+    use crate::tokenizer::SimpleTokenizer;
+    use crate::tokenizer::TextAnalyzer;
     use std::iter;
 
     #[test]
@@ -1543,22 +1560,22 @@ mod tests {
     }
 
     fn folding_helper(text: &str) -> Vec<String> {
-        let tokens = analyzer_builder(SimpleTokenizer)
-            .filter(AsciiFolding::new())
-            .build()
+        let mut tokens = Vec::new();
+        TextAnalyzer::from(SimpleTokenizer)
+            .filter(AsciiFoldingFilter)
             .token_stream(text)
-            .map(|token| token.text.clone())
-            .collect();
+            .process(&mut |token| {
+                tokens.push(token.text.clone());
+            });
         tokens
     }
 
     fn folding_using_raw_tokenizer_helper(text: &str) -> String {
-        let mut token_stream = analyzer_builder(RawTokenizer)
-            .filter(AsciiFolding::new())
-            .build()
+        let mut token_stream = TextAnalyzer::from(RawTokenizer)
+            .filter(AsciiFoldingFilter)
             .token_stream(text);
-        let Token { text, .. } = token_stream.next().unwrap();
-        text
+        token_stream.advance();
+        token_stream.token().text.clone()
     }
 
     #[test]
@@ -1609,9 +1626,9 @@ mod tests {
 
     #[test]
     fn test_to_ascii() {
-        let input = "Rámon".to_string();
+        let mut input = "Rámon".to_string();
         let mut buffer = String::new();
-        to_ascii(&input, &mut buffer);
+        to_ascii(&mut input, &mut buffer);
         assert_eq!("Ramon", buffer);
     }
 

src/tokenizer/facet_tokenizer.rs

@@ -1,4 +1,4 @@
-use super::{Token, Tokenizer};
+use super::{BoxTokenStream, Token, TokenStream, Tokenizer};
 use crate::schema::FACET_SEP_BYTE;
 
 /// The `FacetTokenizer` process a `Facet` binary representation
@@ -9,63 +9,72 @@ use crate::schema::FACET_SEP_BYTE;
 ///     - `/america/north_america/canada`
 ///     - `/america/north_america`
 ///     - `/america`
-#[derive(Clone, Debug, Default)]
+#[derive(Clone)]
 pub struct FacetTokenizer;
 
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 enum State {
     RootFacetNotEmitted,
     UpToPosition(usize), //< we already emitted facet prefix up to &text[..cursor]
     Terminated,
 }
 
-#[derive(Clone, Debug)]
-pub struct FacetTokenStream {
-    text: String,
+pub struct FacetTokenStream<'a> {
+    text: &'a str,
     state: State,
     token: Token,
 }
 
 impl Tokenizer for FacetTokenizer {
-    type Iter = FacetTokenStream;
-    fn token_stream(&self, text: &str) -> Self::Iter {
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
         FacetTokenStream {
-            text: text.to_string(),
+            text,
             state: State::RootFacetNotEmitted, //< pos is the first char that has not been processed yet.
             token: Token::default(),
         }
+        .into()
     }
 }
 
-impl Iterator for FacetTokenStream {
-    type Item = Token;
-    fn next(&mut self) -> Option<Self::Item> {
-        self.state = match self.state {
+impl<'a> TokenStream for FacetTokenStream<'a> {
+    fn advance(&mut self) -> bool {
+        match self.state {
             State::RootFacetNotEmitted => {
-                if self.text.is_empty() {
+                self.state = if self.text.is_empty() {
                     State::Terminated
                 } else {
                     State::UpToPosition(0)
-                }
+                };
+                true
             }
             State::UpToPosition(cursor) => {
-                if let Some(next_sep_pos) = self.text.as_bytes()[cursor + 1..]
+                let bytes: &[u8] = self.text.as_bytes();
+                if let Some(next_sep_pos) = bytes[cursor + 1..]
                     .iter()
-                    .position(|&b| b == FACET_SEP_BYTE)
+                    .cloned()
+                    .position(|b| b == FACET_SEP_BYTE)
                     .map(|pos| cursor + 1 + pos)
                 {
                     let facet_part = &self.text[cursor..next_sep_pos];
                     self.token.text.push_str(facet_part);
-                    State::UpToPosition(next_sep_pos)
+                    self.state = State::UpToPosition(next_sep_pos);
                 } else {
                     let facet_part = &self.text[cursor..];
                     self.token.text.push_str(facet_part);
-                    State::Terminated
+                    self.state = State::Terminated;
                 }
+                true
             }
-            State::Terminated => return None,
-        };
-        Some(self.token.clone())
+            State::Terminated => false,
+        }
+    }
+
+    fn token(&self) -> &Token {
+        &self.token
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        &mut self.token
     }
 }
 
@@ -74,19 +83,21 @@ mod tests {
 
     use super::FacetTokenizer;
     use crate::schema::Facet;
-    use crate::tokenizer::Tokenizer;
+    use crate::tokenizer::{Token, Tokenizer};
 
     #[test]
     fn test_facet_tokenizer() {
         let facet = Facet::from_path(vec!["top", "a", "b"]);
-        let tokens: Vec<_> = FacetTokenizer
-            .token_stream(facet.encoded_str())
-            .map(|token| {
-                Facet::from_encoded(token.text.as_bytes().to_owned())
-                    .unwrap()
-                    .to_string()
-            })
-            .collect();
+        let mut tokens = vec![];
+        {
+            let mut add_token = |token: &Token| {
+                let facet = Facet::from_encoded(token.text.as_bytes().to_owned()).unwrap();
+                tokens.push(format!("{}", facet));
+            };
+            FacetTokenizer
+                .token_stream(facet.encoded_str())
+                .process(&mut add_token);
+        }
         assert_eq!(tokens.len(), 4);
         assert_eq!(tokens[0], "/");
         assert_eq!(tokens[1], "/top");
@@ -97,14 +108,16 @@ mod tests {
     #[test]
     fn test_facet_tokenizer_root_facets() {
         let facet = Facet::root();
-        let tokens: Vec<_> = FacetTokenizer
-            .token_stream(facet.encoded_str())
-            .map(|token| {
-                Facet::from_encoded(token.text.as_bytes().to_owned())
-                    .unwrap()
-                    .to_string()
-            })
-            .collect();
+        let mut tokens = vec![];
+        {
+            let mut add_token = |token: &Token| {
+                let facet = Facet::from_encoded(token.text.as_bytes().to_owned()).unwrap(); // ok test
+                tokens.push(format!("{}", facet));
+            };
+            FacetTokenizer
+                .token_stream(facet.encoded_str()) // ok test
+                .process(&mut add_token);
+        }
         assert_eq!(tokens.len(), 1);
         assert_eq!(tokens[0], "/");
     }

src/tokenizer/lower_caser.rs

@@ -1,36 +1,27 @@
-use super::{Token, TokenFilter};
+use super::{Token, TokenFilter, TokenStream};
+use crate::tokenizer::BoxTokenStream;
 use std::mem;
 
 impl TokenFilter for LowerCaser {
-    fn transform(&mut self, mut token: Token) -> Option<Token> {
-        if token.text.is_ascii() {
-            // fast track for ascii.
-            token.text.make_ascii_lowercase();
-        } else {
-            to_lowercase_unicode(&token.text, &mut self.buffer);
-            mem::swap(&mut token.text, &mut self.buffer);
-        }
-        Some(token)
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(LowerCaserTokenStream {
+            tail: token_stream,
+            buffer: String::with_capacity(100),
+        })
     }
 }
 
 /// Token filter that lowercase terms.
-#[derive(Clone, Debug, Default)]
-pub struct LowerCaser {
-    buffer: String,
-}
+#[derive(Clone)]
+pub struct LowerCaser;
 
-impl LowerCaser {
-    /// Initialize the `LowerCaser`
-    pub fn new() -> Self {
-        LowerCaser {
-            buffer: String::with_capacity(100),
-        }
-    }
+pub struct LowerCaserTokenStream<'a> {
+    buffer: String,
+    tail: BoxTokenStream<'a>,
 }
 
 // writes a lowercased version of text into output.
-fn to_lowercase_unicode(text: &String, output: &mut String) {
+fn to_lowercase_unicode(text: &mut String, output: &mut String) {
     output.clear();
     for c in text.chars() {
         // Contrary to the std, we do not take care of sigma special case.
@@ -39,31 +30,57 @@ fn to_lowercase_unicode(text: &String, output: &mut String) {
     }
 }
 
+impl<'a> TokenStream for LowerCaserTokenStream<'a> {
+    fn advance(&mut self) -> bool {
+        if !self.tail.advance() {
+            return false;
+        }
+        if self.token_mut().text.is_ascii() {
+            // fast track for ascii.
+            self.token_mut().text.make_ascii_lowercase();
+        } else {
+            to_lowercase_unicode(&mut self.tail.token_mut().text, &mut self.buffer);
+            mem::swap(&mut self.tail.token_mut().text, &mut self.buffer);
+        }
+        true
+    }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+}
+
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use crate::tokenizer::{analyzer_builder, LowerCaser, SimpleTokenizer, TextAnalyzerT};
+    use crate::tokenizer::{LowerCaser, SimpleTokenizer, TextAnalyzer};
 
     #[test]
     fn test_to_lower_case() {
-        assert_eq!(lowercase_helper("Русский текст"), vec!["русский", "текст"]);
+        assert_eq!(
+            lowercase_helper("Русский текст"),
+            vec!["русский".to_string(), "текст".to_string()]
+        );
     }
 
     fn lowercase_helper(text: &str) -> Vec<String> {
-        analyzer_builder(SimpleTokenizer)
-            .filter(LowerCaser::new())
-            .build()
-            .token_stream(text)
-            .map(|token| {
-                let Token { text, .. } = token;
-                text
-            })
-            .collect()
+        let mut tokens = vec![];
+        let mut token_stream = TextAnalyzer::from(SimpleTokenizer)
+            .filter(LowerCaser)
+            .token_stream(text);
+        while token_stream.advance() {
+            let token_text = token_stream.token().text.clone();
+            tokens.push(token_text);
+        }
+        tokens
     }
 
     #[test]
     fn test_lowercaser() {
-        assert_eq!(lowercase_helper("Tree"), vec!["tree"]);
-        assert_eq!(lowercase_helper("Русский"), vec!["русский"]);
+        assert_eq!(lowercase_helper("Tree"), vec!["tree".to_string()]);
+        assert_eq!(lowercase_helper("Русский"), vec!["русский".to_string()]);
     }
 }

src/tokenizer/mod.rs

@@ -64,10 +64,10 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let en_stem = analyzer_builder(SimpleTokenizer)
+//! let en_stem = TextAnalyzer::from(SimpleTokenizer)
 //!     .filter(RemoveLongFilter::limit(40))
-//!     .filter(LowerCaser::new())
-//!     .filter(Stemmer::new(Language::English)).build();
+//!     .filter(LowerCaser)
+//!     .filter(Stemmer::new(Language::English));
 //! ```
 //!
 //! Once your tokenizer is defined, you need to
@@ -109,9 +109,9 @@
 //! let index = Index::create_in_ram(schema);
 //!
 //! // We need to register our tokenizer :
-//! let custom_en_tokenizer = analyzer_builder(SimpleTokenizer)
+//! let custom_en_tokenizer = TextAnalyzer::from(SimpleTokenizer)
 //!     .filter(RemoveLongFilter::limit(40))
-//!     .filter(LowerCaser::new()).build();
+//!     .filter(LowerCaser);
 //! index
 //!     .tokenizers()
 //!     .register("custom_en", custom_en_tokenizer);
@@ -133,7 +133,7 @@ mod tokenizer;
 mod tokenizer_manager;
 
 pub use self::alphanum_only::AlphaNumOnlyFilter;
-pub use self::ascii_folding_filter::AsciiFolding;
+pub use self::ascii_folding_filter::AsciiFoldingFilter;
 pub use self::facet_tokenizer::FacetTokenizer;
 pub use self::lower_caser::LowerCaser;
 pub use self::ngram_tokenizer::NgramTokenizer;
@@ -142,11 +142,11 @@ pub use self::remove_long::RemoveLongFilter;
 pub use self::simple_tokenizer::SimpleTokenizer;
 pub use self::stemmer::{Language, Stemmer};
 pub use self::stop_word_filter::StopWordFilter;
-pub(crate) use self::token_stream_chain::{DynTokenStreamChain, TokenStreamChain};
+pub(crate) use self::token_stream_chain::TokenStreamChain;
 
 pub use self::tokenized_string::{PreTokenizedStream, PreTokenizedString};
 pub use self::tokenizer::{
-    analyzer_builder, Identity, TextAnalyzer, TextAnalyzerT, Token, TokenFilter, Tokenizer,
+    BoxTokenFilter, BoxTokenStream, TextAnalyzer, Token, TokenFilter, TokenStream, Tokenizer,
 };
 
 pub use self::tokenizer_manager::TokenizerManager;
@@ -160,7 +160,10 @@ pub const MAX_TOKEN_LEN: usize = u16::max_value() as usize - 4;
 
 #[cfg(test)]
 pub mod tests {
-    use super::*;
+    use super::{
+        Language, LowerCaser, RemoveLongFilter, SimpleTokenizer, Stemmer, Token, TokenizerManager,
+    };
+    use crate::tokenizer::TextAnalyzer;
 
     /// This is a function that can be used in tests and doc tests
     /// to assert a token's correctness.
@@ -187,9 +190,15 @@ pub mod tests {
     fn test_raw_tokenizer() {
         let tokenizer_manager = TokenizerManager::default();
         let en_tokenizer = tokenizer_manager.get("raw").unwrap();
-        let tokens: Vec<Token> = en_tokenizer
-            .token_stream("Hello, happy tax payer!")
-            .collect();
+        let mut tokens: Vec<Token> = vec![];
+        {
+            let mut add_token = |token: &Token| {
+                tokens.push(token.clone());
+            };
+            en_tokenizer
+                .token_stream("Hello, happy tax payer!")
+                .process(&mut add_token);
+        }
         assert_eq!(tokens.len(), 1);
         assert_token(&tokens[0], 0, "Hello, happy tax payer!", 0, 23);
     }
@@ -199,9 +208,15 @@ pub mod tests {
         let tokenizer_manager = TokenizerManager::default();
         assert!(tokenizer_manager.get("en_doesnotexist").is_none());
         let en_tokenizer = tokenizer_manager.get("en_stem").unwrap();
-        let tokens: Vec<Token> = en_tokenizer
-            .token_stream("Hello, happy tax payer!")
-            .collect();
+        let mut tokens: Vec<Token> = vec![];
+        {
+            let mut add_token = |token: &Token| {
+                tokens.push(token.clone());
+            };
+            en_tokenizer
+                .token_stream("Hello, happy tax payer!")
+                .process(&mut add_token);
+        }
 
         assert_eq!(tokens.len(), 4);
         assert_token(&tokens[0], 0, "hello", 0, 5);
@@ -215,16 +230,21 @@ pub mod tests {
         let tokenizer_manager = TokenizerManager::default();
         tokenizer_manager.register(
             "el_stem",
-            analyzer_builder(SimpleTokenizer)
+            TextAnalyzer::from(SimpleTokenizer)
                 .filter(RemoveLongFilter::limit(40))
-                .filter(LowerCaser::new())
-                .filter(Stemmer::new(Language::Greek))
-                .build(),
+                .filter(LowerCaser)
+                .filter(Stemmer::new(Language::Greek)),
         );
         let en_tokenizer = tokenizer_manager.get("el_stem").unwrap();
-        let tokens: Vec<Token> = en_tokenizer
-            .token_stream("Καλημέρα, χαρούμενε φορολογούμενε!")
-            .collect();
+        let mut tokens: Vec<Token> = vec![];
+        {
+            let mut add_token = |token: &Token| {
+                tokens.push(token.clone());
+            };
+            en_tokenizer
+                .token_stream("Καλημέρα, χαρούμενε φορολογούμενε!")
+                .process(&mut add_token);
+        }
 
         assert_eq!(tokens.len(), 3);
         assert_token(&tokens[0], 0, "καλημερ", 0, 16);
@@ -236,9 +256,25 @@ pub mod tests {
     fn test_tokenizer_empty() {
         let tokenizer_manager = TokenizerManager::default();
         let en_tokenizer = tokenizer_manager.get("en_stem").unwrap();
-        let tokens: Vec<Token> = en_tokenizer.token_stream(" ").collect();
-        assert!(tokens.is_empty());
-        let tokens: Vec<Token> = en_tokenizer.token_stream(" ").collect();
-        assert!(tokens.is_empty());
+        {
+            let mut tokens: Vec<Token> = vec![];
+            {
+                let mut add_token = |token: &Token| {
+                    tokens.push(token.clone());
+                };
+                en_tokenizer.token_stream(" ").process(&mut add_token);
+            }
+            assert!(tokens.is_empty());
+        }
+        {
+            let mut tokens: Vec<Token> = vec![];
+            {
+                let mut add_token = |token: &Token| {
+                    tokens.push(token.clone());
+                };
+                en_tokenizer.token_stream(" ").process(&mut add_token);
+            }
+            assert!(tokens.is_empty());
+        }
     }
 }

src/tokenizer/ngram_tokenizer.rs

@@ -1,4 +1,5 @@
-use super::{Token, Tokenizer};
+use super::{Token, TokenStream, Tokenizer};
+use crate::tokenizer::BoxTokenStream;
 
 /// Tokenize the text by splitting words into n-grams of the given size(s)
 ///
@@ -78,7 +79,7 @@ use super::{Token, Tokenizer};
 /// }
 /// assert!(stream.next().is_none());
 /// ```
-#[derive(Clone, Debug, Default)]
+#[derive(Clone)]
 pub struct NgramTokenizer {
     /// min size of the n-gram
     min_gram: usize,
@@ -118,48 +119,54 @@ impl NgramTokenizer {
 }
 
 /// TokenStream associate to the `NgramTokenizer`
-pub struct NgramTokenStream {
+pub struct NgramTokenStream<'a> {
     /// parameters
-    ngram_charidx_iterator: StutteringIterator<CodepointFrontiers>,
+    ngram_charidx_iterator: StutteringIterator<CodepointFrontiers<'a>>,
     /// true if the NgramTokenStream is in prefix mode.
     prefix_only: bool,
     /// input
-    text: String,
+    text: &'a str,
     /// output
     token: Token,
 }
 
 impl Tokenizer for NgramTokenizer {
-    type Iter = NgramTokenStream;
-    fn token_stream(&self, text: &str) -> Self::Iter {
-        NgramTokenStream {
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+        From::from(NgramTokenStream {
             ngram_charidx_iterator: StutteringIterator::new(
                 CodepointFrontiers::for_str(text),
                 self.min_gram,
                 self.max_gram,
             ),
             prefix_only: self.prefix_only,
-            text: text.to_string(),
+            text,
             token: Token::default(),
-        }
+        })
     }
 }
 
-impl Iterator for NgramTokenStream {
-    type Item = Token;
-    fn next(&mut self) -> Option<Self::Item> {
+impl<'a> TokenStream for NgramTokenStream<'a> {
+    fn advance(&mut self) -> bool {
         if let Some((offset_from, offset_to)) = self.ngram_charidx_iterator.next() {
             if self.prefix_only && offset_from > 0 {
-                return None;
+                return false;
             }
             self.token.position = 0;
             self.token.offset_from = offset_from;
             self.token.offset_to = offset_to;
             self.token.text.clear();
             self.token.text.push_str(&self.text[offset_from..offset_to]);
-            return Some(self.token.clone());
-        };
-        None
+            true
+        } else {
+            false
+        }
+    }
+
+    fn token(&self) -> &Token {
+        &self.token
+    }
+    fn token_mut(&mut self) -> &mut Token {
+        &mut self.token
     }
 }
 
@@ -250,21 +257,21 @@ where
 /// or a codepoint ends.
 ///
 /// By convention, we emit [0] for the empty string.
-struct CodepointFrontiers {
-    s: String,
+struct CodepointFrontiers<'a> {
+    s: &'a str,
     next_el: Option<usize>,
 }
 
-impl CodepointFrontiers {
-    fn for_str(s: &str) -> Self {
+impl<'a> CodepointFrontiers<'a> {
+    fn for_str(s: &'a str) -> Self {
         CodepointFrontiers {
-            s: s.to_string(),
+            s,
             next_el: Some(0),
         }
     }
 }
 
-impl<'a> Iterator for CodepointFrontiers {
+impl<'a> Iterator for CodepointFrontiers<'a> {
     type Item = usize;
 
     fn next(&mut self) -> Option<usize> {
@@ -273,7 +280,7 @@ impl<'a> Iterator for CodepointFrontiers {
                 self.next_el = None;
             } else {
                 let first_codepoint_width = utf8_codepoint_width(self.s.as_bytes()[0]);
-                self.s = (&self.s[first_codepoint_width..]).to_string();
+                self.s = &self.s[first_codepoint_width..];
                 self.next_el = Some(offset + first_codepoint_width);
             }
             offset
@@ -294,8 +301,20 @@ fn utf8_codepoint_width(b: u8) -> usize {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
+
+    use super::utf8_codepoint_width;
+    use super::CodepointFrontiers;
+    use super::NgramTokenizer;
+    use super::StutteringIterator;
     use crate::tokenizer::tests::assert_token;
+    use crate::tokenizer::tokenizer::Tokenizer;
+    use crate::tokenizer::{BoxTokenStream, Token};
+
+    fn test_helper(mut tokenizer: BoxTokenStream) -> Vec<Token> {
+        let mut tokens: Vec<Token> = vec![];
+        tokenizer.process(&mut |token: &Token| tokens.push(token.clone()));
+        tokens
+    }
 
     #[test]
     fn test_utf8_codepoint_width() {
@@ -332,9 +351,7 @@ mod tests {
 
     #[test]
     fn test_ngram_tokenizer_1_2_false() {
-        let tokens: Vec<_> = NgramTokenizer::all_ngrams(1, 2)
-            .token_stream("hello")
-            .collect();
+        let tokens = test_helper(NgramTokenizer::all_ngrams(1, 2).token_stream("hello"));
         assert_eq!(tokens.len(), 9);
         assert_token(&tokens[0], 0, "h", 0, 1);
         assert_token(&tokens[1], 0, "he", 0, 2);
@@ -349,9 +366,7 @@ mod tests {
 
     #[test]
     fn test_ngram_tokenizer_min_max_equal() {
-        let tokens: Vec<_> = NgramTokenizer::all_ngrams(3, 3)
-            .token_stream("hello")
-            .collect();
+        let tokens = test_helper(NgramTokenizer::all_ngrams(3, 3).token_stream("hello"));
         assert_eq!(tokens.len(), 3);
         assert_token(&tokens[0], 0, "hel", 0, 3);
         assert_token(&tokens[1], 0, "ell", 1, 4);
@@ -360,9 +375,7 @@ mod tests {
 
     #[test]
     fn test_ngram_tokenizer_2_5_prefix() {
-        let tokens: Vec<_> = NgramTokenizer::prefix_only(2, 5)
-            .token_stream("frankenstein")
-            .collect();
+        let tokens = test_helper(NgramTokenizer::prefix_only(2, 5).token_stream("frankenstein"));
         assert_eq!(tokens.len(), 4);
         assert_token(&tokens[0], 0, "fr", 0, 2);
         assert_token(&tokens[1], 0, "fra", 0, 3);
@@ -372,9 +385,7 @@ mod tests {
 
     #[test]
     fn test_ngram_non_ascii_1_2() {
-        let tokens: Vec<_> = NgramTokenizer::all_ngrams(1, 2)
-            .token_stream("hεllo")
-            .collect();
+        let tokens = test_helper(NgramTokenizer::all_ngrams(1, 2).token_stream("hεllo"));
         assert_eq!(tokens.len(), 9);
         assert_token(&tokens[0], 0, "h", 0, 1);
         assert_token(&tokens[1], 0, "hε", 0, 3);
@@ -389,9 +400,7 @@ mod tests {
 
     #[test]
     fn test_ngram_non_ascii_2_5_prefix() {
-        let tokens: Vec<_> = NgramTokenizer::prefix_only(2, 5)
-            .token_stream("hεllo")
-            .collect();
+        let tokens = test_helper(NgramTokenizer::prefix_only(2, 5).token_stream("hεllo"));
         assert_eq!(tokens.len(), 4);
         assert_token(&tokens[0], 0, "hε", 0, 3);
         assert_token(&tokens[1], 0, "hεl", 0, 4);
@@ -401,16 +410,16 @@ mod tests {
 
     #[test]
     fn test_ngram_empty() {
-        let tokens: Vec<_> = NgramTokenizer::all_ngrams(1, 5).token_stream("").collect();
+        let tokens = test_helper(NgramTokenizer::all_ngrams(1, 5).token_stream(""));
         assert!(tokens.is_empty());
-        let tokens: Vec<_> = NgramTokenizer::all_ngrams(2, 5).token_stream("").collect();
+        let tokens = test_helper(NgramTokenizer::all_ngrams(2, 5).token_stream(""));
         assert!(tokens.is_empty());
     }
 
     #[test]
     #[should_panic(expected = "min_gram must be greater than 0")]
     fn test_ngram_min_max_interval_empty() {
-        NgramTokenizer::all_ngrams(0, 2).token_stream("hellossss");
+        test_helper(NgramTokenizer::all_ngrams(0, 2).token_stream("hellossss"));
     }
 
     #[test]

src/tokenizer/raw_tokenizer.rs

@@ -1,17 +1,17 @@
-use super::{Token, Tokenizer};
+use super::{Token, TokenStream, Tokenizer};
+use crate::tokenizer::BoxTokenStream;
 
 /// For each value of the field, emit a single unprocessed token.
-#[derive(Clone, Debug, Default)]
+#[derive(Clone)]
 pub struct RawTokenizer;
 
-#[derive(Clone, Debug)]
 pub struct RawTokenStream {
-    token: Option<Token>,
+    token: Token,
+    has_token: bool,
 }
 
 impl Tokenizer for RawTokenizer {
-    type Iter = RawTokenStream;
-    fn token_stream(&self, text: &str) -> Self::Iter {
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
         let token = Token {
             offset_from: 0,
             offset_to: text.len(),
@@ -19,13 +19,26 @@ impl Tokenizer for RawTokenizer {
             text: text.to_string(),
             position_length: 1,
         };
-        RawTokenStream { token: Some(token) }
+        RawTokenStream {
+            token,
+            has_token: true,
+        }
+        .into()
     }
 }
 
-impl Iterator for RawTokenStream {
-    type Item = Token;
-    fn next(&mut self) -> Option<Token> {
-        self.token.take()
+impl TokenStream for RawTokenStream {
+    fn advance(&mut self) -> bool {
+        let result = self.has_token;
+        self.has_token = false;
+        result
+    }
+
+    fn token(&self) -> &Token {
+        &self.token
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        &mut self.token
     }
 }

src/tokenizer/remove_long.rs

@@ -2,8 +2,8 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let tokenizer = analyzer_builder(SimpleTokenizer)
-//!   .filter(RemoveLongFilter::limit(5)).build();
+//! let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+//!   .filter(RemoveLongFilter::limit(5));
 //!
 //! let mut stream = tokenizer.token_stream("toolong nice");
 //! // because `toolong` is more than 5 characters, it is filtered
@@ -12,30 +12,61 @@
 //! assert!(stream.next().is_none());
 //! ```
 //!
-use super::{Token, TokenFilter};
+use super::{Token, TokenFilter, TokenStream};
+use crate::tokenizer::BoxTokenStream;
 
 /// `RemoveLongFilter` removes tokens that are longer
 /// than a given number of bytes (in UTF-8 representation).
 ///
 /// It is especially useful when indexing unconstrained content.
 /// e.g. Mail containing base-64 encoded pictures etc.
-#[derive(Clone, Debug)]
+#[derive(Clone)]
 pub struct RemoveLongFilter {
-    limit: usize,
+    length_limit: usize,
 }
 
 impl RemoveLongFilter {
     /// Creates a `RemoveLongFilter` given a limit in bytes of the UTF-8 representation.
-    pub fn limit(limit: usize) -> RemoveLongFilter {
-        RemoveLongFilter { limit }
+    pub fn limit(length_limit: usize) -> RemoveLongFilter {
+        RemoveLongFilter { length_limit }
+    }
+}
+
+impl<'a> RemoveLongFilterStream<'a> {
+    fn predicate(&self, token: &Token) -> bool {
+        token.text.len() < self.token_length_limit
     }
 }
 
 impl TokenFilter for RemoveLongFilter {
-    fn transform(&mut self, token: Token) -> Option<Token> {
-        if token.text.len() >= self.limit {
-            return None;
-        }
-        Some(token)
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(RemoveLongFilterStream {
+            token_length_limit: self.length_limit,
+            tail: token_stream,
+        })
+    }
+}
+
+pub struct RemoveLongFilterStream<'a> {
+    token_length_limit: usize,
+    tail: BoxTokenStream<'a>,
+}
+
+impl<'a> TokenStream for RemoveLongFilterStream<'a> {
+    fn advance(&mut self) -> bool {
+        while self.tail.advance() {
+            if self.predicate(self.tail.token()) {
+                return true;
+            }
+        }
+        false
+    }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
     }
 }

src/tokenizer/simple_tokenizer.rs

@@ -1,74 +1,59 @@
-use super::{Token, Tokenizer};
+use super::BoxTokenStream;
+use super::{Token, TokenStream, Tokenizer};
+use std::str::CharIndices;
 
 /// Tokenize the text by splitting on whitespaces and punctuation.
-#[derive(Clone, Debug)]
+#[derive(Clone)]
 pub struct SimpleTokenizer;
+
+pub struct SimpleTokenStream<'a> {
+    text: &'a str,
+    chars: CharIndices<'a>,
+    token: Token,
+}
+
 impl Tokenizer for SimpleTokenizer {
-    type Iter = SimpleTokenizerStream;
-    fn token_stream(&self, text: &str) -> Self::Iter {
-        let vec: Vec<_> = text.char_indices().collect();
-        SimpleTokenizerStream {
-            text: text.to_string(),
-            chars: vec.into_iter(),
-            position: usize::max_value(),
-        }
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(SimpleTokenStream {
+            text,
+            chars: text.char_indices(),
+            token: Token::default(),
+        })
     }
 }
 
-#[derive(Clone, Debug)]
-pub struct SimpleTokenizerStream {
-    text: String,
-    chars: std::vec::IntoIter<(usize, char)>,
-    position: usize,
-}
-
-impl SimpleTokenizerStream {
+impl<'a> SimpleTokenStream<'a> {
     // search for the end of the current token.
     fn search_token_end(&mut self) -> usize {
         (&mut self.chars)
-            .filter(|&(_, c)| !c.is_alphanumeric())
+            .filter(|&(_, ref c)| !c.is_alphanumeric())
             .map(|(offset, _)| offset)
             .next()
             .unwrap_or_else(|| self.text.len())
     }
 }
 
-impl Iterator for SimpleTokenizerStream {
-    type Item = Token;
-    fn next(&mut self) -> Option<Self::Item> {
-        self.position = self.position.wrapping_add(1);
+impl<'a> TokenStream for SimpleTokenStream<'a> {
+    fn advance(&mut self) -> bool {
+        self.token.text.clear();
+        self.token.position = self.token.position.wrapping_add(1);
         while let Some((offset_from, c)) = self.chars.next() {
             if c.is_alphanumeric() {
                 let offset_to = self.search_token_end();
-                let token = Token {
-                    text: self.text[offset_from..offset_to].into(),
-                    offset_from,
-                    offset_to,
-                    position: self.position,
-                    ..Default::default()
-                };
-                return Some(token);
+                self.token.offset_from = offset_from;
+                self.token.offset_to = offset_to;
+                self.token.text.push_str(&self.text[offset_from..offset_to]);
+                return true;
             }
         }
-        None
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_empty() {
-        let mut empty = SimpleTokenizer.token_stream("");
-        assert_eq!(empty.next(), None);
-    }
-
-    #[test]
-    fn simple_tokenizer() {
-        let mut simple = SimpleTokenizer.token_stream("tokenizer hello world");
-        assert_eq!(simple.next().unwrap().text, "tokenizer");
-        assert_eq!(simple.next().unwrap().text, "hello");
-        assert_eq!(simple.next().unwrap().text, "world");
+        false
+    }
+
+    fn token(&self) -> &Token {
+        &self.token
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        &mut self.token
     }
 }

src/tokenizer/stemmer.rs

@@ -1,6 +1,5 @@
-use std::sync::Arc;
-
-use super::{Token, TokenFilter};
+use super::{Token, TokenFilter, TokenStream};
+use crate::tokenizer::BoxTokenStream;
 use rust_stemmers::{self, Algorithm};
 use serde::{Deserialize, Serialize};
 
@@ -59,15 +58,14 @@ impl Language {
 /// Tokens are expected to be lowercased beforehand.
 #[derive(Clone)]
 pub struct Stemmer {
-    stemmer: Arc<rust_stemmers::Stemmer>,
+    stemmer_algorithm: Algorithm,
 }
 
 impl Stemmer {
     /// Creates a new Stemmer `TokenFilter` for a given language algorithm.
     pub fn new(language: Language) -> Stemmer {
-        let stemmer = rust_stemmers::Stemmer::create(language.algorithm());
         Stemmer {
-            stemmer: Arc::new(stemmer),
+            stemmer_algorithm: language.algorithm(),
         }
     }
 }
@@ -80,12 +78,37 @@ impl Default for Stemmer {
 }
 
 impl TokenFilter for Stemmer {
-    fn transform(&mut self, mut token: Token) -> Option<Token> {
-        // TODO remove allocation
-        let stemmed_str: String = self.stemmer.stem(&token.text).into_owned();
-        // TODO remove clear
-        token.text.clear();
-        token.text.push_str(&stemmed_str);
-        Some(token)
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        let inner_stemmer = rust_stemmers::Stemmer::create(self.stemmer_algorithm);
+        BoxTokenStream::from(StemmerTokenStream {
+            tail: token_stream,
+            stemmer: inner_stemmer,
+        })
+    }
+}
+
+pub struct StemmerTokenStream<'a> {
+    tail: BoxTokenStream<'a>,
+    stemmer: rust_stemmers::Stemmer,
+}
+
+impl<'a> TokenStream for StemmerTokenStream<'a> {
+    fn advance(&mut self) -> bool {
+        if !self.tail.advance() {
+            return false;
+        }
+        // TODO remove allocation
+        let stemmed_str: String = self.stemmer.stem(&self.token().text).into_owned();
+        self.token_mut().text.clear();
+        self.token_mut().text.push_str(&stemmed_str);
+        true
+    }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
     }
 }

src/tokenizer/stop_word_filter.rs

@@ -2,15 +2,16 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let tokenizer = analyzer_builder(SimpleTokenizer)
-//!   .filter(StopWordFilter::remove(vec!["the".to_string(), "is".to_string()])).build();
+//! let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+//!   .filter(StopWordFilter::remove(vec!["the".to_string(), "is".to_string()]));
 //!
 //! let mut stream = tokenizer.token_stream("the fox is crafty");
 //! assert_eq!(stream.next().unwrap().text, "fox");
 //! assert_eq!(stream.next().unwrap().text, "crafty");
 //! assert!(stream.next().is_none());
 //! ```
-use super::{Token, TokenFilter};
+use super::{Token, TokenFilter, TokenStream};
+use crate::tokenizer::BoxTokenStream;
 use fnv::FnvHasher;
 use std::collections::HashSet;
 use std::hash::BuildHasherDefault;
@@ -48,12 +49,42 @@ impl StopWordFilter {
     }
 }
 
+pub struct StopWordFilterStream<'a> {
+    words: StopWordHashSet,
+    tail: BoxTokenStream<'a>,
+}
+
 impl TokenFilter for StopWordFilter {
-    fn transform(&mut self, token: Token) -> Option<Token> {
-        if self.words.contains(&token.text) {
-            return None;
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(StopWordFilterStream {
+            words: self.words.clone(),
+            tail: token_stream,
+        })
+    }
+}
+
+impl<'a> StopWordFilterStream<'a> {
+    fn predicate(&self, token: &Token) -> bool {
+        !self.words.contains(&token.text)
+    }
+}
+
+impl<'a> TokenStream for StopWordFilterStream<'a> {
+    fn advance(&mut self) -> bool {
+        while self.tail.advance() {
+            if self.predicate(self.tail.token()) {
+                return true;
+            }
         }
-        Some(token)
+        false
+    }
+
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
     }
 }
 

src/tokenizer/token_stream_chain.rs

@@ -1,121 +1,95 @@
-use crate::tokenizer::Token;
+use crate::tokenizer::{BoxTokenStream, Token, TokenStream};
+use std::ops::DerefMut;
 
 const POSITION_GAP: usize = 2;
 
-pub(crate) struct TokenStreamChain<Inner, Outer> {
-    streams_with_offsets: Outer,
-    current: Option<(Inner, usize)>,
-    position: usize,
+pub(crate) struct TokenStreamChain<'a> {
+    offsets: Vec<usize>,
+    token_streams: Vec<BoxTokenStream<'a>>,
     position_shift: usize,
+    stream_idx: usize,
+    token: Token,
 }
 
-impl<'a, Inner, Outer> TokenStreamChain<Inner, Outer>
-where
-    Inner: Iterator<Item = Token>,
-    Outer: Iterator<Item = (Inner, usize)>,
-{
-    pub fn new(mut streams_with_offsets: Outer) -> TokenStreamChain<Inner, Outer> {
-        let current = streams_with_offsets.next();
+impl<'a> TokenStreamChain<'a> {
+    pub fn new(
+        offsets: Vec<usize>,
+        token_streams: Vec<BoxTokenStream<'a>>,
+    ) -> TokenStreamChain<'a> {
         TokenStreamChain {
-            streams_with_offsets: streams_with_offsets,
-            current,
-            position: usize::max_value(),
+            offsets,
+            stream_idx: 0,
+            token_streams,
             position_shift: 0,
+            token: Token::default(),
         }
     }
 }
 
-impl<'a, Inner, Outer> Iterator for TokenStreamChain<Inner, Outer>
-where
-    Inner: Iterator<Item = Token>,
-    Outer: Iterator<Item = (Inner, usize)>,
-{
-    type Item = Token;
-    fn next(&mut self) -> Option<Token> {
-        while let Some((ref mut token_stream, offset_offset)) = self.current {
-            if let Some(mut token) = token_stream.next() {
-                token.offset_from += offset_offset;
-                token.offset_to += offset_offset;
-                token.position += self.position_shift;
-                self.position = token.position;
-                return Some(token);
+impl<'a> TokenStream for TokenStreamChain<'a> {
+    fn advance(&mut self) -> bool {
+        while self.stream_idx < self.token_streams.len() {
+            let token_stream = self.token_streams[self.stream_idx].deref_mut();
+            if token_stream.advance() {
+                let token = token_stream.token();
+                let offset_offset = self.offsets[self.stream_idx];
+                self.token.offset_from = token.offset_from + offset_offset;
+                self.token.offset_to = token.offset_to + offset_offset;
+                self.token.position = token.position + self.position_shift;
+                self.token.text.clear();
+                self.token.text.push_str(token.text.as_str());
+                return true;
+            } else {
+                self.stream_idx += 1;
+                self.position_shift = self.token.position.wrapping_add(POSITION_GAP);
             }
-            self.position_shift = self.position.wrapping_add(POSITION_GAP);
-            self.current = self.streams_with_offsets.next();
         }
-        None
+        false
     }
-}
 
-impl DynTokenStreamChain {
-    pub fn from_vec(
-        streams_with_offsets: Vec<(Box<dyn Iterator<Item = Token>>, usize)>,
-    ) -> impl Iterator<Item = Token> {
-        DynTokenStreamChain {
-            streams_with_offsets,
-            idx: 0,
-            position: usize::max_value(),
-            position_shift: 0,
-        }
+    fn token(&self) -> &Token {
+        assert!(
+            self.stream_idx <= self.token_streams.len(),
+            "You called .token(), after the end of the token stream has been reached"
+        );
+        &self.token
     }
-}
 
-pub(crate) struct DynTokenStreamChain {
-    streams_with_offsets: Vec<(Box<dyn Iterator<Item = Token>>, usize)>,
-    idx: usize,
-    position: usize,
-    position_shift: usize,
-}
-
-impl Iterator for DynTokenStreamChain {
-    type Item = Token;
-    fn next(&mut self) -> Option<Token> {
-        while let Some((token_stream, offset_offset)) = self.streams_with_offsets.get_mut(self.idx)
-        {
-            if let Some(mut token) = token_stream.next() {
-                token.offset_from += *offset_offset;
-                token.offset_to += *offset_offset;
-                token.position += self.position_shift;
-                self.position = token.position;
-                return Some(token);
-            }
-            self.idx += 1;
-            self.position_shift = self.position.wrapping_add(POSITION_GAP);
-        }
-        None
+    fn token_mut(&mut self) -> &mut Token {
+        assert!(
+            self.stream_idx <= self.token_streams.len(),
+            "You called .token(), after the end of the token stream has been reached"
+        );
+        &mut self.token
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use super::super::tokenizer::Tokenizer;
-    use super::super::SimpleTokenizer;
-    use super::*;
+    use super::super::{SimpleTokenizer, TokenStream, Tokenizer};
+    use super::TokenStreamChain;
+    use super::POSITION_GAP;
 
     #[test]
     fn test_chain_first_emits_no_tokens() {
         let token_streams = vec![
-            (SimpleTokenizer.token_stream(""), 0),
-            (SimpleTokenizer.token_stream("hello world"), 0),
+            SimpleTokenizer.token_stream(""),
+            SimpleTokenizer.token_stream("hello world"),
         ];
-        let mut token_chain = TokenStreamChain::new(token_streams.into_iter());
-        let token = token_chain.next();
+        let mut token_chain = TokenStreamChain::new(vec![0, 0], token_streams);
 
-        let expect = Token {
-            offset_from: 0,
-            offset_to: 5,
-            position: POSITION_GAP - 1,
-            text: "hello".into(),
-            ..Token::default()
-        };
-        assert_eq!(token.unwrap(), expect);
+        assert!(token_chain.advance());
+        assert_eq!(token_chain.token().text, "hello");
+        assert_eq!(token_chain.token().offset_from, 0);
+        assert_eq!(token_chain.token().offset_to, 5);
+        assert_eq!(token_chain.token().position, POSITION_GAP - 1);
 
-        let token = token_chain.next().unwrap();
-        assert_eq!(token.text, "world");
-        assert_eq!(token.offset_from, 6);
-        assert_eq!(token.offset_to, 11);
-        assert_eq!(token.position, POSITION_GAP);
+        assert!(token_chain.advance());
+        assert_eq!(token_chain.token().text, "world");
+        assert_eq!(token_chain.token().offset_from, 6);
+        assert_eq!(token_chain.token().offset_to, 11);
+        assert_eq!(token_chain.token().position, POSITION_GAP);
 
-        assert!(token_chain.next().is_none());
+        assert!(!token_chain.advance());
     }
 }

src/tokenizer/tokenized_string.rs

@@ -1,4 +1,4 @@
-use crate::tokenizer::{Token, TokenStreamChain};
+use crate::tokenizer::{BoxTokenStream, Token, TokenStream, TokenStreamChain};
 use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
 
@@ -26,14 +26,14 @@ impl PartialOrd for PreTokenizedString {
 /// TokenStream implementation which wraps PreTokenizedString
 pub struct PreTokenizedStream {
     tokenized_string: PreTokenizedString,
-    current_token: usize,
+    current_token: i64,
 }
 
 impl From<PreTokenizedString> for PreTokenizedStream {
     fn from(s: PreTokenizedString) -> PreTokenizedStream {
         PreTokenizedStream {
             tokenized_string: s,
-            current_token: 0,
+            current_token: -1,
         }
     }
 }
@@ -41,28 +41,49 @@ impl From<PreTokenizedString> for PreTokenizedStream {
 impl PreTokenizedStream {
     /// Creates a TokenStream from PreTokenizedString array
     pub fn chain_tokenized_strings<'a>(
-        tok_strings: &'a [&PreTokenizedString],
-    ) -> impl Iterator<Item = Token> + 'a {
-        let streams_with_offsets = tok_strings.iter().scan(0, |total_offset, tok_string| {
-            let next = Some((
-                PreTokenizedStream::from((*tok_string).to_owned()),
-                *total_offset,
-            ));
-            if let Some(last_token) = tok_string.tokens.last() {
-                *total_offset += last_token.offset_to;
+        tok_strings: &'a [&'a PreTokenizedString],
+    ) -> BoxTokenStream {
+        if tok_strings.len() == 1 {
+            PreTokenizedStream::from((*tok_strings[0]).clone()).into()
+        } else {
+            let mut offsets = vec![];
+            let mut total_offset = 0;
+            for &tok_string in tok_strings {
+                offsets.push(total_offset);
+                if let Some(last_token) = tok_string.tokens.last() {
+                    total_offset += last_token.offset_to;
+                }
             }
-            next
-        });
-        TokenStreamChain::new(streams_with_offsets)
+            // TODO remove the string cloning.
+            let token_streams: Vec<BoxTokenStream<'static>> = tok_strings
+                .iter()
+                .map(|&tok_string| PreTokenizedStream::from((*tok_string).clone()).into())
+                .collect();
+            TokenStreamChain::new(offsets, token_streams).into()
+        }
     }
 }
 
-impl Iterator for PreTokenizedStream {
-    type Item = Token;
-    fn next(&mut self) -> Option<Token> {
-        let token = self.tokenized_string.tokens.get(self.current_token)?;
+impl TokenStream for PreTokenizedStream {
+    fn advance(&mut self) -> bool {
         self.current_token += 1;
-        Some(token.clone())
+        self.current_token < self.tokenized_string.tokens.len() as i64
+    }
+
+    fn token(&self) -> &Token {
+        assert!(
+            self.current_token >= 0,
+            "TokenStream not initialized. You should call advance() at least once."
+        );
+        &self.tokenized_string.tokens[self.current_token as usize]
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        assert!(
+            self.current_token >= 0,
+            "TokenStream not initialized. You should call advance() at least once."
+        );
+        &mut self.tokenized_string.tokens[self.current_token as usize]
     }
 }
 
@@ -98,9 +119,10 @@ mod tests {
         let mut token_stream = PreTokenizedStream::from(tok_text.clone());
 
         for expected_token in tok_text.tokens {
-            assert_eq!(token_stream.next().unwrap(), expected_token);
+            assert!(token_stream.advance());
+            assert_eq!(token_stream.token(), &expected_token);
         }
-        assert!(token_stream.next().is_none());
+        assert!(!token_stream.advance());
     }
 
     #[test]
@@ -161,8 +183,9 @@ mod tests {
         ];
 
         for expected_token in expected_tokens {
-            assert_eq!(token_stream.next().unwrap(), expected_token);
+            assert!(token_stream.advance());
+            assert_eq!(token_stream.token(), &expected_token);
         }
-        assert!(token_stream.next().is_none());
+        assert!(!token_stream.advance());
     }
 }

src/tokenizer/tokenizer.rs

@@ -2,23 +2,8 @@ use crate::tokenizer::TokenStreamChain;
 use serde::{Deserialize, Serialize};
 /// The tokenizer module contains all of the tools used to process
 /// text in `tantivy`.
-
-pub trait TextAnalyzerClone {
-    fn box_clone(&self) -> Box<dyn TextAnalyzerT>;
-}
-
-/// 'Top-level' trait hiding concrete types, below which static dispatch occurs.
-pub trait TextAnalyzerT: 'static + Send + Sync + TextAnalyzerClone {
-    /// 'Top-level' dynamic dispatch function hiding concrete types of the staticly
-    /// dispatched `token_stream` from the `Tokenizer` trait.
-    fn token_stream(&self, text: &str) -> Box<dyn Iterator<Item = Token>>;
-}
-
-impl Clone for Box<dyn TextAnalyzerT> {
-    fn clone(&self) -> Self {
-        (**self).box_clone()
-    }
-}
+use std::borrow::{Borrow, BorrowMut};
+use std::ops::{Deref, DerefMut};
 
 /// Token
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
@@ -50,116 +35,35 @@ impl Default for Token {
     }
 }
 
-/// Trait for the pluggable components of `Tokenizer`s.
-pub trait TokenFilter: 'static + Send + Sync + Clone {
-    /// Take a `Token` and transform it or return `None` if it's to be removed
-    /// from the output stream.
-    fn transform(&mut self, token: Token) -> Option<Token>;
-}
-
-/// `Tokenizer` are in charge of splitting text into a stream of token
-/// before indexing.
-///
-/// See the [module documentation](./index.html) for more detail.
-pub trait Tokenizer: 'static + Send + Sync + Clone {
-    /// An iteratable type is returned.
-    type Iter: Iterator<Item = Token>;
-    /// Creates a token stream for a given `str`.
-    fn token_stream(&self, text: &str) -> Self::Iter;
-    /// Tokenize an array`&str`
-    ///
-    /// The resulting `Token` stream is equivalent to what would be obtained if the &str were
-    /// one concatenated `&str`, with an artificial position gap of `2` between the different fields
-    /// to prevent accidental `PhraseQuery` to match accross two terms.
-    fn token_stream_texts<'a>(&'a self, texts: &'a [&str]) -> Box<dyn Iterator<Item = Token> + 'a> {
-        let streams_with_offsets = texts.iter().scan(0, move |total_offset, &text| {
-            let temp = *total_offset;
-            *total_offset += text.len();
-            Some((self.token_stream(text), temp))
-        });
-        Box::new(TokenStreamChain::new(streams_with_offsets))
-    }
-}
-
-/// `TextAnalyzer` wraps the tokenization of an input text and its modification by any filters applied onto it.
+/// `TextAnalyzer` tokenizes an input text into tokens and modifies the resulting `TokenStream`.
 ///
 /// It simply wraps a `Tokenizer` and a list of `TokenFilter` that are applied sequentially.
-#[derive(Clone, Debug, Default)]
-pub struct TextAnalyzer<T>(T);
+pub struct TextAnalyzer {
+    tokenizer: Box<dyn Tokenizer>,
+    token_filters: Vec<BoxTokenFilter>,
+}
 
-impl<T: Tokenizer> From<T> for TextAnalyzer<T> {
-    fn from(src: T) -> TextAnalyzer<T> {
-        TextAnalyzer(src)
+impl<T: Tokenizer> From<T> for TextAnalyzer {
+    fn from(tokenizer: T) -> Self {
+        TextAnalyzer::new(tokenizer, Vec::new())
     }
 }
 
-impl<T: Tokenizer> TextAnalyzerClone for TextAnalyzer<T> {
-    fn box_clone(&self) -> Box<dyn TextAnalyzerT> {
-        Box::new(TextAnalyzer(self.0.clone()))
-    }
-}
-
-impl<T: Tokenizer> TextAnalyzerT for TextAnalyzer<T> {
-    fn token_stream(&self, text: &str) -> Box<dyn Iterator<Item = Token>> {
-        Box::new(self.0.token_stream(text))
-    }
-}
-
-/// Identity `TokenFilter`
-#[derive(Clone, Debug, Default)]
-pub struct Identity;
-
-impl TokenFilter for Identity {
-    fn transform(&mut self, token: Token) -> Option<Token> {
-        Some(token)
-    }
-}
-
-/// `Filter` is a wrapper around a `Token` stream and a `TokenFilter` which modifies it.
-#[derive(Clone, Default, Debug)]
-pub struct Filter<I, F> {
-    iter: I,
-    f: F,
-}
-
-impl<I, F> Iterator for Filter<I, F>
-where
-    I: Iterator<Item = Token>,
-    F: TokenFilter,
-{
-    type Item = Token;
-    fn next(&mut self) -> Option<Token> {
-        while let Some(token) = self.iter.next() {
-            if let Some(tok) = self.f.transform(token) {
-                return Some(tok);
-            }
+impl TextAnalyzer {
+    /// Creates a new `TextAnalyzer` given a tokenizer and a vector of `BoxTokenFilter`.
+    ///
+    /// When creating a `TextAnalyzer` from a `Tokenizer` alone, prefer using
+    /// `TextAnalyzer::from(tokenizer)`.
+    pub fn new<T: Tokenizer>(tokenizer: T, token_filters: Vec<BoxTokenFilter>) -> TextAnalyzer {
+        TextAnalyzer {
+            tokenizer: Box::new(tokenizer),
+            token_filters,
         }
-        None
     }
-}
 
-#[derive(Clone, Debug, Default)]
-pub struct AnalyzerBuilder<T, F> {
-    tokenizer: T,
-    f: F,
-}
-
-/// Construct an `AnalyzerBuilder` on which to apply `TokenFilter`.
-pub fn analyzer_builder<T: Tokenizer>(tokenizer: T) -> AnalyzerBuilder<T, Identity> {
-    AnalyzerBuilder {
-        tokenizer,
-        f: Identity,
-    }
-}
-
-impl<T, F> AnalyzerBuilder<T, F>
-where
-    T: Tokenizer,
-    F: TokenFilter,
-{
     /// Appends a token filter to the current tokenizer.
     ///
-    /// The method consumes the current `Token` and returns a
+    /// The method consumes the current `TokenStream` and returns a
     /// new one.
     ///
     /// # Example
@@ -167,35 +71,248 @@ where
     /// ```rust
     /// use tantivy::tokenizer::*;
     ///
-    /// let en_stem = analyzer_builder(SimpleTokenizer)
+    /// let en_stem = TextAnalyzer::from(SimpleTokenizer)
     ///     .filter(RemoveLongFilter::limit(40))
-    ///     .filter(LowerCaser::new())
-    ///     .filter(Stemmer::default()).build();
+    ///     .filter(LowerCaser)
+    ///     .filter(Stemmer::default());
     /// ```
     ///
-    pub fn filter<G: TokenFilter>(self, f: G) -> AnalyzerBuilder<AnalyzerBuilder<T, F>, G> {
-        AnalyzerBuilder { tokenizer: self, f }
+    pub fn filter<F: Into<BoxTokenFilter>>(mut self, token_filter: F) -> Self {
+        self.token_filters.push(token_filter.into());
+        self
     }
-    /// Finalize the build process.
-    pub fn build(self) -> TextAnalyzer<AnalyzerBuilder<T, F>> {
-        TextAnalyzer(self)
+
+    /// Tokenize an array`&str`
+    ///
+    /// The resulting `BoxTokenStream` is equivalent to what would be obtained if the &str were
+    /// one concatenated `&str`, with an artificial position gap of `2` between the different fields
+    /// to prevent accidental `PhraseQuery` to match accross two terms.
+    pub fn token_stream_texts<'a>(&self, texts: &'a [&'a str]) -> BoxTokenStream<'a> {
+        assert!(!texts.is_empty());
+        if texts.len() == 1 {
+            self.token_stream(texts[0])
+        } else {
+            let mut offsets = vec![];
+            let mut total_offset = 0;
+            for &text in texts {
+                offsets.push(total_offset);
+                total_offset += text.len();
+            }
+            let token_streams: Vec<BoxTokenStream<'a>> = texts
+                .iter()
+                .cloned()
+                .map(|text| self.token_stream(text))
+                .collect();
+            From::from(TokenStreamChain::new(offsets, token_streams))
+        }
+    }
+
+    /// Creates a token stream for a given `str`.
+    pub fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+        let mut token_stream = self.tokenizer.token_stream(text);
+        for token_filter in &self.token_filters {
+            token_stream = token_filter.transform(token_stream);
+        }
+        token_stream
     }
 }
 
-impl<T: Tokenizer, F: TokenFilter> Tokenizer for AnalyzerBuilder<T, F> {
-    type Iter = Filter<T::Iter, F>;
-    fn token_stream(&self, text: &str) -> Self::Iter {
-        Filter {
-            iter: self.tokenizer.token_stream(text),
-            f: self.f.clone(),
+impl Clone for TextAnalyzer {
+    fn clone(&self) -> Self {
+        TextAnalyzer {
+            tokenizer: self.tokenizer.box_clone(),
+            token_filters: self
+                .token_filters
+                .iter()
+                .map(|token_filter| token_filter.box_clone())
+                .collect(),
         }
     }
 }
 
+/// `Tokenizer` are in charge of splitting text into a stream of token
+/// before indexing.
+///
+/// See the [module documentation](./index.html) for more detail.
+///
+/// # Warning
+///
+/// This API may change to use associated types.
+pub trait Tokenizer: 'static + Send + Sync + TokenizerClone {
+    /// Creates a token stream for a given `str`.
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a>;
+}
+
+pub trait TokenizerClone {
+    fn box_clone(&self) -> Box<dyn Tokenizer>;
+}
+
+impl<T: Tokenizer + Clone> TokenizerClone for T {
+    fn box_clone(&self) -> Box<dyn Tokenizer> {
+        Box::new(self.clone())
+    }
+}
+
+impl<'a> TokenStream for Box<dyn TokenStream + 'a> {
+    fn advance(&mut self) -> bool {
+        let token_stream: &mut dyn TokenStream = self.borrow_mut();
+        token_stream.advance()
+    }
+
+    fn token<'b>(&'b self) -> &'b Token {
+        let token_stream: &'b (dyn TokenStream + 'a) = self.borrow();
+        token_stream.token()
+    }
+
+    fn token_mut<'b>(&'b mut self) -> &'b mut Token {
+        let token_stream: &'b mut (dyn TokenStream + 'a) = self.borrow_mut();
+        token_stream.token_mut()
+    }
+}
+
+/// Simple wrapper of `Box<dyn TokenStream + 'a>`.
+///
+/// See `TokenStream` for more information.
+pub struct BoxTokenStream<'a>(Box<dyn TokenStream + 'a>);
+
+impl<'a, T> From<T> for BoxTokenStream<'a>
+where
+    T: TokenStream + 'a,
+{
+    fn from(token_stream: T) -> BoxTokenStream<'a> {
+        BoxTokenStream(Box::new(token_stream))
+    }
+}
+
+impl<'a> Deref for BoxTokenStream<'a> {
+    type Target = dyn TokenStream + 'a;
+
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}
+impl<'a> DerefMut for BoxTokenStream<'a> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut *self.0
+    }
+}
+
+/// Simple wrapper of `Box<dyn TokenFilter + 'a>`.
+///
+/// See `TokenStream` for more information.
+pub struct BoxTokenFilter(Box<dyn TokenFilter>);
+
+impl Deref for BoxTokenFilter {
+    type Target = dyn TokenFilter;
+
+    fn deref(&self) -> &dyn TokenFilter {
+        &*self.0
+    }
+}
+
+impl<T: TokenFilter> From<T> for BoxTokenFilter {
+    fn from(tokenizer: T) -> BoxTokenFilter {
+        BoxTokenFilter(Box::new(tokenizer))
+    }
+}
+
+/// `TokenStream` is the result of the tokenization.
+///
+/// It consists consumable stream of `Token`s.
+///
+/// # Example
+///
+/// ```
+/// use tantivy::tokenizer::*;
+///
+/// let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+///        .filter(RemoveLongFilter::limit(40))
+///        .filter(LowerCaser);
+/// let mut token_stream = tokenizer.token_stream("Hello, happy tax payer");
+/// {
+///     let token = token_stream.next().unwrap();
+///     assert_eq!(&token.text, "hello");
+///     assert_eq!(token.offset_from, 0);
+///     assert_eq!(token.offset_to, 5);
+///     assert_eq!(token.position, 0);
+/// }
+/// {
+///     let token = token_stream.next().unwrap();
+///     assert_eq!(&token.text, "happy");
+///     assert_eq!(token.offset_from, 7);
+///     assert_eq!(token.offset_to, 12);
+///     assert_eq!(token.position, 1);
+/// }
+/// ```
+///
+pub trait TokenStream {
+    /// Advance to the next token
+    ///
+    /// Returns false if there are no other tokens.
+    fn advance(&mut self) -> bool;
+
+    /// Returns a reference to the current token.
+    fn token(&self) -> &Token;
+
+    /// Returns a mutable reference to the current token.
+    fn token_mut(&mut self) -> &mut Token;
+
+    /// Helper to iterate over tokens. It
+    /// simply combines a call to `.advance()`
+    /// and `.token()`.
+    ///
+    /// ```
+    /// use tantivy::tokenizer::*;
+    ///
+    /// let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+    ///       .filter(RemoveLongFilter::limit(40))
+    ///       .filter(LowerCaser);
+    /// let mut token_stream = tokenizer.token_stream("Hello, happy tax payer");
+    /// while let Some(token) = token_stream.next() {
+    ///     println!("Token {:?}", token.text);
+    /// }
+    /// ```
+    fn next(&mut self) -> Option<&Token> {
+        if self.advance() {
+            Some(self.token())
+        } else {
+            None
+        }
+    }
+
+    /// Helper function to consume the entire `TokenStream`
+    /// and push the tokens to a sink function.
+    ///
+    /// Remove this.
+    fn process(&mut self, sink: &mut dyn FnMut(&Token)) -> u32 {
+        let mut num_tokens_pushed = 0u32;
+        while self.advance() {
+            sink(self.token());
+            num_tokens_pushed += 1u32;
+        }
+        num_tokens_pushed
+    }
+}
+
+pub trait TokenFilterClone {
+    fn box_clone(&self) -> BoxTokenFilter;
+}
+
+/// Trait for the pluggable components of `Tokenizer`s.
+pub trait TokenFilter: 'static + Send + Sync + TokenFilterClone {
+    /// Wraps a token stream and returns the modified one.
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a>;
+}
+
+impl<T: TokenFilter + Clone> TokenFilterClone for T {
+    fn box_clone(&self) -> BoxTokenFilter {
+        BoxTokenFilter::from(self.clone())
+    }
+}
+
 #[cfg(test)]
 mod test {
-    use super::*;
-    use crate::tokenizer::SimpleTokenizer;
+    use super::Token;
 
     #[test]
     fn clone() {
@@ -213,15 +330,4 @@ mod test {
         assert_eq!(t1.offset_to, t2.offset_to);
         assert_eq!(t1.text, t2.text);
     }
-
-    #[test]
-    fn text_analyzer() {
-        let mut stream = SimpleTokenizer.token_stream("tokenizer hello world");
-        dbg!(stream.next());
-        dbg!(stream.next());
-        dbg!(stream.next());
-        dbg!(stream.next());
-        dbg!(stream.next());
-        dbg!(stream.next());
-    }
 }

src/tokenizer/tokenizer_manager.rs

@@ -1,5 +1,5 @@
 use crate::tokenizer::stemmer::Language;
-use crate::tokenizer::tokenizer::{analyzer_builder, TextAnalyzer, TextAnalyzerT, Tokenizer};
+use crate::tokenizer::tokenizer::TextAnalyzer;
 use crate::tokenizer::LowerCaser;
 use crate::tokenizer::RawTokenizer;
 use crate::tokenizer::RemoveLongFilter;
@@ -22,23 +22,24 @@ use std::sync::{Arc, RwLock};
 ///  search engine.
 #[derive(Clone)]
 pub struct TokenizerManager {
-    tokenizers: Arc<RwLock<HashMap<String, Box<dyn TextAnalyzerT>>>>,
+    tokenizers: Arc<RwLock<HashMap<String, TextAnalyzer>>>,
 }
 
 impl TokenizerManager {
     /// Registers a new tokenizer associated with a given name.
-    pub fn register<U: Tokenizer, T>(&self, tokenizer_name: &str, tokenizer: T)
+    pub fn register<T>(&self, tokenizer_name: &str, tokenizer: T)
     where
-        T: Into<TextAnalyzer<U>>,
+        TextAnalyzer: From<T>,
     {
+        let boxed_tokenizer: TextAnalyzer = TextAnalyzer::from(tokenizer);
         self.tokenizers
             .write()
             .expect("Acquiring the lock should never fail")
-            .insert(tokenizer_name.to_string(), Box::new(tokenizer.into()));
+            .insert(tokenizer_name.to_string(), boxed_tokenizer);
     }
 
     /// Accessing a tokenizer given its name.
-    pub fn get(&self, tokenizer_name: &str) -> Option<Box<dyn TextAnalyzerT>> {
+    pub fn get(&self, tokenizer_name: &str) -> Option<TextAnalyzer> {
         self.tokenizers
             .read()
             .expect("Acquiring the lock should never fail")
@@ -53,25 +54,23 @@ impl Default for TokenizerManager {
     /// - simple
     /// - en_stem
     /// - ja
-    fn default() -> Self {
+    fn default() -> TokenizerManager {
         let manager = TokenizerManager {
             tokenizers: Arc::new(RwLock::new(HashMap::new())),
         };
         manager.register("raw", RawTokenizer);
         manager.register(
             "default",
-            analyzer_builder(SimpleTokenizer)
+            TextAnalyzer::from(SimpleTokenizer)
                 .filter(RemoveLongFilter::limit(40))
-                .filter(LowerCaser::new())
-                .build(),
+                .filter(LowerCaser),
         );
         manager.register(
             "en_stem",
-            analyzer_builder(SimpleTokenizer)
+            TextAnalyzer::from(SimpleTokenizer)
                 .filter(RemoveLongFilter::limit(40))
-                .filter(LowerCaser::new())
-                .filter(Stemmer::new(Language::English))
-                .build(),
+                .filter(LowerCaser)
+                .filter(Stemmer::new(Language::English)),
         );
         manager
     }