Remove forgotten code.

- Added proptest
- Added comment to Lemire blog post.

CHANGELOG.md

@@ -9,6 +9,10 @@ Tantivy 0.14.0
 - Bugfix in `Query::explain`
 - Removed dependency on `notify` #924. Replaced with `FileWatcher` struct that polls meta file every 500ms in background thread. (@halvorboe @guilload)
 - Added `FilterCollector`, which wraps another collector and filters docs using a predicate over a fast field (@barrotsteindev)
+- Simplified the encoding of the skip reader struct. BlockWAND max tf is now encoded over a single byte. (@pmasurel)
+- `FilterCollector` now supports all Fast Field value types (@barrotsteindev)
+
+This version breaks compatibility and requires users to reindex everything.
 
 Tantivy 0.13.2
 ===================

src/collector/facet_collector.rs

@@ -368,9 +368,12 @@ impl SegmentCollector for FacetSegmentCollector {
             }
             let mut facet = vec![];
             let facet_ord = self.collapse_facet_ords[collapsed_facet_ord];
-            facet_dict.ord_to_term(facet_ord as u64, &mut facet);
-            // TODO
-            facet_counts.insert(Facet::from_encoded(facet).unwrap(), count);
+            // TODO handle errors.
+            if facet_dict.ord_to_term(facet_ord as u64, &mut facet).is_ok() {
+                if let Ok(facet) = Facet::from_encoded(facet) {
+                    facet_counts.insert(facet, count);
+                }
+            }
         }
         FacetCounts { facet_counts }
     }

src/collector/filter_collector_wrapper.rs

@@ -9,8 +9,10 @@
 
 // ---
 // Importing tantivy...
+use std::marker::PhantomData;
+
 use crate::collector::{Collector, SegmentCollector};
-use crate::fastfield::FastFieldReader;
+use crate::fastfield::{FastFieldReader, FastValue};
 use crate::schema::Field;
 use crate::{Score, SegmentReader, TantivyError};
 
@@ -41,78 +43,104 @@ use crate::{Score, SegmentReader, TantivyError};
 ///
 /// let query_parser = QueryParser::for_index(&index, vec![title]);
 /// let query = query_parser.parse_query("diary").unwrap();
-/// let no_filter_collector = FilterCollector::new(price, &|value| value > 20_120u64, TopDocs::with_limit(2));
+/// let no_filter_collector = FilterCollector::new(price, &|value: u64| value > 20_120u64, TopDocs::with_limit(2));
 /// let top_docs = searcher.search(&query, &no_filter_collector).unwrap();
 ///
 /// assert_eq!(top_docs.len(), 1);
 /// assert_eq!(top_docs[0].1, DocAddress(0, 1));
 ///
-/// let filter_all_collector = FilterCollector::new(price, &|value| value < 5u64, TopDocs::with_limit(2));
+/// let filter_all_collector: FilterCollector<_, _, u64> = FilterCollector::new(price, &|value| value < 5u64, TopDocs::with_limit(2));
 /// let filtered_top_docs = searcher.search(&query, &filter_all_collector).unwrap();
 ///
 /// assert_eq!(filtered_top_docs.len(), 0);
 /// ```
-pub struct FilterCollector<TCollector, TPredicate>
+pub struct FilterCollector<TCollector, TPredicate, TPredicateValue: FastValue>
 where
     TPredicate: 'static,
 {
     field: Field,
     collector: TCollector,
     predicate: &'static TPredicate,
+    t_predicate_value: PhantomData<TPredicateValue>,
 }
 
-impl<TCollector, TPredicate> FilterCollector<TCollector, TPredicate>
+impl<TCollector, TPredicate, TPredicateValue: FastValue>
+    FilterCollector<TCollector, TPredicate, TPredicateValue>
 where
     TCollector: Collector + Send + Sync,
-    TPredicate: Fn(u64) -> bool + Send + Sync,
+    TPredicate: Fn(TPredicateValue) -> bool + Send + Sync,
 {
     /// Create a new FilterCollector.
     pub fn new(
         field: Field,
         predicate: &'static TPredicate,
         collector: TCollector,
-    ) -> FilterCollector<TCollector, TPredicate> {
+    ) -> FilterCollector<TCollector, TPredicate, TPredicateValue> {
         FilterCollector {
             field,
             predicate,
             collector,
+            t_predicate_value: PhantomData,
         }
     }
 }
 
-impl<TCollector, TPredicate> Collector for FilterCollector<TCollector, TPredicate>
+impl<TCollector, TPredicate, TPredicateValue: FastValue> Collector
+    for FilterCollector<TCollector, TPredicate, TPredicateValue>
 where
     TCollector: Collector + Send + Sync,
-    TPredicate: 'static + Fn(u64) -> bool + Send + Sync,
+    TPredicate: 'static + Fn(TPredicateValue) -> bool + Send + Sync,
+    TPredicateValue: 'static + FastValue,
 {
     // That's the type of our result.
     // Our standard deviation will be a float.
     type Fruit = TCollector::Fruit;
 
-    type Child = FilterSegmentCollector<TCollector::Child, TPredicate>;
+    type Child = FilterSegmentCollector<TCollector::Child, TPredicate, TPredicateValue>;
 
     fn for_segment(
         &self,
         segment_local_id: u32,
         segment_reader: &SegmentReader,
-    ) -> crate::Result<FilterSegmentCollector<TCollector::Child, TPredicate>> {
+    ) -> crate::Result<FilterSegmentCollector<TCollector::Child, TPredicate, TPredicateValue>> {
+        let schema = segment_reader.schema();
+        let field_entry = schema.get_field_entry(self.field);
+        if !field_entry.is_fast() {
+            return Err(TantivyError::SchemaError(format!(
+                "Field {:?} is not a fast field.",
+                field_entry.name()
+            )));
+        }
+        let requested_type = TPredicateValue::to_type();
+        let field_schema_type = field_entry.field_type().value_type();
+        if requested_type != field_schema_type {
+            return Err(TantivyError::SchemaError(format!(
+                "Field {:?} is of type {:?}!={:?}",
+                field_entry.name(),
+                requested_type,
+                field_schema_type
+            )));
+        }
+
         let fast_field_reader = segment_reader
             .fast_fields()
-            .u64(self.field)
+            .typed_fast_field_reader(self.field)
             .ok_or_else(|| {
-                let field_name = segment_reader.schema().get_field_name(self.field);
                 TantivyError::SchemaError(format!(
-                    "Field {:?} is not a u64 fast field.",
-                    field_name
+                    "{:?} is not declared as a fast field in the schema.",
+                    self.field
                 ))
             })?;
+
         let segment_collector = self
             .collector
             .for_segment(segment_local_id, segment_reader)?;
+
         Ok(FilterSegmentCollector {
             fast_field_reader,
             segment_collector,
             predicate: self.predicate,
+            t_predicate_value: PhantomData,
         })
     }
 
@@ -128,20 +156,23 @@ where
     }
 }
 
-pub struct FilterSegmentCollector<TSegmentCollector, TPredicate>
+pub struct FilterSegmentCollector<TSegmentCollector, TPredicate, TPredicateValue>
 where
     TPredicate: 'static,
+    TPredicateValue: 'static + FastValue,
 {
-    fast_field_reader: FastFieldReader<u64>,
+    fast_field_reader: FastFieldReader<TPredicateValue>,
     segment_collector: TSegmentCollector,
     predicate: &'static TPredicate,
+    t_predicate_value: PhantomData<TPredicateValue>,
 }
 
-impl<TSegmentCollector, TPredicate> SegmentCollector
-    for FilterSegmentCollector<TSegmentCollector, TPredicate>
+impl<TSegmentCollector, TPredicate, TPredicateValue> SegmentCollector
+    for FilterSegmentCollector<TSegmentCollector, TPredicate, TPredicateValue>
 where
     TSegmentCollector: SegmentCollector,
-    TPredicate: 'static + Fn(u64) -> bool + Send + Sync,
+    TPredicate: 'static + Fn(TPredicateValue) -> bool + Send + Sync,
+    TPredicateValue: 'static + FastValue,
 {
     type Fruit = TSegmentCollector::Fruit;
 

src/collector/tests.rs

@@ -8,6 +8,13 @@ use crate::DocId;
 use crate::Score;
 use crate::SegmentLocalId;
 
+use crate::collector::{FilterCollector, TopDocs};
+use crate::query::QueryParser;
+use crate::schema::{Schema, FAST, TEXT};
+use crate::DateTime;
+use crate::{doc, Index};
+use std::str::FromStr;
+
 pub const TEST_COLLECTOR_WITH_SCORE: TestCollector = TestCollector {
     compute_score: true,
 };
@@ -16,6 +23,54 @@ pub const TEST_COLLECTOR_WITHOUT_SCORE: TestCollector = TestCollector {
     compute_score: true,
 };
 
+#[test]
+pub fn test_filter_collector() {
+    let mut schema_builder = Schema::builder();
+    let title = schema_builder.add_text_field("title", TEXT);
+    let price = schema_builder.add_u64_field("price", FAST);
+    let date = schema_builder.add_date_field("date", FAST);
+    let schema = schema_builder.build();
+    let index = Index::create_in_ram(schema);
+
+    let mut index_writer = index.writer_with_num_threads(1, 10_000_000).unwrap();
+    index_writer.add_document(doc!(title => "The Name of the Wind", price => 30_200u64, date => DateTime::from_str("1898-04-09T00:00:00+00:00").unwrap()));
+    index_writer.add_document(doc!(title => "The Diary of Muadib", price => 29_240u64, date => DateTime::from_str("2020-04-09T00:00:00+00:00").unwrap()));
+    index_writer.add_document(doc!(title => "The Diary of Anne Frank", price => 18_240u64, date => DateTime::from_str("2019-04-20T00:00:00+00:00").unwrap()));
+    index_writer.add_document(doc!(title => "A Dairy Cow", price => 21_240u64, date => DateTime::from_str("2019-04-09T00:00:00+00:00").unwrap()));
+    index_writer.add_document(doc!(title => "The Diary of a Young Girl", price => 20_120u64, date => DateTime::from_str("2018-04-09T00:00:00+00:00").unwrap()));
+    assert!(index_writer.commit().is_ok());
+
+    let reader = index.reader().unwrap();
+    let searcher = reader.searcher();
+
+    let query_parser = QueryParser::for_index(&index, vec![title]);
+    let query = query_parser.parse_query("diary").unwrap();
+    let filter_some_collector = FilterCollector::new(
+        price,
+        &|value: u64| value > 20_120u64,
+        TopDocs::with_limit(2),
+    );
+    let top_docs = searcher.search(&query, &filter_some_collector).unwrap();
+
+    assert_eq!(top_docs.len(), 1);
+    assert_eq!(top_docs[0].1, DocAddress(0, 1));
+
+    let filter_all_collector: FilterCollector<_, _, u64> =
+        FilterCollector::new(price, &|value| value < 5u64, TopDocs::with_limit(2));
+    let filtered_top_docs = searcher.search(&query, &filter_all_collector).unwrap();
+
+    assert_eq!(filtered_top_docs.len(), 0);
+
+    fn date_filter(value: DateTime) -> bool {
+        (value - DateTime::from_str("2019-04-09T00:00:00+00:00").unwrap()).num_weeks() > 0
+    }
+
+    let filter_dates_collector = FilterCollector::new(date, &date_filter, TopDocs::with_limit(5));
+    let filtered_date_docs = searcher.search(&query, &filter_dates_collector).unwrap();
+
+    assert_eq!(filtered_date_docs.len(), 2);
+}
+
 /// Stores all of the doc ids.
 /// This collector is only used for tests.
 /// It is unusable in pr

src/collector/top_score_collector.rs

@@ -728,7 +728,7 @@ mod tests {
     }
 
     #[test]
-    fn test_top_collector_not_at_capacity() {
+    fn test_top_collector_not_at_capacity_without_offset() {
         let index = make_index();
         let field = index.schema().get_field("text").unwrap();
         let query_parser = QueryParser::for_index(&index, vec![field]);

src/common/mod.rs

@@ -115,11 +115,16 @@ pub fn u64_to_i64(val: u64) -> i64 {
 /// For simplicity, tantivy internally handles `f64` as `u64`.
 /// The mapping is defined by this function.
 ///
-/// Maps `f64` to `u64` so that lexical order is preserved.
+/// Maps `f64` to `u64` in a monotonic manner, so that bytes lexical order is preserved.
 ///
 /// This is more suited than simply casting (`val as u64`)
 /// which would truncate the result
 ///
+/// # Reference
+///
+/// Daniel Lemire's [blog post](https://lemire.me/blog/2020/12/14/converting-floating-point-numbers-to-integers-while-preserving-order/)
+/// explains the mapping in a clear manner.
+///
 /// # See also
 /// The [reverse mapping is `u64_to_f64`](./fn.u64_to_f64.html).
 #[inline(always)]
@@ -148,6 +153,7 @@ pub(crate) mod test {
     pub use super::minmax;
     pub use super::serialize::test::fixed_size_test;
     use super::{compute_num_bits, f64_to_u64, i64_to_u64, u64_to_f64, u64_to_i64};
+    use proptest::prelude::*;
     use std::f64;
 
     fn test_i64_converter_helper(val: i64) {
@@ -158,6 +164,15 @@ pub(crate) mod test {
         assert_eq!(u64_to_f64(f64_to_u64(val)), val);
     }
 
+    proptest! {
+        #[test]
+        fn test_f64_converter_monotonicity_proptest((left, right) in (proptest::num::f64::NORMAL, proptest::num::f64::NORMAL)) {
+            let left_u64 = f64_to_u64(left);
+            let right_u64 = f64_to_u64(right);
+            assert_eq!(left_u64 < right_u64,  left < right);
+        }
+    }
+
     #[test]
     fn test_i64_converter() {
         assert_eq!(i64_to_u64(i64::min_value()), u64::min_value());

src/core/index.rs

@@ -511,28 +511,28 @@ mod tests {
         }
 
         #[test]
-        fn test_index_manual_policy_mmap() {
+        fn test_index_manual_policy_mmap() -> crate::Result<()> {
             let schema = throw_away_schema();
             let field = schema.get_field("num_likes").unwrap();
-            let mut index = Index::create_from_tempdir(schema).unwrap();
-            let mut writer = index.writer_for_tests().unwrap();
-            writer.commit().unwrap();
+            let mut index = Index::create_from_tempdir(schema)?;
+            let mut writer = index.writer_for_tests()?;
+            writer.commit()?;
             let reader = index
                 .reader_builder()
                 .reload_policy(ReloadPolicy::Manual)
-                .try_into()
-                .unwrap();
+                .try_into()?;
             assert_eq!(reader.searcher().num_docs(), 0);
             writer.add_document(doc!(field=>1u64));
             let (sender, receiver) = crossbeam::channel::unbounded();
             let _handle = index.directory_mut().watch(WatchCallback::new(move || {
                 let _ = sender.send(());
             }));
-            writer.commit().unwrap();
+            writer.commit()?;
             assert!(receiver.recv().is_ok());
             assert_eq!(reader.searcher().num_docs(), 0);
-            reader.reload().unwrap();
+            reader.reload()?;
             assert_eq!(reader.searcher().num_docs(), 1);
+            Ok(())
         }
 
         #[test]

src/core/segment_reader.rs

@@ -310,7 +310,7 @@ impl SegmentReader {
     }
 
     /// Returns an iterator that will iterate over the alive document ids
-    pub fn doc_ids_alive<'a>(&'a self) -> impl Iterator<Item = DocId> + 'a {
+    pub fn doc_ids_alive(&self) -> impl Iterator<Item = DocId> + '_ {
         (0u32..self.max_doc).filter(move |doc| !self.is_deleted(*doc))
     }
 

src/directory/error.rs

@@ -58,7 +58,8 @@ pub enum OpenWriteError {
 }
 
 impl OpenWriteError {
-    pub(crate) fn wrap_io_error(io_error: io::Error, filepath: PathBuf) -> Self {
+    /// Wraps an io error.
+    pub fn wrap_io_error(io_error: io::Error, filepath: PathBuf) -> Self {
         Self::IOError { io_error, filepath }
     }
 }
@@ -143,7 +144,8 @@ pub enum OpenReadError {
 }
 
 impl OpenReadError {
-    pub(crate) fn wrap_io_error(io_error: io::Error, filepath: PathBuf) -> Self {
+    /// Wraps an io error.
+    pub fn wrap_io_error(io_error: io::Error, filepath: PathBuf) -> Self {
         Self::IOError { io_error, filepath }
     }
 }

src/directory/file_watcher.rs

@@ -3,7 +3,7 @@ use crc32fast::Hasher;
 use std::fs;
 use std::io;
 use std::io::BufRead;
-use std::path::PathBuf;
+use std::path::Path;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::thread;
@@ -13,15 +13,15 @@ pub const POLLING_INTERVAL: Duration = Duration::from_millis(if cfg!(test) { 1 }
 
 // Watches a file and executes registered callbacks when the file is modified.
 pub struct FileWatcher {
-    path: Arc<PathBuf>,
+    path: Arc<Path>,
     callbacks: Arc<WatchCallbackList>,
     state: Arc<AtomicUsize>, // 0: new, 1: runnable, 2: terminated
 }
 
 impl FileWatcher {
-    pub fn new(path: &PathBuf) -> FileWatcher {
+    pub fn new(path: &Path) -> FileWatcher {
         FileWatcher {
-            path: Arc::new(path.clone()),
+            path: Arc::from(path),
             callbacks: Default::default(),
             state: Default::default(),
         }
@@ -63,7 +63,7 @@ impl FileWatcher {
         handle
     }
 
-    fn compute_checksum(path: &PathBuf) -> Result<u32, io::Error> {
+    fn compute_checksum(path: &Path) -> Result<u32, io::Error> {
         let reader = match fs::File::open(path) {
             Ok(f) => io::BufReader::new(f),
             Err(e) => {

src/directory/footer.rs

@@ -115,6 +115,18 @@ impl Footer {
                 }
                 Ok(())
             }
+            VersionedFooter::V3 {
+                crc32: _crc,
+                store_compression,
+            } => {
+                if &library_version.store_compression != store_compression {
+                    return Err(Incompatibility::CompressionMismatch {
+                        library_compression_format: library_version.store_compression.to_string(),
+                        index_compression_format: store_compression.to_string(),
+                    });
+                }
+                Ok(())
+            }
             VersionedFooter::UnknownVersion => Err(Incompatibility::IndexMismatch {
                 library_version: library_version.clone(),
                 index_version: self.version.clone(),
@@ -136,24 +148,31 @@ pub enum VersionedFooter {
         crc32: CrcHashU32,
         store_compression: String,
     },
+    // Block wand max termfred on 1 byte
+    V3 {
+        crc32: CrcHashU32,
+        store_compression: String,
+    },
 }
 
 impl BinarySerializable for VersionedFooter {
     fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
         let mut buf = Vec::new();
         match self {
-            VersionedFooter::V2 {
+            VersionedFooter::V3 {
                 crc32,
                 store_compression: compression,
             } => {
                 // Serializes a valid `VersionedFooter` or panics if the version is unknown
                 // [   version    |   crc_hash  | compression_mode ]
                 // [    0..4      |     4..8    |     variable     ]
-                BinarySerializable::serialize(&2u32, &mut buf)?;
+                BinarySerializable::serialize(&3u32, &mut buf)?;
                 BinarySerializable::serialize(crc32, &mut buf)?;
                 BinarySerializable::serialize(compression, &mut buf)?;
             }
-            VersionedFooter::V1 { .. } | VersionedFooter::UnknownVersion => {
+            VersionedFooter::V2 { .. }
+            | VersionedFooter::V1 { .. }
+            | VersionedFooter::UnknownVersion => {
                 return Err(io::Error::new(
                     io::ErrorKind::InvalidInput,
                     "Cannot serialize an unknown versioned footer ",
@@ -182,7 +201,7 @@ impl BinarySerializable for VersionedFooter {
         reader.read_exact(&mut buf[..])?;
         let mut cursor = &buf[..];
         let version = u32::deserialize(&mut cursor)?;
-        if version != 1 && version != 2 {
+        if version > 3 {
             return Ok(VersionedFooter::UnknownVersion);
         }
         let crc32 = u32::deserialize(&mut cursor)?;
@@ -192,12 +211,17 @@ impl BinarySerializable for VersionedFooter {
                 crc32,
                 store_compression,
             }
-        } else {
-            assert_eq!(version, 2);
+        } else if version == 2 {
             VersionedFooter::V2 {
                 crc32,
                 store_compression,
             }
+        } else {
+            assert_eq!(version, 3);
+            VersionedFooter::V3 {
+                crc32,
+                store_compression,
+            }
         })
     }
 }
@@ -205,6 +229,7 @@ impl BinarySerializable for VersionedFooter {
 impl VersionedFooter {
     pub fn crc(&self) -> Option<CrcHashU32> {
         match self {
+            VersionedFooter::V3 { crc32, .. } => Some(*crc32),
             VersionedFooter::V2 { crc32, .. } => Some(*crc32),
             VersionedFooter::V1 { crc32, .. } => Some(*crc32),
             VersionedFooter::UnknownVersion { .. } => None,
@@ -243,7 +268,7 @@ impl<W: TerminatingWrite> Write for FooterProxy<W> {
 impl<W: TerminatingWrite> TerminatingWrite for FooterProxy<W> {
     fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()> {
         let crc32 = self.hasher.take().unwrap().finalize();
-        let footer = Footer::new(VersionedFooter::V2 {
+        let footer = Footer::new(VersionedFooter::V3 {
             crc32,
             store_compression: crate::store::COMPRESSION.to_string(),
         });
@@ -278,7 +303,7 @@ mod tests {
         let footer = Footer::deserialize(&mut &vec[..]).unwrap();
         assert!(matches!(
            footer.versioned_footer,
-           VersionedFooter::V2 { store_compression, .. }
+           VersionedFooter::V3 { store_compression, .. }
            if store_compression == crate::store::COMPRESSION
         ));
         assert_eq!(&footer.version, crate::version());
@@ -288,7 +313,7 @@ mod tests {
     fn test_serialize_deserialize_footer() {
         let mut buffer = Vec::new();
         let crc32 = 123456u32;
-        let footer: Footer = Footer::new(VersionedFooter::V2 {
+        let footer: Footer = Footer::new(VersionedFooter::V3 {
             crc32,
             store_compression: "lz4".to_string(),
         });
@@ -300,7 +325,7 @@ mod tests {
     #[test]
     fn footer_length() {
         let crc32 = 1111111u32;
-        let versioned_footer = VersionedFooter::V2 {
+        let versioned_footer = VersionedFooter::V3 {
             crc32,
             store_compression: "lz4".to_string(),
         };
@@ -321,7 +346,7 @@ mod tests {
             // versionned footer length
             12 | 128,
             // index format version
-            2,
+            3,
             0,
             0,
             0,
@@ -340,7 +365,7 @@ mod tests {
         let versioned_footer = VersionedFooter::deserialize(&mut cursor).unwrap();
         assert!(cursor.is_empty());
         let expected_crc: u32 = LittleEndian::read_u32(&v_footer_bytes[5..9]) as CrcHashU32;
-        let expected_versioned_footer: VersionedFooter = VersionedFooter::V2 {
+        let expected_versioned_footer: VersionedFooter = VersionedFooter::V3 {
             crc32: expected_crc,
             store_compression: "lz4".to_string(),
         };

src/fastfield/facet_reader.rs

@@ -1,4 +1,5 @@
 use super::MultiValueIntFastFieldReader;
+use crate::error::DataCorruption;
 use crate::schema::Facet;
 use crate::termdict::TermDictionary;
 use crate::termdict::TermOrdinal;
@@ -62,12 +63,13 @@ impl FacetReader {
         &mut self,
         facet_ord: TermOrdinal,
         output: &mut Facet,
-    ) -> Result<(), str::Utf8Error> {
+    ) -> crate::Result<()> {
         let found_term = self
             .term_dict
-            .ord_to_term(facet_ord as u64, &mut self.buffer);
+            .ord_to_term(facet_ord as u64, &mut self.buffer)?;
         assert!(found_term, "Term ordinal {} no found.", facet_ord);
-        let facet_str = str::from_utf8(&self.buffer[..])?;
+        let facet_str = str::from_utf8(&self.buffer[..])
+            .map_err(|utf8_err| DataCorruption::comment_only(utf8_err.to_string()))?;
         output.set_facet_str(facet_str);
         Ok(())
     }

src/fastfield/reader.rs

@@ -51,6 +51,15 @@ impl<Item: FastValue> FastFieldReader<Item> {
         }
     }
 
+    pub(crate) fn cast<TFastValue: FastValue>(self) -> FastFieldReader<TFastValue> {
+        FastFieldReader {
+            bit_unpacker: self.bit_unpacker,
+            min_value_u64: self.min_value_u64,
+            max_value_u64: self.max_value_u64,
+            _phantom: PhantomData,
+        }
+    }
+
     /// Return the value associated to the given document.
     ///
     /// This accessor should return as fast as possible.

src/fastfield/readers.rs

@@ -1,6 +1,6 @@
 use crate::common::CompositeFile;
-use crate::fastfield::BytesFastFieldReader;
 use crate::fastfield::MultiValueIntFastFieldReader;
+use crate::fastfield::{BytesFastFieldReader, FastValue};
 use crate::fastfield::{FastFieldNotAvailableError, FastFieldReader};
 use crate::schema::{Cardinality, Field, FieldType, Schema};
 use crate::space_usage::PerFieldSpaceUsage;
@@ -201,6 +201,14 @@ impl FastFieldReaders {
         None
     }
 
+    pub(crate) fn typed_fast_field_reader<TFastValue: FastValue>(
+        &self,
+        field: Field,
+    ) -> Option<FastFieldReader<TFastValue>> {
+        self.u64_lenient(field)
+            .map(|fast_field_reader| fast_field_reader.cast())
+    }
+
     /// Returns the `i64` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a i64 fast field, this method returns `None`.

src/indexer/merger.rs

@@ -503,7 +503,6 @@ impl IndexMerger {
         let mut positions_buffer: Vec<u32> = Vec::with_capacity(1_000);
         let mut delta_computer = DeltaComputer::new();
 
-        let mut field_term_streams = Vec::new();
         let mut max_term_ords: Vec<TermOrdinal> = Vec::new();
 
         let field_readers: Vec<Arc<InvertedIndexReader>> = self
@@ -512,6 +511,7 @@ impl IndexMerger {
             .map(|reader| reader.inverted_index(indexed_field))
             .collect::<crate::Result<Vec<_>>>()?;
 
+        let mut field_term_streams = Vec::new();
         for field_reader in &field_readers {
             let terms = field_reader.terms();
             field_term_streams.push(terms.stream()?);

src/indexer/segment_writer.rs

@@ -11,7 +11,8 @@ use crate::schema::Schema;
 use crate::schema::Term;
 use crate::schema::Value;
 use crate::schema::{Field, FieldEntry};
-use crate::tokenizer::{BoxTokenStream, PreTokenizedStream};
+use crate::tokenizer::PreTokenizedStream;
+use crate::tokenizer::TokenStream;
 use crate::tokenizer::{FacetTokenizer, TextAnalyzer};
 use crate::tokenizer::{TokenStreamChain, Tokenizer};
 use crate::Opstamp;
@@ -141,13 +142,13 @@ impl SegmentWriter {
             }
             let (term_buffer, multifield_postings) =
                 (&mut self.term_buffer, &mut self.multifield_postings);
-            match *field_entry.field_type() {
+            match field_entry.field_type() {
                 FieldType::HierarchicalFacet => {
                     term_buffer.set_field(field);
                     let facets =
                         field_values
                             .iter()
-                            .flat_map(|field_value| match *field_value.value() {
+                            .flat_map(|field_value| match field_value.value() {
                                 Value::Facet(ref facet) => Some(facet.encoded_str()),
                                 _ => {
                                     panic!("Expected hierarchical facet");
@@ -172,37 +173,38 @@ impl SegmentWriter {
                     }
                 }
                 FieldType::Str(_) => {
-                    let mut token_streams: Vec<BoxTokenStream> = vec![];
-                    let mut offsets = vec![];
+                    let mut streams_with_offsets = vec![];
                     let mut total_offset = 0;
 
                     for field_value in field_values {
                         match field_value.value() {
                             Value::PreTokStr(tok_str) => {
-                                offsets.push(total_offset);
+                                streams_with_offsets.push((
+                                    Box::new(PreTokenizedStream::from(tok_str.clone()))
+                                        as Box<dyn TokenStream>,
+                                    total_offset,
+                                ));
                                 if let Some(last_token) = tok_str.tokens.last() {
                                     total_offset += last_token.offset_to;
                                 }
-                                token_streams
-                                    .push(PreTokenizedStream::from(tok_str.clone()).into());
                             }
                             Value::Str(ref text) => {
                                 if let Some(ref mut tokenizer) =
                                     self.tokenizers[field.field_id() as usize]
                                 {
-                                    offsets.push(total_offset);
+                                    streams_with_offsets
+                                        .push((tokenizer.token_stream(text), total_offset));
                                     total_offset += text.len();
-                                    token_streams.push(tokenizer.token_stream(text));
                                 }
                             }
                             _ => (),
                         }
                     }
 
-                    let num_tokens = if token_streams.is_empty() {
+                    let num_tokens = if streams_with_offsets.is_empty() {
                         0
                     } else {
-                        let mut token_stream = TokenStreamChain::new(offsets, token_streams);
+                        let mut token_stream = TokenStreamChain::new(streams_with_offsets);
                         multifield_postings.index_text(
                             doc_id,
                             field,

src/lib.rs

@@ -174,7 +174,7 @@ use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
 /// Index format version.
-const INDEX_FORMAT_VERSION: u32 = 2;
+const INDEX_FORMAT_VERSION: u32 = 3;
 
 /// Structure version for the index.
 #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]

src/positions/reader.rs

@@ -132,7 +132,7 @@ impl PositionReader {
             "offset arguments should be increasing."
         );
         let delta_to_block_offset = offset as i64 - self.block_offset as i64;
-        if delta_to_block_offset < 0 || delta_to_block_offset >= 128 {
+        if !(0..128).contains(&delta_to_block_offset) {
             // The first position is not within the first block.
             // We need to decompress the first block.
             let delta_to_anchor_offset = offset - self.anchor_offset;

src/positions/serializer.rs

@@ -8,7 +8,7 @@ use std::io::{self, Write};
 pub struct PositionSerializer<W: io::Write> {
     bit_packer: BitPacker4x,
     write_stream: CountingWriter<W>,
-    write_skiplist: W,
+    write_skip_index: W,
     block: Vec<u32>,
     buffer: Vec<u8>,
     num_ints: u64,
@@ -16,11 +16,11 @@ pub struct PositionSerializer<W: io::Write> {
 }
 
 impl<W: io::Write> PositionSerializer<W> {
-    pub fn new(write_stream: W, write_skiplist: W) -> PositionSerializer<W> {
+    pub fn new(write_stream: W, write_skip_index: W) -> PositionSerializer<W> {
         PositionSerializer {
             bit_packer: BitPacker4x::new(),
             write_stream: CountingWriter::wrap(write_stream),
-            write_skiplist,
+            write_skip_index,
             block: Vec::with_capacity(128),
             buffer: vec![0u8; 128 * 4],
             num_ints: 0u64,
@@ -52,7 +52,7 @@ impl<W: io::Write> PositionSerializer<W> {
 
     fn flush_block(&mut self) -> io::Result<()> {
         let num_bits = self.bit_packer.num_bits(&self.block[..]);
-        self.write_skiplist.write_all(&[num_bits])?;
+        self.write_skip_index.write_all(&[num_bits])?;
         let written_len = self
             .bit_packer
             .compress(&self.block[..], &mut self.buffer, num_bits);
@@ -70,10 +70,10 @@ impl<W: io::Write> PositionSerializer<W> {
             self.flush_block()?;
         }
         for &long_skip in &self.long_skips {
-            long_skip.serialize(&mut self.write_skiplist)?;
+            long_skip.serialize(&mut self.write_skip_index)?;
         }
-        (self.long_skips.len() as u32).serialize(&mut self.write_skiplist)?;
-        self.write_skiplist.flush()?;
+        (self.long_skips.len() as u32).serialize(&mut self.write_skip_index)?;
+        self.write_skip_index.flush()?;
         self.write_stream.flush()?;
         Ok(())
     }

src/postings/block_search.rs

@@ -109,9 +109,9 @@ impl BlockSearcher {
     /// The results should be equivalent to
     /// ```compile_fail
     /// block[..]
-    //       .iter()
-    //       .take_while(|&&val| val < target)
-    //       .count()
+    ///       .iter()
+    ///       .take_while(|&&val| val < target)
+    ///       .count()
     /// ```
     ///
     /// The `start` argument is just used to hint that the response is

src/postings/mod.rs

@@ -54,7 +54,7 @@ pub mod tests {
     use crate::DocId;
     use crate::HasLen;
     use crate::Score;
-    use std::iter;
+    use std::{iter, mem};
 
     #[test]
     pub fn test_position_write() -> crate::Result<()> {
@@ -71,6 +71,7 @@ pub mod tests {
             field_serializer.write_doc(doc_id, 4, &delta_positions)?;
         }
         field_serializer.close_term()?;
+        mem::drop(field_serializer);
         posting_serializer.close()?;
         let read = segment.open_read(SegmentComponent::POSITIONS)?;
         assert!(read.len() <= 140);
@@ -179,7 +180,7 @@ pub mod tests {
             let inverted_index = segment_reader.inverted_index(text_field)?;
             assert_eq!(inverted_index.terms().num_terms(), 1);
             let mut bytes = vec![];
-            assert!(inverted_index.terms().ord_to_term(0, &mut bytes));
+            assert!(inverted_index.terms().ord_to_term(0, &mut bytes)?);
             assert_eq!(&bytes, b"hello");
         }
         {
@@ -191,7 +192,7 @@ pub mod tests {
             let inverted_index = segment_reader.inverted_index(text_field)?;
             assert_eq!(inverted_index.terms().num_terms(), 1);
             let mut bytes = vec![];
-            assert!(inverted_index.terms().ord_to_term(0, &mut bytes));
+            assert!(inverted_index.terms().ord_to_term(0, &mut bytes)?);
             assert_eq!(&bytes[..], ok_token_text.as_bytes());
         }
         Ok(())

src/postings/skip.rs

@@ -1,32 +1,46 @@
-use crate::common::{read_u32_vint_no_advance, serialize_vint_u32, BinarySerializable};
+use std::convert::TryInto;
+
 use crate::directory::OwnedBytes;
 use crate::postings::compression::{compressed_block_size, COMPRESSION_BLOCK_SIZE};
 use crate::query::BM25Weight;
 use crate::schema::IndexRecordOption;
 use crate::{DocId, Score, TERMINATED};
 
+#[inline(always)]
+fn encode_block_wand_max_tf(max_tf: u32) -> u8 {
+    max_tf.min(u8::MAX as u32) as u8
+}
+
+#[inline(always)]
+fn decode_block_wand_max_tf(max_tf_code: u8) -> u32 {
+    if max_tf_code == u8::MAX {
+        u32::MAX
+    } else {
+        max_tf_code as u32
+    }
+}
+
+#[inline(always)]
+fn read_u32(data: &[u8]) -> u32 {
+    u32::from_le_bytes(data[..4].try_into().unwrap())
+}
+
+#[inline(always)]
+fn write_u32(val: u32, buf: &mut Vec<u8>) {
+    buf.extend_from_slice(&val.to_le_bytes());
+}
+
 pub struct SkipSerializer {
     buffer: Vec<u8>,
-    prev_doc: DocId,
 }
 
 impl SkipSerializer {
     pub fn new() -> SkipSerializer {
-        SkipSerializer {
-            buffer: Vec::new(),
-            prev_doc: 0u32,
-        }
+        SkipSerializer { buffer: Vec::new() }
     }
 
     pub fn write_doc(&mut self, last_doc: DocId, doc_num_bits: u8) {
-        assert!(
-            last_doc > self.prev_doc,
-            "write_doc(...) called with non-increasing doc ids. \
-             Did you forget to call clear maybe?"
-        );
-        let delta_doc = last_doc - self.prev_doc;
-        self.prev_doc = last_doc;
-        delta_doc.serialize(&mut self.buffer).unwrap();
+        write_u32(last_doc, &mut self.buffer);
         self.buffer.push(doc_num_bits);
     }
 
@@ -35,16 +49,13 @@ impl SkipSerializer {
     }
 
     pub fn write_total_term_freq(&mut self, tf_sum: u32) {
-        tf_sum
-            .serialize(&mut self.buffer)
-            .expect("Should never fail");
+        write_u32(tf_sum, &mut self.buffer);
     }
 
     pub fn write_blockwand_max(&mut self, fieldnorm_id: u8, term_freq: u32) {
-        self.buffer.push(fieldnorm_id);
-        let mut buf = [0u8; 8];
-        let bytes = serialize_vint_u32(term_freq, &mut buf);
-        self.buffer.extend_from_slice(bytes);
+        let block_wand_tf = encode_block_wand_max_tf(term_freq);
+        self.buffer
+            .extend_from_slice(&[fieldnorm_id, block_wand_tf]);
     }
 
     pub fn data(&self) -> &[u8] {
@@ -52,7 +63,6 @@ impl SkipSerializer {
     }
 
     pub fn clear(&mut self) {
-        self.prev_doc = 0u32;
         self.buffer.clear();
     }
 }
@@ -159,18 +169,13 @@ impl SkipReader {
     }
 
     fn read_block_info(&mut self) {
-        let doc_delta = {
-            let bytes = self.owned_read.as_slice();
-            let mut buf = [0; 4];
-            buf.copy_from_slice(&bytes[..4]);
-            u32::from_le_bytes(buf)
-        };
-        self.last_doc_in_block += doc_delta as DocId;
-        let doc_num_bits = self.owned_read.as_slice()[4];
-
+        let bytes = self.owned_read.as_slice();
+        let advance_len: usize;
+        self.last_doc_in_block = read_u32(bytes);
+        let doc_num_bits = bytes[4];
         match self.skip_info {
             IndexRecordOption::Basic => {
-                self.owned_read.advance(5);
+                advance_len = 5;
                 self.block_info = BlockInfo::BitPacked {
                     doc_num_bits,
                     tf_num_bits: 0,
@@ -180,11 +185,10 @@ impl SkipReader {
                 };
             }
             IndexRecordOption::WithFreqs => {
-                let bytes = self.owned_read.as_slice();
                 let tf_num_bits = bytes[5];
                 let block_wand_fieldnorm_id = bytes[6];
-                let (block_wand_term_freq, num_bytes) = read_u32_vint_no_advance(&bytes[7..]);
-                self.owned_read.advance(7 + num_bytes);
+                let block_wand_term_freq = decode_block_wand_max_tf(bytes[7]);
+                advance_len = 8;
                 self.block_info = BlockInfo::BitPacked {
                     doc_num_bits,
                     tf_num_bits,
@@ -194,16 +198,11 @@ impl SkipReader {
                 };
             }
             IndexRecordOption::WithFreqsAndPositions => {
-                let bytes = self.owned_read.as_slice();
                 let tf_num_bits = bytes[5];
-                let tf_sum = {
-                    let mut buf = [0; 4];
-                    buf.copy_from_slice(&bytes[6..10]);
-                    u32::from_le_bytes(buf)
-                };
+                let tf_sum = read_u32(&bytes[6..10]);
                 let block_wand_fieldnorm_id = bytes[10];
-                let (block_wand_term_freq, num_bytes) = read_u32_vint_no_advance(&bytes[11..]);
-                self.owned_read.advance(11 + num_bytes);
+                let block_wand_term_freq = decode_block_wand_max_tf(bytes[11]);
+                advance_len = 12;
                 self.block_info = BlockInfo::BitPacked {
                     doc_num_bits,
                     tf_num_bits,
@@ -213,6 +212,7 @@ impl SkipReader {
                 };
             }
         }
+        self.owned_read.advance(advance_len);
     }
 
     pub fn block_info(&self) -> BlockInfo {
@@ -274,6 +274,24 @@ mod tests {
     use crate::directory::OwnedBytes;
     use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
 
+    #[test]
+    fn test_encode_block_wand_max_tf() {
+        for tf in 0..255 {
+            assert_eq!(super::encode_block_wand_max_tf(tf), tf as u8);
+        }
+        for &tf in &[255, 256, 1_000_000, u32::MAX] {
+            assert_eq!(super::encode_block_wand_max_tf(tf), 255);
+        }
+    }
+
+    #[test]
+    fn test_decode_block_wand_max_tf() {
+        for tf in 0..255 {
+            assert_eq!(super::decode_block_wand_max_tf(tf), tf as u32);
+        }
+        assert_eq!(super::decode_block_wand_max_tf(255), u32::MAX);
+    }
+
     #[test]
     fn test_skip_with_freq() {
         let buf = {

src/query/automaton_weight.rs

@@ -20,6 +20,7 @@ pub struct AutomatonWeight<A> {
 impl<A> AutomatonWeight<A>
 where
     A: Automaton + Send + Sync + 'static,
+    A::State: Clone,
 {
     /// Create a new AutomationWeight
     pub fn new<IntoArcA: Into<Arc<A>>>(field: Field, automaton: IntoArcA) -> AutomatonWeight<A> {
@@ -42,6 +43,7 @@ where
 impl<A> Weight for AutomatonWeight<A>
 where
     A: Automaton + Send + Sync + 'static,
+    A::State: Clone,
 {
     fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
         let max_doc = reader.max_doc();
@@ -102,6 +104,7 @@ mod tests {
         index
     }
 
+    #[derive(Clone, Copy)]
     enum State {
         Start,
         NotMatching,

src/schema/facet.rs

@@ -233,6 +233,7 @@ mod tests {
         assert_eq!(Facet::root(), Facet::from("/"));
         assert_eq!(format!("{}", Facet::root()), "/");
         assert!(Facet::root().is_root());
+        assert_eq!(Facet::root().encoded_str(), "");
     }
 
     #[test]

src/store/compression_lz4.rs

@@ -3,7 +3,7 @@ use std::io::{self, Read, Write};
 /// Name of the compression scheme used in the doc store.
 ///
 /// This name is appended to the version string of tantivy.
-pub const COMPRESSION: &'static str = "lz4";
+pub const COMPRESSION: &str = "lz4";
 
 pub fn compress(uncompressed: &[u8], compressed: &mut Vec<u8>) -> io::Result<()> {
     compressed.clear();

src/store/index/skip_index.rs

@@ -35,11 +35,11 @@ struct Layer {
 }
 
 impl Layer {
-    fn cursor<'a>(&'a self) -> impl Iterator<Item = Checkpoint> + 'a {
+    fn cursor(&self) -> impl Iterator<Item = Checkpoint> + '_ {
         self.cursor_at_offset(0u64)
     }
 
-    fn cursor_at_offset<'a>(&'a self, start_offset: u64) -> impl Iterator<Item = Checkpoint> + 'a {
+    fn cursor_at_offset(&self, start_offset: u64) -> impl Iterator<Item = Checkpoint> + '_ {
         let data = &self.data.as_slice();
         LayerCursor {
             remaining: &data[start_offset as usize..],
@@ -59,7 +59,7 @@ pub struct SkipIndex {
 }
 
 impl SkipIndex {
-    pub(crate) fn checkpoints<'a>(&'a self) -> impl Iterator<Item = Checkpoint> + 'a {
+    pub(crate) fn checkpoints(&self) -> impl Iterator<Item = Checkpoint> + '_ {
         self.layers
             .last()
             .into_iter()

src/store/reader.rs

@@ -46,7 +46,7 @@ impl StoreReader {
         })
     }
 
-    pub(crate) fn block_checkpoints<'a>(&'a self) -> impl Iterator<Item = Checkpoint> + 'a {
+    pub(crate) fn block_checkpoints(&self) -> impl Iterator<Item = Checkpoint> + '_ {
         self.skip_index.checkpoints()
     }
 

src/termdict/fst_termdict/termdict.rs

@@ -80,7 +80,6 @@ where
                 .serialize(&mut counting_writer)?;
             let footer_size = counting_writer.written_bytes();
             (footer_size as u64).serialize(&mut counting_writer)?;
-            counting_writer.flush()?;
         }
         Ok(file)
     }
@@ -152,7 +151,7 @@ impl TermDictionary {
     ///
     /// Regardless of whether the term is found or not,
     /// the buffer may be modified.
-    pub fn ord_to_term(&self, mut ord: TermOrdinal, bytes: &mut Vec<u8>) -> bool {
+    pub fn ord_to_term(&self, mut ord: TermOrdinal, bytes: &mut Vec<u8>) -> io::Result<bool> {
         bytes.clear();
         let fst = self.fst_index.as_fst();
         let mut node = fst.root();
@@ -167,10 +166,10 @@ impl TermDictionary {
                 let new_node_addr = transition.addr;
                 node = fst.node(new_node_addr);
             } else {
-                return false;
+                return Ok(false);
             }
         }
-        true
+        Ok(true)
     }
 
     /// Returns the number of terms in the dictionary.

src/termdict/tests.rs

@@ -50,7 +50,7 @@ fn test_term_ordinals() -> crate::Result<()> {
     for (term_ord, term) in COUNTRIES.iter().enumerate() {
         assert_eq!(term_dict.term_ord(term)?, Some(term_ord as u64));
         let mut bytes = vec![];
-        assert!(term_dict.ord_to_term(term_ord as u64, &mut bytes));
+        assert!(term_dict.ord_to_term(term_ord as u64, &mut bytes)?);
         assert_eq!(bytes, term.as_bytes());
     }
     Ok(())
@@ -249,8 +249,7 @@ fn test_empty_string() -> crate::Result<()> {
     Ok(())
 }
 
-#[test]
-fn test_stream_range_boundaries() -> crate::Result<()> {
+fn stream_range_test_dict() -> crate::Result<TermDictionary> {
     let buffer: Vec<u8> = {
         let mut term_dictionary_builder = TermDictionaryBuilder::create(Vec::new())?;
         for i in 0u8..10u8 {
@@ -260,84 +259,96 @@ fn test_stream_range_boundaries() -> crate::Result<()> {
         term_dictionary_builder.finish()?
     };
     let file = FileSlice::from(buffer);
-    let term_dictionary: TermDictionary = TermDictionary::open(file)?;
+    TermDictionary::open(file)
+}
 
-    let value_list = |mut streamer: TermStreamer<'_>, backwards: bool| {
+#[test]
+fn test_stream_range_boundaries_forward() -> crate::Result<()> {
+    let term_dictionary = stream_range_test_dict()?;
+    let value_list = |mut streamer: TermStreamer<'_>| {
         let mut res: Vec<u32> = vec![];
         while let Some((_, ref v)) = streamer.next() {
             res.push(v.doc_freq);
         }
-        if backwards {
-            res.reverse();
-        }
         res
     };
-    {
-        let range = term_dictionary.range().backward().into_stream()?;
-        assert_eq!(
-            value_list(range, true),
-            vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
-        );
-    }
     {
         let range = term_dictionary.range().ge([2u8]).into_stream()?;
         assert_eq!(
-            value_list(range, false),
-            vec![2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
-        );
-    }
-    {
-        let range = term_dictionary.range().ge([2u8]).backward().into_stream()?;
-        assert_eq!(
-            value_list(range, true),
+            value_list(range),
             vec![2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
         );
     }
     {
         let range = term_dictionary.range().gt([2u8]).into_stream()?;
         assert_eq!(
-            value_list(range, false),
-            vec![3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
-        );
-    }
-    {
-        let range = term_dictionary.range().gt([2u8]).backward().into_stream()?;
-        assert_eq!(
-            value_list(range, true),
+            value_list(range),
             vec![3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
         );
     }
     {
         let range = term_dictionary.range().lt([6u8]).into_stream()?;
-        assert_eq!(
-            value_list(range, false),
-            vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32]
-        );
-    }
-    {
-        let range = term_dictionary.range().lt([6u8]).backward().into_stream()?;
-        assert_eq!(
-            value_list(range, true),
-            vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32]
-        );
+        assert_eq!(value_list(range), vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32]);
     }
     {
         let range = term_dictionary.range().le([6u8]).into_stream()?;
         assert_eq!(
-            value_list(range, false),
-            vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32]
-        );
-    }
-    {
-        let range = term_dictionary.range().le([6u8]).backward().into_stream()?;
-        assert_eq!(
-            value_list(range, true),
+            value_list(range),
             vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32]
         );
     }
     {
         let range = term_dictionary.range().ge([0u8]).lt([5u8]).into_stream()?;
-        assert_eq!(value_list(range, false), vec![0u32, 1u32, 2u32, 3u32, 4u32]);
+        assert_eq!(value_list(range), vec![0u32, 1u32, 2u32, 3u32, 4u32]);
+    }
+    Ok(())
+}
+
+#[test]
+fn test_stream_range_boundaries_backward() -> crate::Result<()> {
+    let term_dictionary = stream_range_test_dict()?;
+    let value_list_backward = |mut streamer: TermStreamer<'_>| {
+        let mut res: Vec<u32> = vec![];
+        while let Some((_, ref v)) = streamer.next() {
+            res.push(v.doc_freq);
+        }
+        res.reverse();
+        res
+    };
+    {
+        let range = term_dictionary.range().backward().into_stream()?;
+        assert_eq!(
+            value_list_backward(range),
+            vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
+        );
+    }
+    {
+        let range = term_dictionary.range().ge([2u8]).backward().into_stream()?;
+        assert_eq!(
+            value_list_backward(range),
+            vec![2u32, 3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
+        );
+    }
+    {
+        let range = term_dictionary.range().gt([2u8]).backward().into_stream()?;
+        assert_eq!(
+            value_list_backward(range),
+            vec![3u32, 4u32, 5u32, 6u32, 7u32, 8u32, 9u32]
+        );
+    }
+    {
+        let range = term_dictionary.range().lt([6u8]).backward().into_stream()?;
+        assert_eq!(
+            value_list_backward(range),
+            vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32]
+        );
+    }
+    {
+        let range = term_dictionary.range().le([6u8]).backward().into_stream()?;
+        assert_eq!(
+            value_list_backward(range),
+            vec![0u32, 1u32, 2u32, 3u32, 4u32, 5u32, 6u32]
+        );
     }
     {
         let range = term_dictionary
@@ -346,11 +357,38 @@ fn test_stream_range_boundaries() -> crate::Result<()> {
             .lt([5u8])
             .backward()
             .into_stream()?;
-        assert_eq!(value_list(range, true), vec![0u32, 1u32, 2u32, 3u32, 4u32]);
+        assert_eq!(
+            value_list_backward(range),
+            vec![0u32, 1u32, 2u32, 3u32, 4u32]
+        );
     }
     Ok(())
 }
 
+#[test]
+fn test_ord_to_term() -> crate::Result<()> {
+    let termdict = stream_range_test_dict()?;
+    let mut bytes = vec![];
+    for b in 0u8..10u8 {
+        termdict.ord_to_term(b as u64, &mut bytes)?;
+        assert_eq!(&bytes, &[b]);
+    }
+    Ok(())
+}
+
+#[test]
+fn test_stream_term_ord() -> crate::Result<()> {
+    let termdict = stream_range_test_dict()?;
+    let mut stream = termdict.stream()?;
+    for b in 0u8..10u8 {
+        assert!(stream.advance(), true);
+        assert_eq!(stream.term_ord(), b as u64);
+        assert_eq!(stream.key(), &[b]);
+    }
+    assert!(!stream.advance());
+    Ok(())
+}
+
 #[test]
 fn test_automaton_search() -> crate::Result<()> {
     use crate::query::DFAWrapper;

src/tokenizer/alphanum_only.rs

@@ -19,7 +19,7 @@
 //! // the "emoji" is dropped because its not an alphanum
 //! assert!(stream.next().is_none());
 //! ```
-use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
+use super::{Token, TokenFilter, TokenStream};
 
 /// `TokenFilter` that removes all tokens that contain non
 /// ascii alphanumeric characters.
@@ -27,7 +27,7 @@ use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
 pub struct AlphaNumOnlyFilter;
 
 pub struct AlphaNumOnlyFilterStream<'a> {
-    tail: BoxTokenStream<'a>,
+    tail: Box<dyn TokenStream + 'a>,
 }
 
 impl<'a> AlphaNumOnlyFilterStream<'a> {
@@ -37,8 +37,8 @@ impl<'a> AlphaNumOnlyFilterStream<'a> {
 }
 
 impl TokenFilter for AlphaNumOnlyFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(AlphaNumOnlyFilterStream { tail: token_stream })
+    fn transform<'a>(&self, token_stream: Box<dyn TokenStream + 'a>) -> Box<dyn TokenStream + 'a> {
+        Box::new(AlphaNumOnlyFilterStream { tail: token_stream })
     }
 }
 

src/tokenizer/ascii_folding_filter.rs

@@ -1,4 +1,4 @@
-use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
+use super::{Token, TokenFilter, TokenStream};
 use std::mem;
 
 /// This class converts alphabetic, numeric, and symbolic Unicode characters
@@ -8,8 +8,8 @@ use std::mem;
 pub struct AsciiFoldingFilter;
 
 impl TokenFilter for AsciiFoldingFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        From::from(AsciiFoldingFilterTokenStream {
+    fn transform<'a>(&self, token_stream: Box<dyn TokenStream + 'a>) -> Box<dyn TokenStream + 'a> {
+        Box::new(AsciiFoldingFilterTokenStream {
             tail: token_stream,
             buffer: String::with_capacity(100),
         })
@@ -18,7 +18,7 @@ impl TokenFilter for AsciiFoldingFilter {
 
 pub struct AsciiFoldingFilterTokenStream<'a> {
     buffer: String,
-    tail: BoxTokenStream<'a>,
+    tail: Box<dyn TokenStream + 'a>,
 }
 
 impl<'a> TokenStream for AsciiFoldingFilterTokenStream<'a> {

src/tokenizer/facet_tokenizer.rs

@@ -1,4 +1,4 @@
-use super::{BoxTokenStream, Token, TokenStream, Tokenizer};
+use super::{Token, TokenStream, Tokenizer};
 use crate::schema::FACET_SEP_BYTE;
 
 /// The `FacetTokenizer` process a `Facet` binary representation
@@ -26,13 +26,12 @@ pub struct FacetTokenStream<'a> {
 }
 
 impl Tokenizer for FacetTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
-        FacetTokenStream {
+    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
+        Box::new(FacetTokenStream {
             text,
             state: State::RootFacetNotEmitted, //< pos is the first char that has not been processed yet.
             token: Token::default(),
-        }
-        .into()
+        })
     }
 }
 

src/tokenizer/lower_caser.rs

@@ -1,10 +1,9 @@
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;
 use std::mem;
 
 impl TokenFilter for LowerCaser {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(LowerCaserTokenStream {
+    fn transform<'a>(&self, token_stream: Box<dyn TokenStream + 'a>) -> Box<dyn TokenStream + 'a> {
+        Box::new(LowerCaserTokenStream {
             tail: token_stream,
             buffer: String::with_capacity(100),
         })
@@ -17,7 +16,7 @@ pub struct LowerCaser;
 
 pub struct LowerCaserTokenStream<'a> {
     buffer: String,
-    tail: BoxTokenStream<'a>,
+    tail: Box<dyn TokenStream + 'a>,
 }
 
 // writes a lowercased version of text into output.

src/tokenizer/mod.rs

@@ -145,9 +145,7 @@ pub use self::stop_word_filter::StopWordFilter;
 pub(crate) use self::token_stream_chain::TokenStreamChain;
 
 pub use self::tokenized_string::{PreTokenizedStream, PreTokenizedString};
-pub use self::tokenizer::{
-    BoxTokenFilter, BoxTokenStream, TextAnalyzer, Token, TokenFilter, TokenStream, Tokenizer,
-};
+pub use self::tokenizer::{TextAnalyzer, Token, TokenFilter, TokenStream, Tokenizer};
 
 pub use self::tokenizer_manager::TokenizerManager;
 

src/tokenizer/ngram_tokenizer.rs

@@ -1,5 +1,4 @@
 use super::{Token, TokenStream, Tokenizer};
-use crate::tokenizer::BoxTokenStream;
 
 /// Tokenize the text by splitting words into n-grams of the given size(s)
 ///
@@ -131,8 +130,8 @@ pub struct NgramTokenStream<'a> {
 }
 
 impl Tokenizer for NgramTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
-        From::from(NgramTokenStream {
+    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
+        Box::new(NgramTokenStream {
             ngram_charidx_iterator: StutteringIterator::new(
                 CodepointFrontiers::for_str(text),
                 self.min_gram,
@@ -308,9 +307,9 @@ mod tests {
     use super::StutteringIterator;
     use crate::tokenizer::tests::assert_token;
     use crate::tokenizer::tokenizer::Tokenizer;
-    use crate::tokenizer::{BoxTokenStream, Token};
+    use crate::tokenizer::{Token, TokenStream};
 
-    fn test_helper(mut tokenizer: BoxTokenStream) -> Vec<Token> {
+    fn test_helper(mut tokenizer: Box<dyn TokenStream>) -> Vec<Token> {
         let mut tokens: Vec<Token> = vec![];
         tokenizer.process(&mut |token: &Token| tokens.push(token.clone()));
         tokens

src/tokenizer/raw_tokenizer.rs

@@ -1,5 +1,4 @@
 use super::{Token, TokenStream, Tokenizer};
-use crate::tokenizer::BoxTokenStream;
 
 /// For each value of the field, emit a single unprocessed token.
 #[derive(Clone)]
@@ -11,7 +10,7 @@ pub struct RawTokenStream {
 }
 
 impl Tokenizer for RawTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
         let token = Token {
             offset_from: 0,
             offset_to: text.len(),
@@ -19,11 +18,10 @@ impl Tokenizer for RawTokenizer {
             text: text.to_string(),
             position_length: 1,
         };
-        RawTokenStream {
+        Box::new(RawTokenStream {
             token,
             has_token: true,
-        }
-        .into()
+        })
     }
 }
 

src/tokenizer/remove_long.rs

@@ -13,7 +13,6 @@
 //! ```
 //!
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;
 
 /// `RemoveLongFilter` removes tokens that are longer
 /// than a given number of bytes (in UTF-8 representation).
@@ -39,8 +38,8 @@ impl<'a> RemoveLongFilterStream<'a> {
 }
 
 impl TokenFilter for RemoveLongFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(RemoveLongFilterStream {
+    fn transform<'a>(&self, token_stream: Box<dyn TokenStream + 'a>) -> Box<dyn TokenStream + 'a> {
+        Box::new(RemoveLongFilterStream {
             token_length_limit: self.length_limit,
             tail: token_stream,
         })
@@ -49,7 +48,7 @@ impl TokenFilter for RemoveLongFilter {
 
 pub struct RemoveLongFilterStream<'a> {
     token_length_limit: usize,
-    tail: BoxTokenStream<'a>,
+    tail: Box<dyn TokenStream + 'a>,
 }
 
 impl<'a> TokenStream for RemoveLongFilterStream<'a> {

src/tokenizer/simple_tokenizer.rs

@@ -1,4 +1,3 @@
-use super::BoxTokenStream;
 use super::{Token, TokenStream, Tokenizer};
 use std::str::CharIndices;
 
@@ -13,8 +12,8 @@ pub struct SimpleTokenStream<'a> {
 }
 
 impl Tokenizer for SimpleTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(SimpleTokenStream {
+    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
+        Box::new(SimpleTokenStream {
             text,
             chars: text.char_indices(),
             token: Token::default(),

src/tokenizer/stemmer.rs

@@ -1,5 +1,4 @@
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;
 use rust_stemmers::{self, Algorithm};
 use serde::{Deserialize, Serialize};
 
@@ -78,9 +77,9 @@ impl Default for Stemmer {
 }
 
 impl TokenFilter for Stemmer {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+    fn transform<'a>(&self, token_stream: Box<dyn TokenStream + 'a>) -> Box<dyn TokenStream + 'a> {
         let inner_stemmer = rust_stemmers::Stemmer::create(self.stemmer_algorithm);
-        BoxTokenStream::from(StemmerTokenStream {
+        Box::new(StemmerTokenStream {
             tail: token_stream,
             stemmer: inner_stemmer,
         })
@@ -88,7 +87,7 @@ impl TokenFilter for Stemmer {
 }
 
 pub struct StemmerTokenStream<'a> {
-    tail: BoxTokenStream<'a>,
+    tail: Box<dyn TokenStream + 'a>,
     stemmer: rust_stemmers::Stemmer,
 }
 

src/tokenizer/stop_word_filter.rs

@@ -11,7 +11,6 @@
 //! assert!(stream.next().is_none());
 //! ```
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;
 use fnv::FnvHasher;
 use std::collections::HashSet;
 use std::hash::BuildHasherDefault;
@@ -51,12 +50,12 @@ impl StopWordFilter {
 
 pub struct StopWordFilterStream<'a> {
     words: StopWordHashSet,
-    tail: BoxTokenStream<'a>,
+    tail: Box<dyn TokenStream + 'a>,
 }
 
 impl TokenFilter for StopWordFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(StopWordFilterStream {
+    fn transform<'a>(&self, token_stream: Box<dyn TokenStream + 'a>) -> Box<dyn TokenStream + 'a> {
+        Box::new(StopWordFilterStream {
             words: self.words.clone(),
             tail: token_stream,
         })

src/tokenizer/token_stream_chain.rs

@@ -1,11 +1,9 @@
-use crate::tokenizer::{BoxTokenStream, Token, TokenStream};
-use std::ops::DerefMut;
+use crate::tokenizer::{Token, TokenStream};
 
 const POSITION_GAP: usize = 2;
 
 pub(crate) struct TokenStreamChain<'a> {
-    offsets: Vec<usize>,
-    token_streams: Vec<BoxTokenStream<'a>>,
+    streams_with_offsets: Vec<(Box<dyn TokenStream + 'a>, usize)>,
     position_shift: usize,
     stream_idx: usize,
     token: Token,
@@ -13,13 +11,11 @@ pub(crate) struct TokenStreamChain<'a> {
 
 impl<'a> TokenStreamChain<'a> {
     pub fn new(
-        offsets: Vec<usize>,
-        token_streams: Vec<BoxTokenStream<'a>>,
+        streams_with_offsets: Vec<(Box<dyn TokenStream + 'a>, usize)>,
     ) -> TokenStreamChain<'a> {
         TokenStreamChain {
-            offsets,
+            streams_with_offsets,
             stream_idx: 0,
-            token_streams,
             position_shift: 0,
             token: Token::default(),
         }
@@ -28,11 +24,10 @@ impl<'a> TokenStreamChain<'a> {
 
 impl<'a> TokenStream for TokenStreamChain<'a> {
     fn advance(&mut self) -> bool {
-        while self.stream_idx < self.token_streams.len() {
-            let token_stream = self.token_streams[self.stream_idx].deref_mut();
+        while self.stream_idx < self.streams_with_offsets.len() {
+            let (ref mut token_stream, offset_offset) = self.streams_with_offsets[self.stream_idx];
             if token_stream.advance() {
                 let token = token_stream.token();
-                let offset_offset = self.offsets[self.stream_idx];
                 self.token.offset_from = token.offset_from + offset_offset;
                 self.token.offset_to = token.offset_to + offset_offset;
                 self.token.position = token.position + self.position_shift;
@@ -49,7 +44,7 @@ impl<'a> TokenStream for TokenStreamChain<'a> {
 
     fn token(&self) -> &Token {
         assert!(
-            self.stream_idx <= self.token_streams.len(),
+            self.stream_idx <= self.streams_with_offsets.len(),
             "You called .token(), after the end of the token stream has been reached"
         );
         &self.token
@@ -57,7 +52,7 @@ impl<'a> TokenStream for TokenStreamChain<'a> {
 
     fn token_mut(&mut self) -> &mut Token {
         assert!(
-            self.stream_idx <= self.token_streams.len(),
+            self.stream_idx <= self.streams_with_offsets.len(),
             "You called .token(), after the end of the token stream has been reached"
         );
         &mut self.token
@@ -73,10 +68,10 @@ mod tests {
     #[test]
     fn test_chain_first_emits_no_tokens() {
         let token_streams = vec![
-            SimpleTokenizer.token_stream(""),
-            SimpleTokenizer.token_stream("hello world"),
+            (SimpleTokenizer.token_stream(""), 0),
+            (SimpleTokenizer.token_stream("hello world"), 0),
         ];
-        let mut token_chain = TokenStreamChain::new(vec![0, 0], token_streams);
+        let mut token_chain = TokenStreamChain::new(token_streams);
 
         assert!(token_chain.advance());
         assert_eq!(token_chain.token().text, "hello");

src/tokenizer/tokenized_string.rs

@@ -1,4 +1,4 @@
-use crate::tokenizer::{BoxTokenStream, Token, TokenStream, TokenStreamChain};
+use crate::tokenizer::{Token, TokenStream, TokenStreamChain};
 use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
 
@@ -42,24 +42,23 @@ impl PreTokenizedStream {
     /// Creates a TokenStream from PreTokenizedString array
     pub fn chain_tokenized_strings<'a>(
         tok_strings: &'a [&'a PreTokenizedString],
-    ) -> BoxTokenStream {
+    ) -> Box<dyn TokenStream> {
         if tok_strings.len() == 1 {
-            PreTokenizedStream::from((*tok_strings[0]).clone()).into()
+            Box::new(PreTokenizedStream::from(tok_strings[0].to_owned()))
         } else {
-            let mut offsets = vec![];
+            let mut streams_with_offsets = vec![];
             let mut total_offset = 0;
             for &tok_string in tok_strings {
-                offsets.push(total_offset);
+                streams_with_offsets.push((
+                    Box::new(PreTokenizedStream::from(tok_string.to_owned()))
+                        as Box<dyn TokenStream>,
+                    total_offset,
+                ));
                 if let Some(last_token) = tok_string.tokens.last() {
                     total_offset += last_token.offset_to;
                 }
             }
-            // TODO remove the string cloning.
-            let token_streams: Vec<BoxTokenStream<'static>> = tok_strings
-                .iter()
-                .map(|&tok_string| PreTokenizedStream::from((*tok_string).clone()).into())
-                .collect();
-            TokenStreamChain::new(offsets, token_streams).into()
+            Box::new(TokenStreamChain::new(streams_with_offsets))
         }
     }
 }

src/tokenizer/tokenizer.rs

@@ -2,8 +2,6 @@ use crate::tokenizer::TokenStreamChain;
 use serde::{Deserialize, Serialize};
 /// The tokenizer module contains all of the tools used to process
 /// text in `tantivy`.
-use std::borrow::{Borrow, BorrowMut};
-use std::ops::{Deref, DerefMut};
 
 /// Token
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
@@ -40,7 +38,7 @@ impl Default for Token {
 /// It simply wraps a `Tokenizer` and a list of `TokenFilter` that are applied sequentially.
 pub struct TextAnalyzer {
     tokenizer: Box<dyn Tokenizer>,
-    token_filters: Vec<BoxTokenFilter>,
+    token_filters: Vec<Box<dyn TokenFilter>>,
 }
 
 impl<T: Tokenizer> From<T> for TextAnalyzer {
@@ -50,11 +48,14 @@ impl<T: Tokenizer> From<T> for TextAnalyzer {
 }
 
 impl TextAnalyzer {
-    /// Creates a new `TextAnalyzer` given a tokenizer and a vector of `BoxTokenFilter`.
+    /// Creates a new `TextAnalyzer` given a tokenizer and a vector of `Box<dyn TokenFilter>`.
     ///
     /// When creating a `TextAnalyzer` from a `Tokenizer` alone, prefer using
     /// `TextAnalyzer::from(tokenizer)`.
-    pub fn new<T: Tokenizer>(tokenizer: T, token_filters: Vec<BoxTokenFilter>) -> TextAnalyzer {
+    pub fn new<T: Tokenizer>(
+        tokenizer: T,
+        token_filters: Vec<Box<dyn TokenFilter>>,
+    ) -> TextAnalyzer {
         TextAnalyzer {
             tokenizer: Box::new(tokenizer),
             token_filters,
@@ -77,8 +78,8 @@ impl TextAnalyzer {
     ///     .filter(Stemmer::default());
     /// ```
     ///
-    pub fn filter<F: Into<BoxTokenFilter>>(mut self, token_filter: F) -> Self {
-        self.token_filters.push(token_filter.into());
+    pub fn filter<F: TokenFilter>(mut self, token_filter: F) -> Self {
+        self.token_filters.push(Box::new(token_filter));
         self
     }
 
@@ -87,28 +88,19 @@ impl TextAnalyzer {
     /// The resulting `BoxTokenStream` is equivalent to what would be obtained if the &str were
     /// one concatenated `&str`, with an artificial position gap of `2` between the different fields
     /// to prevent accidental `PhraseQuery` to match accross two terms.
-    pub fn token_stream_texts<'a>(&self, texts: &'a [&'a str]) -> BoxTokenStream<'a> {
-        assert!(!texts.is_empty());
-        if texts.len() == 1 {
-            self.token_stream(texts[0])
-        } else {
-            let mut offsets = vec![];
-            let mut total_offset = 0;
-            for &text in texts {
-                offsets.push(total_offset);
-                total_offset += text.len();
-            }
-            let token_streams: Vec<BoxTokenStream<'a>> = texts
-                .iter()
-                .cloned()
-                .map(|text| self.token_stream(text))
-                .collect();
-            From::from(TokenStreamChain::new(offsets, token_streams))
+    pub fn token_stream_texts<'a>(&self, texts: &'a [&str]) -> Box<dyn TokenStream + 'a> {
+        debug_assert!(!texts.is_empty());
+        let mut streams_with_offsets = vec![];
+        let mut total_offset = 0;
+        for &text in texts {
+            streams_with_offsets.push((self.token_stream(text), total_offset));
+            total_offset += text.len();
         }
+        Box::new(TokenStreamChain::new(streams_with_offsets))
     }
 
     /// Creates a token stream for a given `str`.
-    pub fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+    pub fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
         let mut token_stream = self.tokenizer.token_stream(text);
         for token_filter in &self.token_filters {
             token_stream = token_filter.transform(token_stream);
@@ -140,7 +132,7 @@ impl Clone for TextAnalyzer {
 /// This API may change to use associated types.
 pub trait Tokenizer: 'static + Send + Sync + TokenizerClone {
     /// Creates a token stream for a given `str`.
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a>;
+    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a>;
 }
 
 pub trait TokenizerClone {
@@ -153,69 +145,6 @@ impl<T: Tokenizer + Clone> TokenizerClone for T {
     }
 }
 
-impl<'a> TokenStream for Box<dyn TokenStream + 'a> {
-    fn advance(&mut self) -> bool {
-        let token_stream: &mut dyn TokenStream = self.borrow_mut();
-        token_stream.advance()
-    }
-
-    fn token<'b>(&'b self) -> &'b Token {
-        let token_stream: &'b (dyn TokenStream + 'a) = self.borrow();
-        token_stream.token()
-    }
-
-    fn token_mut<'b>(&'b mut self) -> &'b mut Token {
-        let token_stream: &'b mut (dyn TokenStream + 'a) = self.borrow_mut();
-        token_stream.token_mut()
-    }
-}
-
-/// Simple wrapper of `Box<dyn TokenStream + 'a>`.
-///
-/// See `TokenStream` for more information.
-pub struct BoxTokenStream<'a>(Box<dyn TokenStream + 'a>);
-
-impl<'a, T> From<T> for BoxTokenStream<'a>
-where
-    T: TokenStream + 'a,
-{
-    fn from(token_stream: T) -> BoxTokenStream<'a> {
-        BoxTokenStream(Box::new(token_stream))
-    }
-}
-
-impl<'a> Deref for BoxTokenStream<'a> {
-    type Target = dyn TokenStream + 'a;
-
-    fn deref(&self) -> &Self::Target {
-        &*self.0
-    }
-}
-impl<'a> DerefMut for BoxTokenStream<'a> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut *self.0
-    }
-}
-
-/// Simple wrapper of `Box<dyn TokenFilter + 'a>`.
-///
-/// See `TokenStream` for more information.
-pub struct BoxTokenFilter(Box<dyn TokenFilter>);
-
-impl Deref for BoxTokenFilter {
-    type Target = dyn TokenFilter;
-
-    fn deref(&self) -> &dyn TokenFilter {
-        &*self.0
-    }
-}
-
-impl<T: TokenFilter> From<T> for BoxTokenFilter {
-    fn from(tokenizer: T) -> BoxTokenFilter {
-        BoxTokenFilter(Box::new(tokenizer))
-    }
-}
-
 /// `TokenStream` is the result of the tokenization.
 ///
 /// It consists consumable stream of `Token`s.
@@ -295,18 +224,18 @@ pub trait TokenStream {
 }
 
 pub trait TokenFilterClone {
-    fn box_clone(&self) -> BoxTokenFilter;
+    fn box_clone(&self) -> Box<dyn TokenFilter>;
 }
 
 /// Trait for the pluggable components of `Tokenizer`s.
 pub trait TokenFilter: 'static + Send + Sync + TokenFilterClone {
     /// Wraps a token stream and returns the modified one.
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a>;
+    fn transform<'a>(&self, token_stream: Box<dyn TokenStream + 'a>) -> Box<dyn TokenStream + 'a>;
 }
 
 impl<T: TokenFilter + Clone> TokenFilterClone for T {
-    fn box_clone(&self) -> BoxTokenFilter {
-        BoxTokenFilter::from(self.clone())
+    fn box_clone(&self) -> Box<dyn TokenFilter> {
+        Box::new(self.clone())
     }
 }