Fail fast if the skip index being written is broken.

Issue/969

Cargo.toml

@@ -53,10 +53,11 @@ lru = "0.6"
 winapi = "0.3"
 
 [dev-dependencies]
-rand = "0.7"
+rand = "0.8"
 maplit = "1"
 matches = "0.1.8"
 proptest = "0.10"
+criterion = "0.3"
 
 [dev-dependencies.fail]
 version = "0.4"
@@ -97,3 +98,7 @@ travis-ci = { repository = "tantivy-search/tantivy" }
 name = "failpoints"
 path = "tests/failpoints/mod.rs"
 required-features = ["fail/failpoints"]
+
+[[bench]]
+name = "analyzer"
+harness = false

benches/analyzer.rs

@@ -0,0 +1,22 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use tantivy::tokenizer::TokenizerManager;
+
+const ALICE_TXT: &'static str = include_str!("alice.txt");
+
+pub fn criterion_benchmark(c: &mut Criterion) {
+    let tokenizer_manager = TokenizerManager::default();
+    let tokenizer = tokenizer_manager.get("default").unwrap();
+    c.bench_function("default-tokenize-alice", |b| {
+        b.iter(|| {
+            let mut word_count = 0;
+            let mut token_stream = tokenizer.token_stream(ALICE_TXT);
+            while token_stream.advance() {
+                word_count += 1;
+            }
+            assert_eq!(word_count, 30_731);
+        })
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

src/core/segment_reader.rs

@@ -310,7 +310,7 @@ impl SegmentReader {
     }
 
     /// Returns an iterator that will iterate over the alive document ids
-    pub fn doc_ids_alive(&self) -> impl Iterator<Item = DocId> + '_ {
+    pub fn doc_ids_alive<'a>(&'a self) -> impl Iterator<Item = DocId> + 'a {
         (0u32..self.max_doc).filter(move |doc| !self.is_deleted(*doc))
     }
 

src/directory/ram_directory.rs

@@ -44,12 +44,12 @@ impl VecWriter {
 
 impl Drop for VecWriter {
     fn drop(&mut self) {
-        if !self.is_flushed {
-            panic!(
-                "You forgot to flush {:?} before its writter got Drop. Do not rely on drop.",
-                self.path
-            )
-        }
+        // if !self.is_flushed {
+        //     panic!(
+        //         "You forgot to flush {:?} before its writter got Drop. Do not rely on drop.",
+        //         self.path
+        //     )
+        // }
     }
 }
 

src/functional_test.rs

@@ -1,45 +1,93 @@
-use rand::thread_rng;
-use std::collections::HashSet;
-
-use crate::schema::*;
 use crate::Index;
 use crate::Searcher;
+use crate::{doc, schema::*};
+use rand::thread_rng;
 use rand::Rng;
+use std::collections::HashSet;
 
-fn check_index_content(searcher: &Searcher, vals: &HashSet<u64>) {
+fn check_index_content(searcher: &Searcher, vals: &[u64]) -> crate::Result<()> {
     assert!(searcher.segment_readers().len() < 20);
     assert_eq!(searcher.num_docs() as usize, vals.len());
+    for segment_reader in searcher.segment_readers() {
+        let store_reader = segment_reader.get_store_reader()?;
+        for doc_id in 0..segment_reader.max_doc() {
+            let _doc = store_reader.get(doc_id)?;
+        }
+    }
+    Ok(())
 }
 
 #[test]
 #[ignore]
-fn test_indexing() {
+fn test_functional_store() -> crate::Result<()> {
+    let mut schema_builder = Schema::builder();
+
+    let id_field = schema_builder.add_u64_field("id", INDEXED | STORED);
+    let schema = schema_builder.build();
+
+    let index = Index::create_in_ram(schema);
+    let reader = index.reader()?;
+
+    let mut rng = thread_rng();
+
+    let mut index_writer = index.writer_with_num_threads(3, 12_000_000)?;
+
+    let mut doc_set: Vec<u64> = Vec::new();
+
+    let mut doc_id = 0u64;
+    for iteration in 0..500 {
+        dbg!(iteration);
+        let num_docs: usize = rng.gen_range(0..4);
+        if doc_set.len() >= 1 {
+            let doc_to_remove_id = rng.gen_range(0..doc_set.len());
+            let removed_doc_id = doc_set.swap_remove(doc_to_remove_id);
+            index_writer.delete_term(Term::from_field_u64(id_field, removed_doc_id));
+        }
+        for _ in 0..num_docs {
+            doc_set.push(doc_id);
+            index_writer.add_document(doc!(id_field=>doc_id));
+            doc_id += 1;
+        }
+        index_writer.commit()?;
+        reader.reload()?;
+        let searcher = reader.searcher();
+        check_index_content(&searcher, &doc_set)?;
+    }
+    Ok(())
+}
+
+#[test]
+#[ignore]
+fn test_functional_indexing() -> crate::Result<()> {
     let mut schema_builder = Schema::builder();
 
     let id_field = schema_builder.add_u64_field("id", INDEXED);
     let multiples_field = schema_builder.add_u64_field("multiples", INDEXED);
     let schema = schema_builder.build();
 
-    let index = Index::create_from_tempdir(schema).unwrap();
-    let reader = index.reader().unwrap();
+    let index = Index::create_from_tempdir(schema)?;
+    let reader = index.reader()?;
 
     let mut rng = thread_rng();
 
-    let mut index_writer = index.writer_with_num_threads(3, 120_000_000).unwrap();
+    let mut index_writer = index.writer_with_num_threads(3, 120_000_000)?;
 
     let mut committed_docs: HashSet<u64> = HashSet::new();
     let mut uncommitted_docs: HashSet<u64> = HashSet::new();
 
     for _ in 0..200 {
-        let random_val = rng.gen_range(0, 20);
+        let random_val = rng.gen_range(0..20);
         if random_val == 0 {
-            index_writer.commit().expect("Commit failed");
+            index_writer.commit()?;
             committed_docs.extend(&uncommitted_docs);
             uncommitted_docs.clear();
-            reader.reload().unwrap();
+            reader.reload()?;
             let searcher = reader.searcher();
             // check that everything is correct.
-            check_index_content(&searcher, &committed_docs);
+            check_index_content(
+                &searcher,
+                &committed_docs.iter().cloned().collect::<Vec<u64>>(),
+            )?;
         } else {
             if committed_docs.remove(&random_val) || uncommitted_docs.remove(&random_val) {
                 let doc_id_term = Term::from_field_u64(id_field, random_val);
@@ -55,4 +103,5 @@ fn test_indexing() {
             }
         }
     }
+    Ok(())
 }

src/indexer/log_merge_policy.rs

@@ -8,7 +8,7 @@ const DEFAULT_MIN_LAYER_SIZE: u32 = 10_000;
 const DEFAULT_MIN_MERGE_SIZE: usize = 8;
 const DEFAULT_MAX_MERGE_SIZE: usize = 10_000_000;
 
-/// `LogMergePolicy` tries tries to merge segments that have a similar number of
+/// `LogMergePolicy` tries to merge segments that have a similar number of
 /// documents.
 #[derive(Debug, Clone)]
 pub struct LogMergePolicy {

src/indexer/segment_updater.rs

@@ -25,9 +25,10 @@ use futures::future::Future;
 use futures::future::TryFutureExt;
 use std::borrow::BorrowMut;
 use std::collections::HashSet;
-use std::io::Write;
+use std::io::{self, Write};
 use std::ops::Deref;
 use std::path::PathBuf;
+use std::process;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::sync::RwLock;
@@ -409,6 +410,13 @@ impl SegmentUpdater {
                     let _send_result = merging_future_send.send(segment_meta);
                 }
                 Err(e) => {
+                    if let crate::TantivyError::IOError(ref io_err) = &e {
+                        if io_err.kind() == io::ErrorKind::InvalidData {
+                            println!(" SEGMENTS THAT CAUSE THE BUG {:?}", merge_operation.segment_ids());
+                            error!(" SEGMENTS THAT CAUSE THE BUG {:?}", merge_operation.segment_ids());
+                            process::exit(1);
+                        }
+                    }
                     warn!(
                         "Merge of {:?} was cancelled: {:?}",
                         merge_operation.segment_ids().to_vec(),
@@ -423,7 +431,9 @@ impl SegmentUpdater {
         });
 
         Ok(merging_future_recv
-            .unwrap_or_else(|_| Err(crate::TantivyError::SystemError("Merge failed".to_string()))))
+            .unwrap_or_else(|e| {
+                Err(crate::TantivyError::SystemError("Merge failed".to_string()))
+            }))
     }
 
     async fn consider_merge_options(&self) {

src/indexer/segment_writer.rs

@@ -11,8 +11,7 @@ use crate::schema::Schema;
 use crate::schema::Term;
 use crate::schema::Value;
 use crate::schema::{Field, FieldEntry};
-use crate::tokenizer::PreTokenizedStream;
-use crate::tokenizer::TokenStream;
+use crate::tokenizer::{BoxTokenStream, PreTokenizedStream};
 use crate::tokenizer::{FacetTokenizer, TextAnalyzer};
 use crate::tokenizer::{TokenStreamChain, Tokenizer};
 use crate::Opstamp;
@@ -142,13 +141,13 @@ impl SegmentWriter {
             }
             let (term_buffer, multifield_postings) =
                 (&mut self.term_buffer, &mut self.multifield_postings);
-            match field_entry.field_type() {
+            match *field_entry.field_type() {
                 FieldType::HierarchicalFacet => {
                     term_buffer.set_field(field);
                     let facets =
                         field_values
                             .iter()
-                            .flat_map(|field_value| match field_value.value() {
+                            .flat_map(|field_value| match *field_value.value() {
                                 Value::Facet(ref facet) => Some(facet.encoded_str()),
                                 _ => {
                                     panic!("Expected hierarchical facet");
@@ -173,38 +172,37 @@ impl SegmentWriter {
                     }
                 }
                 FieldType::Str(_) => {
-                    let mut streams_with_offsets = vec![];
+                    let mut token_streams: Vec<BoxTokenStream> = vec![];
+                    let mut offsets = vec![];
                     let mut total_offset = 0;
 
                     for field_value in field_values {
                         match field_value.value() {
                             Value::PreTokStr(tok_str) => {
-                                streams_with_offsets.push((
-                                    Box::new(PreTokenizedStream::from(tok_str.clone()))
-                                        as Box<dyn TokenStream>,
-                                    total_offset,
-                                ));
+                                offsets.push(total_offset);
                                 if let Some(last_token) = tok_str.tokens.last() {
                                     total_offset += last_token.offset_to;
                                 }
+                                token_streams
+                                    .push(PreTokenizedStream::from(tok_str.clone()).into());
                             }
                             Value::Str(ref text) => {
                                 if let Some(ref mut tokenizer) =
                                     self.tokenizers[field.field_id() as usize]
                                 {
-                                    streams_with_offsets
-                                        .push((tokenizer.token_stream(text), total_offset));
+                                    offsets.push(total_offset);
                                     total_offset += text.len();
+                                    token_streams.push(tokenizer.token_stream(text));
                                 }
                             }
                             _ => (),
                         }
                     }
 
-                    let num_tokens = if streams_with_offsets.is_empty() {
+                    let num_tokens = if token_streams.is_empty() {
                         0
                     } else {
-                        let mut token_stream = TokenStreamChain::new(streams_with_offsets);
+                        let mut token_stream = TokenStreamChain::new(offsets, token_streams);
                         multifield_postings.index_text(
                             doc_id,
                             field,

src/positions/reader.rs

@@ -132,7 +132,7 @@ impl PositionReader {
             "offset arguments should be increasing."
         );
         let delta_to_block_offset = offset as i64 - self.block_offset as i64;
-        if !(0..128).contains(&delta_to_block_offset) {
+        if delta_to_block_offset < 0 || delta_to_block_offset >= 128 {
             // The first position is not within the first block.
             // We need to decompress the first block.
             let delta_to_anchor_offset = offset - self.anchor_offset;

src/postings/block_search.rs

@@ -109,9 +109,9 @@ impl BlockSearcher {
     /// The results should be equivalent to
     /// ```compile_fail
     /// block[..]
-    ///       .iter()
-    ///       .take_while(|&&val| val < target)
-    ///       .count()
+    //       .iter()
+    //       .take_while(|&&val| val < target)
+    //       .count()
     /// ```
     ///
     /// The `start` argument is just used to hint that the response is

src/query/term_query/term_scorer.rs

@@ -302,7 +302,7 @@ mod tests {
         let mut rng = rand::thread_rng();
         writer.set_merge_policy(Box::new(NoMergePolicy));
         for _ in 0..3_000 {
-            let term_freq = rng.gen_range(1, 10000);
+            let term_freq = rng.gen_range(1..10000);
             let words: Vec<&str> = std::iter::repeat("bbbb").take(term_freq).collect();
             let text = words.join(" ");
             writer.add_document(doc!(text_field=>text));

src/schema/named_field_document.rs

@@ -1,5 +1,5 @@
 use crate::schema::Value;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 use std::collections::BTreeMap;
 
 /// Internal representation of a document used for JSON
@@ -8,5 +8,5 @@ use std::collections::BTreeMap;
 /// A `NamedFieldDocument` is a simple representation of a document
 /// as a `BTreeMap<String, Vec<Value>>`.
 ///
-#[derive(Serialize)]
+#[derive(Debug, Deserialize, Serialize)]
 pub struct NamedFieldDocument(pub BTreeMap<String, Vec<Value>>);

src/store/index/block.rs

@@ -43,6 +43,9 @@ impl CheckpointBlock {
 
     /// Adding another checkpoint in the block.
     pub fn push(&mut self, checkpoint: Checkpoint) {
+        if let Some(prev_checkpoint) = self.checkpoints.last() {
+            assert!(checkpoint.follows(prev_checkpoint));
+        }
         self.checkpoints.push(checkpoint);
     }
 

src/store/index/mod.rs

@@ -26,6 +26,12 @@ pub struct Checkpoint {
     pub end_offset: u64,
 }
 
+impl Checkpoint {
+    pub(crate) fn follows(&self, other: &Checkpoint) -> bool {
+        (self.start_doc == other.end_doc) && (self.start_offset == other.end_offset)
+    }
+}
+
 impl fmt::Debug for Checkpoint {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(
@@ -39,13 +45,16 @@ impl fmt::Debug for Checkpoint {
 #[cfg(test)]
 mod tests {
 
-    use std::io;
+    use std::{io, iter};
 
+    use futures::executor::block_on;
     use proptest::strategy::{BoxedStrategy, Strategy};
 
     use crate::directory::OwnedBytes;
+    use crate::indexer::NoMergePolicy;
+    use crate::schema::{SchemaBuilder, STORED, STRING};
     use crate::store::index::Checkpoint;
-    use crate::DocId;
+    use crate::{DocAddress, DocId, Index, Term};
 
     use super::{SkipIndex, SkipIndexBuilder};
 
@@ -54,7 +63,7 @@ mod tests {
         let mut output: Vec<u8> = Vec::new();
         let skip_index_builder: SkipIndexBuilder = SkipIndexBuilder::new();
         skip_index_builder.write(&mut output)?;
-        let skip_index: SkipIndex = SkipIndex::from(OwnedBytes::new(output));
+        let skip_index: SkipIndex = SkipIndex::open(OwnedBytes::new(output));
         let mut skip_cursor = skip_index.checkpoints();
         assert!(skip_cursor.next().is_none());
         Ok(())
@@ -72,7 +81,7 @@ mod tests {
         };
         skip_index_builder.insert(checkpoint);
         skip_index_builder.write(&mut output)?;
-        let skip_index: SkipIndex = SkipIndex::from(OwnedBytes::new(output));
+        let skip_index: SkipIndex = SkipIndex::open(OwnedBytes::new(output));
         let mut skip_cursor = skip_index.checkpoints();
         assert_eq!(skip_cursor.next(), Some(checkpoint));
         assert_eq!(skip_cursor.next(), None);
@@ -86,7 +95,7 @@ mod tests {
             Checkpoint {
                 start_doc: 0,
                 end_doc: 3,
-                start_offset: 4,
+                start_offset: 0,
                 end_offset: 9,
             },
             Checkpoint {
@@ -121,7 +130,7 @@ mod tests {
         }
         skip_index_builder.write(&mut output)?;
 
-        let skip_index: SkipIndex = SkipIndex::from(OwnedBytes::new(output));
+        let skip_index: SkipIndex = SkipIndex::open(OwnedBytes::new(output));
         assert_eq!(
             &skip_index.checkpoints().collect::<Vec<_>>()[..],
             &checkpoints[..]
@@ -133,6 +142,40 @@ mod tests {
         (doc as u64) * (doc as u64)
     }
 
+    #[test]
+    fn test_merge_store_with_stacking_reproducing_issue969() -> crate::Result<()> {
+        let mut schema_builder = SchemaBuilder::default();
+        let text = schema_builder.add_text_field("text", STORED | STRING);
+        let body = schema_builder.add_text_field("body", STORED);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_for_tests()?;
+        index_writer.set_merge_policy(Box::new(NoMergePolicy));
+        let long_text: String = iter::repeat("abcdefghijklmnopqrstuvwxyz")
+            .take(1_000)
+            .collect();
+        for _ in 0..20 {
+            index_writer.add_document(doc!(body=>long_text.clone()));
+        }
+        index_writer.commit()?;
+        index_writer.add_document(doc!(text=>"testb"));
+        for _ in 0..10 {
+            index_writer.add_document(doc!(text=>"testd", body=>long_text.clone()));
+        }
+        index_writer.commit()?;
+        index_writer.delete_term(Term::from_field_text(text, "testb"));
+        index_writer.commit()?;
+        let segment_ids = index.searchable_segment_ids()?;
+        block_on(index_writer.merge(&segment_ids))?;
+        let reader = index.reader()?;
+        let searcher = reader.searcher();
+        assert_eq!(searcher.num_docs(), 30);
+        for i in 0..searcher.num_docs() as u32 {
+            let _doc = searcher.doc(DocAddress(0u32, i))?;
+        }
+        Ok(())
+    }
+
     #[test]
     fn test_skip_index_long() -> io::Result<()> {
         let mut output: Vec<u8> = Vec::new();
@@ -150,26 +193,28 @@ mod tests {
         }
         skip_index_builder.write(&mut output)?;
         assert_eq!(output.len(), 4035);
-        let resulting_checkpoints: Vec<Checkpoint> = SkipIndex::from(OwnedBytes::new(output))
+        let resulting_checkpoints: Vec<Checkpoint> = SkipIndex::open(OwnedBytes::new(output))
             .checkpoints()
             .collect();
         assert_eq!(&resulting_checkpoints, &checkpoints);
         Ok(())
     }
 
-    fn integrate_delta(mut vals: Vec<u64>) -> Vec<u64> {
+    fn integrate_delta(vals: Vec<u64>) -> Vec<u64> {
+        let mut output = Vec::with_capacity(vals.len() + 1);
+        output.push(0u64);
         let mut prev = 0u64;
-        for val in vals.iter_mut() {
-            let new_val = *val + prev;
+        for val in vals {
+            let new_val = val + prev;
             prev = new_val;
-            *val = new_val;
+            output.push(new_val);
         }
-        vals
+        output
     }
 
     // Generates a sequence of n valid checkpoints, with n < max_len.
     fn monotonic_checkpoints(max_len: usize) -> BoxedStrategy<Vec<Checkpoint>> {
-        (1..max_len)
+        (0..max_len)
             .prop_flat_map(move |len: usize| {
                 (
                     proptest::collection::vec(1u64..20u64, len as usize).prop_map(integrate_delta),
@@ -221,7 +266,7 @@ mod tests {
              }
              let mut buffer = Vec::new();
              skip_index_builder.write(&mut buffer).unwrap();
-             let skip_index = SkipIndex::from(OwnedBytes::new(buffer));
+             let skip_index = SkipIndex::open(OwnedBytes::new(buffer));
              let iter_checkpoints: Vec<Checkpoint> = skip_index.checkpoints().collect();
              assert_eq!(&checkpoints[..], &iter_checkpoints[..]);
              test_skip_index_aux(skip_index, &checkpoints[..]);

src/store/index/skip_index.rs

@@ -35,11 +35,11 @@ struct Layer {
 }
 
 impl Layer {
-    fn cursor(&self) -> impl Iterator<Item = Checkpoint> + '_ {
+    fn cursor<'a>(&'a self) -> impl Iterator<Item = Checkpoint> + 'a {
         self.cursor_at_offset(0u64)
     }
 
-    fn cursor_at_offset(&self, start_offset: u64) -> impl Iterator<Item = Checkpoint> + '_ {
+    fn cursor_at_offset<'a>(&'a self, start_offset: u64) -> impl Iterator<Item = Checkpoint> + 'a {
         let data = &self.data.as_slice();
         LayerCursor {
             remaining: &data[start_offset as usize..],
@@ -59,7 +59,47 @@ pub struct SkipIndex {
 }
 
 impl SkipIndex {
-    pub(crate) fn checkpoints(&self) -> impl Iterator<Item = Checkpoint> + '_ {
+    pub fn open(mut data: OwnedBytes) -> SkipIndex {
+        let offsets: Vec<u64> = Vec::<VInt>::deserialize(&mut data)
+            .unwrap()
+            .into_iter()
+            .map(|el| el.0)
+            .collect();
+        let mut start_offset = 0;
+        let mut layers = Vec::new();
+        for end_offset in offsets {
+            let layer = Layer {
+                data: data.slice(start_offset as usize, end_offset as usize),
+            };
+            layers.push(layer);
+            start_offset = end_offset;
+        }
+        SkipIndex { layers }
+    }
+
+    pub fn is_valid(&self) -> bool {
+        let checkpoints: Vec<Checkpoint> = self.checkpoints().collect();
+        let mut prev_checkpoint = Checkpoint {
+            start_doc: 0u32,
+            end_doc: 0u32,
+            start_offset: 0u64,
+            end_offset: 0u64,
+        };
+        for checkpoint in checkpoints {
+            if !checkpoint.follows(&prev_checkpoint) {
+                return false;
+            }
+            prev_checkpoint = checkpoint;
+        }
+        true
+    }
+
+    pub(crate) fn from_bytes(data: &[u8]) -> SkipIndex {
+        let data = OwnedBytes::new(data.to_owned());
+        SkipIndex::open(data)
+    }
+
+    pub(crate) fn checkpoints<'a>(&'a self) -> impl Iterator<Item = Checkpoint> + 'a {
         self.layers
             .last()
             .into_iter()
@@ -90,22 +130,3 @@ impl SkipIndex {
         Some(cur_checkpoint)
     }
 }
-
-impl From<OwnedBytes> for SkipIndex {
-    fn from(mut data: OwnedBytes) -> SkipIndex {
-        let offsets: Vec<u64> = Vec::<VInt>::deserialize(&mut data)
-            .unwrap()
-            .into_iter()
-            .map(|el| el.0)
-            .collect();
-        let mut start_offset = 0;
-        let mut layers = Vec::new();
-        for end_offset in offsets {
-            layers.push(Layer {
-                data: data.slice(start_offset as usize, end_offset as usize),
-            });
-            start_offset = end_offset;
-        }
-        SkipIndex { layers }
-    }
-}

src/store/index/skip_index_builder.rs

@@ -1,6 +1,6 @@
 use crate::common::{BinarySerializable, VInt};
 use crate::store::index::block::CheckpointBlock;
-use crate::store::index::{Checkpoint, CHECKPOINT_PERIOD};
+use crate::store::index::{Checkpoint, SkipIndex, CHECKPOINT_PERIOD};
 use std::io;
 use std::io::Write;
 
@@ -28,18 +28,20 @@ impl LayerBuilder {
     ///
     /// If the block was empty to begin with, simply return None.
     fn flush_block(&mut self) -> Option<Checkpoint> {
-        self.block.doc_interval().map(|(start_doc, end_doc)| {
+        if let Some((start_doc, end_doc)) = self.block.doc_interval() {
             let start_offset = self.buffer.len() as u64;
             self.block.serialize(&mut self.buffer);
             let end_offset = self.buffer.len() as u64;
             self.block.clear();
-            Checkpoint {
+            Some(Checkpoint {
                 start_doc,
                 end_doc,
                 start_offset,
                 end_offset,
-            }
-        })
+            })
+        } else {
+            None
+        }
     }
 
     fn push(&mut self, checkpoint: Checkpoint) {
@@ -48,7 +50,7 @@ impl LayerBuilder {
 
     fn insert(&mut self, checkpoint: Checkpoint) -> Option<Checkpoint> {
         self.push(checkpoint);
-        let emit_skip_info = (self.block.len() % CHECKPOINT_PERIOD) == 0;
+        let emit_skip_info = self.block.len() >= CHECKPOINT_PERIOD;
         if emit_skip_info {
             self.flush_block()
         } else {
@@ -85,7 +87,8 @@ impl SkipIndexBuilder {
         }
     }
 
-    pub fn write<W: Write>(mut self, output: &mut W) -> io::Result<()> {
+    pub fn write<W: Write>(mut self, real_output: &mut W) -> io::Result<()> {
+        let mut output: Vec<u8> = Vec::new();
         let mut last_pointer = None;
         for skip_layer in self.layers.iter_mut() {
             if let Some(checkpoint) = last_pointer {
@@ -106,10 +109,14 @@ impl SkipIndexBuilder {
             layer_offset += layer_buffer.len() as u64;
             layer_sizes.push(VInt(layer_offset));
         }
-        layer_sizes.serialize(output)?;
+        layer_sizes.serialize(&mut output)?;
         for layer_buffer in layer_buffers {
             output.write_all(&layer_buffer[..])?;
         }
+        if !SkipIndex::from_bytes(&output).is_valid() {
+            return Err(io::Error::new(io::ErrorKind::InvalidData, "about to write invalid skip index"));
+        }
+        real_output.write_all(&output)?;
         Ok(())
     }
 }

src/store/reader.rs

@@ -35,7 +35,7 @@ impl StoreReader {
         let (data_file, offset_index_file) = split_file(store_file)?;
         let index_data = offset_index_file.read_bytes()?;
         let space_usage = StoreSpaceUsage::new(data_file.len(), offset_index_file.len());
-        let skip_index = SkipIndex::from(index_data);
+        let skip_index = SkipIndex::open(index_data);
         Ok(StoreReader {
             data: data_file,
             cache: Arc::new(Mutex::new(LruCache::new(LRU_CACHE_CAPACITY))),
@@ -46,7 +46,7 @@ impl StoreReader {
         })
     }
 
-    pub(crate) fn block_checkpoints(&self) -> impl Iterator<Item = Checkpoint> + '_ {
+    pub(crate) fn block_checkpoints<'a>(&'a self) -> impl Iterator<Item = Checkpoint> + 'a {
         self.skip_index.checkpoints()
     }
 

src/store/writer.rs

@@ -72,6 +72,7 @@ impl StoreWriter {
         if !self.current_block.is_empty() {
             self.write_and_compress_block()?;
         }
+        assert_eq!(self.first_doc_in_block, self.doc);
         let doc_shift = self.doc;
         let start_shift = self.writer.written_bytes() as u64;
 
@@ -86,12 +87,17 @@ impl StoreWriter {
             checkpoint.end_doc += doc_shift;
             checkpoint.start_offset += start_shift;
             checkpoint.end_offset += start_shift;
-            self.offset_index_writer.insert(checkpoint);
-            self.doc = checkpoint.end_doc;
+            self.register_checkpoint(checkpoint);
         }
         Ok(())
     }
 
+    fn register_checkpoint(&mut self, checkpoint: Checkpoint) {
+        self.offset_index_writer.insert(checkpoint);
+        self.first_doc_in_block = checkpoint.end_doc;
+        self.doc = checkpoint.end_doc;
+    }
+
     fn write_and_compress_block(&mut self) -> io::Result<()> {
         assert!(self.doc > 0);
         self.intermediary_buffer.clear();
@@ -100,14 +106,13 @@ impl StoreWriter {
         self.writer.write_all(&self.intermediary_buffer)?;
         let end_offset = self.writer.written_bytes();
         let end_doc = self.doc;
-        self.offset_index_writer.insert(Checkpoint {
+        self.register_checkpoint(Checkpoint {
             start_doc: self.first_doc_in_block,
             end_doc,
             start_offset,
             end_offset,
         });
         self.current_block.clear();
-        self.first_doc_in_block = self.doc;
         Ok(())
     }
 

src/tokenizer/alphanum_only.rs

@@ -19,7 +19,7 @@
 //! // the "emoji" is dropped because its not an alphanum
 //! assert!(stream.next().is_none());
 //! ```
-use super::{Token, TokenFilter, TokenStream};
+use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
 
 /// `TokenFilter` that removes all tokens that contain non
 /// ascii alphanumeric characters.
@@ -27,7 +27,7 @@ use super::{Token, TokenFilter, TokenStream};
 pub struct AlphaNumOnlyFilter;
 
 pub struct AlphaNumOnlyFilterStream<'a> {
-    tail: Box<dyn TokenStream + 'a>,
+    tail: BoxTokenStream<'a>,
 }
 
 impl<'a> AlphaNumOnlyFilterStream<'a> {
@@ -37,8 +37,8 @@ impl<'a> AlphaNumOnlyFilterStream<'a> {
 }
 
 impl TokenFilter for AlphaNumOnlyFilter {
-    fn transform<'a>(&self, token_stream: Box<dyn TokenStream + 'a>) -> Box<dyn TokenStream + 'a> {
-        Box::new(AlphaNumOnlyFilterStream { tail: token_stream })
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(AlphaNumOnlyFilterStream { tail: token_stream })
     }
 }
 

src/tokenizer/ascii_folding_filter.rs

@@ -1,4 +1,4 @@
-use super::{Token, TokenFilter, TokenStream};
+use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
 use std::mem;
 
 /// This class converts alphabetic, numeric, and symbolic Unicode characters
@@ -8,8 +8,8 @@ use std::mem;
 pub struct AsciiFoldingFilter;
 
 impl TokenFilter for AsciiFoldingFilter {
-    fn transform<'a>(&self, token_stream: Box<dyn TokenStream + 'a>) -> Box<dyn TokenStream + 'a> {
-        Box::new(AsciiFoldingFilterTokenStream {
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        From::from(AsciiFoldingFilterTokenStream {
             tail: token_stream,
             buffer: String::with_capacity(100),
         })
@@ -18,7 +18,7 @@ impl TokenFilter for AsciiFoldingFilter {
 
 pub struct AsciiFoldingFilterTokenStream<'a> {
     buffer: String,
-    tail: Box<dyn TokenStream + 'a>,
+    tail: BoxTokenStream<'a>,
 }
 
 impl<'a> TokenStream for AsciiFoldingFilterTokenStream<'a> {

src/tokenizer/facet_tokenizer.rs

@@ -1,4 +1,4 @@
-use super::{Token, TokenStream, Tokenizer};
+use super::{BoxTokenStream, Token, TokenStream, Tokenizer};
 use crate::schema::FACET_SEP_BYTE;
 
 /// The `FacetTokenizer` process a `Facet` binary representation
@@ -26,12 +26,13 @@ pub struct FacetTokenStream<'a> {
 }
 
 impl Tokenizer for FacetTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
-        Box::new(FacetTokenStream {
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+        FacetTokenStream {
             text,
             state: State::RootFacetNotEmitted, //< pos is the first char that has not been processed yet.
             token: Token::default(),
-        })
+        }
+        .into()
     }
 }
 

src/tokenizer/lower_caser.rs

@@ -1,9 +1,10 @@
 use super::{Token, TokenFilter, TokenStream};
+use crate::tokenizer::BoxTokenStream;
 use std::mem;
 
 impl TokenFilter for LowerCaser {
-    fn transform<'a>(&self, token_stream: Box<dyn TokenStream + 'a>) -> Box<dyn TokenStream + 'a> {
-        Box::new(LowerCaserTokenStream {
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(LowerCaserTokenStream {
             tail: token_stream,
             buffer: String::with_capacity(100),
         })
@@ -16,7 +17,7 @@ pub struct LowerCaser;
 
 pub struct LowerCaserTokenStream<'a> {
     buffer: String,
-    tail: Box<dyn TokenStream + 'a>,
+    tail: BoxTokenStream<'a>,
 }
 
 // writes a lowercased version of text into output.

src/tokenizer/mod.rs

@@ -145,7 +145,9 @@ pub use self::stop_word_filter::StopWordFilter;
 pub(crate) use self::token_stream_chain::TokenStreamChain;
 
 pub use self::tokenized_string::{PreTokenizedStream, PreTokenizedString};
-pub use self::tokenizer::{TextAnalyzer, Token, TokenFilter, TokenStream, Tokenizer};
+pub use self::tokenizer::{
+    BoxTokenFilter, BoxTokenStream, TextAnalyzer, Token, TokenFilter, TokenStream, Tokenizer,
+};
 
 pub use self::tokenizer_manager::TokenizerManager;
 

src/tokenizer/ngram_tokenizer.rs

@@ -1,4 +1,5 @@
 use super::{Token, TokenStream, Tokenizer};
+use crate::tokenizer::BoxTokenStream;
 
 /// Tokenize the text by splitting words into n-grams of the given size(s)
 ///
@@ -130,8 +131,8 @@ pub struct NgramTokenStream<'a> {
 }
 
 impl Tokenizer for NgramTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
-        Box::new(NgramTokenStream {
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+        From::from(NgramTokenStream {
             ngram_charidx_iterator: StutteringIterator::new(
                 CodepointFrontiers::for_str(text),
                 self.min_gram,
@@ -307,9 +308,9 @@ mod tests {
     use super::StutteringIterator;
     use crate::tokenizer::tests::assert_token;
     use crate::tokenizer::tokenizer::Tokenizer;
-    use crate::tokenizer::{Token, TokenStream};
+    use crate::tokenizer::{BoxTokenStream, Token};
 
-    fn test_helper(mut tokenizer: Box<dyn TokenStream>) -> Vec<Token> {
+    fn test_helper(mut tokenizer: BoxTokenStream) -> Vec<Token> {
         let mut tokens: Vec<Token> = vec![];
         tokenizer.process(&mut |token: &Token| tokens.push(token.clone()));
         tokens

src/tokenizer/raw_tokenizer.rs

@@ -1,4 +1,5 @@
 use super::{Token, TokenStream, Tokenizer};
+use crate::tokenizer::BoxTokenStream;
 
 /// For each value of the field, emit a single unprocessed token.
 #[derive(Clone)]
@@ -10,7 +11,7 @@ pub struct RawTokenStream {
 }
 
 impl Tokenizer for RawTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
         let token = Token {
             offset_from: 0,
             offset_to: text.len(),
@@ -18,10 +19,11 @@ impl Tokenizer for RawTokenizer {
             text: text.to_string(),
             position_length: 1,
         };
-        Box::new(RawTokenStream {
+        RawTokenStream {
             token,
             has_token: true,
-        })
+        }
+        .into()
     }
 }
 

src/tokenizer/remove_long.rs

@@ -13,6 +13,7 @@
 //! ```
 //!
 use super::{Token, TokenFilter, TokenStream};
+use crate::tokenizer::BoxTokenStream;
 
 /// `RemoveLongFilter` removes tokens that are longer
 /// than a given number of bytes (in UTF-8 representation).
@@ -38,8 +39,8 @@ impl<'a> RemoveLongFilterStream<'a> {
 }
 
 impl TokenFilter for RemoveLongFilter {
-    fn transform<'a>(&self, token_stream: Box<dyn TokenStream + 'a>) -> Box<dyn TokenStream + 'a> {
-        Box::new(RemoveLongFilterStream {
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(RemoveLongFilterStream {
             token_length_limit: self.length_limit,
             tail: token_stream,
         })
@@ -48,7 +49,7 @@ impl TokenFilter for RemoveLongFilter {
 
 pub struct RemoveLongFilterStream<'a> {
     token_length_limit: usize,
-    tail: Box<dyn TokenStream + 'a>,
+    tail: BoxTokenStream<'a>,
 }
 
 impl<'a> TokenStream for RemoveLongFilterStream<'a> {

src/tokenizer/simple_tokenizer.rs

@@ -1,3 +1,4 @@
+use super::BoxTokenStream;
 use super::{Token, TokenStream, Tokenizer};
 use std::str::CharIndices;
 
@@ -12,8 +13,8 @@ pub struct SimpleTokenStream<'a> {
 }
 
 impl Tokenizer for SimpleTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
-        Box::new(SimpleTokenStream {
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(SimpleTokenStream {
             text,
             chars: text.char_indices(),
             token: Token::default(),

src/tokenizer/stemmer.rs

@@ -1,4 +1,5 @@
 use super::{Token, TokenFilter, TokenStream};
+use crate::tokenizer::BoxTokenStream;
 use rust_stemmers::{self, Algorithm};
 use serde::{Deserialize, Serialize};
 
@@ -77,9 +78,9 @@ impl Default for Stemmer {
 }
 
 impl TokenFilter for Stemmer {
-    fn transform<'a>(&self, token_stream: Box<dyn TokenStream + 'a>) -> Box<dyn TokenStream + 'a> {
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
         let inner_stemmer = rust_stemmers::Stemmer::create(self.stemmer_algorithm);
-        Box::new(StemmerTokenStream {
+        BoxTokenStream::from(StemmerTokenStream {
             tail: token_stream,
             stemmer: inner_stemmer,
         })
@@ -87,7 +88,7 @@ impl TokenFilter for Stemmer {
 }
 
 pub struct StemmerTokenStream<'a> {
-    tail: Box<dyn TokenStream + 'a>,
+    tail: BoxTokenStream<'a>,
     stemmer: rust_stemmers::Stemmer,
 }
 

src/tokenizer/stop_word_filter.rs

@@ -11,6 +11,7 @@
 //! assert!(stream.next().is_none());
 //! ```
 use super::{Token, TokenFilter, TokenStream};
+use crate::tokenizer::BoxTokenStream;
 use fnv::FnvHasher;
 use std::collections::HashSet;
 use std::hash::BuildHasherDefault;
@@ -50,12 +51,12 @@ impl StopWordFilter {
 
 pub struct StopWordFilterStream<'a> {
     words: StopWordHashSet,
-    tail: Box<dyn TokenStream + 'a>,
+    tail: BoxTokenStream<'a>,
 }
 
 impl TokenFilter for StopWordFilter {
-    fn transform<'a>(&self, token_stream: Box<dyn TokenStream + 'a>) -> Box<dyn TokenStream + 'a> {
-        Box::new(StopWordFilterStream {
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+        BoxTokenStream::from(StopWordFilterStream {
             words: self.words.clone(),
             tail: token_stream,
         })

src/tokenizer/token_stream_chain.rs

@@ -1,9 +1,11 @@
-use crate::tokenizer::{Token, TokenStream};
+use crate::tokenizer::{BoxTokenStream, Token, TokenStream};
+use std::ops::DerefMut;
 
 const POSITION_GAP: usize = 2;
 
 pub(crate) struct TokenStreamChain<'a> {
-    streams_with_offsets: Vec<(Box<dyn TokenStream + 'a>, usize)>,
+    offsets: Vec<usize>,
+    token_streams: Vec<BoxTokenStream<'a>>,
     position_shift: usize,
     stream_idx: usize,
     token: Token,
@@ -11,11 +13,13 @@ pub(crate) struct TokenStreamChain<'a> {
 
 impl<'a> TokenStreamChain<'a> {
     pub fn new(
-        streams_with_offsets: Vec<(Box<dyn TokenStream + 'a>, usize)>,
+        offsets: Vec<usize>,
+        token_streams: Vec<BoxTokenStream<'a>>,
     ) -> TokenStreamChain<'a> {
         TokenStreamChain {
-            streams_with_offsets,
+            offsets,
             stream_idx: 0,
+            token_streams,
             position_shift: 0,
             token: Token::default(),
         }
@@ -24,10 +28,11 @@ impl<'a> TokenStreamChain<'a> {
 
 impl<'a> TokenStream for TokenStreamChain<'a> {
     fn advance(&mut self) -> bool {
-        while self.stream_idx < self.streams_with_offsets.len() {
-            let (ref mut token_stream, offset_offset) = self.streams_with_offsets[self.stream_idx];
+        while self.stream_idx < self.token_streams.len() {
+            let token_stream = self.token_streams[self.stream_idx].deref_mut();
             if token_stream.advance() {
                 let token = token_stream.token();
+                let offset_offset = self.offsets[self.stream_idx];
                 self.token.offset_from = token.offset_from + offset_offset;
                 self.token.offset_to = token.offset_to + offset_offset;
                 self.token.position = token.position + self.position_shift;
@@ -44,7 +49,7 @@ impl<'a> TokenStream for TokenStreamChain<'a> {
 
     fn token(&self) -> &Token {
         assert!(
-            self.stream_idx <= self.streams_with_offsets.len(),
+            self.stream_idx <= self.token_streams.len(),
             "You called .token(), after the end of the token stream has been reached"
         );
         &self.token
@@ -52,7 +57,7 @@ impl<'a> TokenStream for TokenStreamChain<'a> {
 
     fn token_mut(&mut self) -> &mut Token {
         assert!(
-            self.stream_idx <= self.streams_with_offsets.len(),
+            self.stream_idx <= self.token_streams.len(),
             "You called .token(), after the end of the token stream has been reached"
         );
         &mut self.token
@@ -68,10 +73,10 @@ mod tests {
     #[test]
     fn test_chain_first_emits_no_tokens() {
         let token_streams = vec![
-            (SimpleTokenizer.token_stream(""), 0),
-            (SimpleTokenizer.token_stream("hello world"), 0),
+            SimpleTokenizer.token_stream(""),
+            SimpleTokenizer.token_stream("hello world"),
         ];
-        let mut token_chain = TokenStreamChain::new(token_streams);
+        let mut token_chain = TokenStreamChain::new(vec![0, 0], token_streams);
 
         assert!(token_chain.advance());
         assert_eq!(token_chain.token().text, "hello");

src/tokenizer/tokenized_string.rs

@@ -1,4 +1,4 @@
-use crate::tokenizer::{Token, TokenStream, TokenStreamChain};
+use crate::tokenizer::{BoxTokenStream, Token, TokenStream, TokenStreamChain};
 use serde::{Deserialize, Serialize};
 use std::cmp::Ordering;
 
@@ -42,23 +42,24 @@ impl PreTokenizedStream {
     /// Creates a TokenStream from PreTokenizedString array
     pub fn chain_tokenized_strings<'a>(
         tok_strings: &'a [&'a PreTokenizedString],
-    ) -> Box<dyn TokenStream> {
+    ) -> BoxTokenStream {
         if tok_strings.len() == 1 {
-            Box::new(PreTokenizedStream::from(tok_strings[0].to_owned()))
+            PreTokenizedStream::from((*tok_strings[0]).clone()).into()
         } else {
-            let mut streams_with_offsets = vec![];
+            let mut offsets = vec![];
             let mut total_offset = 0;
             for &tok_string in tok_strings {
-                streams_with_offsets.push((
-                    Box::new(PreTokenizedStream::from(tok_string.to_owned()))
-                        as Box<dyn TokenStream>,
-                    total_offset,
-                ));
+                offsets.push(total_offset);
                 if let Some(last_token) = tok_string.tokens.last() {
                     total_offset += last_token.offset_to;
                 }
             }
-            Box::new(TokenStreamChain::new(streams_with_offsets))
+            // TODO remove the string cloning.
+            let token_streams: Vec<BoxTokenStream<'static>> = tok_strings
+                .iter()
+                .map(|&tok_string| PreTokenizedStream::from((*tok_string).clone()).into())
+                .collect();
+            TokenStreamChain::new(offsets, token_streams).into()
         }
     }
 }

src/tokenizer/tokenizer.rs

@@ -2,6 +2,8 @@ use crate::tokenizer::TokenStreamChain;
 use serde::{Deserialize, Serialize};
 /// The tokenizer module contains all of the tools used to process
 /// text in `tantivy`.
+use std::borrow::{Borrow, BorrowMut};
+use std::ops::{Deref, DerefMut};
 
 /// Token
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
@@ -38,7 +40,7 @@ impl Default for Token {
 /// It simply wraps a `Tokenizer` and a list of `TokenFilter` that are applied sequentially.
 pub struct TextAnalyzer {
     tokenizer: Box<dyn Tokenizer>,
-    token_filters: Vec<Box<dyn TokenFilter>>,
+    token_filters: Vec<BoxTokenFilter>,
 }
 
 impl<T: Tokenizer> From<T> for TextAnalyzer {
@@ -48,14 +50,11 @@ impl<T: Tokenizer> From<T> for TextAnalyzer {
 }
 
 impl TextAnalyzer {
-    /// Creates a new `TextAnalyzer` given a tokenizer and a vector of `Box<dyn TokenFilter>`.
+    /// Creates a new `TextAnalyzer` given a tokenizer and a vector of `BoxTokenFilter`.
     ///
     /// When creating a `TextAnalyzer` from a `Tokenizer` alone, prefer using
     /// `TextAnalyzer::from(tokenizer)`.
-    pub fn new<T: Tokenizer>(
-        tokenizer: T,
-        token_filters: Vec<Box<dyn TokenFilter>>,
-    ) -> TextAnalyzer {
+    pub fn new<T: Tokenizer>(tokenizer: T, token_filters: Vec<BoxTokenFilter>) -> TextAnalyzer {
         TextAnalyzer {
             tokenizer: Box::new(tokenizer),
             token_filters,
@@ -78,8 +77,8 @@ impl TextAnalyzer {
     ///     .filter(Stemmer::default());
     /// ```
     ///
-    pub fn filter<F: TokenFilter>(mut self, token_filter: F) -> Self {
-        self.token_filters.push(Box::new(token_filter));
+    pub fn filter<F: Into<BoxTokenFilter>>(mut self, token_filter: F) -> Self {
+        self.token_filters.push(token_filter.into());
         self
     }
 
@@ -88,19 +87,28 @@ impl TextAnalyzer {
     /// The resulting `BoxTokenStream` is equivalent to what would be obtained if the &str were
     /// one concatenated `&str`, with an artificial position gap of `2` between the different fields
     /// to prevent accidental `PhraseQuery` to match accross two terms.
-    pub fn token_stream_texts<'a>(&self, texts: &'a [&str]) -> Box<dyn TokenStream + 'a> {
-        debug_assert!(!texts.is_empty());
-        let mut streams_with_offsets = vec![];
-        let mut total_offset = 0;
-        for &text in texts {
-            streams_with_offsets.push((self.token_stream(text), total_offset));
-            total_offset += text.len();
+    pub fn token_stream_texts<'a>(&self, texts: &'a [&'a str]) -> BoxTokenStream<'a> {
+        assert!(!texts.is_empty());
+        if texts.len() == 1 {
+            self.token_stream(texts[0])
+        } else {
+            let mut offsets = vec![];
+            let mut total_offset = 0;
+            for &text in texts {
+                offsets.push(total_offset);
+                total_offset += text.len();
+            }
+            let token_streams: Vec<BoxTokenStream<'a>> = texts
+                .iter()
+                .cloned()
+                .map(|text| self.token_stream(text))
+                .collect();
+            From::from(TokenStreamChain::new(offsets, token_streams))
         }
-        Box::new(TokenStreamChain::new(streams_with_offsets))
     }
 
     /// Creates a token stream for a given `str`.
-    pub fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
+    pub fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
         let mut token_stream = self.tokenizer.token_stream(text);
         for token_filter in &self.token_filters {
             token_stream = token_filter.transform(token_stream);
@@ -132,7 +140,7 @@ impl Clone for TextAnalyzer {
 /// This API may change to use associated types.
 pub trait Tokenizer: 'static + Send + Sync + TokenizerClone {
     /// Creates a token stream for a given `str`.
-    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a>;
+    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a>;
 }
 
 pub trait TokenizerClone {
@@ -145,6 +153,69 @@ impl<T: Tokenizer + Clone> TokenizerClone for T {
     }
 }
 
+impl<'a> TokenStream for Box<dyn TokenStream + 'a> {
+    fn advance(&mut self) -> bool {
+        let token_stream: &mut dyn TokenStream = self.borrow_mut();
+        token_stream.advance()
+    }
+
+    fn token<'b>(&'b self) -> &'b Token {
+        let token_stream: &'b (dyn TokenStream + 'a) = self.borrow();
+        token_stream.token()
+    }
+
+    fn token_mut<'b>(&'b mut self) -> &'b mut Token {
+        let token_stream: &'b mut (dyn TokenStream + 'a) = self.borrow_mut();
+        token_stream.token_mut()
+    }
+}
+
+/// Simple wrapper of `Box<dyn TokenStream + 'a>`.
+///
+/// See `TokenStream` for more information.
+pub struct BoxTokenStream<'a>(Box<dyn TokenStream + 'a>);
+
+impl<'a, T> From<T> for BoxTokenStream<'a>
+where
+    T: TokenStream + 'a,
+{
+    fn from(token_stream: T) -> BoxTokenStream<'a> {
+        BoxTokenStream(Box::new(token_stream))
+    }
+}
+
+impl<'a> Deref for BoxTokenStream<'a> {
+    type Target = dyn TokenStream + 'a;
+
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}
+impl<'a> DerefMut for BoxTokenStream<'a> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut *self.0
+    }
+}
+
+/// Simple wrapper of `Box<dyn TokenFilter + 'a>`.
+///
+/// See `TokenStream` for more information.
+pub struct BoxTokenFilter(Box<dyn TokenFilter>);
+
+impl Deref for BoxTokenFilter {
+    type Target = dyn TokenFilter;
+
+    fn deref(&self) -> &dyn TokenFilter {
+        &*self.0
+    }
+}
+
+impl<T: TokenFilter> From<T> for BoxTokenFilter {
+    fn from(tokenizer: T) -> BoxTokenFilter {
+        BoxTokenFilter(Box::new(tokenizer))
+    }
+}
+
 /// `TokenStream` is the result of the tokenization.
 ///
 /// It consists consumable stream of `Token`s.
@@ -224,18 +295,18 @@ pub trait TokenStream {
 }
 
 pub trait TokenFilterClone {
-    fn box_clone(&self) -> Box<dyn TokenFilter>;
+    fn box_clone(&self) -> BoxTokenFilter;
 }
 
 /// Trait for the pluggable components of `Tokenizer`s.
 pub trait TokenFilter: 'static + Send + Sync + TokenFilterClone {
     /// Wraps a token stream and returns the modified one.
-    fn transform<'a>(&self, token_stream: Box<dyn TokenStream + 'a>) -> Box<dyn TokenStream + 'a>;
+    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a>;
 }
 
 impl<T: TokenFilter + Clone> TokenFilterClone for T {
-    fn box_clone(&self) -> Box<dyn TokenFilter> {
-        Box::new(self.clone())
+    fn box_clone(&self) -> BoxTokenFilter {
+        BoxTokenFilter::from(self.clone())
     }
 }