extra commit

Added ReadOnlyDirectory and implemented Bundle Directory
Added serialize to bundle in the RAMDirectory.
2026-01-01 15:02:55 +00:00 · 2019-12-27 22:53:04 +09:00 · 2019-12-27 12:05:39 +09:00 · 2019-12-26 10:06:52 +09:00
47 changed files with 850 additions and 659 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,13 +1,3 @@
-Tantivy 0.12.0
-======================
- Removing static dispatch in tokenizers for simplicity. (#762)
-
-## How to update?
-
-Crates relying on custom tokenizer, or registering tokenizer in the manager will require some 
-minor changes. Check https://github.com/tantivy-search/tantivy/blob/master/examples/custom_tokenizer.rs
-to check for some code sample.
-
 Tantivy 0.11.3
 =======================
 - Fixed DateTime as a fast field (#735)
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,7 +18,7 @@ byteorder = "1.0"
 crc32fast = "1.2.0"
 once_cell = "1.0"
 regex ={version = "1.3.0", default-features = false, features = ["std"]}
-tantivy-fst = "0.2"
+tantivy-fst = "0.1"
 memmap = {version = "0.7", optional=true}
 lz4 = {version="1.20", optional=true}
 snap = {version="0.2"}
--- a/examples/pre_tokenized_text.rs
+++ b/examples/pre_tokenized_text.rs
@@ -9,7 +9,7 @@
 // - import tokenized text straight from json,
 // - perform a search on documents with pre-tokenized text

-use tantivy::tokenizer::{PreTokenizedString, SimpleTokenizer, Token, Tokenizer};
+use tantivy::tokenizer::{PreTokenizedString, SimpleTokenizer, Token, TokenStream, Tokenizer};

 use tantivy::collector::{Count, TopDocs};
 use tantivy::query::TermQuery;
--- a/examples/stop_words.rs
+++ b/examples/stop_words.rs
@@ -50,7 +50,7 @@ fn main() -> tantivy::Result<()> {

    // This tokenizer lowers all of the text (to help with stop word matching)
    // then removes all instances of `the` and `and` from the corpus
-    let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+    let tokenizer = SimpleTokenizer
        .filter(LowerCaser)
        .filter(StopWordFilter::remove(vec![
            "the".to_string(),
--- a/src/collector/top_score_collector.rs
+++ b/src/collector/top_score_collector.rs
@@ -6,7 +6,6 @@ use crate::collector::tweak_score_top_collector::TweakedScoreTopCollector;
 use crate::collector::{
    CustomScorer, CustomSegmentScorer, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector,
 };
-use crate::fastfield::FastFieldReader;
 use crate::schema::Field;
 use crate::DocAddress;
 use crate::DocId;
@@ -62,34 +61,6 @@ impl fmt::Debug for TopDocs {
    }
 }

-struct ScorerByFastFieldReader {
-    ff_reader: FastFieldReader<u64>,
-}
-
-impl CustomSegmentScorer<u64> for ScorerByFastFieldReader {
-    fn score(&self, doc: DocId) -> u64 {
-        self.ff_reader.get_u64(u64::from(doc))
-    }
-}
-
-struct ScorerByField {
-    field: Field,
-}
-
-impl CustomScorer<u64> for ScorerByField {
-    type Child = ScorerByFastFieldReader;
-
-    fn segment_scorer(&self, segment_reader: &SegmentReader) -> crate::Result<Self::Child> {
-        let ff_reader = segment_reader
-            .fast_fields()
-            .u64(self.field)
-            .ok_or_else(|| {
-                crate::Error::SchemaError(format!("Field requested is not a i64/u64 fast field."))
-            })?;
-        Ok(ScorerByFastFieldReader { ff_reader })
-    }
-}
-
 impl TopDocs {
    /// Creates a top score collector, with a number of documents equal to "limit".
    ///
@@ -172,7 +143,14 @@ impl TopDocs {
        self,
        field: Field,
    ) -> impl Collector<Fruit = Vec<(u64, DocAddress)>> {
-        self.custom_score(ScorerByField { field })
+        self.custom_score(move |segment_reader: &SegmentReader| {
+            let ff_reader = segment_reader
+                .fast_fields()
+                .u64(field)
+                .expect("Field requested is not a i64/u64 fast field.");
+            //TODO error message missmatch actual behavior for i64
+            move |doc: DocId| ff_reader.get(doc)
+        })
    }

    /// Ranks the documents using a custom score.
@@ -594,6 +572,7 @@ mod tests {
    }

    #[test]
+    #[should_panic(expected = "Field requested is not a i64/u64 fast field")]
    fn test_field_not_fast_field() {
        let mut schema_builder = Schema::builder();
        let title = schema_builder.add_text_field(TITLE, TEXT);
@@ -608,12 +587,7 @@ mod tests {
        let searcher = index.reader().unwrap().searcher();
        let segment = searcher.segment_reader(0);
        let top_collector = TopDocs::with_limit(4).order_by_u64_field(size);
-        let err = top_collector.for_segment(0, segment);
-        if let Err(crate::Error::SchemaError(msg)) = err {
-            assert_eq!(msg, "Field requested is not a i64/u64 fast field.");
-        } else {
-            assert!(false);
-        }
+        assert!(top_collector.for_segment(0, segment).is_ok());
    }

    fn index(
--- a/src/common/composite_file.rs
+++ b/src/common/composite_file.rs
@@ -186,7 +186,7 @@ mod test {
    use super::{CompositeFile, CompositeWrite};
    use crate::common::BinarySerializable;
    use crate::common::VInt;
-    use crate::directory::{Directory, RAMDirectory};
+    use crate::directory::{Directory, RAMDirectory, ReadOnlyDirectory};
    use crate::schema::Field;
    use std::io::Write;
    use std::path::Path;
--- a/src/core/index.rs
+++ b/src/core/index.rs
@@ -20,7 +20,8 @@ use crate::reader::IndexReaderBuilder;
 use crate::schema::Field;
 use crate::schema::FieldType;
 use crate::schema::Schema;
-use crate::tokenizer::{TextAnalyzer, TokenizerManager};
+use crate::tokenizer::BoxedTokenizer;
+use crate::tokenizer::TokenizerManager;
 use crate::IndexWriter;
 use crate::Result;
 use num_cpus;
@@ -172,11 +173,11 @@ impl Index {
    }

    /// Helper to access the tokenizer associated to a specific field.
-    pub fn tokenizer_for_field(&self, field: Field) -> Result<TextAnalyzer> {
+    pub fn tokenizer_for_field(&self, field: Field) -> Result<BoxedTokenizer> {
        let field_entry = self.schema.get_field_entry(field);
        let field_type = field_entry.field_type();
        let tokenizer_manager: &TokenizerManager = self.tokenizers();
-        let tokenizer_name_opt: Option<TextAnalyzer> = match field_type {
+        let tokenizer_name_opt: Option<BoxedTokenizer> = match field_type {
            FieldType::Str(text_options) => text_options
                .get_indexing_options()
                .map(|text_indexing_options| text_indexing_options.tokenizer().to_string())
@@ -337,7 +338,7 @@ impl Index {

    /// Creates a new segment.
    pub fn new_segment(&self) -> Segment {
-        let segment_meta = self
+        let mut segment_meta = self
            .inventory
            .new_segment_meta(SegmentId::generate_random(), 0);
        self.segment(segment_meta)
--- a/src/core/index_meta.rs
+++ b/src/core/index_meta.rs
@@ -35,6 +35,7 @@ impl SegmentMetaInventory {
            segment_id,
            max_doc,
            deletes: None,
+            bundled: false,
        };
        SegmentMeta::from(self.inventory.track(inner))
    }
@@ -81,6 +82,19 @@ impl SegmentMeta {
        self.tracked.segment_id
    }

+    pub fn with_bundled(self) -> SegmentMeta {
+        SegmentMeta::from(self.tracked.map(|inner| InnerSegmentMeta {
+            segment_id: inner.segment_id,
+            max_doc: inner.max_doc,
+            deletes: inner.deletes.clone(),
+            bundled: true,
+        }))
+    }
+
+    pub fn is_bundled(&self) -> bool {
+        self.tracked.bundled
+    }
+
    /// Returns the number of deleted documents.
    pub fn num_deleted_docs(&self) -> u32 {
        self.tracked
@@ -107,8 +121,12 @@ impl SegmentMeta {
    /// It just joins the segment id with the extension
    /// associated to a segment component.
    pub fn relative_path(&self, component: SegmentComponent) -> PathBuf {
-        let mut path = self.id().uuid_string();
-        path.push_str(&*match component {
+        let suffix = self.suffix(component);
+        self.relative_path_from_suffix(&suffix)
+    }
+
+    fn suffix(&self, component: SegmentComponent) -> String {
+        match component {
            SegmentComponent::POSTINGS => ".idx".to_string(),
            SegmentComponent::POSITIONS => ".pos".to_string(),
            SegmentComponent::POSITIONSSKIP => ".posidx".to_string(),
@@ -117,7 +135,17 @@ impl SegmentMeta {
            SegmentComponent::FASTFIELDS => ".fast".to_string(),
            SegmentComponent::FIELDNORMS => ".fieldnorm".to_string(),
            SegmentComponent::DELETE => format!(".{}.del", self.delete_opstamp().unwrap_or(0)),
-        });
+        }
+    }
+
+    /// Returns the relative path of a component of our segment.
+    ///
+    /// It just joins the segment id with the extension
+    /// associated to a segment component.
+    pub fn relative_path_from_suffix(&self, suffix: &str) -> PathBuf {
+        let mut path = self.id().uuid_string();
+        path.push_str(".");
+        path.push_str(&suffix);
        PathBuf::from(path)
    }

@@ -161,6 +189,7 @@ impl SegmentMeta {
            segment_id: inner_meta.segment_id,
            max_doc,
            deletes: None,
+            bundled: inner_meta.bundled,
        });
        SegmentMeta { tracked }
    }
@@ -175,6 +204,7 @@ impl SegmentMeta {
            segment_id: inner_meta.segment_id,
            max_doc: inner_meta.max_doc,
            deletes: Some(delete_meta),
+            bundled: inner_meta.bundled,
        });
        SegmentMeta { tracked }
    }
@@ -185,6 +215,7 @@ struct InnerSegmentMeta {
    segment_id: SegmentId,
    max_doc: u32,
    deletes: Option<DeleteMeta>,
+    bundled: bool,
 }

 impl InnerSegmentMeta {
--- a/src/core/segment.rs
+++ b/src/core/segment.rs
@@ -4,14 +4,12 @@ use crate::core::SegmentId;
 use crate::core::SegmentMeta;
 use crate::directory::error::{OpenReadError, OpenWriteError};
 use crate::directory::Directory;
-use crate::directory::{ReadOnlySource, WritePtr};
+use crate::directory::{ReadOnlyDirectory, ReadOnlySource, WritePtr};
 use crate::indexer::segment_serializer::SegmentSerializer;
 use crate::schema::Schema;
 use crate::Opstamp;
-use crate::Result;
 use std::fmt;
 use std::path::PathBuf;
-use std::result;

 /// A segment is a piece of the index.
 #[derive(Clone)]
@@ -83,23 +81,30 @@ impl Segment {
    }

    /// Open one of the component file for a *regular* read.
-    pub fn open_read(
-        &self,
-        component: SegmentComponent,
-    ) -> result::Result<ReadOnlySource, OpenReadError> {
+    pub fn open_read(&self, component: SegmentComponent) -> Result<ReadOnlySource, OpenReadError> {
        let path = self.relative_path(component);
        let source = self.index.directory().open_read(&path)?;
        Ok(source)
    }

    /// Open one of the component file for *regular* write.
-    pub fn open_write(
+    pub fn open_write(&mut self, component: SegmentComponent) -> Result<WritePtr, OpenWriteError> {
+        let path = self.relative_path(component);
+        self.index.directory_mut().open_write(&path)
+    }
+
+    pub fn open_bundle_writer(&mut self) -> Result<WritePtr, OpenWriteError> {
+        let path = self.meta.relative_path_from_suffix("bundle");
+        self.index.directory_mut().open_write(&path)
+    }
+
+    pub(crate) fn open_write_in_directory(
        &mut self,
        component: SegmentComponent,
-    ) -> result::Result<WritePtr, OpenWriteError> {
+        directory: &mut dyn Directory,
+    ) -> Result<WritePtr, OpenWriteError> {
        let path = self.relative_path(component);
-        let write = self.index.directory_mut().open_write(&path)?;
-        Ok(write)
+        directory.open_write(&path)
    }
 }

@@ -109,5 +114,5 @@ pub trait SerializableSegment {
    ///
    /// # Returns
    /// The number of documents in the segment.
-    fn write(&self, serializer: SegmentSerializer) -> Result<u32>;
+    fn write(&self, serializer: SegmentSerializer) -> crate::Result<u32>;
 }
--- a/src/directory/bundle_directory.rs
+++ b/src/directory/bundle_directory.rs
@@ -0,0 +1,97 @@
+use crate::directory::directory::ReadOnlyDirectory;
+use crate::directory::error::OpenReadError;
+use crate::directory::ReadOnlySource;
+use crate::error::DataCorruption;
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+#[derive(Clone)]
+struct BundleDirectory {
+    source_map: Arc<HashMap<PathBuf, ReadOnlySource>>,
+}
+
+impl BundleDirectory {
+    pub fn from_source(source: ReadOnlySource) -> Result<BundleDirectory, DataCorruption> {
+        let mut index_offset_buf = [0u8; 8];
+        let (body_idx, footer_offset) = source.split_from_end(8);
+        index_offset_buf.copy_from_slice(footer_offset.as_slice());
+        let offset = u64::from_le_bytes(index_offset_buf);
+        let (body_source, idx_source) = body_idx.split(offset as usize);
+        let idx: HashMap<PathBuf, (u64, u64)> = serde_json::from_slice(idx_source.as_slice())
+            .map_err(|err| {
+                let msg = format!("Failed to read index from bundle. {:?}", err);
+                DataCorruption::comment_only(msg)
+            })?;
+        let source_map: HashMap<PathBuf, ReadOnlySource> = idx
+            .into_iter()
+            .map(|(path, (start, stop))| {
+                let source = body_source.slice(start as usize, stop as usize);
+                (path, source)
+            })
+            .collect();
+        Ok(BundleDirectory {
+            source_map: Arc::new(source_map),
+        })
+    }
+}
+
+impl ReadOnlyDirectory for BundleDirectory {
+    fn open_read(&self, path: &Path) -> Result<ReadOnlySource, OpenReadError> {
+        self.source_map
+            .get(path)
+            .cloned()
+            .ok_or_else(|| OpenReadError::FileDoesNotExist(path.to_path_buf()))
+    }
+
+    fn exists(&self, path: &Path) -> bool {
+        self.source_map.contains_key(path)
+    }
+
+    fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
+        let source = self
+            .source_map
+            .get(path)
+            .ok_or_else(|| OpenReadError::FileDoesNotExist(path.to_path_buf()))?;
+        Ok(source.as_slice().to_vec())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::BundleDirectory;
+    use crate::directory::{RAMDirectory, ReadOnlyDirectory, TerminatingWrite};
+    use crate::Directory;
+    use std::io::Write;
+    use std::path::Path;
+
+    #[test]
+    fn test_bundle_directory() {
+        let mut ram_directory = RAMDirectory::default();
+        let test_path_atomic = Path::new("testpath_atomic");
+        let test_path_wrt = Path::new("testpath_wrt");
+        assert!(ram_directory
+            .atomic_write(test_path_atomic, b"titi")
+            .is_ok());
+        {
+            let mut test_wrt = ram_directory.open_write(test_path_wrt).unwrap();
+            assert!(test_wrt.write_all(b"toto").is_ok());
+            assert!(test_wrt.terminate().is_ok());
+        }
+        let mut dest_directory = RAMDirectory::default();
+        let bundle_path = Path::new("bundle");
+        let mut wrt = dest_directory.open_write(bundle_path).unwrap();
+        assert!(ram_directory.serialize_bundle(&mut wrt).is_ok());
+        assert!(wrt.terminate().is_ok());
+        let source = dest_directory.open_read(bundle_path).unwrap();
+        let bundle_directory = BundleDirectory::from_source(source).unwrap();
+        assert_eq!(
+            &bundle_directory.atomic_read(test_path_atomic).unwrap()[..],
+            b"titi"
+        );
+        assert_eq!(
+            &bundle_directory.open_read(test_path_wrt).unwrap()[..],
+            b"toto"
+        );
+    }
+}
--- a/src/directory/directory.rs
+++ b/src/directory/directory.rs
@@ -100,17 +100,7 @@ fn retry_policy(is_blocking: bool) -> RetryPolicy {
    }
 }

-/// Write-once read many (WORM) abstraction for where
-/// tantivy's data should be stored.
-///
-/// There are currently two implementations of `Directory`
-///
-/// - The [`MMapDirectory`](struct.MmapDirectory.html), this
-/// should be your default choice.
-/// - The [`RAMDirectory`](struct.RAMDirectory.html), which
-/// should be used mostly for tests.
-///
-pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
+pub trait ReadOnlyDirectory {
    /// Opens a virtual file for read.
    ///
    /// Once a virtual file is open, its data may not
@@ -122,6 +112,31 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
    /// You should only use this to read files create with [Directory::open_write].
    fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError>;

+    /// Returns true iff the file exists
+    fn exists(&self, path: &Path) -> bool;
+
+    /// Reads the full content file that has been written using
+    /// atomic_write.
+    ///
+    /// This should only be used for small files.
+    ///
+    /// You should only use this to read files create with [Directory::atomic_write].
+    fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError>;
+}
+
+/// Write-once read many (WORM) abstraction for where
+/// tantivy's data should be stored.
+///
+/// There are currently two implementations of `Directory`
+///
+/// - The [`MMapDirectory`](struct.MmapDirectory.html), this
+/// should be your default choice.
+/// - The [`RAMDirectory`](struct.RAMDirectory.html), which
+/// should be used mostly for tests.
+///
+pub trait Directory:
+    DirectoryClone + ReadOnlyDirectory + fmt::Debug + Send + Sync + 'static
+{
    /// Removes a file
    ///
    /// Removing a file will not affect an eventual
@@ -131,9 +146,6 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
    /// `DeleteError::DoesNotExist`.
    fn delete(&self, path: &Path) -> result::Result<(), DeleteError>;

-    /// Returns true iff the file exists
-    fn exists(&self, path: &Path) -> bool;
-
    /// Opens a writer for the *virtual file* associated with
    /// a Path.
    ///
@@ -155,14 +167,6 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
    /// The file may not previously exist.
    fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError>;

-    /// Reads the full content file that has been written using
-    /// atomic_write.
-    ///
-    /// This should only be used for small files.
-    ///
-    /// You should only use this to read files create with [Directory::atomic_write].
-    fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError>;
-
    /// Atomically replace the content of a file with data.
    ///
    /// This calls ensure that reads can never *observe*
--- a/src/directory/managed_directory.rs
+++ b/src/directory/managed_directory.rs
@@ -10,6 +10,7 @@ use crate::directory::{WatchCallback, WatchHandle};
 use crate::error::DataCorruption;
 use crate::Directory;

+use crate::directory::directory::ReadOnlyDirectory;
 use crc32fast::Hasher;
 use serde_json;
 use std::collections::HashSet;
@@ -264,14 +265,6 @@ impl ManagedDirectory {
 }

 impl Directory for ManagedDirectory {
-    fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
-        let read_only_source = self.directory.open_read(path)?;
-        let (footer, reader) = Footer::extract_footer(read_only_source)
-            .map_err(|err| IOError::with_path(path.to_path_buf(), err))?;
-        footer.is_compatible()?;
-        Ok(reader)
-    }
-
    fn open_write(&mut self, path: &Path) -> result::Result<WritePtr, OpenWriteError> {
        self.register_file_as_managed(path)
            .map_err(|e| IOError::with_path(path.to_owned(), e))?;
@@ -289,18 +282,10 @@ impl Directory for ManagedDirectory {
        self.directory.atomic_write(path, data)
    }

-    fn atomic_read(&self, path: &Path) -> result::Result<Vec<u8>, OpenReadError> {
-        self.directory.atomic_read(path)
-    }
-
    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
        self.directory.delete(path)
    }

-    fn exists(&self, path: &Path) -> bool {
-        self.directory.exists(path)
-    }
-
    fn acquire_lock(&self, lock: &Lock) -> result::Result<DirectoryLock, LockError> {
        self.directory.acquire_lock(lock)
    }
@@ -310,6 +295,24 @@ impl Directory for ManagedDirectory {
    }
 }

+impl ReadOnlyDirectory for ManagedDirectory {
+    fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
+        let read_only_source = self.directory.open_read(path)?;
+        let (footer, reader) = Footer::extract_footer(read_only_source)
+            .map_err(|err| IOError::with_path(path.to_path_buf(), err))?;
+        footer.is_compatible()?;
+        Ok(reader)
+    }
+
+    fn exists(&self, path: &Path) -> bool {
+        self.directory.exists(path)
+    }
+
+    fn atomic_read(&self, path: &Path) -> result::Result<Vec<u8>, OpenReadError> {
+        self.directory.atomic_read(path)
+    }
+}
+
 impl Clone for ManagedDirectory {
    fn clone(&self) -> ManagedDirectory {
        ManagedDirectory {
@@ -323,7 +326,9 @@ impl Clone for ManagedDirectory {
 #[cfg(test)]
 mod tests_mmap_specific {

-    use crate::directory::{Directory, ManagedDirectory, MmapDirectory, TerminatingWrite};
+    use crate::directory::{
+        Directory, ManagedDirectory, MmapDirectory, ReadOnlyDirectory, TerminatingWrite,
+    };
    use std::collections::HashSet;
    use std::fs::OpenOptions;
    use std::io::Write;
--- a/src/directory/mmap_directory.rs
+++ b/src/directory/mmap_directory.rs
@@ -6,6 +6,7 @@ use self::notify::RawEvent;
 use self::notify::RecursiveMode;
 use self::notify::Watcher;
 use crate::core::META_FILEPATH;
+use crate::directory::directory::ReadOnlyDirectory;
 use crate::directory::error::LockError;
 use crate::directory::error::{
    DeleteError, IOError, OpenDirectoryError, OpenReadError, OpenWriteError,
@@ -141,19 +142,11 @@ impl MmapCache {
    }
 }

-pub enum WatcherMode {
-    Event,
-    Poll
-}
-
 struct WatcherWrapper {
    _watcher: Mutex<notify::RecommendedWatcher>,
    watcher_router: Arc<WatchCallbackList>,
-    watcher_mode: WatcherMode,
 }

-
-
 impl WatcherWrapper {
    pub fn new(path: &Path) -> Result<Self, OpenDirectoryError> {
        let (tx, watcher_recv): (Sender<RawEvent>, Receiver<RawEvent>) = channel();
@@ -171,57 +164,33 @@ impl WatcherWrapper {
            })?;
        let watcher_router: Arc<WatchCallbackList> = Default::default();
        let watcher_router_clone = watcher_router.clone();
-        let path_clone = path.clone();
-        let meta_path = path_clone.join(*META_FILEPATH);
        thread::Builder::new()
            .name("meta-file-watch-thread".to_string())
            .spawn(move || {
-                let mut old_content = String::new();
-                let mode = WatcherMode::Event;
                loop {
-                    match mode {
-                        WatcherMode::Event => {
-                            match watcher_recv.recv().map(|evt| evt.path) {
-                                Ok(Some(changed_path)) => {
-                                    // ... Actually subject to false positive.
-                                    // We might want to be more accurate than this at one point.
-                                    if let Some(filename) = changed_path.file_name() {
-                                        if filename == *META_FILEPATH {
-                                            let _ = watcher_router_clone.broadcast();
-                                        }
-                                    }
-                                }
-                                Ok(None) => {
-                                    // not an event we are interested in.
-                                }
-                                Err(_e) => {
-                                    // the watch send channel was dropped
-                                    break;
+                    match watcher_recv.recv().map(|evt| evt.path) {
+                        Ok(Some(changed_path)) => {
+                            // ... Actually subject to false positive.
+                            // We might want to be more accurate than this at one point.
+                            if let Some(filename) = changed_path.file_name() {
+                                if filename == *META_FILEPATH {
+                                    let _ = watcher_router_clone.broadcast();
                                }
                            }
                        }
-                        WatcherMode::Poll => {
-                            let mut file = match File::open(&meta_path) {
-                                Err(why) => panic!("open: nope"),
-                                Ok(file) => file,
-                            };
-                            let mut new_content = String::new();
-                            match file.read_to_string(&mut new_content) {
-                                Err(why) => panic!("read: nope"),
-                                Ok(_) => {},
-                            }
-                            if old_content != new_content {
-                                let _ = watcher_router_clone.broadcast();
-                                old_content = new_content;
-                            }
+                        Ok(None) => {
+                            // not an event we are interested in.
                        }
-                    };
+                        Err(_e) => {
+                            // the watch send channel was dropped
+                            break;
+                        }
+                    }
                }
            })?;
        Ok(WatcherWrapper {
            _watcher: Mutex::new(watcher),
            watcher_router,
-            watcher_mode: WatcherMode::Event,
        })
    }

@@ -439,24 +408,6 @@ impl TerminatingWrite for SafeFileWriter {
 }

 impl Directory for MmapDirectory {
-    fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
-        debug!("Open Read {:?}", path);
-        let full_path = self.resolve_path(path);
-
-        let mut mmap_cache = self.inner.mmap_cache.write().map_err(|_| {
-            let msg = format!(
-                "Failed to acquired write lock \
-                 on mmap cache while reading {:?}",
-                path
-            );
-            IOError::with_path(path.to_owned(), make_io_err(msg))
-        })?;
-        Ok(mmap_cache
-            .get_mmap(&full_path)?
-            .map(ReadOnlySource::from)
-            .unwrap_or_else(ReadOnlySource::empty))
-    }
-
    /// Any entry associated to the path in the mmap will be
    /// removed before the file is deleted.
    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
@@ -475,11 +426,6 @@ impl Directory for MmapDirectory {
        }
    }

-    fn exists(&self, path: &Path) -> bool {
-        let full_path = self.resolve_path(path);
-        full_path.exists()
-    }
-
    fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
        debug!("Open Write {:?}", path);
        let full_path = self.resolve_path(path);
@@ -510,25 +456,6 @@ impl Directory for MmapDirectory {
        Ok(BufWriter::new(Box::new(writer)))
    }

-    fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
-        let full_path = self.resolve_path(path);
-        let mut buffer = Vec::new();
-        match File::open(&full_path) {
-            Ok(mut file) => {
-                file.read_to_end(&mut buffer)
-                    .map_err(|e| IOError::with_path(path.to_owned(), e))?;
-                Ok(buffer)
-            }
-            Err(e) => {
-                if e.kind() == io::ErrorKind::NotFound {
-                    Err(OpenReadError::FileDoesNotExist(path.to_owned()))
-                } else {
-                    Err(IOError::with_path(path.to_owned(), e).into())
-                }
-            }
-        }
-    }
-
    fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
        debug!("Atomic Write {:?}", path);
        let full_path = self.resolve_path(path);
@@ -562,6 +489,50 @@ impl Directory for MmapDirectory {
    }
 }

+impl ReadOnlyDirectory for MmapDirectory {
+    fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
+        debug!("Open Read {:?}", path);
+        let full_path = self.resolve_path(path);
+
+        let mut mmap_cache = self.inner.mmap_cache.write().map_err(|_| {
+            let msg = format!(
+                "Failed to acquired write lock \
+                 on mmap cache while reading {:?}",
+                path
+            );
+            IOError::with_path(path.to_owned(), make_io_err(msg))
+        })?;
+        Ok(mmap_cache
+            .get_mmap(&full_path)?
+            .map(ReadOnlySource::from)
+            .unwrap_or_else(ReadOnlySource::empty))
+    }
+
+    fn exists(&self, path: &Path) -> bool {
+        let full_path = self.resolve_path(path);
+        full_path.exists()
+    }
+
+    fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
+        let full_path = self.resolve_path(path);
+        let mut buffer = Vec::new();
+        match File::open(&full_path) {
+            Ok(mut file) => {
+                file.read_to_end(&mut buffer)
+                    .map_err(|e| IOError::with_path(path.to_owned(), e))?;
+                Ok(buffer)
+            }
+            Err(e) => {
+                if e.kind() == io::ErrorKind::NotFound {
+                    Err(OpenReadError::FileDoesNotExist(path.to_owned()))
+                } else {
+                    Err(IOError::with_path(path.to_owned(), e).into())
+                }
+            }
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {

--- a/src/directory/mod.rs
+++ b/src/directory/mod.rs
@@ -7,6 +7,7 @@ WORM directory abstraction.
 #[cfg(feature = "mmap")]
 mod mmap_directory;

+mod bundle_directory;
 mod directory;
 mod directory_lock;
 mod footer;
@@ -19,7 +20,7 @@ mod watch_event_router;
 pub mod error;

 pub use self::directory::DirectoryLock;
-pub use self::directory::{Directory, DirectoryClone};
+pub use self::directory::{Directory, DirectoryClone, ReadOnlyDirectory};
 pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};
 pub use self::ram_directory::RAMDirectory;
 pub use self::read_only_source::ReadOnlySource;
--- a/src/directory/ram_directory.rs
+++ b/src/directory/ram_directory.rs
@@ -1,4 +1,6 @@
+use crate::common::CountingWriter;
 use crate::core::META_FILEPATH;
+use crate::directory::directory::ReadOnlyDirectory;
 use crate::directory::error::{DeleteError, OpenReadError, OpenWriteError};
 use crate::directory::AntiCallToken;
 use crate::directory::WatchCallbackList;
@@ -115,6 +117,22 @@ impl InnerDirectory {
    fn total_mem_usage(&self) -> usize {
        self.fs.values().map(|f| f.len()).sum()
    }
+
+    fn serialize_bundle(&self, wrt: &mut WritePtr) -> io::Result<()> {
+        let mut counting_writer = CountingWriter::wrap(wrt);
+        let mut file_index: HashMap<PathBuf, (u64, u64)> = HashMap::default();
+        for (path, source) in &self.fs {
+            let start = counting_writer.written_bytes();
+            counting_writer.write_all(source.as_slice())?;
+            let stop = counting_writer.written_bytes();
+            file_index.insert(path.to_path_buf(), (start, stop));
+        }
+        let index_offset = counting_writer.written_bytes();
+        serde_json::to_writer(&mut counting_writer, &file_index)?;
+        let index_offset_buffer = index_offset.to_le_bytes();
+        counting_writer.write_all(&index_offset_buffer[..])?;
+        Ok(())
+    }
 }

 impl fmt::Debug for RAMDirectory {
@@ -144,13 +162,18 @@ impl RAMDirectory {
    pub fn total_mem_usage(&self) -> usize {
        self.fs.read().unwrap().total_mem_usage()
    }
+
+    /// Serialize the RAMDirectory into a bundle.
+    ///
+    /// This method will fail, write nothing, and return an error if a
+    /// clone of this repository exists.
+    pub fn serialize_bundle(self, wrt: &mut WritePtr) -> io::Result<()> {
+        let inner_directory_rlock = self.fs.read().unwrap();
+        inner_directory_rlock.serialize_bundle(wrt)
+    }
 }

 impl Directory for RAMDirectory {
-    fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
-        self.fs.read().unwrap().open_read(path)
-    }
-
    fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
        fail_point!("RAMDirectory::delete", |_| {
            use crate::directory::error::IOError;
@@ -160,10 +183,6 @@ impl Directory for RAMDirectory {
        self.fs.write().unwrap().delete(path)
    }

-    fn exists(&self, path: &Path) -> bool {
-        self.fs.read().unwrap().exists(path)
-    }
-
    fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
        let mut fs = self.fs.write().unwrap();
        let path_buf = PathBuf::from(path);
@@ -177,10 +196,6 @@ impl Directory for RAMDirectory {
        }
    }

-    fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
-        Ok(self.open_read(path)?.as_slice().to_owned())
-    }
-
    fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
        fail_point!("RAMDirectory::atomic_write", |msg| Err(io::Error::new(
            io::ErrorKind::Other,
@@ -204,3 +219,17 @@ impl Directory for RAMDirectory {
        Ok(self.fs.write().unwrap().watch(watch_callback))
    }
 }
+
+impl ReadOnlyDirectory for RAMDirectory {
+    fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError> {
+        self.fs.read().unwrap().open_read(path)
+    }
+
+    fn exists(&self, path: &Path) -> bool {
+        self.fs.read().unwrap().exists(path)
+    }
+
+    fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
+        Ok(self.open_read(path)?.as_slice().to_owned())
+    }
+}
--- a/src/error.rs
+++ b/src/error.rs
@@ -25,10 +25,10 @@ impl DataCorruption {
        }
    }

-    pub fn comment_only(comment: String) -> DataCorruption {
+    pub fn comment_only<TS: ToString>(comment: TS) -> DataCorruption {
        DataCorruption {
            filepath: None,
-            comment,
+            comment: comment.to_string(),
        }
    }
 }
--- a/src/fastfield/mod.rs
+++ b/src/fastfield/mod.rs
@@ -179,7 +179,7 @@ mod tests {

    use super::*;
    use crate::common::CompositeFile;
-    use crate::directory::{Directory, RAMDirectory, WritePtr};
+    use crate::directory::{Directory, RAMDirectory, ReadOnlyDirectory, WritePtr};
    use crate::fastfield::FastFieldReader;
    use crate::merge_policy::NoMergePolicy;
    use crate::schema::Field;
--- a/src/fastfield/reader.rs
+++ b/src/fastfield/reader.rs
@@ -4,7 +4,7 @@ use crate::common::compute_num_bits;
 use crate::common::BinarySerializable;
 use crate::common::CompositeFile;
 use crate::directory::ReadOnlySource;
-use crate::directory::{Directory, RAMDirectory, WritePtr};
+use crate::directory::{Directory, RAMDirectory, ReadOnlyDirectory, WritePtr};
 use crate::fastfield::{FastFieldSerializer, FastFieldsWriter};
 use crate::schema::Schema;
 use crate::schema::FAST;
--- a/src/indexer/operation.rs
+++ b/src/indexer/operation.rs
@@ -19,8 +19,6 @@ pub struct AddOperation {
 /// UserOperation is an enum type that encapsulates other operation types.
 #[derive(Eq, PartialEq, Debug)]
 pub enum UserOperation {
-    /// Add operation
    Add(Document),
-    /// Delete operation
    Delete(Term),
 }
--- a/src/indexer/segment_serializer.rs
+++ b/src/indexer/segment_serializer.rs
@@ -1,10 +1,13 @@
-use crate::Result;
+use crate::Directory;

 use crate::core::Segment;
 use crate::core::SegmentComponent;
+use crate::directory::error::OpenWriteError;
+use crate::directory::{DirectoryClone, RAMDirectory, TerminatingWrite, WritePtr};
 use crate::fastfield::FastFieldSerializer;
 use crate::fieldnorm::FieldNormsSerializer;
 use crate::postings::InvertedIndexSerializer;
+use crate::schema::Schema;
 use crate::store::StoreWriter;

 /// Segment serializer is in charge of laying out on disk
@@ -14,25 +17,50 @@ pub struct SegmentSerializer {
    fast_field_serializer: FastFieldSerializer,
    fieldnorms_serializer: FieldNormsSerializer,
    postings_serializer: InvertedIndexSerializer,
+    bundle_writer: Option<(RAMDirectory, WritePtr)>,
+}
+
+pub(crate) struct SegmentSerializerWriters {
+    postings_wrt: WritePtr,
+    positions_skip_wrt: WritePtr,
+    positions_wrt: WritePtr,
+    terms_wrt: WritePtr,
+    fast_field_wrt: WritePtr,
+    fieldnorms_wrt: WritePtr,
+    store_wrt: WritePtr,
+}
+
+impl SegmentSerializerWriters {
+    pub(crate) fn for_segment(segment: &mut Segment) -> Result<Self, OpenWriteError> {
+        Ok(SegmentSerializerWriters {
+            postings_wrt: segment.open_write(SegmentComponent::POSTINGS)?,
+            positions_skip_wrt: segment.open_write(SegmentComponent::POSITIONS)?,
+            positions_wrt: segment.open_write(SegmentComponent::POSITIONSSKIP)?,
+            terms_wrt: segment.open_write(SegmentComponent::TERMS)?,
+            fast_field_wrt: segment.open_write(SegmentComponent::FASTFIELDS)?,
+            fieldnorms_wrt: segment.open_write(SegmentComponent::FIELDNORMS)?,
+            store_wrt: segment.open_write(SegmentComponent::STORE)?,
+        })
+    }
 }

 impl SegmentSerializer {
-    /// Creates a new `SegmentSerializer`.
-    pub fn for_segment(segment: &mut Segment) -> Result<SegmentSerializer> {
-        let store_write = segment.open_write(SegmentComponent::STORE)?;
-
-        let fast_field_write = segment.open_write(SegmentComponent::FASTFIELDS)?;
-        let fast_field_serializer = FastFieldSerializer::from_write(fast_field_write)?;
-
-        let fieldnorms_write = segment.open_write(SegmentComponent::FIELDNORMS)?;
-        let fieldnorms_serializer = FieldNormsSerializer::from_write(fieldnorms_write)?;
-
-        let postings_serializer = InvertedIndexSerializer::open(segment)?;
+    pub(crate) fn new(schema: Schema, writers: SegmentSerializerWriters) -> crate::Result<Self> {
+        let fast_field_serializer = FastFieldSerializer::from_write(writers.fast_field_wrt)?;
+        let fieldnorms_serializer = FieldNormsSerializer::from_write(writers.fieldnorms_wrt)?;
+        let postings_serializer = InvertedIndexSerializer::open(
+            schema,
+            writers.terms_wrt,
+            writers.postings_wrt,
+            writers.positions_wrt,
+            writers.positions_skip_wrt,
+        );
        Ok(SegmentSerializer {
-            store_writer: StoreWriter::new(store_write),
+            store_writer: StoreWriter::new(writers.store_wrt),
            fast_field_serializer,
            fieldnorms_serializer,
            postings_serializer,
+            bundle_writer: None,
        })
    }

@@ -57,11 +85,15 @@ impl SegmentSerializer {
    }

    /// Finalize the segment serialization.
-    pub fn close(self) -> Result<()> {
+    pub fn close(mut self) -> crate::Result<()> {
        self.fast_field_serializer.close()?;
        self.postings_serializer.close()?;
        self.store_writer.close()?;
        self.fieldnorms_serializer.close()?;
+        if let Some((ram_directory, mut bundle_wrt)) = self.bundle_writer.take() {
+            ram_directory.serialize_bundle(&mut bundle_wrt)?;
+            bundle_wrt.terminate()?;
+        }
        Ok(())
    }
 }
--- a/src/indexer/segment_updater.rs
+++ b/src/indexer/segment_updater.rs
@@ -12,6 +12,7 @@ use crate::indexer::index_writer::advance_deletes;
 use crate::indexer::merge_operation::MergeOperationInventory;
 use crate::indexer::merger::IndexMerger;
 use crate::indexer::segment_manager::SegmentsStatus;
+use crate::indexer::segment_serializer::SegmentSerializerWriters;
 use crate::indexer::stamper::Stamper;
 use crate::indexer::SegmentEntry;
 use crate::indexer::SegmentSerializer;
@@ -132,7 +133,9 @@ fn merge(
    let merger: IndexMerger = IndexMerger::open(index.schema(), &segments[..])?;

    // ... we just serialize this index merger in our new segment to merge the two segments.
-    let segment_serializer = SegmentSerializer::for_segment(&mut merged_segment)?;
+    let segment_serializer_wrts = SegmentSerializerWriters::for_segment(&mut merged_segment)?;
+    let segment_serializer =
+        SegmentSerializer::new(merged_segment.schema(), segment_serializer_wrts)?;

    let num_docs = merger.write(segment_serializer)?;

--- a/src/indexer/segment_writer.rs
+++ b/src/indexer/segment_writer.rs
@@ -3,7 +3,7 @@ use crate::core::Segment;
 use crate::core::SerializableSegment;
 use crate::fastfield::FastFieldsWriter;
 use crate::fieldnorm::FieldNormsWriter;
-use crate::indexer::segment_serializer::SegmentSerializer;
+use crate::indexer::segment_serializer::{SegmentSerializer, SegmentSerializerWriters};
 use crate::postings::compute_table_size;
 use crate::postings::MultiFieldPostingsWriter;
 use crate::schema::FieldType;
@@ -11,9 +11,10 @@ use crate::schema::Schema;
 use crate::schema::Term;
 use crate::schema::Value;
 use crate::schema::{Field, FieldEntry};
-use crate::tokenizer::{BoxTokenStream, PreTokenizedStream};
-use crate::tokenizer::{FacetTokenizer, TextAnalyzer};
-use crate::tokenizer::{TokenStreamChain, Tokenizer};
+use crate::tokenizer::BoxedTokenizer;
+use crate::tokenizer::FacetTokenizer;
+use crate::tokenizer::PreTokenizedStream;
+use crate::tokenizer::{TokenStream, TokenStreamChain, Tokenizer};
 use crate::DocId;
 use crate::Opstamp;
 use crate::Result;
@@ -49,7 +50,7 @@ pub struct SegmentWriter {
    fast_field_writers: FastFieldsWriter,
    fieldnorms_writer: FieldNormsWriter,
    doc_opstamps: Vec<Opstamp>,
-    tokenizers: Vec<Option<TextAnalyzer>>,
+    tokenizers: Vec<Option<BoxedTokenizer>>,
 }

 impl SegmentWriter {
@@ -68,7 +69,8 @@ impl SegmentWriter {
        schema: &Schema,
    ) -> Result<SegmentWriter> {
        let table_num_bits = initial_table_size(memory_budget)?;
-        let segment_serializer = SegmentSerializer::for_segment(&mut segment)?;
+        let segment_serializer_wrts = SegmentSerializerWriters::for_segment(&mut segment)?;
+        let segment_serializer = SegmentSerializer::new(segment.schema(), segment_serializer_wrts)?;
        let multifield_postings = MultiFieldPostingsWriter::new(schema, table_num_bits);
        let tokenizers = schema
            .fields()
@@ -158,7 +160,7 @@ impl SegmentWriter {
                    }
                }
                FieldType::Str(_) => {
-                    let mut token_streams: Vec<BoxTokenStream> = vec![];
+                    let mut token_streams: Vec<Box<dyn TokenStream>> = vec![];
                    let mut offsets = vec![];
                    let mut total_offset = 0;

@@ -171,7 +173,7 @@ impl SegmentWriter {
                                }

                                token_streams
-                                    .push(PreTokenizedStream::from(tok_str.clone()).into());
+                                    .push(Box::new(PreTokenizedStream::from(tok_str.clone())));
                            }
                            Value::Str(ref text) => {
                                if let Some(ref mut tokenizer) =
@@ -190,7 +192,8 @@ impl SegmentWriter {
                    let num_tokens = if token_streams.is_empty() {
                        0
                    } else {
-                        let mut token_stream = TokenStreamChain::new(offsets, token_streams);
+                        let mut token_stream: Box<dyn TokenStream> =
+                            Box::new(TokenStreamChain::new(offsets, token_streams));
                        self.multifield_postings
                            .index_text(doc_id, field, &mut token_stream)
                    };
--- a/src/indexer/stamper.rs
+++ b/src/indexer/stamper.rs
@@ -1,76 +1,18 @@
 use crate::Opstamp;
 use std::ops::Range;
-use std::sync::atomic::Ordering;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;

-#[cfg(not(target_arch = "arm"))]
-mod atomic_impl {
-
-    use crate::Opstamp;
-    use std::sync::atomic::{AtomicU64, Ordering};
-
-    #[derive(Default)]
-    pub struct AtomicU64Wrapper(AtomicU64);
-
-    impl AtomicU64Wrapper {
-        pub fn new(first_opstamp: Opstamp) -> AtomicU64Wrapper {
-            AtomicU64Wrapper(AtomicU64::new(first_opstamp as u64))
-        }
-
-        pub fn fetch_add(&self, val: u64, order: Ordering) -> u64 {
-            self.0.fetch_add(val as u64, order) as u64
-        }
-
-        pub fn revert(&self, val: u64, order: Ordering) -> u64 {
-            self.0.store(val, order);
-            val
-        }
-    }
-}
-
-#[cfg(target_arch = "arm")]
-mod atomic_impl {
-
-    use crate::Opstamp;
-    /// Under other architecture, we rely on a mutex.
-    use std::sync::atomic::Ordering;
-    use std::sync::RwLock;
-
-    #[derive(Default)]
-    pub struct AtomicU64Wrapper(RwLock<u64>);
-
-    impl AtomicU64Wrapper {
-        pub fn new(first_opstamp: Opstamp) -> AtomicU64Wrapper {
-            AtomicU64Wrapper(RwLock::new(first_opstamp))
-        }
-
-        pub fn fetch_add(&self, incr: u64, _order: Ordering) -> u64 {
-            let mut lock = self.0.write().unwrap();
-            let previous_val = *lock;
-            *lock = previous_val + incr;
-            previous_val
-        }
-
-        pub fn revert(&self, val: u64, _order: Ordering) -> u64 {
-            let mut lock = self.0.write().unwrap();
-            *lock = val;
-            val
-        }
-    }
-}
-
-use self::atomic_impl::AtomicU64Wrapper;
-
 /// Stamper provides Opstamps, which is just an auto-increment id to label
 /// an operation.
 ///
 /// Cloning does not "fork" the stamp generation. The stamper actually wraps an `Arc`.
 #[derive(Clone, Default)]
-pub struct Stamper(Arc<AtomicU64Wrapper>);
+pub struct Stamper(Arc<AtomicU64>);

 impl Stamper {
    pub fn new(first_opstamp: Opstamp) -> Stamper {
-        Stamper(Arc::new(AtomicU64Wrapper::new(first_opstamp)))
+        Stamper(Arc::new(AtomicU64::new(first_opstamp)))
    }

    pub fn stamp(&self) -> Opstamp {
@@ -89,7 +31,8 @@ impl Stamper {

    /// Reverts the stamper to a given `Opstamp` value and returns it
    pub fn revert(&self, to_opstamp: Opstamp) -> Opstamp {
-        self.0.revert(to_opstamp, Ordering::SeqCst)
+        self.0.store(to_opstamp, Ordering::SeqCst);
+        to_opstamp
    }
 }

--- a/src/lib.rs
+++ b/src/lib.rs
@@ -165,7 +165,6 @@ pub use crate::core::SegmentComponent;
 pub use crate::core::{Index, IndexMeta, Searcher, Segment, SegmentId, SegmentMeta};
 pub use crate::core::{InvertedIndexReader, SegmentReader};
 pub use crate::directory::Directory;
-pub use crate::indexer::operation::UserOperation;
 pub use crate::indexer::IndexWriter;
 pub use crate::postings::Postings;
 pub use crate::reader::LeasedItem;
--- a/src/postings/mod.rs
+++ b/src/postings/mod.rs
@@ -75,7 +75,7 @@ pub mod tests {
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
        let mut segment = index.new_segment();
-        let mut posting_serializer = InvertedIndexSerializer::open(&mut segment).unwrap();
+        let mut posting_serializer = InvertedIndexSerializer::for_segment(&mut segment).unwrap();
        {
            let mut field_serializer = posting_serializer.new_field(text_field, 120 * 4).unwrap();
            field_serializer.new_term("abc".as_bytes()).unwrap();
--- a/src/postings/serializer.rs
+++ b/src/postings/serializer.rs
@@ -10,8 +10,8 @@ use crate::postings::USE_SKIP_INFO_LIMIT;
 use crate::schema::Schema;
 use crate::schema::{Field, FieldEntry, FieldType};
 use crate::termdict::{TermDictionaryBuilder, TermOrdinal};
-use crate::DocId;
 use crate::Result;
+use crate::{Directory, DocId};
 use std::io::{self, Write};

 /// `InvertedIndexSerializer` is in charge of serializing
@@ -54,33 +54,36 @@ pub struct InvertedIndexSerializer {
 }

 impl InvertedIndexSerializer {
-    /// Open a new `InvertedIndexSerializer` for the given segment
-    fn create(
-        terms_write: CompositeWrite<WritePtr>,
-        postings_write: CompositeWrite<WritePtr>,
-        positions_write: CompositeWrite<WritePtr>,
-        positionsidx_write: CompositeWrite<WritePtr>,
-        schema: Schema,
-    ) -> Result<InvertedIndexSerializer> {
-        Ok(InvertedIndexSerializer {
-            terms_write,
-            postings_write,
-            positions_write,
-            positionsidx_write,
+    pub(crate) fn for_segment(segment: &mut Segment) -> crate::Result<Self> {
+        let schema = segment.schema();
+        use crate::core::SegmentComponent;
+        let terms_wrt = segment.open_write(SegmentComponent::TERMS)?;
+        let postings_wrt = segment.open_write(SegmentComponent::POSTINGS)?;
+        let positions_wrt = segment.open_write(SegmentComponent::POSITIONS)?;
+        let positions_idx_wrt = segment.open_write(SegmentComponent::POSITIONSSKIP)?;
+        Ok(Self::open(
            schema,
-        })
+            terms_wrt,
+            postings_wrt,
+            positions_wrt,
+            positions_idx_wrt,
+        ))
    }
-
    /// Open a new `PostingsSerializer` for the given segment
-    pub fn open(segment: &mut Segment) -> Result<InvertedIndexSerializer> {
-        use crate::SegmentComponent::{POSITIONS, POSITIONSSKIP, POSTINGS, TERMS};
-        InvertedIndexSerializer::create(
-            CompositeWrite::wrap(segment.open_write(TERMS)?),
-            CompositeWrite::wrap(segment.open_write(POSTINGS)?),
-            CompositeWrite::wrap(segment.open_write(POSITIONS)?),
-            CompositeWrite::wrap(segment.open_write(POSITIONSSKIP)?),
-            segment.schema(),
-        )
+    pub(crate) fn open(
+        schema: Schema,
+        terms_wrt: WritePtr,
+        postings_wrt: WritePtr,
+        positions_wrt: WritePtr,
+        positions_idx_wrt: WritePtr,
+    ) -> InvertedIndexSerializer {
+        InvertedIndexSerializer {
+            terms_write: CompositeWrite::wrap(terms_wrt),
+            postings_write: CompositeWrite::wrap(postings_wrt),
+            positions_write: CompositeWrite::wrap(positions_wrt),
+            positionsidx_write: CompositeWrite::wrap(positions_idx_wrt),
+            schema,
+        }
    }

    /// Must be called before starting pushing terms of
--- a/src/query/query_parser/query_parser.rs
+++ b/src/query/query_parser/query_parser.rs
@@ -533,7 +533,7 @@ mod test {
    use crate::schema::{IndexRecordOption, TextFieldIndexing, TextOptions};
    use crate::schema::{Schema, Term, INDEXED, STORED, STRING, TEXT};
    use crate::tokenizer::{
-        LowerCaser, SimpleTokenizer, StopWordFilter, TextAnalyzer, TokenizerManager,
+        LowerCaser, SimpleTokenizer, StopWordFilter, Tokenizer, TokenizerManager,
    };
    use crate::Index;
    use matches::assert_matches;
@@ -563,7 +563,7 @@ mod test {
        let tokenizer_manager = TokenizerManager::default();
        tokenizer_manager.register(
            "en_with_stop_words",
-            TextAnalyzer::from(SimpleTokenizer)
+            SimpleTokenizer
                .filter(LowerCaser)
                .filter(StopWordFilter::remove(vec!["the".to_string()])),
        );
--- a/src/reader/mod.rs
+++ b/src/reader/mod.rs
@@ -11,7 +11,6 @@ use crate::Result;
 use crate::Searcher;
 use crate::SegmentReader;
 use std::sync::Arc;
-use std::thread;

 /// Defines when a new version of the index should be reloaded.
 ///
--- a/src/snippet/mod.rs
+++ b/src/snippet/mod.rs
@@ -1,7 +1,8 @@
 use crate::query::Query;
 use crate::schema::Field;
 use crate::schema::Value;
-use crate::tokenizer::{TextAnalyzer, Token};
+use crate::tokenizer::BoxedTokenizer;
+use crate::tokenizer::{Token, TokenStream};
 use crate::Document;
 use crate::Result;
 use crate::Searcher;
@@ -141,7 +142,7 @@ impl Snippet {
 /// Fragments must be valid in the sense that `&text[fragment.start..fragment.stop]`\
 /// has to be a valid string.
 fn search_fragments<'a>(
-    tokenizer: &TextAnalyzer,
+    tokenizer: &BoxedTokenizer,
    text: &'a str,
    terms: &BTreeMap<String, f32>,
    max_num_chars: usize,
@@ -250,7 +251,7 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str)
 /// ```
 pub struct SnippetGenerator {
    terms_text: BTreeMap<String, f32>,
-    tokenizer: TextAnalyzer,
+    tokenizer: BoxedTokenizer,
    field: Field,
    max_num_chars: usize,
 }
@@ -346,11 +347,12 @@ Survey in 2016, 2017, and 2018."#;

    #[test]
    fn test_snippet() {
+        let boxed_tokenizer = SimpleTokenizer.into();
        let terms = btreemap! {
            String::from("rust") => 1.0,
            String::from("language") => 0.9
        };
-        let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 100);
+        let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 100);
        assert_eq!(fragments.len(), 7);
        {
            let first = &fragments[0];
@@ -372,12 +374,13 @@ Survey in 2016, 2017, and 2018."#;

    #[test]
    fn test_snippet_scored_fragment() {
+        let boxed_tokenizer = SimpleTokenizer.into();
        {
            let terms = btreemap! {
                String::from("rust") =>1.0f32,
                String::from("language") => 0.9f32
            };
-            let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 20);
+            let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 20);
            {
                let first = &fragments[0];
                assert_eq!(first.score, 1.0);
@@ -386,12 +389,13 @@ Survey in 2016, 2017, and 2018."#;
            let snippet = select_best_fragment_combination(&fragments[..], &TEST_TEXT);
            assert_eq!(snippet.to_html(), "<b>Rust</b> is a systems")
        }
+        let boxed_tokenizer = SimpleTokenizer.into();
        {
            let terms = btreemap! {
                String::from("rust") =>0.9f32,
                String::from("language") => 1.0f32
            };
-            let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 20);
+            let fragments = search_fragments(&boxed_tokenizer, TEST_TEXT, &terms, 20);
            //assert_eq!(fragments.len(), 7);
            {
                let first = &fragments[0];
@@ -405,12 +409,14 @@ Survey in 2016, 2017, and 2018."#;

    #[test]
    fn test_snippet_in_second_fragment() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
        let text = "a b c d e f g";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("c"), 1.0);

-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);

        assert_eq!(fragments.len(), 1);
        {
@@ -427,12 +433,14 @@ Survey in 2016, 2017, and 2018."#;

    #[test]
    fn test_snippet_with_term_at_the_end_of_fragment() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
        let text = "a b c d e f f g";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("f"), 1.0);

-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);

        assert_eq!(fragments.len(), 2);
        {
@@ -449,13 +457,15 @@ Survey in 2016, 2017, and 2018."#;

    #[test]
    fn test_snippet_with_second_fragment_has_the_highest_score() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
        let text = "a b c d e f g";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("f"), 1.0);
        terms.insert(String::from("a"), 0.9);

-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 7);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 7);

        assert_eq!(fragments.len(), 2);
        {
@@ -472,12 +482,14 @@ Survey in 2016, 2017, and 2018."#;

    #[test]
    fn test_snippet_with_term_not_in_text() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
        let text = "a b c d";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("z"), 1.0);

-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);

        assert_eq!(fragments.len(), 0);

@@ -488,10 +500,12 @@ Survey in 2016, 2017, and 2018."#;

    #[test]
    fn test_snippet_with_no_terms() {
+        let boxed_tokenizer = SimpleTokenizer.into();
+
        let text = "a b c d";

        let terms = BTreeMap::new();
-        let fragments = search_fragments(&From::from(SimpleTokenizer), &text, &terms, 3);
+        let fragments = search_fragments(&boxed_tokenizer, &text, &terms, 3);
        assert_eq!(fragments.len(), 0);

        let snippet = select_best_fragment_combination(&fragments[..], &text);
--- a/src/store/mod.rs
+++ b/src/store/mod.rs
@@ -57,7 +57,7 @@ use self::compression_snap::{compress, decompress};
 pub mod tests {

    use super::*;
-    use crate::directory::{Directory, RAMDirectory, WritePtr};
+    use crate::directory::{Directory, RAMDirectory, ReadOnlyDirectory, WritePtr};
    use crate::schema::Document;
    use crate::schema::FieldValue;
    use crate::schema::Schema;
--- a/src/termdict/mod.rs
+++ b/src/termdict/mod.rs
@@ -36,7 +36,7 @@ pub use self::termdict::{TermDictionary, TermDictionaryBuilder};
 mod tests {
    use super::{TermDictionary, TermDictionaryBuilder, TermStreamer};
    use crate::core::Index;
-    use crate::directory::{Directory, RAMDirectory, ReadOnlySource};
+    use crate::directory::{Directory, RAMDirectory, ReadOnlyDirectory, ReadOnlySource};
    use crate::postings::TermInfo;
    use crate::schema::{Document, FieldType, Schema, TEXT};
    use std::path::PathBuf;
--- a/src/tokenizer/alphanum_only.rs
+++ b/src/tokenizer/alphanum_only.rs
@@ -2,7 +2,7 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let tokenizer = TextAnalyzer::from(RawTokenizer)
+//! let tokenizer = RawTokenizer
 //!   .filter(AlphaNumOnlyFilter);
 //!
 //! let mut stream = tokenizer.token_stream("hello there");
@@ -10,7 +10,7 @@
 //! // contains a space
 //! assert!(stream.next().is_none());
 //!
-//! let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+//! let tokenizer = SimpleTokenizer
 //!   .filter(AlphaNumOnlyFilter);
 //!
 //! let mut stream = tokenizer.token_stream("hello there 💣");
@@ -19,30 +19,56 @@
 //! // the "emoji" is dropped because its not an alphanum
 //! assert!(stream.next().is_none());
 //! ```
-use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
+use super::{Token, TokenFilter, TokenStream};

 /// `TokenFilter` that removes all tokens that contain non
 /// ascii alphanumeric characters.
 #[derive(Clone)]
 pub struct AlphaNumOnlyFilter;

-pub struct AlphaNumOnlyFilterStream<'a> {
-    tail: BoxTokenStream<'a>,
+pub struct AlphaNumOnlyFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    tail: TailTokenStream,
 }

-impl<'a> AlphaNumOnlyFilterStream<'a> {
+impl<TailTokenStream> AlphaNumOnlyFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
    fn predicate(&self, token: &Token) -> bool {
        token.text.chars().all(|c| c.is_ascii_alphanumeric())
    }
-}

-impl TokenFilter for AlphaNumOnlyFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(AlphaNumOnlyFilterStream { tail: token_stream })
+    fn wrap(tail: TailTokenStream) -> AlphaNumOnlyFilterStream<TailTokenStream> {
+        AlphaNumOnlyFilterStream { tail }
    }
 }

-impl<'a> TokenStream for AlphaNumOnlyFilterStream<'a> {
+impl<TailTokenStream> TokenFilter<TailTokenStream> for AlphaNumOnlyFilter
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = AlphaNumOnlyFilterStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+        AlphaNumOnlyFilterStream::wrap(token_stream)
+    }
+}
+
+impl<TailTokenStream> TokenStream for AlphaNumOnlyFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+
    fn advance(&mut self) -> bool {
        while self.tail.advance() {
            if self.predicate(self.tail.token()) {
@@ -52,12 +78,4 @@ impl<'a> TokenStream for AlphaNumOnlyFilterStream<'a> {

        false
    }
-
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
-    }
 }
--- a/src/tokenizer/ascii_folding_filter.rs
+++ b/src/tokenizer/ascii_folding_filter.rs
@@ -1,4 +1,4 @@
-use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
+use super::{Token, TokenFilter, TokenStream};
 use std::mem;

 /// This class converts alphabetic, numeric, and symbolic Unicode characters
@@ -7,21 +7,26 @@ use std::mem;
 #[derive(Clone)]
 pub struct AsciiFoldingFilter;

-impl TokenFilter for AsciiFoldingFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        From::from(AsciiFoldingFilterTokenStream {
-            tail: token_stream,
-            buffer: String::with_capacity(100),
-        })
+impl<TailTokenStream> TokenFilter<TailTokenStream> for AsciiFoldingFilter
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = AsciiFoldingFilterTokenStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+        AsciiFoldingFilterTokenStream::wrap(token_stream)
    }
 }

-pub struct AsciiFoldingFilterTokenStream<'a> {
+pub struct AsciiFoldingFilterTokenStream<TailTokenStream> {
    buffer: String,
-    tail: BoxTokenStream<'a>,
+    tail: TailTokenStream,
 }

-impl<'a> TokenStream for AsciiFoldingFilterTokenStream<'a> {
+impl<TailTokenStream> TokenStream for AsciiFoldingFilterTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
    fn advance(&mut self) -> bool {
        if !self.tail.advance() {
            return false;
@@ -43,6 +48,18 @@ impl<'a> TokenStream for AsciiFoldingFilterTokenStream<'a> {
    }
 }

+impl<TailTokenStream> AsciiFoldingFilterTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn wrap(tail: TailTokenStream) -> AsciiFoldingFilterTokenStream<TailTokenStream> {
+        AsciiFoldingFilterTokenStream {
+            tail,
+            buffer: String::with_capacity(100),
+        }
+    }
+}
+
 // Returns a string that represents the ascii folded version of
 // the character. If the `char` does not require ascii folding
 // (e.g. simple ASCII chars like `A`) or if the `char`
@@ -1544,7 +1561,8 @@ mod tests {
    use crate::tokenizer::AsciiFoldingFilter;
    use crate::tokenizer::RawTokenizer;
    use crate::tokenizer::SimpleTokenizer;
-    use crate::tokenizer::TextAnalyzer;
+    use crate::tokenizer::TokenStream;
+    use crate::tokenizer::Tokenizer;
    use std::iter;

    #[test]
@@ -1561,7 +1579,7 @@ mod tests {

    fn folding_helper(text: &str) -> Vec<String> {
        let mut tokens = Vec::new();
-        TextAnalyzer::from(SimpleTokenizer)
+        SimpleTokenizer
            .filter(AsciiFoldingFilter)
            .token_stream(text)
            .process(&mut |token| {
@@ -1571,9 +1589,7 @@ mod tests {
    }

    fn folding_using_raw_tokenizer_helper(text: &str) -> String {
-        let mut token_stream = TextAnalyzer::from(RawTokenizer)
-            .filter(AsciiFoldingFilter)
-            .token_stream(text);
+        let mut token_stream = RawTokenizer.filter(AsciiFoldingFilter).token_stream(text);
        token_stream.advance();
        token_stream.token().text.clone()
    }
--- a/src/tokenizer/facet_tokenizer.rs
+++ b/src/tokenizer/facet_tokenizer.rs
@@ -1,4 +1,4 @@
-use super::{BoxTokenStream, Token, TokenStream, Tokenizer};
+use super::{Token, TokenStream, Tokenizer};
 use crate::schema::FACET_SEP_BYTE;

 /// The `FacetTokenizer` process a `Facet` binary representation
@@ -25,14 +25,15 @@ pub struct FacetTokenStream<'a> {
    token: Token,
 }

-impl Tokenizer for FacetTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+impl<'a> Tokenizer<'a> for FacetTokenizer {
+    type TokenStreamImpl = FacetTokenStream<'a>;
+
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
        FacetTokenStream {
            text,
            state: State::RootFacetNotEmitted, //< pos is the first char that has not been processed yet.
            token: Token::default(),
        }
-        .into()
    }
 }

@@ -83,7 +84,7 @@ mod tests {

    use super::FacetTokenizer;
    use crate::schema::Facet;
-    use crate::tokenizer::{Token, Tokenizer};
+    use crate::tokenizer::{Token, TokenStream, Tokenizer};

    #[test]
    fn test_facet_tokenizer() {
--- a/src/tokenizer/lower_caser.rs
+++ b/src/tokenizer/lower_caser.rs
@@ -1,23 +1,24 @@
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;
 use std::mem;

-impl TokenFilter for LowerCaser {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(LowerCaserTokenStream {
-            tail: token_stream,
-            buffer: String::with_capacity(100),
-        })
-    }
-}
-
 /// Token filter that lowercase terms.
 #[derive(Clone)]
 pub struct LowerCaser;

-pub struct LowerCaserTokenStream<'a> {
+impl<TailTokenStream> TokenFilter<TailTokenStream> for LowerCaser
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = LowerCaserTokenStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+        LowerCaserTokenStream::wrap(token_stream)
+    }
+}
+
+pub struct LowerCaserTokenStream<TailTokenStream> {
    buffer: String,
-    tail: BoxTokenStream<'a>,
+    tail: TailTokenStream,
 }

 // writes a lowercased version of text into output.
@@ -30,7 +31,18 @@ fn to_lowercase_unicode(text: &mut String, output: &mut String) {
    }
 }

-impl<'a> TokenStream for LowerCaserTokenStream<'a> {
+impl<TailTokenStream> TokenStream for LowerCaserTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+
    fn advance(&mut self) -> bool {
        if !self.tail.advance() {
            return false;
@@ -44,19 +56,26 @@ impl<'a> TokenStream for LowerCaserTokenStream<'a> {
        }
        true
    }
+}

-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
+impl<TailTokenStream> LowerCaserTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn wrap(tail: TailTokenStream) -> LowerCaserTokenStream<TailTokenStream> {
+        LowerCaserTokenStream {
+            tail,
+            buffer: String::with_capacity(100),
+        }
    }
 }

 #[cfg(test)]
 mod tests {
-    use crate::tokenizer::{LowerCaser, SimpleTokenizer, TextAnalyzer};
+    use crate::tokenizer::LowerCaser;
+    use crate::tokenizer::SimpleTokenizer;
+    use crate::tokenizer::TokenStream;
+    use crate::tokenizer::Tokenizer;

    #[test]
    fn test_to_lower_case() {
@@ -68,9 +87,7 @@ mod tests {

    fn lowercase_helper(text: &str) -> Vec<String> {
        let mut tokens = vec![];
-        let mut token_stream = TextAnalyzer::from(SimpleTokenizer)
-            .filter(LowerCaser)
-            .token_stream(text);
+        let mut token_stream = SimpleTokenizer.filter(LowerCaser).token_stream(text);
        while token_stream.advance() {
            let token_text = token_stream.token().text.clone();
            tokens.push(token_text);
--- a/src/tokenizer/mod.rs
+++ b/src/tokenizer/mod.rs
@@ -64,7 +64,7 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let en_stem = TextAnalyzer::from(SimpleTokenizer)
+//! let en_stem = SimpleTokenizer
 //!     .filter(RemoveLongFilter::limit(40))
 //!     .filter(LowerCaser)
 //!     .filter(Stemmer::new(Language::English));
@@ -109,7 +109,7 @@
 //! let index = Index::create_in_ram(schema);
 //!
 //! // We need to register our tokenizer :
-//! let custom_en_tokenizer = TextAnalyzer::from(SimpleTokenizer)
+//! let custom_en_tokenizer = SimpleTokenizer
 //!     .filter(RemoveLongFilter::limit(40))
 //!     .filter(LowerCaser);
 //! index
@@ -143,11 +143,10 @@ pub use self::simple_tokenizer::SimpleTokenizer;
 pub use self::stemmer::{Language, Stemmer};
 pub use self::stop_word_filter::StopWordFilter;
 pub(crate) use self::token_stream_chain::TokenStreamChain;
+pub use self::tokenizer::BoxedTokenizer;

 pub use self::tokenized_string::{PreTokenizedStream, PreTokenizedString};
-pub use self::tokenizer::{
-    BoxTokenFilter, BoxTokenStream, TextAnalyzer, Token, TokenFilter, TokenStream, Tokenizer,
-};
+pub use self::tokenizer::{Token, TokenFilter, TokenStream, Tokenizer};

 pub use self::tokenizer_manager::TokenizerManager;

@@ -161,9 +160,9 @@ pub const MAX_TOKEN_LEN: usize = u16::max_value() as usize - 4;
 #[cfg(test)]
 pub mod tests {
    use super::{
-        Language, LowerCaser, RemoveLongFilter, SimpleTokenizer, Stemmer, Token, TokenizerManager,
+        Language, LowerCaser, RemoveLongFilter, SimpleTokenizer, Stemmer, Token, Tokenizer,
+        TokenizerManager,
    };
-    use crate::tokenizer::TextAnalyzer;

    /// This is a function that can be used in tests and doc tests
    /// to assert a token's correctness.
@@ -230,7 +229,7 @@ pub mod tests {
        let tokenizer_manager = TokenizerManager::default();
        tokenizer_manager.register(
            "el_stem",
-            TextAnalyzer::from(SimpleTokenizer)
+            SimpleTokenizer
                .filter(RemoveLongFilter::limit(40))
                .filter(LowerCaser)
                .filter(Stemmer::new(Language::Greek)),
--- a/src/tokenizer/ngram_tokenizer.rs
+++ b/src/tokenizer/ngram_tokenizer.rs
@@ -1,5 +1,4 @@
 use super::{Token, TokenStream, Tokenizer};
-use crate::tokenizer::BoxTokenStream;

 /// Tokenize the text by splitting words into n-grams of the given size(s)
 ///
@@ -130,9 +129,11 @@ pub struct NgramTokenStream<'a> {
    token: Token,
 }

-impl Tokenizer for NgramTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
-        From::from(NgramTokenStream {
+impl<'a> Tokenizer<'a> for NgramTokenizer {
+    type TokenStreamImpl = NgramTokenStream<'a>;
+
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
+        NgramTokenStream {
            ngram_charidx_iterator: StutteringIterator::new(
                CodepointFrontiers::for_str(text),
                self.min_gram,
@@ -141,7 +142,7 @@ impl Tokenizer for NgramTokenizer {
            prefix_only: self.prefix_only,
            text,
            token: Token::default(),
-        })
+        }
    }
 }

@@ -307,10 +308,10 @@ mod tests {
    use super::NgramTokenizer;
    use super::StutteringIterator;
    use crate::tokenizer::tests::assert_token;
-    use crate::tokenizer::tokenizer::Tokenizer;
-    use crate::tokenizer::{BoxTokenStream, Token};
+    use crate::tokenizer::tokenizer::{TokenStream, Tokenizer};
+    use crate::tokenizer::Token;

-    fn test_helper(mut tokenizer: BoxTokenStream) -> Vec<Token> {
+    fn test_helper<T: TokenStream>(mut tokenizer: T) -> Vec<Token> {
        let mut tokens: Vec<Token> = vec![];
        tokenizer.process(&mut |token: &Token| tokens.push(token.clone()));
        tokens
--- a/src/tokenizer/raw_tokenizer.rs
+++ b/src/tokenizer/raw_tokenizer.rs
@@ -1,5 +1,4 @@
 use super::{Token, TokenStream, Tokenizer};
-use crate::tokenizer::BoxTokenStream;

 /// For each value of the field, emit a single unprocessed token.
 #[derive(Clone)]
@@ -10,8 +9,10 @@ pub struct RawTokenStream {
    has_token: bool,
 }

-impl Tokenizer for RawTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+impl<'a> Tokenizer<'a> for RawTokenizer {
+    type TokenStreamImpl = RawTokenStream;
+
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
        let token = Token {
            offset_from: 0,
            offset_to: text.len(),
@@ -23,7 +24,6 @@ impl Tokenizer for RawTokenizer {
            token,
            has_token: true,
        }
-        .into()
    }
 }

--- a/src/tokenizer/remove_long.rs
+++ b/src/tokenizer/remove_long.rs
@@ -2,7 +2,7 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+//! let tokenizer = SimpleTokenizer
 //!   .filter(RemoveLongFilter::limit(5));
 //!
 //! let mut stream = tokenizer.token_stream("toolong nice");
@@ -13,7 +13,6 @@
 //! ```
 //!
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;

 /// `RemoveLongFilter` removes tokens that are longer
 /// than a given number of bytes (in UTF-8 representation).
@@ -32,27 +31,56 @@ impl RemoveLongFilter {
    }
 }

-impl<'a> RemoveLongFilterStream<'a> {
+impl<TailTokenStream> RemoveLongFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
    fn predicate(&self, token: &Token) -> bool {
        token.text.len() < self.token_length_limit
    }
-}

-impl TokenFilter for RemoveLongFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(RemoveLongFilterStream {
-            token_length_limit: self.length_limit,
-            tail: token_stream,
-        })
+    fn wrap(
+        token_length_limit: usize,
+        tail: TailTokenStream,
+    ) -> RemoveLongFilterStream<TailTokenStream> {
+        RemoveLongFilterStream {
+            token_length_limit,
+            tail,
+        }
    }
 }

-pub struct RemoveLongFilterStream<'a> {
-    token_length_limit: usize,
-    tail: BoxTokenStream<'a>,
+impl<TailTokenStream> TokenFilter<TailTokenStream> for RemoveLongFilter
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = RemoveLongFilterStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+        RemoveLongFilterStream::wrap(self.length_limit, token_stream)
+    }
 }

-impl<'a> TokenStream for RemoveLongFilterStream<'a> {
+pub struct RemoveLongFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    token_length_limit: usize,
+    tail: TailTokenStream,
+}
+
+impl<TailTokenStream> TokenStream for RemoveLongFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+
    fn advance(&mut self) -> bool {
        while self.tail.advance() {
            if self.predicate(self.tail.token()) {
@@ -61,12 +89,4 @@ impl<'a> TokenStream for RemoveLongFilterStream<'a> {
        }
        false
    }
-
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
-    }
 }
--- a/src/tokenizer/simple_tokenizer.rs
+++ b/src/tokenizer/simple_tokenizer.rs
@@ -1,4 +1,3 @@
-use super::BoxTokenStream;
 use super::{Token, TokenStream, Tokenizer};
 use std::str::CharIndices;

@@ -12,13 +11,15 @@ pub struct SimpleTokenStream<'a> {
    token: Token,
 }

-impl Tokenizer for SimpleTokenizer {
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(SimpleTokenStream {
+impl<'a> Tokenizer<'a> for SimpleTokenizer {
+    type TokenStreamImpl = SimpleTokenStream<'a>;
+
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
+        SimpleTokenStream {
            text,
            chars: text.char_indices(),
            token: Token::default(),
-        })
+        }
    }
 }

--- a/src/tokenizer/stemmer.rs
+++ b/src/tokenizer/stemmer.rs
@@ -1,5 +1,4 @@
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;
 use rust_stemmers::{self, Algorithm};

 /// Available stemmer languages.
@@ -76,22 +75,38 @@ impl Default for Stemmer {
    }
 }

-impl TokenFilter for Stemmer {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
+impl<TailTokenStream> TokenFilter<TailTokenStream> for Stemmer
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = StemmerTokenStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
        let inner_stemmer = rust_stemmers::Stemmer::create(self.stemmer_algorithm);
-        BoxTokenStream::from(StemmerTokenStream {
-            tail: token_stream,
-            stemmer: inner_stemmer,
-        })
+        StemmerTokenStream::wrap(inner_stemmer, token_stream)
    }
 }

-pub struct StemmerTokenStream<'a> {
-    tail: BoxTokenStream<'a>,
+pub struct StemmerTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    tail: TailTokenStream,
    stemmer: rust_stemmers::Stemmer,
 }

-impl<'a> TokenStream for StemmerTokenStream<'a> {
+impl<TailTokenStream> TokenStream for StemmerTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+
    fn advance(&mut self) -> bool {
        if !self.tail.advance() {
            return false;
@@ -102,12 +117,16 @@ impl<'a> TokenStream for StemmerTokenStream<'a> {
        self.token_mut().text.push_str(&stemmed_str);
        true
    }
+}

-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
+impl<TailTokenStream> StemmerTokenStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn wrap(
+        stemmer: rust_stemmers::Stemmer,
+        tail: TailTokenStream,
+    ) -> StemmerTokenStream<TailTokenStream> {
+        StemmerTokenStream { tail, stemmer }
    }
 }
--- a/src/tokenizer/stop_word_filter.rs
+++ b/src/tokenizer/stop_word_filter.rs
@@ -2,7 +2,7 @@
 //! ```rust
 //! use tantivy::tokenizer::*;
 //!
-//! let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+//! let tokenizer = SimpleTokenizer
 //!   .filter(StopWordFilter::remove(vec!["the".to_string(), "is".to_string()]));
 //!
 //! let mut stream = tokenizer.token_stream("the fox is crafty");
@@ -11,7 +11,6 @@
 //! assert!(stream.next().is_none());
 //! ```
 use super::{Token, TokenFilter, TokenStream};
-use crate::tokenizer::BoxTokenStream;
 use fnv::FnvHasher;
 use std::collections::HashSet;
 use std::hash::BuildHasherDefault;
@@ -49,27 +48,53 @@ impl StopWordFilter {
    }
 }

-pub struct StopWordFilterStream<'a> {
+pub struct StopWordFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
    words: StopWordHashSet,
-    tail: BoxTokenStream<'a>,
+    tail: TailTokenStream,
 }

-impl TokenFilter for StopWordFilter {
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
-        BoxTokenStream::from(StopWordFilterStream {
-            words: self.words.clone(),
-            tail: token_stream,
-        })
+impl<TailTokenStream> TokenFilter<TailTokenStream> for StopWordFilter
+where
+    TailTokenStream: TokenStream,
+{
+    type ResultTokenStream = StopWordFilterStream<TailTokenStream>;
+
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream {
+        StopWordFilterStream::wrap(self.words.clone(), token_stream)
    }
 }

-impl<'a> StopWordFilterStream<'a> {
+impl<TailTokenStream> StopWordFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
    fn predicate(&self, token: &Token) -> bool {
        !self.words.contains(&token.text)
    }
+
+    fn wrap(
+        words: StopWordHashSet,
+        tail: TailTokenStream,
+    ) -> StopWordFilterStream<TailTokenStream> {
+        StopWordFilterStream { words, tail }
+    }
 }

-impl<'a> TokenStream for StopWordFilterStream<'a> {
+impl<TailTokenStream> TokenStream for StopWordFilterStream<TailTokenStream>
+where
+    TailTokenStream: TokenStream,
+{
+    fn token(&self) -> &Token {
+        self.tail.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.tail.token_mut()
+    }
+
    fn advance(&mut self) -> bool {
        while self.tail.advance() {
            if self.predicate(self.tail.token()) {
@@ -78,14 +103,6 @@ impl<'a> TokenStream for StopWordFilterStream<'a> {
        }
        false
    }
-
-    fn token(&self) -> &Token {
-        self.tail.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.tail.token_mut()
-    }
 }

 impl Default for StopWordFilter {
--- a/src/tokenizer/token_stream_chain.rs
+++ b/src/tokenizer/token_stream_chain.rs
@@ -1,21 +1,23 @@
-use crate::tokenizer::{BoxTokenStream, Token, TokenStream};
-use std::ops::DerefMut;
+use crate::tokenizer::{Token, TokenStream};

 const POSITION_GAP: usize = 2;

-pub(crate) struct TokenStreamChain<'a> {
+pub(crate) struct TokenStreamChain<TTokenStream: TokenStream> {
    offsets: Vec<usize>,
-    token_streams: Vec<BoxTokenStream<'a>>,
+    token_streams: Vec<TTokenStream>,
    position_shift: usize,
    stream_idx: usize,
    token: Token,
 }

-impl<'a> TokenStreamChain<'a> {
+impl<'a, TTokenStream> TokenStreamChain<TTokenStream>
+where
+    TTokenStream: TokenStream,
+{
    pub fn new(
        offsets: Vec<usize>,
-        token_streams: Vec<BoxTokenStream<'a>>,
-    ) -> TokenStreamChain<'a> {
+        token_streams: Vec<TTokenStream>,
+    ) -> TokenStreamChain<TTokenStream> {
        TokenStreamChain {
            offsets,
            stream_idx: 0,
@@ -26,10 +28,13 @@ impl<'a> TokenStreamChain<'a> {
    }
 }

-impl<'a> TokenStream for TokenStreamChain<'a> {
+impl<'a, TTokenStream> TokenStream for TokenStreamChain<TTokenStream>
+where
+    TTokenStream: TokenStream,
+{
    fn advance(&mut self) -> bool {
        while self.stream_idx < self.token_streams.len() {
-            let token_stream = self.token_streams[self.stream_idx].deref_mut();
+            let token_stream = &mut self.token_streams[self.stream_idx];
            if token_stream.advance() {
                let token = token_stream.token();
                let offset_offset = self.offsets[self.stream_idx];
--- a/src/tokenizer/tokenized_string.rs
+++ b/src/tokenizer/tokenized_string.rs
@@ -1,4 +1,4 @@
-use crate::tokenizer::{BoxTokenStream, Token, TokenStream, TokenStreamChain};
+use crate::tokenizer::{Token, TokenStream, TokenStreamChain};
 use std::cmp::Ordering;

 /// Struct representing pre-tokenized text
@@ -41,9 +41,9 @@ impl PreTokenizedStream {
    /// Creates a TokenStream from PreTokenizedString array
    pub fn chain_tokenized_strings<'a>(
        tok_strings: &'a [&'a PreTokenizedString],
-    ) -> BoxTokenStream {
+    ) -> Box<dyn TokenStream + 'a> {
        if tok_strings.len() == 1 {
-            PreTokenizedStream::from((*tok_strings[0]).clone()).into()
+            Box::new(PreTokenizedStream::from((*tok_strings[0]).clone()))
        } else {
            let mut offsets = vec![];
            let mut total_offset = 0;
@@ -53,12 +53,11 @@ impl PreTokenizedStream {
                    total_offset += last_token.offset_to;
                }
            }
-            // TODO remove the string cloning.
-            let token_streams: Vec<BoxTokenStream<'static>> = tok_strings
+            let token_streams: Vec<_> = tok_strings
                .iter()
-                .map(|&tok_string| PreTokenizedStream::from((*tok_string).clone()).into())
+                .map(|tok_string| PreTokenizedStream::from((*tok_string).clone()))
                .collect();
-            TokenStreamChain::new(offsets, token_streams).into()
+            Box::new(TokenStreamChain::new(offsets, token_streams))
        }
    }
 }
--- a/src/tokenizer/tokenizer.rs
+++ b/src/tokenizer/tokenizer.rs
@@ -2,7 +2,6 @@ use crate::tokenizer::TokenStreamChain;
 /// The tokenizer module contains all of the tools used to process
 /// text in `tantivy`.
 use std::borrow::{Borrow, BorrowMut};
-use std::ops::{Deref, DerefMut};

 /// Token
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
@@ -34,31 +33,20 @@ impl Default for Token {
    }
 }

-/// `TextAnalyzer` tokenizes an input text into tokens and modifies the resulting `TokenStream`.
+/// `Tokenizer` are in charge of splitting text into a stream of token
+/// before indexing.
 ///
-/// It simply wraps a `Tokenizer` and a list of `TokenFilter` that are applied sequentially.
-pub struct TextAnalyzer {
-    tokenizer: Box<dyn Tokenizer>,
-    token_filters: Vec<BoxTokenFilter>,
-}
+/// See the [module documentation](./index.html) for more detail.
+///
+/// # Warning
+///
+/// This API may change to use associated types.
+pub trait Tokenizer<'a>: Sized + Clone {
+    /// Type associated to the resulting tokenstream tokenstream.
+    type TokenStreamImpl: TokenStream;

-impl<T: Tokenizer> From<T> for TextAnalyzer {
-    fn from(tokenizer: T) -> Self {
-        TextAnalyzer::new(tokenizer, Vec::new())
-    }
-}
-
-impl TextAnalyzer {
-    /// Creates a new `TextAnalyzer` given a tokenizer and a vector of `BoxTokenFilter`.
-    ///
-    /// When creating a `TextAnalyzer` from a `Tokenizer` alone, prefer using
-    /// `TextAnalyzer::from(tokenizer)`.
-    pub fn new<T: Tokenizer>(tokenizer: T, token_filters: Vec<BoxTokenFilter>) -> TextAnalyzer {
-        TextAnalyzer {
-            tokenizer: Box::new(tokenizer),
-            token_filters,
-        }
-    }
+    /// Creates a token stream for a given `str`.
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl;

    /// Appends a token filter to the current tokenizer.
    ///
@@ -70,26 +58,90 @@ impl TextAnalyzer {
    /// ```rust
    /// use tantivy::tokenizer::*;
    ///
-    /// let en_stem = TextAnalyzer::from(SimpleTokenizer)
+    /// let en_stem = SimpleTokenizer
    ///     .filter(RemoveLongFilter::limit(40))
    ///     .filter(LowerCaser)
    ///     .filter(Stemmer::default());
    /// ```
    ///
-    pub fn filter<F: Into<BoxTokenFilter>>(mut self, token_filter: F) -> Self {
-        self.token_filters.push(token_filter.into());
-        self
+    fn filter<NewFilter>(self, new_filter: NewFilter) -> ChainTokenizer<NewFilter, Self>
+    where
+        NewFilter: TokenFilter<<Self as Tokenizer<'a>>::TokenStreamImpl>,
+    {
+        ChainTokenizer {
+            head: new_filter,
+            tail: self,
+        }
+    }
+}
+
+/// A boxed tokenizer
+trait BoxedTokenizerTrait: Send + Sync {
+    /// Tokenize a `&str`
+    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a>;
+
+    /// Tokenize an array`&str`
+    ///
+    /// The resulting `TokenStream` is equivalent to what would be obtained if the &str were
+    /// one concatenated `&str`, with an artificial position gap of `2` between the different fields
+    /// to prevent accidental `PhraseQuery` to match accross two terms.
+    fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<dyn TokenStream + 'b>;
+
+    /// Return a boxed clone of the tokenizer
+    fn boxed_clone(&self) -> BoxedTokenizer;
+}
+
+/// A boxed tokenizer
+pub struct BoxedTokenizer(Box<dyn BoxedTokenizerTrait>);
+
+impl<T> From<T> for BoxedTokenizer
+where
+    T: 'static + Send + Sync + for<'a> Tokenizer<'a>,
+{
+    fn from(tokenizer: T) -> BoxedTokenizer {
+        BoxedTokenizer(Box::new(BoxableTokenizer(tokenizer)))
+    }
+}
+
+impl BoxedTokenizer {
+    /// Tokenize a `&str`
+    pub fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
+        self.0.token_stream(text)
    }

    /// Tokenize an array`&str`
    ///
-    /// The resulting `BoxTokenStream` is equivalent to what would be obtained if the &str were
+    /// The resulting `TokenStream` is equivalent to what would be obtained if the &str were
    /// one concatenated `&str`, with an artificial position gap of `2` between the different fields
    /// to prevent accidental `PhraseQuery` to match accross two terms.
-    pub fn token_stream_texts<'a>(&self, texts: &'a [&'a str]) -> BoxTokenStream<'a> {
+    pub fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<dyn TokenStream + 'b> {
+        self.0.token_stream_texts(texts)
+    }
+}
+
+impl Clone for BoxedTokenizer {
+    fn clone(&self) -> BoxedTokenizer {
+        self.0.boxed_clone()
+    }
+}
+
+#[derive(Clone)]
+struct BoxableTokenizer<A>(A)
+where
+    A: for<'a> Tokenizer<'a> + Send + Sync;
+
+impl<A> BoxedTokenizerTrait for BoxableTokenizer<A>
+where
+    A: 'static + Send + Sync + for<'a> Tokenizer<'a>,
+{
+    fn token_stream<'a>(&self, text: &'a str) -> Box<dyn TokenStream + 'a> {
+        Box::new(self.0.token_stream(text))
+    }
+
+    fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<dyn TokenStream + 'b> {
        assert!(!texts.is_empty());
        if texts.len() == 1 {
-            self.token_stream(texts[0])
+            Box::new(self.0.token_stream(texts[0]))
        } else {
            let mut offsets = vec![];
            let mut total_offset = 0;
@@ -97,124 +149,34 @@ impl TextAnalyzer {
                offsets.push(total_offset);
                total_offset += text.len();
            }
-            let token_streams: Vec<BoxTokenStream<'a>> = texts
-                .iter()
-                .cloned()
-                .map(|text| self.token_stream(text))
-                .collect();
-            From::from(TokenStreamChain::new(offsets, token_streams))
+            let token_streams: Vec<_> =
+                texts.iter().map(|text| self.0.token_stream(text)).collect();
+            Box::new(TokenStreamChain::new(offsets, token_streams))
        }
    }

-    /// Creates a token stream for a given `str`.
-    pub fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
-        let mut token_stream = self.tokenizer.token_stream(text);
-        for token_filter in &self.token_filters {
-            token_stream = token_filter.transform(token_stream);
-        }
-        token_stream
+    fn boxed_clone(&self) -> BoxedTokenizer {
+        self.0.clone().into()
    }
 }

-impl Clone for TextAnalyzer {
-    fn clone(&self) -> Self {
-        TextAnalyzer {
-            tokenizer: self.tokenizer.box_clone(),
-            token_filters: self
-                .token_filters
-                .iter()
-                .map(|token_filter| token_filter.box_clone())
-                .collect(),
-        }
-    }
-}
-
-/// `Tokenizer` are in charge of splitting text into a stream of token
-/// before indexing.
-///
-/// See the [module documentation](./index.html) for more detail.
-///
-/// # Warning
-///
-/// This API may change to use associated types.
-pub trait Tokenizer: 'static + Send + Sync + TokenizerClone {
-    /// Creates a token stream for a given `str`.
-    fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a>;
-}
-
-pub trait TokenizerClone {
-    fn box_clone(&self) -> Box<dyn Tokenizer>;
-}
-
-impl<T: Tokenizer + Clone> TokenizerClone for T {
-    fn box_clone(&self) -> Box<dyn Tokenizer> {
-        Box::new(self.clone())
-    }
-}
-
-impl<'a> TokenStream for Box<dyn TokenStream + 'a> {
+impl<'b> TokenStream for Box<dyn TokenStream + 'b> {
    fn advance(&mut self) -> bool {
        let token_stream: &mut dyn TokenStream = self.borrow_mut();
        token_stream.advance()
    }

-    fn token<'b>(&'b self) -> &'b Token {
-        let token_stream: &'b (dyn TokenStream + 'a) = self.borrow();
+    fn token(&self) -> &Token {
+        let token_stream: &dyn TokenStream = self.borrow();
        token_stream.token()
    }

-    fn token_mut<'b>(&'b mut self) -> &'b mut Token {
-        let token_stream: &'b mut (dyn TokenStream + 'a) = self.borrow_mut();
+    fn token_mut(&mut self) -> &mut Token {
+        let token_stream: &mut dyn TokenStream = self.borrow_mut();
        token_stream.token_mut()
    }
 }

-/// Simple wrapper of `Box<dyn TokenStream + 'a>`.
-///
-/// See `TokenStream` for more information.
-pub struct BoxTokenStream<'a>(Box<dyn TokenStream + 'a>);
-
-impl<'a, T> From<T> for BoxTokenStream<'a>
-where
-    T: TokenStream + 'a,
-{
-    fn from(token_stream: T) -> BoxTokenStream<'a> {
-        BoxTokenStream(Box::new(token_stream))
-    }
-}
-
-impl<'a> Deref for BoxTokenStream<'a> {
-    type Target = dyn TokenStream + 'a;
-
-    fn deref(&self) -> &Self::Target {
-        &*self.0
-    }
-}
-impl<'a> DerefMut for BoxTokenStream<'a> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut *self.0
-    }
-}
-
-/// Simple wrapper of `Box<dyn TokenFilter + 'a>`.
-///
-/// See `TokenStream` for more information.
-pub struct BoxTokenFilter(Box<dyn TokenFilter>);
-
-impl Deref for BoxTokenFilter {
-    type Target = dyn TokenFilter;
-
-    fn deref(&self) -> &dyn TokenFilter {
-        &*self.0
-    }
-}
-
-impl<T: TokenFilter> From<T> for BoxTokenFilter {
-    fn from(tokenizer: T) -> BoxTokenFilter {
-        BoxTokenFilter(Box::new(tokenizer))
-    }
-}
-
 /// `TokenStream` is the result of the tokenization.
 ///
 /// It consists consumable stream of `Token`s.
@@ -224,7 +186,7 @@ impl<T: TokenFilter> From<T> for BoxTokenFilter {
 /// ```
 /// use tantivy::tokenizer::*;
 ///
-/// let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+/// let tokenizer = SimpleTokenizer
 ///        .filter(RemoveLongFilter::limit(40))
 ///        .filter(LowerCaser);
 /// let mut token_stream = tokenizer.token_stream("Hello, happy tax payer");
@@ -263,7 +225,7 @@ pub trait TokenStream {
    /// ```
    /// use tantivy::tokenizer::*;
    ///
-    /// let tokenizer = TextAnalyzer::from(SimpleTokenizer)
+    /// let tokenizer = SimpleTokenizer
    ///       .filter(RemoveLongFilter::limit(40))
    ///       .filter(LowerCaser);
    /// let mut token_stream = tokenizer.token_stream("Hello, happy tax payer");
@@ -281,8 +243,6 @@ pub trait TokenStream {

    /// Helper function to consume the entire `TokenStream`
    /// and push the tokens to a sink function.
-    ///
-    /// Remove this.
    fn process(&mut self, sink: &mut dyn FnMut(&Token)) -> u32 {
        let mut num_tokens_pushed = 0u32;
        while self.advance() {
@@ -293,20 +253,33 @@ pub trait TokenStream {
    }
 }

-pub trait TokenFilterClone {
-    fn box_clone(&self) -> BoxTokenFilter;
+#[derive(Clone)]
+pub struct ChainTokenizer<HeadTokenFilterFactory, TailTokenizer> {
+    head: HeadTokenFilterFactory,
+    tail: TailTokenizer,
+}
+
+impl<'a, HeadTokenFilterFactory, TailTokenizer> Tokenizer<'a>
+    for ChainTokenizer<HeadTokenFilterFactory, TailTokenizer>
+where
+    HeadTokenFilterFactory: TokenFilter<TailTokenizer::TokenStreamImpl>,
+    TailTokenizer: Tokenizer<'a>,
+{
+    type TokenStreamImpl = HeadTokenFilterFactory::ResultTokenStream;
+
+    fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
+        let tail_token_stream = self.tail.token_stream(text);
+        self.head.transform(tail_token_stream)
+    }
 }

 /// Trait for the pluggable components of `Tokenizer`s.
-pub trait TokenFilter: 'static + Send + Sync + TokenFilterClone {
-    /// Wraps a token stream and returns the modified one.
-    fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a>;
-}
+pub trait TokenFilter<TailTokenStream: TokenStream>: Clone {
+    /// The resulting `TokenStream` type.
+    type ResultTokenStream: TokenStream;

-impl<T: TokenFilter + Clone> TokenFilterClone for T {
-    fn box_clone(&self) -> BoxTokenFilter {
-        BoxTokenFilter::from(self.clone())
-    }
+    /// Wraps a token stream and returns the modified one.
+    fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream;
 }

 #[cfg(test)]
--- a/src/tokenizer/tokenizer_manager.rs
+++ b/src/tokenizer/tokenizer_manager.rs
@@ -1,10 +1,11 @@
 use crate::tokenizer::stemmer::Language;
-use crate::tokenizer::tokenizer::TextAnalyzer;
+use crate::tokenizer::BoxedTokenizer;
 use crate::tokenizer::LowerCaser;
 use crate::tokenizer::RawTokenizer;
 use crate::tokenizer::RemoveLongFilter;
 use crate::tokenizer::SimpleTokenizer;
 use crate::tokenizer::Stemmer;
+use crate::tokenizer::Tokenizer;
 use std::collections::HashMap;
 use std::sync::{Arc, RwLock};

@@ -22,16 +23,16 @@ use std::sync::{Arc, RwLock};
 ///  search engine.
 #[derive(Clone)]
 pub struct TokenizerManager {
-    tokenizers: Arc<RwLock<HashMap<String, TextAnalyzer>>>,
+    tokenizers: Arc<RwLock<HashMap<String, BoxedTokenizer>>>,
 }

 impl TokenizerManager {
    /// Registers a new tokenizer associated with a given name.
-    pub fn register<T>(&self, tokenizer_name: &str, tokenizer: T)
+    pub fn register<A>(&self, tokenizer_name: &str, tokenizer: A)
    where
-        TextAnalyzer: From<T>,
+        A: Into<BoxedTokenizer>,
    {
-        let boxed_tokenizer: TextAnalyzer = TextAnalyzer::from(tokenizer);
+        let boxed_tokenizer = tokenizer.into();
        self.tokenizers
            .write()
            .expect("Acquiring the lock should never fail")
@@ -39,7 +40,7 @@ impl TokenizerManager {
    }

    /// Accessing a tokenizer given its name.
-    pub fn get(&self, tokenizer_name: &str) -> Option<TextAnalyzer> {
+    pub fn get(&self, tokenizer_name: &str) -> Option<BoxedTokenizer> {
        self.tokenizers
            .read()
            .expect("Acquiring the lock should never fail")
@@ -61,13 +62,13 @@ impl Default for TokenizerManager {
        manager.register("raw", RawTokenizer);
        manager.register(
            "default",
-            TextAnalyzer::from(SimpleTokenizer)
+            SimpleTokenizer
                .filter(RemoveLongFilter::limit(40))
                .filter(LowerCaser),
        );
        manager.register(
            "en_stem",
-            TextAnalyzer::from(SimpleTokenizer)
+            SimpleTokenizer
                .filter(RemoveLongFilter::limit(40))
                .filter(LowerCaser)
                .filter(Stemmer::new(Language::English)),
--- a/tests/failpoints/mod.rs
+++ b/tests/failpoints/mod.rs
@@ -1,6 +1,8 @@
 use fail;
 use std::path::Path;
-use tantivy::directory::{Directory, ManagedDirectory, RAMDirectory, TerminatingWrite};
+use tantivy::directory::{
+    Directory, ManagedDirectory, RAMDirectory, ReadOnlyDirectory, TerminatingWrite,
+};
 use tantivy::doc;
 use tantivy::schema::{Schema, TEXT};
 use tantivy::{Index, Term};
Author	SHA1	Message	Date
Paul Masurel	d7973892a2	extra commit	2019-12-27 22:53:04 +09:00
Paul Masurel	cd7484c035	Added ReadOnlyDirectory and implemented Bundle Directory	2019-12-27 12:05:39 +09:00
Paul Masurel	7ed6bc8718	Added serialize to bundle in the RAMDirectory.	2019-12-26 10:06:52 +09:00