Rebased

src/core/index.rs

@@ -282,7 +282,7 @@ impl Index {
                 TantivyError::LockFailure(
                     err,
                     Some(
-                        "Failed to acquire index lock. If you are using\
+                        "Failed to acquire index lock. If you are using \
                          a regular directory, this means there is already an \
                          `IndexWriter` working on this `Directory`, in this process \
                          or in a different process."
@@ -438,7 +438,7 @@ mod tests {
     }
 
     #[test]
-    fn create_should_wipeoff_existing() {
+    fn create_should_wipe_off_existing() {
         let directory = RAMDirectory::create();
         assert!(Index::create(directory.clone(), throw_away_schema()).is_ok());
         assert!(Index::exists(&directory));
@@ -545,7 +545,12 @@ mod tests {
         }
     }
 
-    fn test_index_on_commit_reload_policy_aux(field: Field, mut reader_index: Index, index: &Index, reader: &IndexReader) {
+    fn test_index_on_commit_reload_policy_aux(
+        field: Field,
+        mut reader_index: Index,
+        index: &Index,
+        reader: &IndexReader,
+    ) {
         let (sender, receiver) = crossbeam::channel::unbounded();
         let _watch_handle = reader_index.directory_mut().watch(Box::new(move || {
             let _ = sender.send(());

src/core/segment.rs

@@ -9,7 +9,7 @@ use crate::indexer::segment_serializer::SegmentSerializer;
 use crate::schema::Schema;
 use crate::Opstamp;
 use std::fmt;
-use std::ops::Deref;
+use std::ops::{Deref, DerefMut};
 use std::path::PathBuf;
 
 #[derive(Clone)]

src/indexer/index_writer.rs

@@ -216,9 +216,9 @@ fn index_documents(
     tokenizers: &TokenizerManager,
     mut delete_cursor: DeleteCursor,
 ) -> crate::Result<bool> {
-    let mut segment_writer =
-        SegmentWriter::for_segment(memory_budget, segment.clone(), tokenizers)?;
     let schema = segment.schema();
+    let mut segment_writer =
+        SegmentWriter::for_segment(memory_budget, segment.clone(), &schema, tokenizers)?;
     for document_group in grouped_document_iterator {
         for doc in document_group {
             segment_writer.add_document(doc, &schema)?;

src/indexer/segment_writer.rs

@@ -11,10 +11,10 @@ use crate::schema::Schema;
 use crate::schema::Term;
 use crate::schema::Value;
 use crate::schema::{Field, FieldEntry};
-use crate::tokenizer::FacetTokenizer;
-use crate::tokenizer::PreTokenizedStream;
-use crate::tokenizer::{BoxedTokenizer, TokenizerManager};
-use crate::tokenizer::{TokenStream, TokenStreamChain, Tokenizer};
+use crate::tokenizer::TokenizerManager;
+use crate::tokenizer::{BoxTokenStream, FacetTokenizer};
+use crate::tokenizer::{PreTokenizedStream, TextAnalyzer};
+use crate::tokenizer::{TokenStreamChain, Tokenizer};
 use crate::DocId;
 use crate::Opstamp;
 use std::io;
@@ -65,10 +65,12 @@ impl SegmentWriter {
         memory_budget: usize,
         mut segment: Segment,
         schema: &Schema,
+        tokenizer_manager: &TokenizerManager,
     ) -> crate::Result<SegmentWriter> {
         let table_num_bits = initial_table_size(memory_budget)?;
         let segment_serializer = SegmentSerializer::for_segment(&mut segment)?;
         let multifield_postings = MultiFieldPostingsWriter::new(&schema, table_num_bits);
+
         let tokenizers = schema
             .fields()
             .map(
@@ -77,7 +79,7 @@ impl SegmentWriter {
                         .get_indexing_options()
                         .and_then(|text_index_option| {
                             let tokenizer_name = &text_index_option.tokenizer();
-                            tokenizers.get(tokenizer_name)
+                            tokenizer_manager.get(tokenizer_name)
                         }),
                     _ => None,
                 },

src/lib.rs

@@ -121,6 +121,7 @@ mod functional_test;
 mod macros;
 
 pub use crate::error::TantivyError;
+pub use crate::error::TantivyError as Error;
 pub use chrono;
 
 /// Tantivy result.

src/postings/mod.rs

@@ -220,7 +220,8 @@ pub mod tests {
 
         {
             let mut segment_writer =
-                SegmentWriter::for_segment(3_000_000, segment.clone(), index.tokenizers()).unwrap();
+                SegmentWriter::for_segment(3_000_000, segment.clone(), &schema, index.tokenizers())
+                    .unwrap();
             {
                 let mut doc = Document::default();
                 // checking that position works if the field has two values

src/reader/mod.rs

@@ -5,17 +5,16 @@ mod pool;
 use self::meta_file_reader::MetaFileIndexReader;
 pub use self::meta_file_reader::{IndexReaderBuilder, ReloadPolicy};
 pub use self::pool::LeasedItem;
-use self::pool::Pool;
-use crate::core::Segment;
-use crate::directory::Directory;
-use crate::directory::WatchHandle;
-use crate::directory::META_LOCK;
-use crate::reader::index_writer_reader::NRTReader;
-use crate::Index;
-use crate::Searcher;
-/*
-<<<<<<< HEAD
 
+
+
+
+
+pub(crate) use crate::reader::index_writer_reader::NRTReader;
+
+use crate::Searcher;
+
+/*
 //
 //enum SegmentSource {
 //    FromMetaFile,