add From impl to BoxTokenStream

Cargo.toml

@@ -23,8 +23,8 @@ once_cell = "1.10.0"
 regex = { version = "1.5.5", default-features = false, features = ["std", "unicode"] }
 aho-corasick = "1.0"
 tantivy-fst = "0.4.0"
-memmap2 = { version = "0.7.1", optional = true }
-lz4_flex = { version = "0.11", default-features = false, optional = true }
+memmap2 = { version = "0.6.0", optional = true }
+lz4_flex = { version = "0.10", default-features = false, features = ["checked-decode"], optional = true }
 brotli = { version = "3.3.4", optional = true }
 zstd = { version = "0.12", optional = true, default-features = false }
 snap = { version = "1.0.5", optional = true }
@@ -44,7 +44,7 @@ census = "0.4.0"
 rustc-hash = "1.1.0"
 thiserror = "1.0.30"
 htmlescape = "0.3.1"
-fail = { version = "0.5.0", optional = true }
+fail = "0.5.0"
 murmurhash32 = "0.3.0"
 time = { version = "0.3.10", features = ["serde-well-known"] }
 smallvec = "1.8.0"
@@ -112,7 +112,7 @@ lz4-compression = ["lz4_flex"]
 snappy-compression = ["snap"]
 zstd-compression = ["zstd"]
 
-failpoints = ["fail", "fail/failpoints"]
+failpoints = ["fail/failpoints"]
 unstable = [] # useful for benches.
 
 quickwit = ["sstable", "futures-util"]

bitpacker/src/filter_vec/mod.rs

@@ -1,6 +1,6 @@
 use std::ops::RangeInclusive;
 
-#[cfg(target_arch = "x86_64")]
+#[cfg(any(target_arch = "x86_64"))]
 mod avx2;
 
 mod scalar;

columnar/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 license = "MIT"
 homepage = "https://github.com/quickwit-oss/tantivy"
 repository = "https://github.com/quickwit-oss/tantivy"
-description = "column oriented storage for tantivy"
+desciption = "column oriented storage for tantivy"
 categories = ["database-implementations", "data-structures", "compression"]
 
 [dependencies]

columnar/src/column_values/mod.rs

@@ -2,7 +2,7 @@
 
 //! # `fastfield_codecs`
 //!
-//! - Columnar storage of data for tantivy [`crate::Column`].
+//! - Columnar storage of data for tantivy [`Column`].
 //! - Encode data in different codecs.
 //! - Monotonically map values to u64/u128
 

columnar/src/columnar/merge/merge_mapping.rs

@@ -52,8 +52,8 @@ pub enum MergeRowOrder {
     /// Columnar tables are simply stacked one above the other.
     /// If the i-th columnar_readers has n_rows_i rows, then
     /// in the resulting columnar,
-    /// rows [r0..n_row_0) contains the row of `columnar_readers[0]`, in ordder
-    /// rows [n_row_0..n_row_0 + n_row_1 contains the row of `columnar_readers[1]`, in order.
+    /// rows [r0..n_row_0) contains the row of columnar_readers[0], in ordder
+    /// rows [n_row_0..n_row_0 + n_row_1 contains the row of columnar_readers[1], in order.
     /// ..
     /// No documents is deleted.
     Stack(StackMergeOrder),

examples/custom_tokenizer.rs

@@ -53,7 +53,7 @@ fn main() -> tantivy::Result<()> {
     // this will store tokens of 3 characters each
     index
         .tokenizers()
-        .register("ngram3", NgramTokenizer::new(3, 3, false).unwrap());
+        .register("ngram3", NgramTokenizer::new(3, 3, false));
 
     // To insert document we need an index writer.
     // There must be only one writer at a time.

examples/warmer.rs

@@ -6,14 +6,12 @@ use tantivy::collector::TopDocs;
 use tantivy::query::QueryParser;
 use tantivy::schema::{Schema, FAST, TEXT};
 use tantivy::{
-    doc, DocAddress, DocId, Index, Opstamp, Searcher, SearcherGeneration, SegmentId, SegmentReader,
-    Warmer,
+    doc, DocAddress, DocId, Index, IndexReader, Opstamp, Searcher, SearcherGeneration, SegmentId,
+    SegmentReader, Warmer,
 };
 
 // This example shows how warmers can be used to
-// load values from an external sources and
-// tie their lifecycle to that of the index segments
-// using the Warmer API.
+// load a values from an external sources using the Warmer API.
 //
 // In this example, we assume an e-commerce search engine.
 
@@ -25,11 +23,9 @@ pub trait PriceFetcher: Send + Sync + 'static {
     fn fetch_prices(&self, product_ids: &[ProductId]) -> Vec<Price>;
 }
 
-type SegmentKey = (SegmentId, Option<Opstamp>);
-
 struct DynamicPriceColumn {
     field: String,
-    price_cache: RwLock<HashMap<SegmentKey, Arc<Vec<Price>>>>,
+    price_cache: RwLock<HashMap<(SegmentId, Option<Opstamp>), Arc<Vec<Price>>>>,
     price_fetcher: Box<dyn PriceFetcher>,
 }
 
@@ -50,6 +46,7 @@ impl DynamicPriceColumn {
 impl Warmer for DynamicPriceColumn {
     fn warm(&self, searcher: &Searcher) -> tantivy::Result<()> {
         for segment in searcher.segment_readers() {
+            let key = (segment.segment_id(), segment.delete_opstamp());
             let product_id_reader = segment
                 .fast_fields()
                 .u64(&self.field)?
@@ -58,40 +55,37 @@ impl Warmer for DynamicPriceColumn {
                 .doc_ids_alive()
                 .map(|doc| product_id_reader.get_val(doc))
                 .collect();
-
-            let mut prices = self.price_fetcher.fetch_prices(&product_ids).into_iter();
-
-            let prices: Vec<Price> = (0..segment.max_doc())
-                .map(|doc| {
-                    if !segment.is_deleted(doc) {
-                        prices.next().unwrap()
-                    } else {
-                        0
-                    }
-                })
-                .collect();
-
-            let key = (segment.segment_id(), segment.delete_opstamp());
+            let mut prices_it = self.price_fetcher.fetch_prices(&product_ids).into_iter();
+            let mut price_vals: Vec<Price> = Vec::new();
+            for doc in 0..segment.max_doc() {
+                if segment.is_deleted(doc) {
+                    price_vals.push(0);
+                } else {
+                    price_vals.push(prices_it.next().unwrap())
+                }
+            }
             self.price_cache
                 .write()
                 .unwrap()
-                .insert(key, Arc::new(prices));
+                .insert(key, Arc::new(price_vals));
         }
-
         Ok(())
     }
 
     fn garbage_collect(&self, live_generations: &[&SearcherGeneration]) {
-        let live_keys: HashSet<SegmentKey> = live_generations
-            .iter()
-            .flat_map(|gen| gen.segments())
-            .map(|(&segment_id, &opstamp)| (segment_id, opstamp))
+        let live_segment_id_and_delete_ops: HashSet<(SegmentId, Option<Opstamp>)> =
+            live_generations
+                .iter()
+                .flat_map(|gen| gen.segments())
+                .map(|(&segment_id, &opstamp)| (segment_id, opstamp))
+                .collect();
+        let mut price_cache_wrt = self.price_cache.write().unwrap();
+        // let price_cache = std::mem::take(&mut *price_cache_wrt);
+        // Drain would be nicer here.
+        *price_cache_wrt = std::mem::take(&mut *price_cache_wrt)
+            .into_iter()
+            .filter(|(seg_id_and_op, _)| !live_segment_id_and_delete_ops.contains(seg_id_and_op))
             .collect();
-
-        self.price_cache
-            .write()
-            .unwrap()
-            .retain(|key, _| live_keys.contains(key));
     }
 }
 
@@ -106,17 +100,17 @@ pub struct ExternalPriceTable {
 
 impl ExternalPriceTable {
     pub fn update_price(&self, product_id: ProductId, price: Price) {
-        self.prices.write().unwrap().insert(product_id, price);
+        let mut prices_wrt = self.prices.write().unwrap();
+        prices_wrt.insert(product_id, price);
     }
 }
 
 impl PriceFetcher for ExternalPriceTable {
     fn fetch_prices(&self, product_ids: &[ProductId]) -> Vec<Price> {
-        let prices = self.prices.read().unwrap();
-
+        let prices_read = self.prices.read().unwrap();
         product_ids
             .iter()
-            .map(|product_id| prices.get(product_id).cloned().unwrap_or(0))
+            .map(|product_id| prices_read.get(product_id).cloned().unwrap_or(0))
             .collect()
     }
 }
@@ -149,8 +143,11 @@ fn main() -> tantivy::Result<()> {
     writer.add_document(doc!(product_id=>SNEAKERS, text=>"uber sweet sneakers"))?;
     writer.commit()?;
 
-    let warmers = vec![Arc::downgrade(&price_dynamic_column) as Weak<dyn Warmer>];
-    let reader = index.reader_builder().warmers(warmers).try_into()?;
+    let warmers: Vec<Weak<dyn Warmer>> = vec![Arc::downgrade(
+        &(price_dynamic_column.clone() as Arc<dyn Warmer>),
+    )];
+    let reader: IndexReader = index.reader_builder().warmers(warmers).try_into()?;
+    reader.reload()?;
 
     let query_parser = QueryParser::for_index(&index, vec![text]);
     let query = query_parser.parse_query("cooking")?;

src/aggregation/bucket/mod.rs

@@ -15,12 +15,6 @@
 //! Results of final buckets are [`BucketResult`](super::agg_result::BucketResult).
 //! Results of intermediate buckets are
 //! [`IntermediateBucketResult`](super::intermediate_agg_result::IntermediateBucketResult)
-//!
-//! ## Supported Bucket Aggregations
-//! - [Histogram](HistogramAggregation)
-//! - [DateHistogram](DateHistogramAggregationReq)
-//! - [Range](RangeAggregation)
-//! - [Terms](TermsAggregation)
 
 mod histogram;
 mod range;

src/aggregation/metric/mod.rs

@@ -6,15 +6,6 @@
 //! Some aggregations output a single numeric metric (e.g. Average) and are called
 //! single-value numeric metrics aggregation, others generate multiple metrics (e.g. Stats) and are
 //! called multi-value numeric metrics aggregation.
-//!
-//! ## Supported Metric Aggregations
-//! - [Average](AverageAggregation)
-//! - [Stats](StatsAggregation)
-//! - [Min](MinAggregation)
-//! - [Max](MaxAggregation)
-//! - [Sum](SumAggregation)
-//! - [Count](CountAggregation)
-//! - [Percentiles](PercentilesAggregationReq)
 
 mod average;
 mod count;

src/collector/filter_collector_wrapper.rs

@@ -6,35 +6,32 @@
 //
 // Of course, you can have a look at the tantivy's built-in collectors
 // such as the `CountCollector` for more examples.
-use std::fmt::Debug;
-use std::marker::PhantomData;
 
-use columnar::{BytesColumn, Column, DynamicColumn, HasAssociatedColumnType};
+// ---
+// Importing tantivy...
+use std::marker::PhantomData;
+use std::sync::Arc;
+
+use columnar::{ColumnValues, DynamicColumn, HasAssociatedColumnType};
 
 use crate::collector::{Collector, SegmentCollector};
 use crate::schema::Field;
-use crate::{DocId, Score, SegmentReader, TantivyError};
+use crate::{Score, SegmentReader, TantivyError};
 
 /// The `FilterCollector` filters docs using a fast field value and a predicate.
-///
-/// Only the documents containing at least one value for which the predicate returns `true`
-/// will be passed on to the next collector.
-///
-/// In other words,
-/// - documents with no values are filtered out.
-/// - documents with several values are accepted if at least one value matches the predicate.
-///
+/// Only the documents for which the predicate returned "true" will be passed on to the next
+/// collector.
 ///
 /// ```rust
 /// use tantivy::collector::{TopDocs, FilterCollector};
 /// use tantivy::query::QueryParser;
-/// use tantivy::schema::{Schema, TEXT, FAST};
+/// use tantivy::schema::{Schema, TEXT, INDEXED, FAST};
 /// use tantivy::{doc, DocAddress, Index};
 ///
 /// # fn main() -> tantivy::Result<()> {
 /// let mut schema_builder = Schema::builder();
 /// let title = schema_builder.add_text_field("title", TEXT);
-/// let price = schema_builder.add_u64_field("price", FAST);
+/// let price = schema_builder.add_u64_field("price", INDEXED | FAST);
 /// let schema = schema_builder.build();
 /// let index = Index::create_in_ram(schema);
 ///
@@ -50,24 +47,20 @@ use crate::{DocId, Score, SegmentReader, TantivyError};
 ///
 /// let query_parser = QueryParser::for_index(&index, vec![title]);
 /// let query = query_parser.parse_query("diary")?;
-/// let no_filter_collector = FilterCollector::new(price, |value: u64| value > 20_120u64, TopDocs::with_limit(2));
+/// let no_filter_collector = FilterCollector::new(price, &|value: u64| value > 20_120u64, TopDocs::with_limit(2));
 /// let top_docs = searcher.search(&query, &no_filter_collector)?;
 ///
 /// assert_eq!(top_docs.len(), 1);
 /// assert_eq!(top_docs[0].1, DocAddress::new(0, 1));
 ///
-/// let filter_all_collector: FilterCollector<_, _, u64> = FilterCollector::new(price, |value| value < 5u64, TopDocs::with_limit(2));
+/// let filter_all_collector: FilterCollector<_, _, u64> = FilterCollector::new(price, &|value| value < 5u64, TopDocs::with_limit(2));
 /// let filtered_top_docs = searcher.search(&query, &filter_all_collector)?;
 ///
 /// assert_eq!(filtered_top_docs.len(), 0);
 /// # Ok(())
 /// # }
 /// ```
-///
-/// Note that this is limited to fast fields which implement the
-/// [`FastValue`][crate::fastfield::FastValue] trait, e.g. `u64` but not `&[u8]`.
-/// To filter based on a bytes fast field, use a [`BytesFilterCollector`] instead.
-pub struct FilterCollector<TCollector, TPredicate, TPredicateValue>
+pub struct FilterCollector<TCollector, TPredicate, TPredicateValue: Default>
 where TPredicate: 'static + Clone
 {
     field: Field,
@@ -76,15 +69,19 @@ where TPredicate: 'static + Clone
     t_predicate_value: PhantomData<TPredicateValue>,
 }
 
-impl<TCollector, TPredicate, TPredicateValue>
+impl<TCollector, TPredicate, TPredicateValue: Default>
     FilterCollector<TCollector, TPredicate, TPredicateValue>
 where
     TCollector: Collector + Send + Sync,
     TPredicate: Fn(TPredicateValue) -> bool + Send + Sync + Clone,
 {
-    /// Create a new `FilterCollector`.
-    pub fn new(field: Field, predicate: TPredicate, collector: TCollector) -> Self {
-        Self {
+    /// Create a new FilterCollector.
+    pub fn new(
+        field: Field,
+        predicate: TPredicate,
+        collector: TCollector,
+    ) -> FilterCollector<TCollector, TPredicate, TPredicateValue> {
+        FilterCollector {
             field,
             predicate,
             collector,
@@ -93,7 +90,7 @@ where
     }
 }
 
-impl<TCollector, TPredicate, TPredicateValue> Collector
+impl<TCollector, TPredicate, TPredicateValue: Default> Collector
     for FilterCollector<TCollector, TPredicate, TPredicateValue>
 where
     TCollector: Collector + Send + Sync,
@@ -101,6 +98,8 @@ where
     TPredicateValue: HasAssociatedColumnType,
     DynamicColumn: Into<Option<columnar::Column<TPredicateValue>>>,
 {
+    // That's the type of our result.
+    // Our standard deviation will be a float.
     type Fruit = TCollector::Fruit;
 
     type Child = FilterSegmentCollector<TCollector::Child, TPredicate, TPredicateValue>;
@@ -109,7 +108,7 @@ where
         &self,
         segment_local_id: u32,
         segment_reader: &SegmentReader,
-    ) -> crate::Result<Self::Child> {
+    ) -> crate::Result<FilterSegmentCollector<TCollector::Child, TPredicate, TPredicateValue>> {
         let schema = segment_reader.schema();
         let field_entry = schema.get_field_entry(self.field);
         if !field_entry.is_fast() {
@@ -119,16 +118,16 @@ where
             )));
         }
 
-        let column_opt = segment_reader
+        let fast_field_reader = segment_reader
             .fast_fields()
-            .column_opt(field_entry.name())?;
+            .column_first_or_default(schema.get_field_name(self.field))?;
 
         let segment_collector = self
             .collector
             .for_segment(segment_local_id, segment_reader)?;
 
         Ok(FilterSegmentCollector {
-            column_opt,
+            fast_field_reader,
             segment_collector,
             predicate: self.predicate.clone(),
             t_predicate_value: PhantomData,
@@ -147,208 +146,35 @@ where
     }
 }
 
-pub struct FilterSegmentCollector<TSegmentCollector, TPredicate, TPredicateValue> {
-    column_opt: Option<Column<TPredicateValue>>,
+pub struct FilterSegmentCollector<TSegmentCollector, TPredicate, TPredicateValue>
+where
+    TPredicate: 'static,
+    DynamicColumn: Into<Option<columnar::Column<TPredicateValue>>>,
+{
+    fast_field_reader: Arc<dyn ColumnValues<TPredicateValue>>,
     segment_collector: TSegmentCollector,
     predicate: TPredicate,
     t_predicate_value: PhantomData<TPredicateValue>,
 }
 
-impl<TSegmentCollector, TPredicate, TPredicateValue>
-    FilterSegmentCollector<TSegmentCollector, TPredicate, TPredicateValue>
-where
-    TPredicateValue: PartialOrd + Copy + Debug + Send + Sync + 'static,
-    TPredicate: 'static + Fn(TPredicateValue) -> bool + Send + Sync,
-{
-    #[inline]
-    fn accept_document(&self, doc_id: DocId) -> bool {
-        if let Some(column) = &self.column_opt {
-            for val in column.values_for_doc(doc_id) {
-                if (self.predicate)(val) {
-                    return true;
-                }
-            }
-        }
-        false
-    }
-}
-
 impl<TSegmentCollector, TPredicate, TPredicateValue> SegmentCollector
     for FilterSegmentCollector<TSegmentCollector, TPredicate, TPredicateValue>
 where
     TSegmentCollector: SegmentCollector,
     TPredicateValue: HasAssociatedColumnType,
-    TPredicate: 'static + Fn(TPredicateValue) -> bool + Send + Sync, /* DynamicColumn: Into<Option<columnar::Column<TPredicateValue>>> */
+    TPredicate: 'static + Fn(TPredicateValue) -> bool + Send + Sync,
+    DynamicColumn: Into<Option<columnar::Column<TPredicateValue>>>,
 {
     type Fruit = TSegmentCollector::Fruit;
 
     fn collect(&mut self, doc: u32, score: Score) {
-        if self.accept_document(doc) {
-            self.segment_collector.collect(doc, score);
+        let value = self.fast_field_reader.get_val(doc);
+        if (self.predicate)(value) {
+            self.segment_collector.collect(doc, score)
         }
     }
 
-    fn harvest(self) -> TSegmentCollector::Fruit {
-        self.segment_collector.harvest()
-    }
-}
-
-/// A variant of the [`FilterCollector`] specialized for bytes fast fields, i.e.
-/// it transparently wraps an inner [`Collector`] but filters documents
-/// based on the result of applying the predicate to the bytes fast field.
-///
-/// A document is accepted if and only if the predicate returns `true` for at least one value.
-///
-/// In other words,
-/// - documents with no values are filtered out.
-/// - documents with several values are accepted if at least one value matches the predicate.
-///
-/// ```rust
-/// use tantivy::collector::{TopDocs, BytesFilterCollector};
-/// use tantivy::query::QueryParser;
-/// use tantivy::schema::{Schema, TEXT, FAST};
-/// use tantivy::{doc, DocAddress, Index};
-///
-/// # fn main() -> tantivy::Result<()> {
-/// let mut schema_builder = Schema::builder();
-/// let title = schema_builder.add_text_field("title", TEXT);
-/// let barcode = schema_builder.add_bytes_field("barcode", FAST);
-/// let schema = schema_builder.build();
-/// let index = Index::create_in_ram(schema);
-///
-/// let mut index_writer = index.writer_with_num_threads(1, 10_000_000)?;
-/// index_writer.add_document(doc!(title => "The Name of the Wind", barcode => &b"010101"[..]))?;
-/// index_writer.add_document(doc!(title => "The Diary of Muadib", barcode => &b"110011"[..]))?;
-/// index_writer.add_document(doc!(title => "A Dairy Cow", barcode => &b"110111"[..]))?;
-/// index_writer.add_document(doc!(title => "The Diary of a Young Girl", barcode => &b"011101"[..]))?;
-/// index_writer.add_document(doc!(title => "Bridget Jones's Diary"))?;
-/// index_writer.commit()?;
-///
-/// let reader = index.reader()?;
-/// let searcher = reader.searcher();
-///
-/// let query_parser = QueryParser::for_index(&index, vec![title]);
-/// let query = query_parser.parse_query("diary")?;
-/// let filter_collector = BytesFilterCollector::new(barcode, |bytes: &[u8]| bytes.starts_with(b"01"), TopDocs::with_limit(2));
-/// let top_docs = searcher.search(&query, &filter_collector)?;
-///
-/// assert_eq!(top_docs.len(), 1);
-/// assert_eq!(top_docs[0].1, DocAddress::new(0, 3));
-/// # Ok(())
-/// # }
-/// ```
-pub struct BytesFilterCollector<TCollector, TPredicate>
-where TPredicate: 'static + Clone
-{
-    field: Field,
-    collector: TCollector,
-    predicate: TPredicate,
-}
-
-impl<TCollector, TPredicate> BytesFilterCollector<TCollector, TPredicate>
-where
-    TCollector: Collector + Send + Sync,
-    TPredicate: Fn(&[u8]) -> bool + Send + Sync + Clone,
-{
-    /// Create a new `BytesFilterCollector`.
-    pub fn new(field: Field, predicate: TPredicate, collector: TCollector) -> Self {
-        Self {
-            field,
-            predicate,
-            collector,
-        }
-    }
-}
-
-impl<TCollector, TPredicate> Collector for BytesFilterCollector<TCollector, TPredicate>
-where
-    TCollector: Collector + Send + Sync,
-    TPredicate: 'static + Fn(&[u8]) -> bool + Send + Sync + Clone,
-{
-    type Fruit = TCollector::Fruit;
-
-    type Child = BytesFilterSegmentCollector<TCollector::Child, TPredicate>;
-
-    fn for_segment(
-        &self,
-        segment_local_id: u32,
-        segment_reader: &SegmentReader,
-    ) -> crate::Result<Self::Child> {
-        let schema = segment_reader.schema();
-        let field_name = schema.get_field_name(self.field);
-
-        let column_opt = segment_reader.fast_fields().bytes(field_name)?;
-
-        let segment_collector = self
-            .collector
-            .for_segment(segment_local_id, segment_reader)?;
-
-        Ok(BytesFilterSegmentCollector {
-            column_opt,
-            segment_collector,
-            predicate: self.predicate.clone(),
-            buffer: Vec::new(),
-        })
-    }
-
-    fn requires_scoring(&self) -> bool {
-        self.collector.requires_scoring()
-    }
-
-    fn merge_fruits(
-        &self,
-        segment_fruits: Vec<<TCollector::Child as SegmentCollector>::Fruit>,
-    ) -> crate::Result<TCollector::Fruit> {
-        self.collector.merge_fruits(segment_fruits)
-    }
-}
-
-pub struct BytesFilterSegmentCollector<TSegmentCollector, TPredicate>
-where TPredicate: 'static
-{
-    column_opt: Option<BytesColumn>,
-    segment_collector: TSegmentCollector,
-    predicate: TPredicate,
-    buffer: Vec<u8>,
-}
-
-impl<TSegmentCollector, TPredicate> BytesFilterSegmentCollector<TSegmentCollector, TPredicate>
-where
-    TSegmentCollector: SegmentCollector,
-    TPredicate: 'static + Fn(&[u8]) -> bool + Send + Sync,
-{
-    #[inline]
-    fn accept_document(&mut self, doc_id: DocId) -> bool {
-        if let Some(column) = &self.column_opt {
-            for ord in column.term_ords(doc_id) {
-                self.buffer.clear();
-
-                let found = column.ord_to_bytes(ord, &mut self.buffer).unwrap_or(false);
-
-                if found && (self.predicate)(&self.buffer) {
-                    return true;
-                }
-            }
-        }
-        false
-    }
-}
-
-impl<TSegmentCollector, TPredicate> SegmentCollector
-    for BytesFilterSegmentCollector<TSegmentCollector, TPredicate>
-where
-    TSegmentCollector: SegmentCollector,
-    TPredicate: 'static + Fn(&[u8]) -> bool + Send + Sync,
-{
-    type Fruit = TSegmentCollector::Fruit;
-
-    fn collect(&mut self, doc: u32, score: Score) {
-        if self.accept_document(doc) {
-            self.segment_collector.collect(doc, score);
-        }
-    }
-
-    fn harvest(self) -> TSegmentCollector::Fruit {
+    fn harvest(self) -> <TSegmentCollector as SegmentCollector>::Fruit {
         self.segment_collector.harvest()
     }
 }

src/collector/mod.rs

@@ -112,7 +112,7 @@ mod docset_collector;
 pub use self::docset_collector::DocSetCollector;
 
 mod filter_collector_wrapper;
-pub use self::filter_collector_wrapper::{BytesFilterCollector, FilterCollector};
+pub use self::filter_collector_wrapper::FilterCollector;
 
 /// `Fruit` is the type for the result of our collection.
 /// e.g. `usize` for the `Count` collector.

src/core/json_utils.rs

@@ -259,7 +259,7 @@ pub(crate) fn set_string_and_get_terms(
 
 /// Writes a value of a JSON field to a `Term`.
 /// The Term format is as follows:
-/// `[JSON_TYPE][JSON_PATH][JSON_END_OF_PATH][VALUE_BYTES]`
+/// [JSON_TYPE][JSON_PATH][JSON_END_OF_PATH][VALUE_BYTES]
 pub struct JsonTermWriter<'a> {
     term_buffer: &'a mut Term,
     path_stack: Vec<usize>,

src/core/segment_reader.rs

@@ -2,6 +2,8 @@ use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
 use std::{fmt, io};
 
+use fail::fail_point;
+
 use crate::core::{InvertedIndexReader, Segment, SegmentComponent, SegmentId};
 use crate::directory::{CompositeFile, FileSlice};
 use crate::error::DataCorruption;
@@ -149,7 +151,7 @@ impl SegmentReader {
 
         let store_file = segment.open_read(SegmentComponent::Store)?;
 
-        crate::fail_point!("SegmentReader::open#middle");
+        fail_point!("SegmentReader::open#middle");
 
         let postings_file = segment.open_read(SegmentComponent::Postings)?;
         let postings_composite = CompositeFile::open(&postings_file)?;

src/directory/ram_directory.rs

@@ -5,6 +5,7 @@ use std::sync::{Arc, RwLock};
 use std::{fmt, result};
 
 use common::HasLen;
+use fail::fail_point;
 
 use super::FileHandle;
 use crate::core::META_FILEPATH;
@@ -183,7 +184,7 @@ impl Directory for RamDirectory {
     }
 
     fn delete(&self, path: &Path) -> result::Result<(), DeleteError> {
-        crate::fail_point!("RamDirectory::delete", |_| {
+        fail_point!("RamDirectory::delete", |_| {
             Err(DeleteError::IoError {
                 io_error: Arc::new(io::Error::from(io::ErrorKind::Other)),
                 filepath: path.to_path_buf(),

src/indexer/segment_updater.rs

@@ -6,6 +6,7 @@ use std::path::PathBuf;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};
 
+use fail::fail_point;
 use rayon::{ThreadPool, ThreadPoolBuilder};
 
 use super::segment_manager::SegmentManager;
@@ -42,7 +43,7 @@ pub(crate) fn save_metas(metas: &IndexMeta, directory: &dyn Directory) -> crate:
     let mut buffer = serde_json::to_vec_pretty(metas)?;
     // Just adding a new line at the end of the buffer.
     writeln!(&mut buffer)?;
-    crate::fail_point!("save_metas", |msg| Err(crate::TantivyError::from(
+    fail_point!("save_metas", |msg| Err(crate::TantivyError::from(
         std::io::Error::new(
             std::io::ErrorKind::Other,
             msg.unwrap_or_else(|| "Undefined".to_string())

src/indexer/segment_writer.rs

@@ -1,6 +1,5 @@
 use columnar::MonotonicallyMappableToU64;
 use itertools::Itertools;
-use tokenizer_api::BoxTokenStream;
 
 use super::doc_id_mapping::{get_doc_id_mapping_from_field, DocIdMapping};
 use super::operation::AddOperation;
@@ -16,7 +15,7 @@ use crate::postings::{
 use crate::schema::{FieldEntry, FieldType, Schema, Term, Value, DATE_TIME_PRECISION_INDEXED};
 use crate::store::{StoreReader, StoreWriter};
 use crate::tokenizer::{FacetTokenizer, PreTokenizedStream, TextAnalyzer, Tokenizer};
-use crate::{DocId, Document, Opstamp, SegmentComponent, TantivyError};
+use crate::{DocId, Document, Opstamp, SegmentComponent};
 
 /// Computes the initial size of the hash table.
 ///
@@ -99,18 +98,14 @@ impl SegmentWriter {
                     }
                     _ => None,
                 };
-                let tokenizer_name = text_options
-                    .map(|text_index_option| text_index_option.tokenizer())
-                    .unwrap_or("default");
-
-                tokenizer_manager.get(tokenizer_name).ok_or_else(|| {
-                    TantivyError::SchemaError(format!(
-                        "Error getting tokenizer for field: {}",
-                        field_entry.name()
-                    ))
-                })
+                text_options
+                    .and_then(|text_index_option| {
+                        let tokenizer_name = &text_index_option.tokenizer();
+                        tokenizer_manager.get(tokenizer_name)
+                    })
+                    .unwrap_or_default()
             })
-            .collect::<Result<Vec<_>, _>>()?;
+            .collect();
         Ok(SegmentWriter {
             max_doc: 0,
             ctx: IndexingContext::new(table_size),
@@ -210,7 +205,7 @@ impl SegmentWriter {
                     for value in values {
                         let mut token_stream = match value {
                             Value::PreTokStr(tok_str) => {
-                                BoxTokenStream::new(PreTokenizedStream::from(tok_str.clone()))
+                                PreTokenizedStream::from(tok_str.clone()).into()
                             }
                             Value::Str(ref text) => {
                                 let text_analyzer =
@@ -443,9 +438,7 @@ fn remap_and_write(
 
 #[cfg(test)]
 mod tests {
-    use std::path::{Path, PathBuf};
-
-    use tempfile::TempDir;
+    use std::path::Path;
 
     use super::compute_initial_table_size;
     use crate::collector::Count;
@@ -453,9 +446,7 @@ mod tests {
     use crate::directory::RamDirectory;
     use crate::postings::TermInfo;
     use crate::query::PhraseQuery;
-    use crate::schema::{
-        IndexRecordOption, Schema, TextFieldIndexing, TextOptions, Type, STORED, STRING, TEXT,
-    };
+    use crate::schema::{IndexRecordOption, Schema, Type, STORED, STRING, TEXT};
     use crate::store::{Compressor, StoreReader, StoreWriter};
     use crate::time::format_description::well_known::Rfc3339;
     use crate::time::OffsetDateTime;
@@ -909,32 +900,4 @@ mod tests {
         postings.positions(&mut positions);
         assert_eq!(positions, &[4]); //< as opposed to 3 if we had a position length of 1.
     }
-
-    #[test]
-    fn test_show_error_when_tokenizer_not_registered() {
-        let text_field_indexing = TextFieldIndexing::default()
-            .set_tokenizer("custom_en")
-            .set_index_option(IndexRecordOption::WithFreqsAndPositions);
-        let text_options = TextOptions::default()
-            .set_indexing_options(text_field_indexing)
-            .set_stored();
-        let mut schema_builder = Schema::builder();
-        schema_builder.add_text_field("title", text_options);
-        let schema = schema_builder.build();
-        let tempdir = TempDir::new().unwrap();
-        let tempdir_path = PathBuf::from(tempdir.path());
-        Index::create_in_dir(&tempdir_path, schema).unwrap();
-        let index = Index::open_in_dir(tempdir_path).unwrap();
-        let schema = index.schema();
-        let mut index_writer = index.writer(50_000_000).unwrap();
-        let title = schema.get_field("title").unwrap();
-        let mut document = Document::default();
-        document.add_text(title, "The Old Man and the Sea");
-        index_writer.add_document(document).unwrap();
-        let error = index_writer.commit().unwrap_err();
-        assert_eq!(
-            error.to_string(),
-            "Schema error: 'Error getting tokenizer for field: title'"
-        );
-    }
 }

src/indexer/stamper.rs

@@ -101,7 +101,6 @@ mod test {
 
     use super::Stamper;
 
-    #[allow(clippy::redundant_clone)]
     #[test]
     fn test_stamper() {
         let stamper = Stamper::new(7u64);
@@ -117,7 +116,6 @@ mod test {
         assert_eq!(stamper.stamp(), 15u64);
     }
 
-    #[allow(clippy::redundant_clone)]
     #[test]
     fn test_stamper_revert() {
         let stamper = Stamper::new(7u64);

src/lib.rs

@@ -299,35 +299,6 @@ pub struct DocAddress {
     pub doc_id: DocId,
 }
 
-#[macro_export]
-/// Enable fail_point if feature is enabled.
-macro_rules! fail_point {
-    ($name:expr) => {{
-        #[cfg(feature = "failpoints")]
-        {
-            fail::eval($name, |_| {
-                panic!("Return is not supported for the fail point \"{}\"", $name);
-            });
-        }
-    }};
-    ($name:expr, $e:expr) => {{
-        #[cfg(feature = "failpoints")]
-        {
-            if let Some(res) = fail::eval($name, $e) {
-                return res;
-            }
-        }
-    }};
-    ($name:expr, $cond:expr, $e:expr) => {{
-        #[cfg(feature = "failpoints")]
-        {
-            if $cond {
-                fail::fail_point!($name, $e);
-            }
-        }
-    }};
-}
-
 #[cfg(test)]
 pub mod tests {
     use common::{BinarySerializable, FixedSize};
@@ -905,8 +876,8 @@ pub mod tests {
         }"#,
         )
         .unwrap();
-        let doc = doc!(json_field=>json_val);
-        let index = Index::create_in_ram(schema);
+        let doc = doc!(json_field=>json_val.clone());
+        let index = Index::create_in_ram(schema.clone());
         let mut writer = index.writer_for_tests().unwrap();
         writer.add_document(doc).unwrap();
         writer.commit().unwrap();

src/postings/serializer.rs

@@ -2,6 +2,7 @@ use std::cmp::Ordering;
 use std::io::{self, Write};
 
 use common::{BinarySerializable, CountingWriter, VInt};
+use fail::fail_point;
 
 use super::TermInfo;
 use crate::core::Segment;
@@ -204,7 +205,7 @@ impl<'a> FieldSerializer<'a> {
     /// If the current block is incomplete, it needs to be encoded
     /// using `VInt` encoding.
     pub fn close_term(&mut self) -> io::Result<()> {
-        crate::fail_point!("FieldSerializer::close_term", |msg: Option<String>| {
+        fail_point!("FieldSerializer::close_term", |msg: Option<String>| {
             Err(io::Error::new(io::ErrorKind::Other, format!("{msg:?}")))
         });
         if self.term_open {

src/query/more_like_this/more_like_this.rs

@@ -5,7 +5,7 @@ use crate::query::bm25::idf;
 use crate::query::{BooleanQuery, BoostQuery, Occur, Query, TermQuery};
 use crate::schema::{Field, FieldType, IndexRecordOption, Term, Value};
 use crate::tokenizer::{
-    FacetTokenizer, PreTokenizedStream, TokenStream, Tokenizer,
+    BoxTokenStream, FacetTokenizer, PreTokenizedStream, TokenStream, Tokenizer,
 };
 use crate::{DocAddress, Result, Searcher, TantivyError};
 
@@ -206,8 +206,8 @@ impl MoreLikeThis {
                 for value in values {
                     match value {
                         Value::PreTokStr(tok_str) => {
-                            let mut token_stream =
-                                PreTokenizedStream::from(tok_str.clone());
+                            let mut token_stream: BoxTokenStream =
+                                PreTokenizedStream::from(tok_str.clone()).into();
                             token_stream.process(&mut |token| {
                                 if !self.is_noise_word(token.text.clone()) {
                                     let term = Term::from_field_text(field, &token.text);

src/query/range_query/range_query.rs

@@ -472,7 +472,6 @@ mod tests {
 
     use super::RangeQuery;
     use crate::collector::{Count, TopDocs};
-    use crate::indexer::NoMergePolicy;
     use crate::query::QueryParser;
     use crate::schema::{Document, Field, IntoIpv6Addr, Schema, FAST, INDEXED, STORED, TEXT};
     use crate::{doc, Index};
@@ -548,8 +547,7 @@ mod tests {
 
         let index = Index::create_in_ram(schema);
         {
-            let mut index_writer = index.writer_with_num_threads(1, 60_000_000)?;
-            index_writer.set_merge_policy(Box::new(NoMergePolicy));
+            let mut index_writer = index.writer_with_num_threads(2, 60_000_000)?;
 
             for i in 1..100 {
                 let mut doc = Document::new();
@@ -559,9 +557,6 @@ mod tests {
                     }
                 }
                 index_writer.add_document(doc)?;
-                if i == 10 {
-                    index_writer.commit()?;
-                }
             }
 
             index_writer.commit()?;

src/snippet/mod.rs

@@ -693,7 +693,7 @@ Survey in 2016, 2017, and 2018."#;
         terms.insert(String::from("bc"), 1.0);
 
         let fragments = search_fragments(
-            &mut From::from(NgramTokenizer::all_ngrams(2, 2).unwrap()),
+            &mut From::from(NgramTokenizer::all_ngrams(2, 2)),
             text,
             &terms,
             3,

src/store/reader.rs

@@ -426,7 +426,7 @@ mod tests {
         assert_eq!(store.cache_stats().cache_hits, 1);
         assert_eq!(store.cache_stats().cache_misses, 2);
 
-        assert_eq!(store.cache.peek_lru(), Some(11207));
+        assert_eq!(store.cache.peek_lru(), Some(11163));
 
         Ok(())
     }

src/tokenizer/mod.rs

@@ -154,7 +154,7 @@ pub use self::split_compound_words::SplitCompoundWords;
 pub use self::stemmer::{Language, Stemmer};
 pub use self::stop_word_filter::StopWordFilter;
 pub use self::tokenized_string::{PreTokenizedStream, PreTokenizedString};
-pub use self::tokenizer::{TextAnalyzer, TextAnalyzerBuilder};
+pub use self::tokenizer::TextAnalyzer;
 pub use self::tokenizer_manager::TokenizerManager;
 pub use self::whitespace_tokenizer::WhitespaceTokenizer;
 

src/tokenizer/ngram_tokenizer.rs

@@ -1,5 +1,4 @@
 use super::{Token, TokenStream, Tokenizer};
-use crate::TantivyError;
 
 /// Tokenize the text by splitting words into n-grams of the given size(s)
 ///
@@ -34,7 +33,7 @@ use crate::TantivyError;
 /// ```rust
 /// use tantivy::tokenizer::*;
 ///
-/// let mut tokenizer = NgramTokenizer::new(2, 3, false).unwrap();
+/// let mut tokenizer = NgramTokenizer::new(2, 3, false);
 /// let mut stream = tokenizer.token_stream("hello");
 /// {
 ///     let token = stream.next().unwrap();
@@ -80,7 +79,7 @@ use crate::TantivyError;
 /// }
 /// assert!(stream.next().is_none());
 /// ```
-#[derive(Clone, Debug)]
+#[derive(Clone)]
 pub struct NgramTokenizer {
     /// min size of the n-gram
     min_gram: usize,
@@ -93,39 +92,30 @@ pub struct NgramTokenizer {
 
 impl NgramTokenizer {
     /// Configures a new Ngram tokenizer
-    pub fn new(
-        min_gram: usize,
-        max_gram: usize,
-        prefix_only: bool,
-    ) -> crate::Result<NgramTokenizer> {
-        if min_gram == 0 {
-            return Err(TantivyError::InvalidArgument(
-                "min_gram must be greater than 0".to_string(),
-            ));
-        }
-        if min_gram > max_gram {
-            return Err(TantivyError::InvalidArgument(
-                "min_gram must not be greater than max_gram".to_string(),
-            ));
-        }
-        Ok(NgramTokenizer {
+    pub fn new(min_gram: usize, max_gram: usize, prefix_only: bool) -> NgramTokenizer {
+        assert!(min_gram > 0, "min_gram must be greater than 0");
+        assert!(
+            min_gram <= max_gram,
+            "min_gram must not be greater than max_gram"
+        );
+        NgramTokenizer {
             min_gram,
             max_gram,
             prefix_only,
             token: Token::default(),
-        })
+        }
     }
 
     /// Create a `NGramTokenizer` which generates tokens for all inner ngrams.
     ///
     /// This is as opposed to only prefix ngrams    .
-    pub fn all_ngrams(min_gram: usize, max_gram: usize) -> crate::Result<NgramTokenizer> {
+    pub fn all_ngrams(min_gram: usize, max_gram: usize) -> NgramTokenizer {
         Self::new(min_gram, max_gram, false)
     }
 
     /// Create a `NGramTokenizer` which only generates tokens for the
     /// prefix ngrams.
-    pub fn prefix_only(min_gram: usize, max_gram: usize) -> crate::Result<NgramTokenizer> {
+    pub fn prefix_only(min_gram: usize, max_gram: usize) -> NgramTokenizer {
         Self::new(min_gram, max_gram, true)
     }
 }
@@ -359,11 +349,7 @@ mod tests {
 
     #[test]
     fn test_ngram_tokenizer_1_2_false() {
-        let tokens = test_helper(
-            NgramTokenizer::all_ngrams(1, 2)
-                .unwrap()
-                .token_stream("hello"),
-        );
+        let tokens = test_helper(NgramTokenizer::all_ngrams(1, 2).token_stream("hello"));
         assert_eq!(tokens.len(), 9);
         assert_token(&tokens[0], 0, "h", 0, 1);
         assert_token(&tokens[1], 0, "he", 0, 2);
@@ -378,11 +364,7 @@ mod tests {
 
     #[test]
     fn test_ngram_tokenizer_min_max_equal() {
-        let tokens = test_helper(
-            NgramTokenizer::all_ngrams(3, 3)
-                .unwrap()
-                .token_stream("hello"),
-        );
+        let tokens = test_helper(NgramTokenizer::all_ngrams(3, 3).token_stream("hello"));
         assert_eq!(tokens.len(), 3);
         assert_token(&tokens[0], 0, "hel", 0, 3);
         assert_token(&tokens[1], 0, "ell", 1, 4);
@@ -391,11 +373,7 @@ mod tests {
 
     #[test]
     fn test_ngram_tokenizer_2_5_prefix() {
-        let tokens = test_helper(
-            NgramTokenizer::prefix_only(2, 5)
-                .unwrap()
-                .token_stream("frankenstein"),
-        );
+        let tokens = test_helper(NgramTokenizer::prefix_only(2, 5).token_stream("frankenstein"));
         assert_eq!(tokens.len(), 4);
         assert_token(&tokens[0], 0, "fr", 0, 2);
         assert_token(&tokens[1], 0, "fra", 0, 3);
@@ -405,11 +383,7 @@ mod tests {
 
     #[test]
     fn test_ngram_non_ascii_1_2() {
-        let tokens = test_helper(
-            NgramTokenizer::all_ngrams(1, 2)
-                .unwrap()
-                .token_stream("hεllo"),
-        );
+        let tokens = test_helper(NgramTokenizer::all_ngrams(1, 2).token_stream("hεllo"));
         assert_eq!(tokens.len(), 9);
         assert_token(&tokens[0], 0, "h", 0, 1);
         assert_token(&tokens[1], 0, "hε", 0, 3);
@@ -424,11 +398,7 @@ mod tests {
 
     #[test]
     fn test_ngram_non_ascii_2_5_prefix() {
-        let tokens = test_helper(
-            NgramTokenizer::prefix_only(2, 5)
-                .unwrap()
-                .token_stream("hεllo"),
-        );
+        let tokens = test_helper(NgramTokenizer::prefix_only(2, 5).token_stream("hεllo"));
         assert_eq!(tokens.len(), 4);
         assert_token(&tokens[0], 0, "hε", 0, 3);
         assert_token(&tokens[1], 0, "hεl", 0, 4);
@@ -438,26 +408,22 @@ mod tests {
 
     #[test]
     fn test_ngram_empty() {
-        let tokens = test_helper(NgramTokenizer::all_ngrams(1, 5).unwrap().token_stream(""));
+        let tokens = test_helper(NgramTokenizer::all_ngrams(1, 5).token_stream(""));
         assert!(tokens.is_empty());
-        let tokens = test_helper(NgramTokenizer::all_ngrams(2, 5).unwrap().token_stream(""));
+        let tokens = test_helper(NgramTokenizer::all_ngrams(2, 5).token_stream(""));
         assert!(tokens.is_empty());
     }
 
     #[test]
     #[should_panic(expected = "min_gram must be greater than 0")]
     fn test_ngram_min_max_interval_empty() {
-        test_helper(
-            NgramTokenizer::all_ngrams(0, 2)
-                .unwrap()
-                .token_stream("hellossss"),
-        );
+        test_helper(NgramTokenizer::all_ngrams(0, 2).token_stream("hellossss"));
     }
 
     #[test]
     #[should_panic(expected = "min_gram must not be greater than max_gram")]
     fn test_invalid_interval_should_panic_if_smaller() {
-        NgramTokenizer::all_ngrams(2, 1).unwrap();
+        NgramTokenizer::all_ngrams(2, 1);
     }
 
     #[test]

src/tokenizer/tokenizer.rs

@@ -9,26 +9,6 @@ pub struct TextAnalyzer {
     tokenizer: Box<dyn BoxableTokenizer>,
 }
 
-impl Tokenizer for Box<dyn BoxableTokenizer> {
-    type TokenStream<'a> = BoxTokenStream<'a>;
-
-    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
-        self.box_token_stream(text)
-    }
-}
-
-impl Clone for Box<dyn BoxableTokenizer> {
-    fn clone(&self) -> Self {
-        self.box_clone()
-    }
-}
-
-fn add_filter<F: TokenFilter>(tokenizer: Box<dyn BoxableTokenizer>, filter: F) -> Box<dyn BoxableTokenizer> {
-    let filtered_tokenizer = filter.transform(tokenizer);
-    Box::new(filtered_tokenizer)
-}
-
-
 /// A boxable `Tokenizer`, with its `TokenStream` type erased.
 trait BoxableTokenizer: 'static + Send + Sync {
     /// Creates a boxed token stream for a given `str`.
@@ -39,7 +19,7 @@ trait BoxableTokenizer: 'static + Send + Sync {
 
 impl<T: Tokenizer> BoxableTokenizer for T {
     fn box_token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a> {
-        BoxTokenStream::new(self.token_stream(text))
+        self.token_stream(text).into()
     }
     fn box_clone(&self) -> Box<dyn BoxableTokenizer> {
         Box::new(self.clone())

sstable/Cargo.toml

@@ -7,7 +7,7 @@ homepage = "https://github.com/quickwit-oss/tantivy"
 repository = "https://github.com/quickwit-oss/tantivy"
 keywords = ["search", "information", "retrieval", "sstable"]
 categories = ["database-implementations", "data-structures", "compression"]
-description = "sstables for tantivy"
+desciption = "sstables for tantivy"
 
 [dependencies]
 common = {version= "0.5", path="../common", package="tantivy-common"}

stacker/src/fastcpy.rs

@@ -44,7 +44,7 @@ pub fn fast_short_slice_copy(src: &[u8], dst: &mut [u8]) {
         return;
     }
 
-    // The code will use the vmovdqu instruction to copy 32 bytes at a time.
+    /// The code will use the vmovdqu instruction to copy 32 bytes at a time.
     #[cfg(target_feature = "avx")]
     {
         if len <= 64 {

tokenizer-api/src/lib.rs

@@ -63,22 +63,17 @@ pub trait Tokenizer: 'static + Clone + Send + Sync {
 /// Simple wrapper of `Box<dyn TokenStream + 'a>`.
 pub struct BoxTokenStream<'a>(Box<dyn TokenStream + 'a>);
 
-impl<'a> TokenStream for BoxTokenStream<'a> {
-    fn advance(&mut self) -> bool {
-        self.0.advance()
-    }
-
-    fn token(&self) -> &Token {
-        self.0.token()
-    }
-
-    fn token_mut(&mut self) -> &mut Token {
-        self.0.token_mut()
+impl<'a> From<BoxTokenStream<'a>> for Box<dyn TokenStream + 'a> {
+    fn from(token_stream: BoxTokenStream<'a>) -> Self {
+        token_stream.0
     }
 }
 
-impl<'a> BoxTokenStream<'a> {
-    pub fn new<T: TokenStream + 'a>(token_stream: T) -> BoxTokenStream<'a> {
+impl<'a, T> From<T> for BoxTokenStream<'a>
+where
+    T: TokenStream + 'a,
+{
+    fn from(token_stream: T) -> BoxTokenStream<'a> {
         BoxTokenStream(Box::new(token_stream))
     }
 }
@@ -157,7 +152,6 @@ pub trait TokenFilter: 'static + Send + Sync {
     fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T>;
 }
 
-
 #[cfg(test)]
 mod test {
     use super::*;