Followup fulmicoton poc.

src/indexer/segment_writer.rs

@@ -1,5 +1,6 @@
 use columnar::MonotonicallyMappableToU64;
 use itertools::Itertools;
+use tokenizer_api::BoxTokenStream;
 
 use super::doc_id_mapping::{get_doc_id_mapping_from_field, DocIdMapping};
 use super::operation::AddOperation;
@@ -209,7 +210,7 @@ impl SegmentWriter {
                     for value in values {
                         let mut token_stream = match value {
                             Value::PreTokStr(tok_str) => {
-                                PreTokenizedStream::from(tok_str.clone()).into()
+                                BoxTokenStream::new(PreTokenizedStream::from(tok_str.clone()))
                             }
                             Value::Str(ref text) => {
                                 let text_analyzer =

src/query/more_like_this/more_like_this.rs

@@ -5,7 +5,7 @@ use crate::query::bm25::idf;
 use crate::query::{BooleanQuery, BoostQuery, Occur, Query, TermQuery};
 use crate::schema::{Field, FieldType, IndexRecordOption, Term, Value};
 use crate::tokenizer::{
-    BoxTokenStream, FacetTokenizer, PreTokenizedStream, TokenStream, Tokenizer,
+    FacetTokenizer, PreTokenizedStream, TokenStream, Tokenizer,
 };
 use crate::{DocAddress, Result, Searcher, TantivyError};
 
@@ -206,8 +206,8 @@ impl MoreLikeThis {
                 for value in values {
                     match value {
                         Value::PreTokStr(tok_str) => {
-                            let mut token_stream: BoxTokenStream =
-                                PreTokenizedStream::from(tok_str.clone()).into();
+                            let mut token_stream =
+                                PreTokenizedStream::from(tok_str.clone());
                             token_stream.process(&mut |token| {
                                 if !self.is_noise_word(token.text.clone()) {
                                     let term = Term::from_field_text(field, &token.text);

src/tokenizer/tokenizer.rs

@@ -9,6 +9,30 @@ pub struct TextAnalyzer {
     tokenizer: Box<dyn BoxableTokenizer>,
 }
 
+/// Wrapper to avoid recursive acalls of `box_token_stream`.
+#[derive(Clone)]
+struct BoxedTokenizer(Box<dyn BoxableTokenizer>);
+
+impl Tokenizer for BoxedTokenizer {
+    type TokenStream<'a> = BoxTokenStream<'a>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        self.0.box_token_stream(text)
+    }
+}
+
+impl Clone for Box<dyn BoxableTokenizer> {
+    fn clone(&self) -> Self {
+        self.box_clone()
+    }
+}
+
+fn add_filter<F: TokenFilter>(tokenizer: BoxedTokenizer, filter: F) -> BoxedTokenizer {
+    let filtered_tokenizer = filter.transform(tokenizer);
+    BoxedTokenizer(Box::new(filtered_tokenizer))
+}
+
+
 /// A boxable `Tokenizer`, with its `TokenStream` type erased.
 trait BoxableTokenizer: 'static + Send + Sync {
     /// Creates a boxed token stream for a given `str`.
@@ -19,7 +43,7 @@ trait BoxableTokenizer: 'static + Send + Sync {
 
 impl<T: Tokenizer> BoxableTokenizer for T {
     fn box_token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a> {
-        self.token_stream(text).into()
+        BoxTokenStream::new(self.token_stream(text))
     }
     fn box_clone(&self) -> Box<dyn BoxableTokenizer> {
         Box::new(self.clone())
@@ -52,6 +76,11 @@ impl TextAnalyzer {
         TextAnalyzerBuilder { tokenizer }
     }
 
+    /// TODO
+    pub fn dynamic_filter_builder<T: Tokenizer>(tokenizer: T) -> DynamicTextAnalyzerBuilder {
+        DynamicTextAnalyzerBuilder::new(tokenizer)
+    }
+
     /// Creates a token stream for a given `str`.
     pub fn token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a> {
         self.tokenizer.box_token_stream(text)
@@ -90,3 +119,57 @@ impl<T: Tokenizer> TextAnalyzerBuilder<T> {
         }
     }
 }
+
+/// Builder helper for [`TextAnalyzer`] with dynamic filters.
+pub struct DynamicTextAnalyzerBuilder {
+    tokenizer: BoxedTokenizer,
+}
+
+impl DynamicTextAnalyzerBuilder {
+    pub fn new<T: Tokenizer>(tokenizer: T) -> DynamicTextAnalyzerBuilder {
+        DynamicTextAnalyzerBuilder { tokenizer: BoxedTokenizer(Box::new(tokenizer)) }
+    }
+
+    pub fn filter<F: TokenFilter>(self, filter: F) -> DynamicTextAnalyzerBuilder {
+        DynamicTextAnalyzerBuilder {
+            tokenizer: add_filter(self.tokenizer, filter),
+        }
+    }
+
+    pub fn build(self) -> TextAnalyzer {
+        TextAnalyzer {
+            tokenizer: self.tokenizer.0,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+    use crate::tokenizer::{AlphaNumOnlyFilter, LowerCaser, RemoveLongFilter, WhitespaceTokenizer};
+
+    #[test]
+    fn test_text_analyzer_builder() {
+        let mut analyzer = TextAnalyzer::builder(WhitespaceTokenizer::default())
+            .filter(AlphaNumOnlyFilter)
+            .filter(RemoveLongFilter::limit(6))
+            .filter(LowerCaser)
+            .build();
+        let mut stream = analyzer.token_stream("- first bullet point");
+        assert_eq!(stream.next().unwrap().text, "first");
+        assert_eq!(stream.next().unwrap().text, "point");
+    }
+
+    #[test]
+    fn test_text_analyzer_with_filters_boxed() {
+        let mut analyzer = TextAnalyzer::dynamic_filter_builder(WhitespaceTokenizer::default())
+            .filter(AlphaNumOnlyFilter)
+            .filter(RemoveLongFilter::limit(6))
+            .filter(LowerCaser)
+            .build();
+        let mut stream = analyzer.token_stream("- first bullet point");
+        assert_eq!(stream.next().unwrap().text, "first");
+        assert_eq!(stream.next().unwrap().text, "point");
+    }
+}

tokenizer-api/src/lib.rs

@@ -63,10 +63,22 @@ pub trait Tokenizer: 'static + Clone + Send + Sync {
 /// Simple wrapper of `Box<dyn TokenStream + 'a>`.
 pub struct BoxTokenStream<'a>(Box<dyn TokenStream + 'a>);
 
-impl<'a, T> From<T> for BoxTokenStream<'a>
-where T: TokenStream + 'a
-{
-    fn from(token_stream: T) -> BoxTokenStream<'a> {
+impl<'a> TokenStream for BoxTokenStream<'a> {
+    fn advance(&mut self) -> bool {
+        self.0.advance()
+    }
+
+    fn token(&self) -> &Token {
+        self.0.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.0.token_mut()
+    }
+}
+
+impl<'a> BoxTokenStream<'a> {
+    pub fn new<T: TokenStream + 'a>(token_stream: T) -> BoxTokenStream<'a> {
         BoxTokenStream(Box::new(token_stream))
     }
 }
@@ -145,6 +157,7 @@ pub trait TokenFilter: 'static + Send + Sync {
     fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T>;
 }
 
+
 #[cfg(test)]
 mod test {
     use super::*;