docs

src/tokenizer/mod.rs

@@ -144,7 +144,7 @@ pub use self::tokenizer::BoxedTokenizer;
 pub use self::tokenizer_manager::TokenizerManager;
 pub use self::simple_tokenizer::SimpleTokenizer;
 pub use self::raw_tokenizer::RawTokenizer;
-pub use self::token_stream_chain::TokenStreamChain;
+pub(crate) use self::token_stream_chain::TokenStreamChain;
 pub use self::japanese_tokenizer::JapaneseTokenizer;
 pub use self::remove_long::RemoveLongFilter;
 pub use self::lower_caser::LowerCaser;

src/tokenizer/token_stream_chain.rs

@@ -1,6 +1,6 @@
 use tokenizer::{TokenStream, Token};
 
-pub struct TokenStreamChain<TTokenStream: TokenStream> {
+pub(crate) struct TokenStreamChain<TTokenStream: TokenStream> {
     offsets: Vec<usize>,
     token_streams: Vec<TTokenStream>,
     position_shift: usize,

src/tokenizer/tokenizer.rs

@@ -205,6 +205,7 @@ pub trait TokenStream {
     /// ```
     /// # extern crate tantivy;
     /// # use tantivy::tokenizer::*;
+    /// #
     /// # fn main() {
     /// # let tokenizer = SimpleTokenizer
     /// #       .filter(RemoveLongFilter::limit(40))