doc

src/collector/mod.rs

@@ -1,6 +1,7 @@
 /*!
-Collector module
+Defines how the documents matching a search query should be processed.
 */
+
 use SegmentReader;
 use SegmentLocalId;
 use DocId;

src/core/index.rs

@@ -33,7 +33,7 @@ fn load_metas(directory: &Directory) -> Result<IndexMeta> {
     serde_json::from_str(&meta_string).chain_err(|| ErrorKind::CorruptedFile(META_FILEPATH.clone()))
 }
 
-/// Tantivy's Search Index
+/// Search Index
 pub struct Index {
     directory: ManagedDirectory,
     schema: Schema,

src/core/segment_id.rs

@@ -5,7 +5,7 @@ use std::cmp::{Ordering, Ord};
 #[cfg(test)]
 use std::sync::atomic;
 
-/// Tantivy `SegmentId`.
+/// Uuid identifying a segment.
 ///
 /// Tantivy's segment are identified
 /// by a UUID which is used to prefix the filenames

src/fastfield/mod.rs

@@ -1,5 +1,5 @@
 /*!
-Fast fields is a column oriented storage.
+Column oriented field storage for tantivy.
 
 It is the equivalent of `Lucene`'s `DocValues`.
 

src/indexer/segment_writer.rs

@@ -155,11 +155,11 @@ impl<'a> SegmentWriter<'a> {
                                 })
                                 .collect();
                             let mut token_stream = tokenizer.token_stream_texts(&texts[..]);
-                            self.multifield_postings.index_text(
-                                doc_id,
-                                field,
-                                &mut token_stream,
-                            )
+                            self.multifield_postings
+                                .index_text(
+                                    doc_id,
+                                    field,
+                                    &mut token_stream)
                         } else {
                             0
                         };

src/query/mod.rs

@@ -1,7 +1,5 @@
 /*!
-Query module
-
-The query module regroups all of tantivy's query objects
+Query
 */
 
 mod query;

src/schema/mod.rs

@@ -1,5 +1,5 @@
 /*!
-
+Schema definition for tantivy's indices.
 
 # Setting your schema in Tantivy
 

src/schema/text_options.rs

@@ -81,6 +81,9 @@ impl TextFieldIndexing {
         self
     }
 
+    /// Returns the indexing options associated to this field.
+    ///
+    /// See [IndexRecordOption](./enum.IndexRecordOption.html) for more detail.
     pub fn index_option(&self) -> IndexRecordOption {
         self.record
     }

src/store/mod.rs

@@ -1,5 +1,5 @@
 /*!
-Tantivy's store is a compressed, row-oriented storage.
+Compressed/slow/row-oriented storage for documents.
 
 A field needs to be marked as stored in the schema in
 order to be handled in the `Store`.

src/tokenizer/mod.rs

@@ -139,7 +139,7 @@ mod token_stream_chain;
 mod raw_tokenizer;
 
 
-pub use self::tokenizer::{box_tokenizer, Tokenizer, Token, TokenFilter, TokenStream};
+pub use self::tokenizer::{Tokenizer, Token, TokenFilter, TokenStream};
 pub use self::tokenizer::BoxedTokenizer;
 pub use self::tokenizer_manager::TokenizerManager;
 pub use self::simple_tokenizer::SimpleTokenizer;

src/tokenizer/tokenizer.rs

@@ -80,9 +80,19 @@ pub trait Tokenizer<'a>: Sized + Clone {
     }
 }
 
+/// A boxed tokenizer
 pub trait BoxedTokenizer: Send + Sync {
+    /// Tokenize a `&str`
     fn token_stream<'a>(&self, text: &'a str) -> Box<TokenStream + 'a>;
+
+    /// Tokenize an array`&str`
+    ///
+    /// The resulting `TokenStream` is equivalent to what would be obtained if the &str were
+    /// one concatenated `&str`, with an artificial position gap of `2` between the different fields
+    /// to prevent accidental `PhraseQuery` to match accross two terms.
     fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<TokenStream + 'b>;
+
+    /// Return a boxed clone of the tokenizer
     fn boxed_clone(&self) -> Box<BoxedTokenizer>;
 }
 
@@ -121,7 +131,7 @@ where
     }
 }
 
-pub fn box_tokenizer<A>(a: A) -> Box<BoxedTokenizer>
+pub(crate) fn box_tokenizer<A>(a: A) -> Box<BoxedTokenizer>
 where
     A: 'static + Send + Sync + for<'a> Tokenizer<'a>,
 {
@@ -180,12 +190,31 @@ impl<'b> TokenStream for Box<TokenStream + 'b> {
 /// ```
 ///
 pub trait TokenStream {
+
+    /// Advance to the next token
+    ///
+    /// Returns false if there are no other tokens.
     fn advance(&mut self) -> bool;
 
+    /// Returns a reference to the current token.
     fn token(&self) -> &Token;
 
+    /// Returns a mutable reference to the current token.
     fn token_mut(&mut self) -> &mut Token;
 
+    /// ```
+    /// # extern crate tantivy;
+    /// # use tantivy::tokenizer::*;
+    /// # fn main() {
+    /// # let tokenizer = SimpleTokenizer
+    /// #       .filter(RemoveLongFilter::limit(40))
+    /// #       .filter(LowerCaser);
+    /// let mut token_stream = tokenizer.token_stream("Hello, happy tax payer");
+    /// while let Some(token) = token_stream.next() {
+    ///     println!("Token {:?}", token.text);
+    /// }
+    /// # }
+    /// ```
     fn next(&mut self) -> Option<&Token> {
         if self.advance() {
             Some(self.token())

src/tokenizer/tokenizer_manager.rs

@@ -2,7 +2,7 @@ use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
 use tokenizer::BoxedTokenizer;
 use tokenizer::Tokenizer;
-use tokenizer::box_tokenizer;
+use tokenizer::tokenizer::box_tokenizer;
 use tokenizer::RawTokenizer;
 use tokenizer::SimpleTokenizer;
 use tokenizer::JapaneseTokenizer;