diff --git a/src/collector/mod.rs b/src/collector/mod.rs index aea552e58..3a45a82d8 100644 --- a/src/collector/mod.rs +++ b/src/collector/mod.rs @@ -1,6 +1,7 @@ /*! -Collector module +Defines how the documents matching a search query should be processed. */ + use SegmentReader; use SegmentLocalId; use DocId; diff --git a/src/core/index.rs b/src/core/index.rs index ede44867c..7e0f5f7dc 100644 --- a/src/core/index.rs +++ b/src/core/index.rs @@ -33,7 +33,7 @@ fn load_metas(directory: &Directory) -> Result { serde_json::from_str(&meta_string).chain_err(|| ErrorKind::CorruptedFile(META_FILEPATH.clone())) } -/// Tantivy's Search Index +/// Search Index pub struct Index { directory: ManagedDirectory, schema: Schema, diff --git a/src/core/segment_id.rs b/src/core/segment_id.rs index b9612b46a..febe74666 100644 --- a/src/core/segment_id.rs +++ b/src/core/segment_id.rs @@ -5,7 +5,7 @@ use std::cmp::{Ordering, Ord}; #[cfg(test)] use std::sync::atomic; -/// Tantivy `SegmentId`. +/// Uuid identifying a segment. /// /// Tantivy's segment are identified /// by a UUID which is used to prefix the filenames diff --git a/src/fastfield/mod.rs b/src/fastfield/mod.rs index 9748d07a1..0dc7f9875 100644 --- a/src/fastfield/mod.rs +++ b/src/fastfield/mod.rs @@ -1,5 +1,5 @@ /*! -Fast fields is a column oriented storage. +Column oriented field storage for tantivy. It is the equivalent of `Lucene`'s `DocValues`. diff --git a/src/indexer/segment_writer.rs b/src/indexer/segment_writer.rs index 5cd20140b..09258c88c 100644 --- a/src/indexer/segment_writer.rs +++ b/src/indexer/segment_writer.rs @@ -155,11 +155,11 @@ impl<'a> SegmentWriter<'a> { }) .collect(); let mut token_stream = tokenizer.token_stream_texts(&texts[..]); - self.multifield_postings.index_text( - doc_id, - field, - &mut token_stream, - ) + self.multifield_postings + .index_text( + doc_id, + field, + &mut token_stream) } else { 0 }; diff --git a/src/query/mod.rs b/src/query/mod.rs index 044a68b77..af31dfe57 100644 --- a/src/query/mod.rs +++ b/src/query/mod.rs @@ -1,7 +1,5 @@ /*! -Query module - -The query module regroups all of tantivy's query objects +Query */ mod query; diff --git a/src/schema/mod.rs b/src/schema/mod.rs index c174cb50c..fa128bd32 100644 --- a/src/schema/mod.rs +++ b/src/schema/mod.rs @@ -1,5 +1,5 @@ /*! - +Schema definition for tantivy's indices. # Setting your schema in Tantivy diff --git a/src/schema/text_options.rs b/src/schema/text_options.rs index e7cc2d992..38841498e 100644 --- a/src/schema/text_options.rs +++ b/src/schema/text_options.rs @@ -81,6 +81,9 @@ impl TextFieldIndexing { self } + /// Returns the indexing options associated to this field. + /// + /// See [IndexRecordOption](./enum.IndexRecordOption.html) for more detail. pub fn index_option(&self) -> IndexRecordOption { self.record } diff --git a/src/store/mod.rs b/src/store/mod.rs index 46138d556..f6663d6d9 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -1,5 +1,5 @@ /*! -Tantivy's store is a compressed, row-oriented storage. +Compressed/slow/row-oriented storage for documents. A field needs to be marked as stored in the schema in order to be handled in the `Store`. diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index d664fd7b9..33f6191c7 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -139,7 +139,7 @@ mod token_stream_chain; mod raw_tokenizer; -pub use self::tokenizer::{box_tokenizer, Tokenizer, Token, TokenFilter, TokenStream}; +pub use self::tokenizer::{Tokenizer, Token, TokenFilter, TokenStream}; pub use self::tokenizer::BoxedTokenizer; pub use self::tokenizer_manager::TokenizerManager; pub use self::simple_tokenizer::SimpleTokenizer; diff --git a/src/tokenizer/tokenizer.rs b/src/tokenizer/tokenizer.rs index 71a01dc40..63dd21ee6 100644 --- a/src/tokenizer/tokenizer.rs +++ b/src/tokenizer/tokenizer.rs @@ -80,9 +80,19 @@ pub trait Tokenizer<'a>: Sized + Clone { } } +/// A boxed tokenizer pub trait BoxedTokenizer: Send + Sync { + /// Tokenize a `&str` fn token_stream<'a>(&self, text: &'a str) -> Box; + + /// Tokenize an array`&str` + /// + /// The resulting `TokenStream` is equivalent to what would be obtained if the &str were + /// one concatenated `&str`, with an artificial position gap of `2` between the different fields + /// to prevent accidental `PhraseQuery` to match accross two terms. fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box; + + /// Return a boxed clone of the tokenizer fn boxed_clone(&self) -> Box; } @@ -121,7 +131,7 @@ where } } -pub fn box_tokenizer(a: A) -> Box +pub(crate) fn box_tokenizer(a: A) -> Box where A: 'static + Send + Sync + for<'a> Tokenizer<'a>, { @@ -180,12 +190,31 @@ impl<'b> TokenStream for Box { /// ``` /// pub trait TokenStream { + + /// Advance to the next token + /// + /// Returns false if there are no other tokens. fn advance(&mut self) -> bool; + /// Returns a reference to the current token. fn token(&self) -> &Token; + /// Returns a mutable reference to the current token. fn token_mut(&mut self) -> &mut Token; + /// ``` + /// # extern crate tantivy; + /// # use tantivy::tokenizer::*; + /// # fn main() { + /// # let tokenizer = SimpleTokenizer + /// # .filter(RemoveLongFilter::limit(40)) + /// # .filter(LowerCaser); + /// let mut token_stream = tokenizer.token_stream("Hello, happy tax payer"); + /// while let Some(token) = token_stream.next() { + /// println!("Token {:?}", token.text); + /// } + /// # } + /// ``` fn next(&mut self) -> Option<&Token> { if self.advance() { Some(self.token()) diff --git a/src/tokenizer/tokenizer_manager.rs b/src/tokenizer/tokenizer_manager.rs index ac2c7a691..24f611a89 100644 --- a/src/tokenizer/tokenizer_manager.rs +++ b/src/tokenizer/tokenizer_manager.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::sync::{Arc, RwLock}; use tokenizer::BoxedTokenizer; use tokenizer::Tokenizer; -use tokenizer::box_tokenizer; +use tokenizer::tokenizer::box_tokenizer; use tokenizer::RawTokenizer; use tokenizer::SimpleTokenizer; use tokenizer::JapaneseTokenizer;