diff --git a/src/core/index.rs b/src/core/index.rs index 0ba36cd08..275ce721a 100644 --- a/src/core/index.rs +++ b/src/core/index.rs @@ -114,6 +114,7 @@ impl Index { Index::create_from_metas(directory, &metas) } + /// Reads the index meta file from the directory. pub fn load_metas(&self) -> Result { load_metas(self.directory()) } diff --git a/src/schema/text_options.rs b/src/schema/text_options.rs index 63aee99ea..18202847e 100644 --- a/src/schema/text_options.rs +++ b/src/schema/text_options.rs @@ -42,6 +42,11 @@ impl Default for TextOptions { } } +/// Configuration defining indexing for a text field. +/// It wraps: +/// +/// * record (See [`IndexRecordOption`](./enum.IndexRecordOption.html)) +/// * tokenizer #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] pub struct TextFieldIndexing { record: IndexRecordOption, diff --git a/src/tokenizer/stemmer.rs b/src/tokenizer/stemmer.rs index 0d6a50dbc..f2f4a8a93 100644 --- a/src/tokenizer/stemmer.rs +++ b/src/tokenizer/stemmer.rs @@ -3,6 +3,7 @@ use super::{Token, TokenFilter, TokenStream}; use rust_stemmers::{self, Algorithm}; /// `Stemmer` token filter. Currently only English is supported. +/// Tokens are expected to be lowercased beforehands. #[derive(Clone)] pub struct Stemmer { stemmer_algorithm: Arc, diff --git a/src/tokenizer/tokenizer.rs b/src/tokenizer/tokenizer.rs index 2120bf98f..2909bcdd1 100644 --- a/src/tokenizer/tokenizer.rs +++ b/src/tokenizer/tokenizer.rs @@ -253,7 +253,10 @@ where } } + +/// Trait for the pluggable components of `Tokenizer`s. pub trait TokenFilter: Clone { + /// The resulting `TokenStream` type. type ResultTokenStream: TokenStream; /// Wraps a token stream and returns the modified one. diff --git a/src/tokenizer/tokenizer_manager.rs b/src/tokenizer/tokenizer_manager.rs index 812660d82..bbc141c92 100644 --- a/src/tokenizer/tokenizer_manager.rs +++ b/src/tokenizer/tokenizer_manager.rs @@ -28,6 +28,7 @@ pub struct TokenizerManager { } impl TokenizerManager { + /// Registers a new tokenizer associated with a given name. pub fn register(&self, tokenizer_name: &str, tokenizer: A) where A: 'static + Send + Sync + for<'a> Tokenizer<'a>, @@ -39,6 +40,7 @@ impl TokenizerManager { .insert(tokenizer_name.to_string(), boxed_tokenizer); } + /// Accessing a tokenizer given its name. pub fn get(&self, tokenizer_name: &str) -> Option> { self.tokenizers .read()