diff --git a/tokenizer-api/Cargo.toml b/tokenizer-api/Cargo.toml index 201f84dec..ecd19e18c 100644 --- a/tokenizer-api/Cargo.toml +++ b/tokenizer-api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tantivy-tokenizer-api" -version = "0.1.0" +version = "0.1.1" license = "MIT" edition = "2021" description = "Tokenizer API of tantivy" diff --git a/tokenizer-api/src/lib.rs b/tokenizer-api/src/lib.rs index adb37a0b4..aa9f8f8f6 100644 --- a/tokenizer-api/src/lib.rs +++ b/tokenizer-api/src/lib.rs @@ -63,6 +63,12 @@ pub trait Tokenizer: 'static + Clone + Send + Sync { /// Simple wrapper of `Box`. pub struct BoxTokenStream<'a>(Box); +impl<'a> From> for Box { + fn from(token_stream: BoxTokenStream<'a>) -> Self { + token_stream.0 + } +} + impl<'a, T> From for BoxTokenStream<'a> where T: TokenStream + 'a {