diff --git a/Cargo.toml b/Cargo.toml index a7c8d04e6..f4b06b497 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,26 @@ fs4 = { version = "0.13.1", optional = true } levenshtein_automata = "0.2.1" uuid = { version = "1.0.0", features = ["v4", "serde"] } crossbeam-channel = "0.5.4" -rust-stemmers = { version = "1.2.0", optional = true } +frostem = { version = "1.20260804.0", optional = true, default-features = false, features = [ + "arabic", + "danish", + "dutch", + "english", + "finnish", + "french", + "german", + "greek", + "hungarian", + "italian", + "norwegian", + "portuguese", + "romanian", + "russian", + "spanish", + "swedish", + "tamil", + "turkish", +] } downcast-rs = "2.0.1" bitpacking = { version = "0.9.3", default-features = false, features = [ "bitpacker4x", @@ -115,7 +134,7 @@ overflow-checks = true [features] default = ["mmap", "stopwords", "lz4-compression", "columnar-zstd-compression", "stemmer"] -stemmer = ["rust-stemmers"] +stemmer = ["frostem"] mmap = ["fs4", "tempfile", "memmap2"] stopwords = [] diff --git a/src/tokenizer/stemmer.rs b/src/tokenizer/stemmer.rs index 764efc2ee..a4c86966c 100644 --- a/src/tokenizer/stemmer.rs +++ b/src/tokenizer/stemmer.rs @@ -1,7 +1,7 @@ use std::borrow::Cow; use std::mem; -use rust_stemmers::Algorithm; +use frostem::Algorithm; use serde::{Deserialize, Serialize}; use super::{Token, TokenFilter, TokenStream, Tokenizer}; @@ -101,7 +101,7 @@ impl Tokenizer for StemmerFilter { type TokenStream<'a> = StemmerTokenStream>; fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { - let stemmer = rust_stemmers::Stemmer::create(self.stemmer_algorithm); + let stemmer = frostem::Stemmer::new(self.stemmer_algorithm); StemmerTokenStream { tail: self.inner.token_stream(text), stemmer, @@ -112,7 +112,7 @@ impl Tokenizer for StemmerFilter { pub struct StemmerTokenStream { tail: T, - stemmer: rust_stemmers::Stemmer, + stemmer: frostem::Stemmer, buffer: String, } @@ -149,7 +149,9 @@ mod tests { use super::*; use crate::tokenizer::tests::assert_token; - use crate::tokenizer::{LowerCaser, SimpleTokenizer, TextAnalyzer, TokenizerManager}; + use crate::tokenizer::{ + LowerCaser, RawTokenizer, SimpleTokenizer, TextAnalyzer, TokenizerManager, + }; #[test] fn test_en_stem() { @@ -198,4 +200,18 @@ mod tests { assert_token(&tokens[1], 1, "χαρουμεν", 18, 36); assert_token(&tokens[2], 2, "φορολογουμεν", 37, 63); } + + /// Regression for a multibyte Greek suffix that could panic in the + /// unmaintained `rust-stemmers` 1.2.0 Greek implementation after the + /// stemmer shortened a word using stale UTF-8 byte offsets. + #[test] + fn test_greek_stemmer_handles_multibyte_suffixes() { + let mut analyzer = TextAnalyzer::builder(RawTokenizer::default()) + .filter(Stemmer::new(Language::Greek)) + .build(); + let mut stream = analyzer.token_stream("αντιθετε"); + + assert!(stream.advance()); + assert_eq!(stream.token().text, "ανετ"); + } }