Replace rust-stemmers with frostem for stemming (#3028)

rust-stemmers is unmaintained and lags behind upstream Snowball.
frostem tracks snowball main and regenerates algorithms automatically.

Cargo.toml

@@ -37,7 +37,26 @@ fs4 = { version = "0.13.1", optional = true }
 levenshtein_automata = "0.2.1"
 uuid = { version = "1.0.0", features = ["v4", "serde"] }
 crossbeam-channel = "0.5.4"
-rust-stemmers = { version = "1.2.0", optional = true }
+frostem = { version = "1.20260804.0", optional = true, default-features = false, features = [
+    "arabic",
+    "danish",
+    "dutch",
+    "english",
+    "finnish",
+    "french",
+    "german",
+    "greek",
+    "hungarian",
+    "italian",
+    "norwegian",
+    "portuguese",
+    "romanian",
+    "russian",
+    "spanish",
+    "swedish",
+    "tamil",
+    "turkish",
+] }
 downcast-rs = "2.0.1"
 bitpacking = { version = "0.9.3", default-features = false, features = [
     "bitpacker4x",
@@ -115,7 +134,7 @@ overflow-checks = true
 
 [features]
 default = ["mmap", "stopwords", "lz4-compression", "columnar-zstd-compression", "stemmer"]
-stemmer = ["rust-stemmers"]
+stemmer = ["frostem"]
 mmap = ["fs4", "tempfile", "memmap2"]
 stopwords = []
 

src/tokenizer/stemmer.rs

@@ -1,7 +1,7 @@
 use std::borrow::Cow;
 use std::mem;
 
-use rust_stemmers::Algorithm;
+use frostem::Algorithm;
 use serde::{Deserialize, Serialize};
 
 use super::{Token, TokenFilter, TokenStream, Tokenizer};
@@ -101,7 +101,7 @@ impl<T: Tokenizer> Tokenizer for StemmerFilter<T> {
     type TokenStream<'a> = StemmerTokenStream<T::TokenStream<'a>>;
 
     fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
-        let stemmer = rust_stemmers::Stemmer::create(self.stemmer_algorithm);
+        let stemmer = frostem::Stemmer::new(self.stemmer_algorithm);
         StemmerTokenStream {
             tail: self.inner.token_stream(text),
             stemmer,
@@ -112,7 +112,7 @@ impl<T: Tokenizer> Tokenizer for StemmerFilter<T> {
 
 pub struct StemmerTokenStream<T> {
     tail: T,
-    stemmer: rust_stemmers::Stemmer,
+    stemmer: frostem::Stemmer,
     buffer: String,
 }
 
@@ -149,7 +149,9 @@ mod tests {
 
     use super::*;
     use crate::tokenizer::tests::assert_token;
-    use crate::tokenizer::{LowerCaser, SimpleTokenizer, TextAnalyzer, TokenizerManager};
+    use crate::tokenizer::{
+        LowerCaser, RawTokenizer, SimpleTokenizer, TextAnalyzer, TokenizerManager,
+    };
 
     #[test]
     fn test_en_stem() {
@@ -198,4 +200,18 @@ mod tests {
         assert_token(&tokens[1], 1, "χαρουμεν", 18, 36);
         assert_token(&tokens[2], 2, "φορολογουμεν", 37, 63);
     }
+
+    /// Regression for a multibyte Greek suffix that could panic in the
+    /// unmaintained `rust-stemmers` 1.2.0 Greek implementation after the
+    /// stemmer shortened a word using stale UTF-8 byte offsets.
+    #[test]
+    fn test_greek_stemmer_handles_multibyte_suffixes() {
+        let mut analyzer = TextAnalyzer::builder(RawTokenizer::default())
+            .filter(Stemmer::new(Language::Greek))
+            .build();
+        let mut stream = analyzer.token_stream("αντιθετε");
+
+        assert!(stream.advance());
+        assert_eq!(stream.token().text, "ανετ");
+    }
 }