diff --git a/src/index/src/fulltext_index/tokenizer.rs b/src/index/src/fulltext_index/tokenizer.rs index 0a1cc47aa8e..f8be197926a 100644 --- a/src/index/src/fulltext_index/tokenizer.rs +++ b/src/index/src/fulltext_index/tokenizer.rs @@ -144,8 +144,8 @@ impl Analyzer { /// Returns the bloom filter hash of each token in the given text. /// /// Equivalent to hashing every token returned by [`Analyzer::analyze_text`] with - /// [`element_hash`]. Only case-insensitive non-ASCII tokens allocate, in `to_lowercase`; - /// case-insensitive ASCII tokens are lowercased in `buf`. + /// [`element_hash`]. Case-insensitive ASCII tokens that already have no uppercase bytes are + /// hashed directly; other case-insensitive ASCII tokens are lowercased in `buf`. pub fn analyze_text_hashes<'a>( &self, text: &'a str, @@ -153,7 +153,9 @@ impl Analyzer { ) -> impl Iterator + use<'a> { let case_sensitive = self.case_sensitive; self.tokenizer.tokenize(text).into_iter().map(move |token| { - if case_sensitive { + if case_sensitive + || (token.is_ascii() && !token.bytes().any(|b| b.is_ascii_uppercase())) + { element_hash(token.as_bytes()) } else if token.is_ascii() { buf.clear(); @@ -189,24 +191,78 @@ mod tests { #[test] fn test_analyze_text_hashes_matches_analyze_text() { - let text = "Hello, WORLD ship_Ship 清洁表面 ÄÖÜ straße İstanbul x"; - for (tokenizer, case_sensitive) in [ - (Box::new(EnglishTokenizer) as Box, false), - (Box::new(EnglishTokenizer), true), - (Box::new(ChineseTokenizer), false), - ] { - let analyzer = Analyzer::new(tokenizer, case_sensitive); - let expected = analyzer - .analyze_text(text) + let texts = [ + "Hello, WORLD ship_Ship 清洁表面 ÄÖÜ straße İstanbul x", + "", + "!!!", + "lowercase 1234 snake_case", + "MiXeD Straße İÉCOLE", + ]; + for text in texts { + for case_sensitive in [false, true] { + for tokenizer in [ + Box::new(EnglishTokenizer) as Box, + Box::new(ChineseTokenizer), + ] { + let analyzer = Analyzer::new(tokenizer, case_sensitive); + let expected = analyzer + .analyze_text(text) + .unwrap() + .iter() + .map(|token| element_hash(token)) + .collect::>(); + let mut buf = b"stale buffer contents".to_vec(); + let actual = analyzer + .analyze_text_hashes(text, &mut buf) + .collect::>(); + assert_eq!( + expected, actual, + "text={text:?}, case_sensitive={case_sensitive}" + ); + } + } + } + } + + #[test] + fn test_analyze_text_hashes_borrows_lowercase_ascii_without_changing_buf() { + let analyzer = Analyzer::new(Box::new(EnglishTokenizer), false); + let mut buf = b"sentinel".to_vec(); + + let hashes = analyzer + .analyze_text_hashes("lowercase 1234 snake_case", &mut buf) + .collect::>(); + assert_eq!(buf, b"sentinel"); + assert_eq!( + hashes, + analyzer + .analyze_text("lowercase 1234 snake_case") .unwrap() .iter() - .map(|t| element_hash(t)) - .collect::>(); - let hashes = analyzer - .analyze_text_hashes(text, &mut Vec::new()) - .collect::>(); - assert_eq!(expected, hashes); - } + .map(|token| element_hash(token)) + .collect::>() + ); + + let hashes = analyzer + .analyze_text_hashes("lowercase UPPER", &mut buf) + .collect::>(); + assert_eq!(buf, b"upper"); + assert_eq!( + hashes, + analyzer + .analyze_text("lowercase UPPER") + .unwrap() + .iter() + .map(|token| element_hash(token)) + .collect::>() + ); + + let case_sensitive = Analyzer::new(Box::new(EnglishTokenizer), true); + buf = b"case-sensitive sentinel".to_vec(); + let _ = case_sensitive + .analyze_text_hashes("UPPER", &mut buf) + .collect::>(); + assert_eq!(buf, b"case-sensitive sentinel"); } #[test]