perf(index): hash already-lowercase ASCII tokens without copying

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

src/index/src/fulltext_index/tokenizer.rs

@@ -144,8 +144,8 @@ impl Analyzer {
     /// Returns the bloom filter hash of each token in the given text.
     ///
     /// Equivalent to hashing every token returned by [`Analyzer::analyze_text`] with
-    /// [`element_hash`]. Only case-insensitive non-ASCII tokens allocate, in `to_lowercase`;
-    /// case-insensitive ASCII tokens are lowercased in `buf`.
+    /// [`element_hash`]. Case-insensitive ASCII tokens that already have no uppercase bytes are
+    /// hashed directly; other case-insensitive ASCII tokens are lowercased in `buf`.
     pub fn analyze_text_hashes<'a>(
         &self,
         text: &'a str,
@@ -153,7 +153,9 @@ impl Analyzer {
     ) -> impl Iterator<Item = u64> + use<'a> {
         let case_sensitive = self.case_sensitive;
         self.tokenizer.tokenize(text).into_iter().map(move |token| {
-            if case_sensitive {
+            if case_sensitive
+                || (token.is_ascii() && !token.bytes().any(|b| b.is_ascii_uppercase()))
+            {
                 element_hash(token.as_bytes())
             } else if token.is_ascii() {
                 buf.clear();
@@ -189,24 +191,78 @@ mod tests {
 
     #[test]
     fn test_analyze_text_hashes_matches_analyze_text() {
-        let text = "Hello, WORLD ship_Ship 清洁表面 ÄÖÜ straße İstanbul x";
-        for (tokenizer, case_sensitive) in [
-            (Box::new(EnglishTokenizer) as Box<dyn Tokenizer>, false),
-            (Box::new(EnglishTokenizer), true),
-            (Box::new(ChineseTokenizer), false),
-        ] {
-            let analyzer = Analyzer::new(tokenizer, case_sensitive);
-            let expected = analyzer
-                .analyze_text(text)
+        let texts = [
+            "Hello, WORLD ship_Ship 清洁表面 ÄÖÜ straße İstanbul x",
+            "",
+            "!!!",
+            "lowercase 1234 snake_case",
+            "MiXeD Straße İÉCOLE",
+        ];
+        for text in texts {
+            for case_sensitive in [false, true] {
+                for tokenizer in [
+                    Box::new(EnglishTokenizer) as Box<dyn Tokenizer>,
+                    Box::new(ChineseTokenizer),
+                ] {
+                    let analyzer = Analyzer::new(tokenizer, case_sensitive);
+                    let expected = analyzer
+                        .analyze_text(text)
+                        .unwrap()
+                        .iter()
+                        .map(|token| element_hash(token))
+                        .collect::<Vec<_>>();
+                    let mut buf = b"stale buffer contents".to_vec();
+                    let actual = analyzer
+                        .analyze_text_hashes(text, &mut buf)
+                        .collect::<Vec<_>>();
+                    assert_eq!(
+                        expected, actual,
+                        "text={text:?}, case_sensitive={case_sensitive}"
+                    );
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn test_analyze_text_hashes_borrows_lowercase_ascii_without_changing_buf() {
+        let analyzer = Analyzer::new(Box::new(EnglishTokenizer), false);
+        let mut buf = b"sentinel".to_vec();
+
+        let hashes = analyzer
+            .analyze_text_hashes("lowercase 1234 snake_case", &mut buf)
+            .collect::<Vec<_>>();
+        assert_eq!(buf, b"sentinel");
+        assert_eq!(
+            hashes,
+            analyzer
+                .analyze_text("lowercase 1234 snake_case")
                 .unwrap()
                 .iter()
-                .map(|t| element_hash(t))
-                .collect::<Vec<_>>();
-            let hashes = analyzer
-                .analyze_text_hashes(text, &mut Vec::new())
-                .collect::<Vec<_>>();
-            assert_eq!(expected, hashes);
-        }
+                .map(|token| element_hash(token))
+                .collect::<Vec<_>>()
+        );
+
+        let hashes = analyzer
+            .analyze_text_hashes("lowercase UPPER", &mut buf)
+            .collect::<Vec<_>>();
+        assert_eq!(buf, b"upper");
+        assert_eq!(
+            hashes,
+            analyzer
+                .analyze_text("lowercase UPPER")
+                .unwrap()
+                .iter()
+                .map(|token| element_hash(token))
+                .collect::<Vec<_>>()
+        );
+
+        let case_sensitive = Analyzer::new(Box::new(EnglishTokenizer), true);
+        buf = b"case-sensitive sentinel".to_vec();
+        let _ = case_sensitive
+            .analyze_text_hashes("UPPER", &mut buf)
+            .collect::<Vec<_>>();
+        assert_eq!(buf, b"case-sensitive sentinel");
     }
 
     #[test]