perf: optimize fulltext zh tokenizer for ascii-only text (#5975)

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>
This commit is contained in:
Zhenchi
2025-04-24 23:31:26 +00:00
committed by GitHub
parent e30753fc31
commit d5026f3491
+5 -1
View File
@@ -46,7 +46,11 @@ pub struct ChineseTokenizer;
impl Tokenizer for ChineseTokenizer {
fn tokenize<'a>(&self, text: &'a str) -> Vec<&'a str> {
JIEBA.cut(text, false)
if text.is_ascii() {
EnglishTokenizer {}.tokenize(text)
} else {
JIEBA.cut(text, false)
}
}
}