mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-10-28 14:46:22 +00:00
perf: optimize fulltext zh tokenizer for ascii-only text (#5975)
Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>
This commit is contained in:
@@ -46,7 +46,11 @@ pub struct ChineseTokenizer;
|
||||
|
||||
impl Tokenizer for ChineseTokenizer {
|
||||
fn tokenize<'a>(&self, text: &'a str) -> Vec<&'a str> {
|
||||
JIEBA.cut(text, false)
|
||||
if text.is_ascii() {
|
||||
EnglishTokenizer {}.tokenize(text)
|
||||
} else {
|
||||
JIEBA.cut(text, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user