diff --git a/src/index/src/fulltext_index/tokenizer.rs b/src/index/src/fulltext_index/tokenizer.rs index b00e7fda9c8..54aa33edc84 100644 --- a/src/index/src/fulltext_index/tokenizer.rs +++ b/src/index/src/fulltext_index/tokenizer.rs @@ -46,7 +46,11 @@ pub struct ChineseTokenizer; impl Tokenizer for ChineseTokenizer { fn tokenize<'a>(&self, text: &'a str) -> Vec<&'a str> { - JIEBA.cut(text, false) + if text.is_ascii() { + EnglishTokenizer {}.tokenize(text) + } else { + JIEBA.cut(text, false) + } } }