From d5026f3491859847242b9a548d4f8d69aa964978 Mon Sep 17 00:00:00 2001
From: Zhenchi <zhongzc_arch@outlook.com>
Date: Fri, 25 Apr 2025 07:31:26 +0800
Subject: [PATCH] perf: optimize fulltext zh tokenizer for ascii-only text
 (#5975)

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>
---
 src/index/src/fulltext_index/tokenizer.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/index/src/fulltext_index/tokenizer.rs b/src/index/src/fulltext_index/tokenizer.rs
index b00e7fda9c8..54aa33edc84 100644
--- a/src/index/src/fulltext_index/tokenizer.rs
+++ b/src/index/src/fulltext_index/tokenizer.rs
@@ -46,7 +46,11 @@ pub struct ChineseTokenizer;
 
 impl Tokenizer for ChineseTokenizer {
     fn tokenize<'a>(&self, text: &'a str) -> Vec<&'a str> {
-        JIEBA.cut(text, false)
+        if text.is_ascii() {
+            EnglishTokenizer {}.tokenize(text)
+        } else {
+            JIEBA.cut(text, false)
+        }
     }
 }