diff --git a/Cargo.lock b/Cargo.lock index e985f483efb..6ec2a2d2764 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1774,15 +1774,6 @@ dependencies = [ "shlex 2.0.1", ] -[[package]] -name = "cedarwood" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0524a528a6a0288df1863c3c20fe92c301875b4941e7b6c4b394ab08c5a4c55" -dependencies = [ - "smallvec", -] - [[package]] name = "census" version = "0.4.2" @@ -1927,7 +1918,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7d8d1efd5109b9c1cd3b7966bd071cdfb53bb6eb0b22a473a68c2f70a11a1eb" dependencies = [ "parse-zoneinfo", - "phf_codegen 0.12.1", + "phf_codegen", "phf_shared 0.12.1", "uncased", ] @@ -7343,25 +7334,19 @@ dependencies = [ [[package]] name = "jieba-macros" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82" -dependencies = [ - "phf_codegen 0.13.1", -] +checksum = "2987cf45b1bfdb4ff1f6a814b1aa2ad6361f4ff43bf353a9bd29f391a9568f70" [[package]] name = "jieba-rs" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9" +checksum = "0a3a7d02ea31e6fc6f2fe10495dc7c5efcf381e1f21d7172f58ba792efa1c2fd" dependencies = [ "bytecount", - "cedarwood", "include-flate", "jieba-macros", - "phf 0.13.1", - "regex", "rustc-hash", ] @@ -10774,20 +10759,10 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efbdcb6f01d193b17f0b9c3360fa7e0e620991b193ff08702f78b3ce365d7e61" dependencies = [ - "phf_generator 0.12.1", + "phf_generator", "phf_shared 0.12.1", ] -[[package]] -name = "phf_codegen" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" -dependencies = [ - "phf_generator 0.13.1", - "phf_shared 0.13.1", -] - [[package]] name = "phf_generator" version = "0.12.1" @@ -10798,23 +10773,13 @@ dependencies = [ "phf_shared 0.12.1", ] -[[package]] -name = "phf_generator" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" -dependencies = [ - "fastrand", - "phf_shared 0.13.1", -] - [[package]] name = "phf_macros" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d713258393a82f091ead52047ca779d37e5766226d009de21696c4e667044368" dependencies = [ - "phf_generator 0.12.1", + "phf_generator", "phf_shared 0.12.1", "proc-macro2", "quote", @@ -14895,9 +14860,9 @@ dependencies = [ [[package]] name = "tantivy-jieba" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3392170e86f1c387170aba7d171a466ffdc98a8b55b006e19ac64b123a7b690a" +checksum = "25cf6c5eb9419d7da9aacf5e184863b9ed476c478de0e196729f16124e716dd1" dependencies = [ "jieba-rs", "lazy_static", diff --git a/src/index/Cargo.toml b/src/index/Cargo.toml index 078f9e42c56..2458a6591af 100644 --- a/src/index/Cargo.toml +++ b/src/index/Cargo.toml @@ -23,7 +23,7 @@ fst.workspace = true futures.workspace = true greptime-proto.workspace = true itertools.workspace = true -jieba-rs = "0.10" +jieba-rs = "0.11" lazy_static.workspace = true mockall.workspace = true pin-project.workspace = true @@ -37,7 +37,7 @@ serde_json.workspace = true snafu.workspace = true store-api.workspace = true tantivy = { version = "0.26", features = ["zstd-compression"] } -tantivy-jieba = "0.20" +tantivy-jieba = "0.21" tokio.workspace = true tokio-util.workspace = true uuid.workspace = true diff --git a/src/index/src/fulltext_index/tokenizer.rs b/src/index/src/fulltext_index/tokenizer.rs index e6752a40420..0a1cc47aa8e 100644 --- a/src/index/src/fulltext_index/tokenizer.rs +++ b/src/index/src/fulltext_index/tokenizer.rs @@ -262,6 +262,8 @@ mod tests { let text = "[2026/04/09/ 13:56:11.031]2026-04-09 13:56:11.031 - [ trace_id=340a6a44b0bd8e37bb7697ss7da61ff0 span_id=085ff5ttf1e0a23b trace_flags=01] - [http-nio-8081-exec-16] INFO c.h.p.xx.web.service.impl.CCCXForwardKKKServiceImpl.pushout(188) - 登录手机号18888888888的动态key:829889AC8 ship_ship ship__ship _ __ __IDENTIFIER__ _ship ship_ EOF"; let tokens = tokenizer.tokenize(text); + // Jieba 0.11 preserves ASCII compounds and emits searchable alphabetic parts. + // The underscore fallback also retains leading, trailing, and repeated underscores. assert_eq!( tokens, vec![ @@ -271,36 +273,34 @@ mod tests { "13", "56", "11.031", - "2026-04", - "09", + "2026-04-09", "13", "56", "11.031", "trace", - "_", "id", + "trace_id", "340a6a44b0bd8e37bb7697ss7da61ff0", "span", - "_", "id", + "span_id", "085ff5ttf1e0a23b", "trace", - "_", "flags", + "trace_flags", "01", "http", - "nio-8081", - "exec-16", + "nio", + "exec", + "http-nio-8081-exec-16", "INFO", - "c", - "h", - "p", "xx", "web", "service", "impl", "CCCXForwardKKKServiceImpl", "pushout", + "c.h.p.xx.web.service.impl.CCCXForwardKKKServiceImpl.pushout", "188", "登录", "手机", @@ -311,8 +311,8 @@ mod tests { "key", "829889AC8", "ship", - "_", "ship", + "ship_ship", "ship", "__", "ship", @@ -367,8 +367,8 @@ mod tests { "_", "id", "trace", - "_", "id", + "trace_id", "手机", "手机号", "_",