diff --git a/Cargo.lock b/Cargo.lock index 77e60390cb0..b0c66cc8b5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2312,6 +2312,7 @@ dependencies = [ "bytes", "common-test-util", "futures", + "icu_properties", "lazy_static", "pin-project", "regex", @@ -2507,7 +2508,6 @@ dependencies = [ "geohash", "h3o", "hyperloglogplus", - "icu_properties", "jsonb", "memchr", "mito-codec", @@ -6954,8 +6954,6 @@ dependencies = [ "futures", "greptime-proto", "itertools 0.14.0", - "jieba-rs", - "lazy_static", "mockall", "pin-project", "prost 0.14.1", diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml index 702171e9d67..6df2283c2cf 100644 --- a/src/common/base/Cargo.toml +++ b/src/common/base/Cargo.toml @@ -14,6 +14,7 @@ async-trait.workspace = true bitvec = "1.0" bytes.workspace = true futures.workspace = true +icu_properties.workspace = true lazy_static.workspace = true pin-project.workspace = true regex.workspace = true diff --git a/src/common/base/src/lib.rs b/src/common/base/src/lib.rs index 04fc7038018..2d5a6fbd894 100644 --- a/src/common/base/src/lib.rs +++ b/src/common/base/src/lib.rs @@ -25,6 +25,7 @@ pub mod readable_size; pub mod regex_pattern; pub mod secrets; pub mod serde; +pub mod term_token; pub type AffectedRows = usize; diff --git a/src/common/base/src/term_token.rs b/src/common/base/src/term_token.rs new file mode 100644 index 00000000000..a6121559a67 --- /dev/null +++ b/src/common/base/src/term_token.rs @@ -0,0 +1,280 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Word boundaries shared by `matches_term` and the fulltext bloom index. +//! +//! Bloom pruning is only sound if the index splits text exactly where +//! `matches_term` sees word boundaries, so both use [`classify_char`], and the +//! query side derives its probes with [`extract_term_probes`] and [`extract_like_probes`]. + +use std::ops::Range; + +use icu_properties::props::Script; +use icu_properties::{CodePointMapData, CodePointMapDataBorrowed}; + +/// Classes of characters that decide where `matches_term` sees a word boundary. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CharClass { + /// `[0-9A-Za-z]`. + AsciiWord, + /// Characters of the Han script, alphanumeric or not. + Han, + /// Non-ASCII alphanumerics outside the Han script, such as `é` or `д`. + UnicodeWord, + /// Everything else, including `_`. + Other, +} + +/// Returns the [`CharClass`] of `c`. [`tokenize`] splits text wherever the class +/// changes and at every [`CharClass::Other`] character. +pub fn classify_char(c: char) -> CharClass { + if c.is_ascii_alphanumeric() { + CharClass::AsciiWord + } else if is_han(c) { + CharClass::Han + } else if c.is_alphanumeric() { + CharClass::UnicodeWord + } else { + CharClass::Other + } +} + +static HAN_SCRIPT_DATA: CodePointMapDataBorrowed<'static, Script> = + CodePointMapData::