From 5c4d9eb4f8f40c424e5b171dc2bdc6a9bfcb8592 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Wed, 14 Oct 2026 08:07:24 +0000 Subject: [PATCH] fix: align fulltext bloom tokens with matches_term and use them for LIKE (#9470) * fix: align fulltext bloom tokens with matches_term and use them for LIKE The bloom fulltext index stored tokens that did not follow matches_term's word boundaries (`_`, ASCII next to Han, Han substrings), so pruning dropped rows matches_term accepts. Tokens and query probes now share the character classes of matches_term, Han runs are indexed as bigrams, and LIKE patterns use the same probes. The bloom blob is written under a new key; blobs from older versions are no longer probed. Tantivy no longer serves matches_term. Signed-off-by: Dennis Zhuang * chore: drop unused lazy_static from index and avoid typo-flagged test tokens Signed-off-by: Dennis Zhuang * docs: link like_probes to tokenize instead of the downstream ScriptTokenizer Signed-off-by: Dennis Zhuang * refactor: name probe extraction with verbs and document CharClass Signed-off-by: Dennis Zhuang * feat: rebuild legacy fulltext bloom indexes and probe lower(col) The manifest records the fulltext bloom version per column, so ADMIN BUILD_INDEX now rebuilds SSTs whose bloom index predates the current tokens, and ssts_index_meta shows the version. Case-insensitive bloom indexes also store the tokens of the whole lowercased text for rows where it can differ (U+0130, U+212A, sigma), so matches_term(lower(col), ..) and lower(col) LIKE .. use the index. Signed-off-by: Dennis Zhuang * test: cover legacy bloom index rewrite by compaction in the compat case Signed-off-by: Dennis Zhuang * chore: log outdated fulltext bloom indexes in the consistency check Signed-off-by: Dennis Zhuang * docs: explain the unrecorded fulltext bloom version Signed-off-by: Dennis Zhuang --------- Signed-off-by: Dennis Zhuang --- Cargo.lock | 4 +- src/common/base/Cargo.toml | 1 + src/common/base/src/lib.rs | 1 + src/common/base/src/term_token.rs | 280 +++++ src/common/function/Cargo.toml | 1 - .../function/src/scalars/matches_term.rs | 38 +- src/index/Cargo.toml | 2 - src/index/benches/index_build_bench.rs | 4 +- src/index/benches/tokenizer_bench.rs | 22 +- .../src/fulltext_index/create/bloom_filter.rs | 10 +- src/index/src/fulltext_index/tokenizer.rs | 592 +-------- src/mito2/src/cache/index/result_cache.rs | 12 +- src/mito2/src/engine/basic_test.rs | 4 +- src/mito2/src/engine/puffin_index.rs | 32 +- src/mito2/src/sst/file.rs | 82 +- src/mito2/src/sst/file_purger.rs | 1 + src/mito2/src/sst/file_ref.rs | 1 + src/mito2/src/sst/index.rs | 10 + src/mito2/src/sst/index/fulltext_index.rs | 11 +- .../src/sst/index/fulltext_index/applier.rs | 334 ++++- .../index/fulltext_index/applier/builder.rs | 296 +++-- .../src/sst/index/fulltext_index/creator.rs | 234 +--- src/mito2/src/sst/index/indexer/finish.rs | 1 + src/query/src/optimizer/constant_term.rs | 11 +- .../admin/build_index_table_restart.result | 2 +- .../function/fulltext_index_pruning.result | 1076 +++++++++++++++++ .../function/fulltext_index_pruning.sql | 296 +++++ .../common/information_schema/ssts.result | 16 +- .../cases/downgrade_compatibility/setup.sql | 11 + .../downgrade_compatibility/verify.result | 18 + .../cases/downgrade_compatibility/verify.sql | 6 + .../cases/fulltext_bloom_tokens/case.toml | 8 + .../cases/fulltext_bloom_tokens/setup.sql | 28 + .../cases/fulltext_bloom_tokens/verify.result | 197 +++ .../cases/fulltext_bloom_tokens/verify.sql | 71 ++ 35 files changed, 2687 insertions(+), 1026 deletions(-) create mode 100644 src/common/base/src/term_token.rs create mode 100644 tests/cases/standalone/common/function/fulltext_index_pruning.result create mode 100644 tests/cases/standalone/common/function/fulltext_index_pruning.sql create mode 100644 tests/compatibility/cases/fulltext_bloom_tokens/case.toml create mode 100644 tests/compatibility/cases/fulltext_bloom_tokens/setup.sql create mode 100644 tests/compatibility/cases/fulltext_bloom_tokens/verify.result create mode 100644 tests/compatibility/cases/fulltext_bloom_tokens/verify.sql diff --git a/Cargo.lock b/Cargo.lock index 77e60390cb0..b0c66cc8b5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2312,6 +2312,7 @@ dependencies = [ "bytes", "common-test-util", "futures", + "icu_properties", "lazy_static", "pin-project", "regex", @@ -2507,7 +2508,6 @@ dependencies = [ "geohash", "h3o", "hyperloglogplus", - "icu_properties", "jsonb", "memchr", "mito-codec", @@ -6954,8 +6954,6 @@ dependencies = [ "futures", "greptime-proto", "itertools 0.14.0", - "jieba-rs", - "lazy_static", "mockall", "pin-project", "prost 0.14.1", diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml index 702171e9d67..6df2283c2cf 100644 --- a/src/common/base/Cargo.toml +++ b/src/common/base/Cargo.toml @@ -14,6 +14,7 @@ async-trait.workspace = true bitvec = "1.0" bytes.workspace = true futures.workspace = true +icu_properties.workspace = true lazy_static.workspace = true pin-project.workspace = true regex.workspace = true diff --git a/src/common/base/src/lib.rs b/src/common/base/src/lib.rs index 04fc7038018..2d5a6fbd894 100644 --- a/src/common/base/src/lib.rs +++ b/src/common/base/src/lib.rs @@ -25,6 +25,7 @@ pub mod readable_size; pub mod regex_pattern; pub mod secrets; pub mod serde; +pub mod term_token; pub type AffectedRows = usize; diff --git a/src/common/base/src/term_token.rs b/src/common/base/src/term_token.rs new file mode 100644 index 00000000000..a6121559a67 --- /dev/null +++ b/src/common/base/src/term_token.rs @@ -0,0 +1,280 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Word boundaries shared by `matches_term` and the fulltext bloom index. +//! +//! Bloom pruning is only sound if the index splits text exactly where +//! `matches_term` sees word boundaries, so both use [`classify_char`], and the +//! query side derives its probes with [`extract_term_probes`] and [`extract_like_probes`]. + +use std::ops::Range; + +use icu_properties::props::Script; +use icu_properties::{CodePointMapData, CodePointMapDataBorrowed}; + +/// Classes of characters that decide where `matches_term` sees a word boundary. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CharClass { + /// `[0-9A-Za-z]`. + AsciiWord, + /// Characters of the Han script, alphanumeric or not. + Han, + /// Non-ASCII alphanumerics outside the Han script, such as `é` or `д`. + UnicodeWord, + /// Everything else, including `_`. + Other, +} + +/// Returns the [`CharClass`] of `c`. [`tokenize`] splits text wherever the class +/// changes and at every [`CharClass::Other`] character. +pub fn classify_char(c: char) -> CharClass { + if c.is_ascii_alphanumeric() { + CharClass::AsciiWord + } else if is_han(c) { + CharClass::Han + } else if c.is_alphanumeric() { + CharClass::UnicodeWord + } else { + CharClass::Other + } +} + +static HAN_SCRIPT_DATA: CodePointMapDataBorrowed<'static, Script> = + CodePointMapData::