fix: align fulltext bloom tokens with matches_term and use them for LIKE (#9470)

* fix: align fulltext bloom tokens with matches_term and use them for LIKE

The bloom fulltext index stored tokens that did not follow
matches_term's word boundaries (`_`, ASCII next to Han, Han substrings),
so pruning dropped rows matches_term accepts.

Tokens and query probes now share the character classes of
matches_term, Han runs are indexed as bigrams, and LIKE patterns use the
same probes. The bloom blob is written under a new key; blobs from older
versions are no longer probed. Tantivy no longer serves matches_term.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* chore: drop unused lazy_static from index and avoid typo-flagged test tokens

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* docs: link like_probes to tokenize instead of the downstream ScriptTokenizer

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* refactor: name probe extraction with verbs and document CharClass

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* feat: rebuild legacy fulltext bloom indexes and probe lower(col)

The manifest records the fulltext bloom version per column, so ADMIN
BUILD_INDEX now rebuilds SSTs whose bloom index predates the current
tokens, and ssts_index_meta shows the version.

Case-insensitive bloom indexes also store the tokens of the whole
lowercased text for rows where it can differ (U+0130, U+212A, sigma),
so matches_term(lower(col), ..) and lower(col) LIKE .. use the index.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* test: cover legacy bloom index rewrite by compaction in the compat case

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* chore: log outdated fulltext bloom indexes in the consistency check

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* docs: explain the unrecorded fulltext bloom version

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

---------

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

Cargo.lock

@@ -2312,6 +2312,7 @@ dependencies = [
  "bytes",
  "common-test-util",
  "futures",
+ "icu_properties",
  "lazy_static",
  "pin-project",
  "regex",
@@ -2507,7 +2508,6 @@ dependencies = [
  "geohash",
  "h3o",
  "hyperloglogplus",
- "icu_properties",
  "jsonb",
  "memchr",
  "mito-codec",
@@ -6954,8 +6954,6 @@ dependencies = [
  "futures",
  "greptime-proto",
  "itertools 0.14.0",
- "jieba-rs",
- "lazy_static",
  "mockall",
  "pin-project",
  "prost 0.14.1",

src/common/base/Cargo.toml

@@ -14,6 +14,7 @@ async-trait.workspace = true
 bitvec = "1.0"
 bytes.workspace = true
 futures.workspace = true
+icu_properties.workspace = true
 lazy_static.workspace = true
 pin-project.workspace = true
 regex.workspace = true

src/common/base/src/lib.rs

@@ -25,6 +25,7 @@ pub mod readable_size;
 pub mod regex_pattern;
 pub mod secrets;
 pub mod serde;
+pub mod term_token;
 
 pub type AffectedRows = usize;
 

src/common/base/src/term_token.rs

@@ -0,0 +1,280 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Word boundaries shared by `matches_term` and the fulltext bloom index.
+//!
+//! Bloom pruning is only sound if the index splits text exactly where
+//! `matches_term` sees word boundaries, so both use [`classify_char`], and the
+//! query side derives its probes with [`extract_term_probes`] and [`extract_like_probes`].
+
+use std::ops::Range;
+
+use icu_properties::props::Script;
+use icu_properties::{CodePointMapData, CodePointMapDataBorrowed};
+
+/// Classes of characters that decide where `matches_term` sees a word boundary.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum CharClass {
+    /// `[0-9A-Za-z]`.
+    AsciiWord,
+    /// Characters of the Han script, alphanumeric or not.
+    Han,
+    /// Non-ASCII alphanumerics outside the Han script, such as `é` or `д`.
+    UnicodeWord,
+    /// Everything else, including `_`.
+    Other,
+}
+
+/// Returns the [`CharClass`] of `c`. [`tokenize`] splits text wherever the class
+/// changes and at every [`CharClass::Other`] character.
+pub fn classify_char(c: char) -> CharClass {
+    if c.is_ascii_alphanumeric() {
+        CharClass::AsciiWord
+    } else if is_han(c) {
+        CharClass::Han
+    } else if c.is_alphanumeric() {
+        CharClass::UnicodeWord
+    } else {
+        CharClass::Other
+    }
+}
+
+static HAN_SCRIPT_DATA: CodePointMapDataBorrowed<'static, Script> =
+    CodePointMapData::<Script>::new();
+
+fn is_han(c: char) -> bool {
+    HAN_SCRIPT_DATA.get(c) == Script::Han
+}
+
+/// Returns whether lowercasing `text` as a whole can produce tokens that lowercasing
+/// each token of `text` doesn't, which `matches_term(lower(col), ..)` and
+/// `lower(col) LIKE ..` rely on. Every other character lowercases to one character
+/// of the same class, but U+0130 and U+212A change length or class, and `Σ`
+/// lowercases depending on the characters after it.
+pub fn has_irregular_lowercase(text: &str) -> bool {
+    !text.is_ascii() && text.contains(['\u{130}', '\u{212A}', 'Σ'])
+}
+
+/// Returns the tokens the fulltext bloom index stores for `text`.
+///
+/// A token is a maximal run of ASCII alphanumerics or of non-Han alphanumerics;
+/// `_` and other symbols only separate tokens. A run of Han characters is
+/// emitted as its bigrams, or as itself when it has a single character, since
+/// `matches_term` matches Han terms as plain substrings.
+pub fn tokenize(text: &str) -> Vec<&str> {
+    let mut tokens = Vec::new();
+    for_each_word_run(text, |class, range| {
+        let run = &text[range];
+        if class == CharClass::Han {
+            push_han_tokens(run, &mut tokens);
+        } else {
+            tokens.push(run);
+        }
+    });
+    tokens
+}
+
+/// Calls `f` on every maximal run of word characters of the same class.
+fn for_each_word_run(text: &str, mut f: impl FnMut(CharClass, Range<usize>)) {
+    if text.is_ascii() {
+        // ASCII text only has `AsciiWord` and `Other` characters, so scan bytes
+        // instead of decoding chars. Most log lines take this path.
+        let mut start = None;
+        for (i, b) in text.bytes().enumerate() {
+            match (b.is_ascii_alphanumeric(), start) {
+                (true, None) => start = Some(i),
+                (false, Some(s)) => {
+                    f(CharClass::AsciiWord, s..i);
+                    start = None;
+                }
+                _ => {}
+            }
+        }
+        if let Some(s) = start {
+            f(CharClass::AsciiWord, s..text.len());
+        }
+        return;
+    }
+
+    let mut run: Option<(CharClass, usize)> = None;
+    for (i, c) in text.char_indices() {
+        let class = classify_char(c);
+        if let Some((run_class, start)) = run {
+            if run_class == class {
+                continue;
+            }
+            f(run_class, start..i);
+            run = None;
+        }
+        if class != CharClass::Other {
+            run = Some((class, i));
+        }
+    }
+    if let Some((run_class, start)) = run {
+        f(run_class, start..text.len());
+    }
+}
+
+fn push_han_tokens<'a>(run: &'a str, tokens: &mut Vec<&'a str>) {
+    let mut starts = run.char_indices().map(|(i, _)| i).skip(1);
+    let Some(mut second) = starts.next() else {
+        tokens.push(run);
+        return;
+    };
+    let mut first = 0;
+    for next in starts.chain(std::iter::once(run.len())) {
+        tokens.push(&run[first..next]);
+        first = second;
+        second = next;
+    }
+}
+
+/// Returns tokens that every text satisfying `matches_term(text, term)` contains,
+/// as produced by [`tokenize`].
+pub fn extract_term_probes(term: &str) -> Vec<String> {
+    // Non-Han terms only match between word boundaries that `tokenize` also
+    // splits at, so their edge runs are whole tokens. Han-containing terms match
+    // as plain substrings and give no such guarantee at their edges.
+    let bounded = !term.chars().any(|c| classify_char(c) == CharClass::Han);
+    let mut probes = Vec::new();
+    push_literal_probes(term, bounded, bounded, &mut probes);
+    probes
+}
+
+/// Returns tokens that every text satisfying `text LIKE pattern` contains, as
+/// produced by [`tokenize`]. `\` escapes the next character, following
+/// arrow's `like` kernel; a trailing `\` is a literal backslash.
+pub fn extract_like_probes(pattern: &str) -> Vec<String> {
+    let mut probes = Vec::new();
+    let mut literal = String::new();
+    // The pattern start and end are anchors; a wildcard can stand for word
+    // characters, so a run touching it may be part of a longer token.
+    let mut anchored_left = true;
+    let mut chars = pattern.chars();
+    while let Some(c) = chars.next() {
+        match c {
+            '%' | '_' => {
+                push_literal_probes(&literal, anchored_left, false, &mut probes);
+                literal.clear();
+                anchored_left = false;
+            }
+            '\\' => literal.push(chars.next().unwrap_or('\\')),
+            c => literal.push(c),
+        }
+    }
+    push_literal_probes(&literal, anchored_left, true, &mut probes);
+    probes
+}
+
+/// Pushes the tokens of `literal` that stay whole tokens wherever `literal`
+/// occurs. A side is bounded when the character beyond it can't extend the
+/// adjacent run.
+fn push_literal_probes(
+    literal: &str,
+    left_bounded: bool,
+    right_bounded: bool,
+    probes: &mut Vec<String>,
+) {
+    for_each_word_run(literal, |class, range| {
+        let whole =
+            (range.start > 0 || left_bounded) && (range.end < literal.len() || right_bounded);
+        let run = &literal[range];
+        if class == CharClass::Han && run.chars().nth(1).is_some() {
+            // Bigrams inside the run occur in every text containing it.
+            let mut tokens = Vec::new();
+            push_han_tokens(run, &mut tokens);
+            probes.extend(tokens.into_iter().map(str::to_string));
+        } else if whole {
+            probes.push(run.to_string());
+        }
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn han_detection_uses_script_not_all_cjk() {
+        assert!(is_han('汉'));
+        assert!(is_han('\u{30000}'));
+        assert!(!is_han('あ'));
+        assert!(!is_han('한'));
+    }
+
+    #[test]
+    fn test_script_tokenizer() {
+        let cases: [(&str, &[&str]); 7] = [
+            ("hello_world __x__", &["hello", "world", "x"]),
+            ("trace_id=abc", &["trace", "id", "abc"]),
+            ("错误error日志", &["错误", "error", "日志"]),
+            ("连接timeout.5次", &["连接", "timeout", "5", "次"]),
+            ("用户user-123登录", &["用户", "user", "123", "登录"]),
+            ("中国农业银行", &["中国", "国农", "农业", "业银", "银行"]),
+            ("Größe日x", &["Gr", "öß", "e", "日", "x"]),
+        ];
+        for (text, expected) in cases {
+            assert_eq!(tokenize(text), expected, "text: {text}");
+        }
+    }
+
+    #[test]
+    fn test_irregular_lowercase_chars() {
+        for c in (0..=0x10FFFF).filter_map(char::from_u32) {
+            let lower = c.to_lowercase().collect::<Vec<_>>();
+            let regular = lower.len() == 1 && classify_char(lower[0]) == classify_char(c);
+            let expect_regular = !matches!(c, '\u{130}' | '\u{212A}');
+            assert_eq!(regular, expect_regular, "U+{:04X}", c as u32);
+        }
+    }
+
+    #[test]
+    fn test_term_probes() {
+        let cases: [(&str, &[&str]); 8] = [
+            ("world", &["world"]),
+            ("hello world", &["hello", "world"]),
+            ("/start", &["start"]),
+            ("naïve", &["na", "ï", "ve"]),
+            // Han terms match as substrings: only bigrams and runs enclosed by
+            // other characters of the term are safe.
+            ("农业", &["农业"]),
+            ("农", &[]),
+            ("机号1888", &["机号"]),
+            ("x错误y日z", &["错误", "y", "日"]),
+        ];
+        for (term, expected) in cases {
+            assert_eq!(extract_term_probes(term), expected, "term: {term}");
+        }
+    }
+
+    #[test]
+    fn test_like_probes() {
+        let cases: [(&str, &[&str]); 11] = [
+            ("%timeout%", &[]),
+            ("% timeout %", &["timeout"]),
+            ("timeout%", &[]),
+            ("timeout: %", &["timeout"]),
+            ("%: timeout", &["timeout"]),
+            ("a_b c", &["c"]),
+            (r"trace\_id=%", &["trace", "id"]),
+            (r"100\% done", &["100", "done"]),
+            (r"x\\y", &["x", "y"]),
+            ("%错误error日%", &["错误", "error"]),
+            ("%错误 x%", &["错误"]),
+        ];
+        for (pattern, expected) in cases {
+            assert_eq!(extract_like_probes(pattern), expected, "pattern: {pattern}");
+        }
+    }
+}

src/common/function/Cargo.toml

@@ -46,7 +46,6 @@ geo-types = { version = "0.7", optional = true }
 geohash = { version = "0.13", optional = true }
 h3o = { version = "0.6", optional = true }
 hyperloglogplus = "0.4"
-icu_properties.workspace = true
 jsonb.workspace = true
 memchr = "2.7"
 mito-codec.workspace = true

src/common/function/src/scalars/matches_term.rs

@@ -15,13 +15,12 @@
 use std::fmt;
 use std::sync::Arc;
 
+use common_base::term_token::{CharClass, classify_char};
 use datafusion_common::arrow::array::{Array, AsArray, BooleanArray, BooleanBuilder};
 use datafusion_common::arrow::compute;
 use datafusion_common::arrow::datatypes::DataType;
 use datafusion_common::{DataFusionError, ScalarValue};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, Volatility};
-use icu_properties::props::Script;
-use icu_properties::{CodePointMapData, CodePointMapDataBorrowed};
 use memchr::memmem;
 
 use crate::function::Function;
@@ -231,14 +230,6 @@ pub struct MatchesTermFinder {
     ends_with_other: bool,
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-enum CharClass {
-    AsciiWord,
-    Han,
-    UnicodeWord,
-    Other,
-}
-
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 enum TermKind {
     AsciiLike,
@@ -246,25 +237,6 @@ enum TermKind {
     HanContaining,
 }
 
-fn classify_char(c: char) -> CharClass {
-    if c.is_ascii_alphanumeric() {
-        CharClass::AsciiWord
-    } else if is_han(c) {
-        CharClass::Han
-    } else if c.is_alphanumeric() {
-        CharClass::UnicodeWord
-    } else {
-        CharClass::Other
-    }
-}
-
-static HAN_SCRIPT_DATA: CodePointMapDataBorrowed<'static, Script> =
-    CodePointMapData::<Script>::new();
-
-fn is_han(c: char) -> bool {
-    HAN_SCRIPT_DATA.get(c) == Script::Han
-}
-
 fn classify_term(term: &str) -> TermKind {
     let mut has_han = false;
     let mut has_unicode_word = false;
@@ -508,12 +480,4 @@ mod tests {
         assert!(MatchesTermFinder::new("行账号").find("中国农业银行账号"));
         assert!(MatchesTermFinder::new("登录").find("登录手机号18888888888的动态key"));
     }
-
-    #[test]
-    fn han_detection_uses_script_not_all_cjk() {
-        assert!(is_han('汉'));
-        assert!(is_han('\u{30000}'));
-        assert!(!is_han('あ'));
-        assert!(!is_han('한'));
-    }
 }

src/index/Cargo.toml

@@ -23,8 +23,6 @@ fst.workspace = true
 futures.workspace = true
 greptime-proto.workspace = true
 itertools.workspace = true
-jieba-rs = "0.11"
-lazy_static.workspace = true
 mockall.workspace = true
 pin-project.workspace = true
 prost.workspace = true

src/index/benches/index_build_bench.rs

@@ -31,7 +31,7 @@ use index::bloom_filter::reader::BloomFilterReaderImpl;
 use index::external_provider::{ExternalTempFileProvider, Reader, Writer};
 use index::fulltext_index::Config;
 use index::fulltext_index::create::{BloomFilterFulltextIndexCreator, FulltextIndexCreator};
-use index::fulltext_index::tokenizer::{Analyzer, EnglishTokenizer};
+use index::fulltext_index::tokenizer::{Analyzer, ScriptTokenizer};
 use index::inverted_index::create::InvertedIndexCreator;
 use index::inverted_index::create::sort::external_sort::ExternalSorter;
 use index::inverted_index::create::sort_create::SortIndexCreator;
@@ -365,7 +365,7 @@ fn bench_bloom_search(c: &mut Criterion) {
     let mut group = c.benchmark_group("bloom_search");
 
     let lines = service_logs(ROWS * 10);
-    let analyzer = Analyzer::new(Box::new(EnglishTokenizer), false);
+    let analyzer = Analyzer::new(Box::new(ScriptTokenizer), false);
     let blob = rt.block_on(async {
         let mut creator = bloom_creator();
         for line in &lines {

src/index/benches/tokenizer_bench.rs

@@ -21,7 +21,7 @@ use async_trait::async_trait;
 use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
 use futures::AsyncRead;
 use index::fulltext_index::create::{FulltextIndexCreator, TantivyFulltextIndexCreator};
-use index::fulltext_index::tokenizer::{ChineseTokenizer, EnglishTokenizer, Tokenizer};
+use index::fulltext_index::tokenizer::{ScriptTokenizer, Tokenizer};
 use index::fulltext_index::{Analyzer, Config};
 use puffin::puffin_manager::{PuffinWriter, PutOptions};
 
@@ -86,8 +86,8 @@ impl PuffinWriter for NoopPuffinWriter {
     }
 }
 
-fn bench_english_tokenizer(c: &mut Criterion) {
-    let tokenizer = EnglishTokenizer;
+fn bench_tokenizer_english_text(c: &mut Criterion) {
+    let tokenizer = ScriptTokenizer;
 
     let texts = vec![
         ("short", "Hello, world! This is a test."),
@@ -125,7 +125,7 @@ fn bench_english_tokenizer(c: &mut Criterion) {
         ),
     ];
 
-    let mut group = c.benchmark_group("english_tokenizer");
+    let mut group = c.benchmark_group("tokenizer_english_text");
 
     for (size, text) in texts {
         group.bench_with_input(BenchmarkId::new("tokenize", size), &text, |b, text| {
@@ -136,7 +136,7 @@ fn bench_english_tokenizer(c: &mut Criterion) {
     group.finish();
 
     // Benchmark with repeated tokenization to simulate real-world usage
-    let mut repeat_group = c.benchmark_group("english_tokenizer_repeated");
+    let mut repeat_group = c.benchmark_group("tokenizer_english_text_repeated");
 
     let sample_text = "The quick brown fox jumps over the lazy dog. This sentence contains most letters of the alphabet.";
 
@@ -157,9 +157,9 @@ fn bench_english_tokenizer(c: &mut Criterion) {
     repeat_group.finish();
 }
 
-fn bench_chinese_tokenizer(c: &mut Criterion) {
-    let tokenizer = ChineseTokenizer;
-    let mut group = c.benchmark_group("chinese_tokenizer");
+fn bench_tokenizer_chinese_text(c: &mut Criterion) {
+    let tokenizer = ScriptTokenizer;
+    let mut group = c.benchmark_group("tokenizer_chinese_text");
 
     for (name, text) in CHINESE_TOKENIZER_TEXTS {
         group.throughput(Throughput::Bytes(text.len() as u64));
@@ -170,7 +170,7 @@ fn bench_chinese_tokenizer(c: &mut Criterion) {
 
     group.finish();
 
-    let mut repeat_group = c.benchmark_group("chinese_tokenizer_repeated");
+    let mut repeat_group = c.benchmark_group("tokenizer_chinese_text_repeated");
     let sample_text = CHINESE_TOKENIZER_TEXTS
         .iter()
         .find(|(name, _)| *name == "mixed_log")
@@ -253,8 +253,8 @@ fn bench_tantivy_chinese_fulltext_index(c: &mut Criterion) {
 
 criterion_group!(
     benches,
-    bench_english_tokenizer,
-    bench_chinese_tokenizer,
+    bench_tokenizer_english_text,
+    bench_tokenizer_chinese_text,
     bench_tantivy_chinese_fulltext_index
 );
 criterion_main!(benches);

src/index/src/fulltext_index/create/bloom_filter.rs

@@ -29,7 +29,7 @@ use crate::fulltext_index::error::{
     AbortedSnafu, BiErrorsSnafu, BloomFilterFinishSnafu, ExternalSnafu, PuffinAddBlobSnafu, Result,
     SerializeToJsonSnafu,
 };
-use crate::fulltext_index::tokenizer::{Analyzer, ChineseTokenizer, EnglishTokenizer};
+use crate::fulltext_index::tokenizer::{Analyzer, ScriptTokenizer};
 use crate::fulltext_index::{Config, KEY_FULLTEXT_CONFIG};
 
 const PIPE_BUFFER_SIZE_FOR_SENDING_BLOB: usize = 8192;
@@ -50,11 +50,9 @@ impl BloomFilterFulltextIndexCreator {
         global_memory_usage: Arc<AtomicUsize>,
         global_memory_usage_threshold: Option<usize>,
     ) -> Self {
-        let tokenizer = match config.analyzer {
-            crate::fulltext_index::Analyzer::English => Box::new(EnglishTokenizer) as _,
-            crate::fulltext_index::Analyzer::Chinese => Box::new(ChineseTokenizer) as _,
-        };
-        let analyzer = Analyzer::new(tokenizer, config.case_sensitive);
+        // Both analyzers tokenize the same way: query probes must be derivable from the
+        // term alone, which a context-dependent segmenter like jieba can't provide.
+        let analyzer = Analyzer::new(Box::new(ScriptTokenizer), config.case_sensitive);
 
         let inner = BloomFilterCreator::new(
             rows_per_segment,

src/index/src/fulltext_index/tokenizer.rs

@@ -12,121 +12,32 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use common_base::term_token::{self, has_irregular_lowercase};
+
 use crate::Bytes;
 use crate::bloom_filter::element_hash;
 use crate::fulltext_index::error::Result;
 
-lazy_static::lazy_static! {
-    static ref JIEBA: jieba_rs::Jieba = jieba_rs::Jieba::new();
-}
-
-/// A-Z, a-z, 0-9, and '_' are true
-const VALID_ASCII_TOKEN: [bool; 256] = [
-    false, false, false, false, false, false, false, false, false, false, false, false, false,
-    false, false, false, false, false, false, false, false, false, false, false, false, false,
-    false, false, false, false, false, false, false, false, false, false, false, false, false,
-    false, false, false, false, false, false, false, false, false, true, true, true, true, true,
-    true, true, true, true, true, false, false, false, false, false, false, false, true, true,
-    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
-    true, true, true, true, true, true, true, true, false, false, false, false, true, false, true,
-    true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true,
-    true, true, true, true, true, true, true, true, true, false, false, false, false, false, false,
-    false, false, false, false, false, false, false, false, false, false, false, false, false,
-    false, false, false, false, false, false, false, false, false, false, false, false, false,
-    false, false, false, false, false, false, false, false, false, false, false, false, false,
-    false, false, false, false, false, false, false, false, false, false, false, false, false,
-    false, false, false, false, false, false, false, false, false, false, false, false, false,
-    false, false, false, false, false, false, false, false, false, false, false, false, false,
-    false, false, false, false, false, false, false, false, false, false, false, false, false,
-    false, false, false, false, false, false, false, false, false, false, false, false, false,
-    false, false, false, false, false, false, false, false, false, false, false, false, false,
-    false, false, false, false, false, false, false, false, false, false,
-];
-
 /// `Tokenizer` tokenizes a text into a list of tokens.
 pub trait Tokenizer: Send {
     fn tokenize<'a>(&self, text: &'a str) -> Vec<&'a str>;
 }
 
-/// `EnglishTokenizer` tokenizes an English text.
-///
-/// It splits the text by non-alphabetic characters.
+/// `ScriptTokenizer` produces the tokens described in [`term_token::tokenize`].
 #[derive(Debug, Default)]
-pub struct EnglishTokenizer;
+pub struct ScriptTokenizer;
 
-impl Tokenizer for EnglishTokenizer {
+impl Tokenizer for ScriptTokenizer {
     fn tokenize<'a>(&self, text: &'a str) -> Vec<&'a str> {
-        if text.is_ascii() {
-            let mut tokens = Vec::new();
-            let mut start = 0;
-            for (i, &byte) in text.as_bytes().iter().enumerate() {
-                if !VALID_ASCII_TOKEN[byte as usize] {
-                    if start < i {
-                        tokens.push(&text[start..i]);
-                    }
-                    start = i + 1;
-                }
-            }
-
-            if start < text.len() {
-                tokens.push(&text[start..]);
-            }
-
-            tokens
-        } else {
-            text.split(|c: char| !c.is_alphanumeric() && c != '_')
-                .filter(|s| !s.is_empty())
-                .collect()
-        }
+        term_token::tokenize(text)
     }
 }
 
-/// `ChineseTokenizer` tokenizes a Chinese text.
-///
-/// It uses Jieba search-mode tokenization to improve recall for Chinese fulltext search.
-/// Enabling HMM also helps merge some unknown fragments into larger tokens, which can reduce
-/// token cardinality versus a fully fragmented output.
-#[derive(Debug, Default)]
-pub struct ChineseTokenizer;
-
-impl Tokenizer for ChineseTokenizer {
-    fn tokenize<'a>(&self, text: &'a str) -> Vec<&'a str> {
-        if text.is_ascii() {
-            EnglishTokenizer {}.tokenize(text)
-        } else {
-            // Search-mode tokenization emits finer-grained searchable terms, while HMM helps
-            // merge some unknown fragments and avoid excessive token fragmentation.
-            let mut tokens = JIEBA
-                .cut_for_search(text, true)
-                .into_iter()
-                .map(|token| token.word)
-                .filter(|token| is_indexable_token(token))
-                .collect::<Vec<_>>();
-
-            let english = EnglishTokenizer {};
-            tokens.extend(
-                english
-                    .tokenize(text)
-                    .into_iter()
-                    .filter(|token| is_ascii_underscore_token(token)),
-            );
-
-            tokens
-        }
-    }
-}
-
-fn is_indexable_token(token: &str) -> bool {
-    token.chars().any(|c| c.is_alphanumeric() || c == '_')
-}
-
-fn is_ascii_underscore_token(token: &str) -> bool {
-    token.is_ascii() && token.chars().any(|c| c == '_')
-}
-
 /// `Analyzer` analyzes a text into a list of tokens.
 ///
 /// It uses a `Tokenizer` to tokenize the text and optionally lowercases the tokens.
+/// When case-insensitive, it also emits the tokens of the whole lowercased text if
+/// they can differ, so that predicates on `lower(col)` can probe the index.
 pub struct Analyzer {
     tokenizer: Box<dyn Tokenizer>,
     case_sensitive: bool,
@@ -152,22 +63,36 @@ impl Analyzer {
         buf: &'a mut Vec<u8>,
     ) -> impl Iterator<Item = u64> + use<'a> {
         let case_sensitive = self.case_sensitive;
-        self.tokenizer.tokenize(text).into_iter().map(move |token| {
-            if case_sensitive {
-                element_hash(token.as_bytes())
-            } else if token.is_ascii() {
-                buf.clear();
-                buf.extend(token.bytes().map(|b| b.to_ascii_lowercase()));
-                element_hash(buf)
-            } else {
-                element_hash(token.to_lowercase().as_bytes())
-            }
-        })
+        let lowercased_text_hashes = self
+            .irregular_lowercased_text(text)
+            .map(|lowered| {
+                self.tokenizer
+                    .tokenize(&lowered)
+                    .into_iter()
+                    .map(|token| element_hash(token.as_bytes()))
+                    .collect::<Vec<_>>()
+            })
+            .unwrap_or_default();
+        self.tokenizer
+            .tokenize(text)
+            .into_iter()
+            .map(move |token| {
+                if case_sensitive {
+                    element_hash(token.as_bytes())
+                } else if token.is_ascii() {
+                    buf.clear();
+                    buf.extend(token.bytes().map(|b| b.to_ascii_lowercase()));
+                    element_hash(buf)
+                } else {
+                    element_hash(token.to_lowercase().as_bytes())
+                }
+            })
+            .chain(lowercased_text_hashes)
     }
 
     /// Analyzes the given text into a list of tokens.
     pub fn analyze_text(&self, text: &str) -> Result<Vec<Bytes>> {
-        let res = self
+        let mut res = self
             .tokenizer
             .tokenize(text)
             .iter()
@@ -178,9 +103,22 @@ impl Analyzer {
                     s.to_lowercase().as_bytes().to_vec()
                 }
             })
-            .collect();
+            .collect::<Vec<_>>();
+        if let Some(lowered) = self.irregular_lowercased_text(text) {
+            res.extend(
+                self.tokenizer
+                    .tokenize(&lowered)
+                    .into_iter()
+                    .map(|token| token.as_bytes().to_vec()),
+            );
+        }
         Ok(res)
     }
+
+    /// Returns the lowercased text whose tokens must be indexed too, if any.
+    fn irregular_lowercased_text(&self, text: &str) -> Option<String> {
+        (!self.case_sensitive && has_irregular_lowercase(text)).then(|| text.to_lowercase())
+    }
 }
 
 #[cfg(test)]
@@ -190,12 +128,8 @@ mod tests {
     #[test]
     fn test_analyze_text_hashes_matches_analyze_text() {
         let text = "Hello, WORLD ship_Ship 清洁表面 ÄÖÜ straße İstanbul x";
-        for (tokenizer, case_sensitive) in [
-            (Box::new(EnglishTokenizer) as Box<dyn Tokenizer>, false),
-            (Box::new(EnglishTokenizer), true),
-            (Box::new(ChineseTokenizer), false),
-        ] {
-            let analyzer = Analyzer::new(tokenizer, case_sensitive);
+        for case_sensitive in [false, true] {
+            let analyzer = Analyzer::new(Box::new(ScriptTokenizer), case_sensitive);
             let expected = analyzer
                 .analyze_text(text)
                 .unwrap()
@@ -209,433 +143,13 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_english_tokenizer() {
-        let tokenizer = EnglishTokenizer;
-        let text = "Hello, world!!! This is a----++   test012_345+67890 ship_ship ship__ship _ __ __IDENTIFIER__ _ship ship_";
-        let tokens = tokenizer.tokenize(text);
-        assert_eq!(
-            tokens,
-            vec![
-                "Hello",
-                "world",
-                "This",
-                "is",
-                "a",
-                "test012_345",
-                "67890",
-                "ship_ship",
-                "ship__ship",
-                "_",
-                "__",
-                "__IDENTIFIER__",
-                "_ship",
-                "ship_"
-            ]
-        );
-    }
-
-    #[test]
-    fn test_english_tokenizer_with_utf8() {
-        let tokenizer = EnglishTokenizer;
-        let text = "💸unfold the 纸巾😣and gently 清洁表😭面";
-        let tokens = tokenizer.tokenize(text);
-        assert_eq!(
-            tokens,
-            // Don't care what happens to non-ASCII characters.
-            // It's kind of a misconfiguration to use EnglishTokenizer on non-ASCII text.
-            vec!["unfold", "the", "纸巾", "and", "gently", "清洁表", "面"]
-        );
-    }
-
-    #[test]
-    fn test_chinese_tokenizer() {
-        let tokenizer = ChineseTokenizer;
-        let text = "我喜欢苹果";
-        let tokens = tokenizer.tokenize(text);
-        assert_eq!(tokens, vec!["我", "喜欢", "苹果"]);
-    }
-
-    #[test]
-    fn test_chinese_tokenizer_issue_7943_sample() {
-        let tokenizer = ChineseTokenizer;
-        let text = "[2026/04/09/ 13:56:11.031]2026-04-09 13:56:11.031 - [ trace_id=340a6a44b0bd8e37bb7697ss7da61ff0 span_id=085ff5ttf1e0a23b trace_flags=01] - [http-nio-8081-exec-16] INFO c.h.p.xx.web.service.impl.CCCXForwardKKKServiceImpl.pushout(188) - 登录手机号18888888888的动态key：829889AC8 ship_ship ship__ship _ __ __IDENTIFIER__ _ship ship_ EOF";
-        let tokens = tokenizer.tokenize(text);
-
-        // Jieba 0.11 preserves ASCII compounds and emits searchable alphabetic parts.
-        // The underscore fallback also retains leading, trailing, and repeated underscores.
-        assert_eq!(
-            tokens,
-            vec![
-                "2026",
-                "04",
-                "09",
-                "13",
-                "56",
-                "11.031",
-                "2026-04-09",
-                "13",
-                "56",
-                "11.031",
-                "trace",
-                "id",
-                "trace_id",
-                "340a6a44b0bd8e37bb7697ss7da61ff0",
-                "span",
-                "id",
-                "span_id",
-                "085ff5ttf1e0a23b",
-                "trace",
-                "flags",
-                "trace_flags",
-                "01",
-                "http",
-                "nio",
-                "exec",
-                "http-nio-8081-exec-16",
-                "INFO",
-                "xx",
-                "web",
-                "service",
-                "impl",
-                "CCCXForwardKKKServiceImpl",
-                "pushout",
-                "c.h.p.xx.web.service.impl.CCCXForwardKKKServiceImpl.pushout",
-                "188",
-                "登录",
-                "手机",
-                "手机号",
-                "18888888888",
-                "的",
-                "动态",
-                "key",
-                "829889AC8",
-                "ship",
-                "ship",
-                "ship_ship",
-                "ship",
-                "__",
-                "ship",
-                "_",
-                "__",
-                "__",
-                "IDENTIFIER",
-                "__",
-                "_",
-                "ship",
-                "ship",
-                "_",
-                "EOF",
-                "trace_id",
-                "span_id",
-                "trace_flags",
-                "ship_ship",
-                "ship__ship",
-                "_",
-                "__",
-                "__IDENTIFIER__",
-                "_ship",
-                "ship_"
-            ]
-        );
-    }
-
-    #[test]
-    fn test_chinese_tokenizer_keeps_ascii_underscore_compounds() {
-        let tokenizer = ChineseTokenizer;
-        let text = "trace_id=abc 登录手机号 dynamic_key=xyz";
-
-        let tokens = tokenizer.tokenize(text);
-
-        assert!(tokens.contains(&"trace_id"));
-        assert!(tokens.contains(&"dynamic_key"));
-        assert!(tokens.contains(&"登录"));
-        assert!(tokens.contains(&"手机号"));
-    }
-
-    #[test]
-    fn test_chinese_tokenizer_skips_non_ascii_underscore_tokens() {
-        let tokenizer = ChineseTokenizer;
-        let text = "登录_id trace_id 手机号_trace";
-
-        let tokens = tokenizer.tokenize(text);
-
-        assert_eq!(
-            tokens,
-            [
-                "登录",
-                "_",
-                "id",
-                "trace",
-                "id",
-                "trace_id",
-                "手机",
-                "手机号",
-                "_",
-                "trace",
-                "trace_id"
-            ]
-        );
-    }
-
-    #[test]
-    fn test_chinese_tokenizer_aggressive_tokenization_probe() {
-        let tokenizer = ChineseTokenizer;
-        let text = "哈基米哦南北绿豆，噢马自立曼波。登录手机号。中国农业银行。装电视台，中国中央广播电视台。压不缩，笑不活。";
-
-        let default_tokens = tokenizer.tokenize(text);
-        let cut_hmm_false = JIEBA
-            .cut(text, false)
-            .into_iter()
-            .map(|token| token.word)
-            .collect::<Vec<_>>();
-        let cut_hmm_true = JIEBA
-            .cut(text, true)
-            .into_iter()
-            .map(|token| token.word)
-            .collect::<Vec<_>>();
-        let cut_for_search_hmm_false = JIEBA
-            .cut_for_search(text, false)
-            .into_iter()
-            .map(|token| token.word)
-            .collect::<Vec<_>>();
-        let cut_for_search_hmm_true = JIEBA
-            .cut_for_search(text, true)
-            .into_iter()
-            .map(|token| token.word)
-            .collect::<Vec<_>>();
-
-        assert_eq!(
-            default_tokens,
-            [
-                "哈基米",
-                "哦",
-                "南北",
-                "绿豆",
-                "噢",
-                "马",
-                "自立",
-                "曼波",
-                "登录",
-                "手机",
-                "手机号",
-                "中国",
-                "农业",
-                "银行",
-                "中国农业银行",
-                "装",
-                "电视",
-                "电视台",
-                "中国",
-                "中央",
-                "广播",
-                "电视",
-                "电视台",
-                "不缩",
-                "压不缩",
-                "笑",
-                "不活",
-            ]
-        );
-        assert_eq!(
-            cut_hmm_false,
-            [
-                "哈",
-                "基",
-                "米",
-                "哦",
-                "南北",
-                "绿豆",
-                "，",
-                "噢",
-                "马",
-                "自立",
-                "曼",
-                "波",
-                "。",
-                "登录",
-                "手机号",
-                "。",
-                "中国农业银行",
-                "。",
-                "装",
-                "电视台",
-                "，",
-                "中国",
-                "中央",
-                "广播",
-                "电视台",
-                "。",
-                "压",
-                "不",
-                "缩",
-                "，",
-                "笑",
-                "不",
-                "活",
-                "。"
-            ]
-        );
-        assert_eq!(
-            cut_hmm_true,
-            [
-                "哈基米",
-                "哦",
-                "南北",
-                "绿豆",
-                "，",
-                "噢",
-                "马",
-                "自立",
-                "曼波",
-                "。",
-                "登录",
-                "手机号",
-                "。",
-                "中国农业银行",
-                "。",
-                "装",
-                "电视台",
-                "，",
-                "中国",
-                "中央",
-                "广播",
-                "电视台",
-                "。",
-                "压不缩",
-                "，",
-                "笑",
-                "不活",
-                "。"
-            ]
-        );
-        assert_eq!(
-            cut_for_search_hmm_false,
-            [
-                "哈",
-                "基",
-                "米",
-                "哦",
-                "南北",
-                "绿豆",
-                "，",
-                "噢",
-                "马",
-                "自立",
-                "曼",
-                "波",
-                "。",
-                "登录",
-                "手机",
-                "手机号",
-                "。",
-                "中国",
-                "农业",
-                "银行",
-                "中国农业银行",
-                "。",
-                "装",
-                "电视",
-                "电视台",
-                "，",
-                "中国",
-                "中央",
-                "广播",
-                "电视",
-                "电视台",
-                "。",
-                "压",
-                "不",
-                "缩",
-                "，",
-                "笑",
-                "不",
-                "活",
-                "。"
-            ]
-        );
-
-        assert_eq!(
-            cut_for_search_hmm_true,
-            [
-                "哈基米",
-                "哦",
-                "南北",
-                "绿豆",
-                "，",
-                "噢",
-                "马",
-                "自立",
-                "曼波",
-                "。",
-                "登录",
-                "手机",
-                "手机号",
-                "。",
-                "中国",
-                "农业",
-                "银行",
-                "中国农业银行",
-                "。",
-                "装",
-                "电视",
-                "电视台",
-                "，",
-                "中国",
-                "中央",
-                "广播",
-                "电视",
-                "电视台",
-                "。",
-                "不缩",
-                "压不缩",
-                "，",
-                "笑",
-                "不活",
-                "。"
-            ]
-        );
-    }
-
-    #[test]
-    fn test_valid_ascii_token_lookup_table() {
-        // Test all ASCII values in a single loop
-        for c in 0u8..=255u8 {
-            let is_valid = VALID_ASCII_TOKEN[c as usize];
-            let should_be_valid = (c as char).is_ascii_alphanumeric() || c == b'_';
-
-            assert_eq!(
-                is_valid,
-                should_be_valid,
-                "Character '{}' (byte {}) validity mismatch: expected {}, got {}",
-                if c.is_ascii() && !c.is_ascii_control() {
-                    c as char
-                } else {
-                    '?'
-                },
-                c,
-                should_be_valid,
-                is_valid
-            );
-        }
-    }
-
     #[test]
     fn test_analyzer() {
-        let tokenizer = EnglishTokenizer;
-        let analyzer = Analyzer::new(Box::new(tokenizer), false);
-        let text = "Hello, world! This is a test.";
-        let tokens = analyzer.analyze_text(text).unwrap();
+        let analyzer = Analyzer::new(Box::new(ScriptTokenizer), false);
+        let tokens = analyzer.analyze_text("Hello, World_Two").unwrap();
         assert_eq!(
             tokens,
-            vec![
-                b"hello".to_vec(),
-                b"world".to_vec(),
-                b"this".to_vec(),
-                b"is".to_vec(),
-                b"a".to_vec(),
-                b"test".to_vec()
-            ]
+            vec![b"hello".to_vec(), b"world".to_vec(), b"two".to_vec()]
         );
     }
 }

src/mito2/src/cache/index/result_cache.rs

@@ -23,7 +23,7 @@ use store_api::storage::{ColumnId, FileId};
 
 use crate::metrics::{CACHE_BYTES, CACHE_EVICTION, CACHE_HIT, CACHE_MISS};
 use crate::sst::index::fulltext_index::applier::builder::{
-    FulltextQuery, FulltextRequest, FulltextTerm,
+    FulltextLike, FulltextQuery, FulltextRequest, FulltextTerm,
 };
 use crate::sst::parquet::row_selection::RowGroupSelection;
 
@@ -176,7 +176,12 @@ impl FulltextIndexKey {
                     .iter()
                     .map(|term| term.term.len() + size_of::<FulltextTerm>())
                     .sum::<usize>();
-                query_size + term_size
+                let like_size = request
+                    .likes
+                    .iter()
+                    .map(|like| like.pattern.len() + size_of::<FulltextLike>())
+                    .sum::<usize>();
+                query_size + term_size + like_size
             })
             .sum();
         Self {
@@ -343,6 +348,7 @@ mod tests {
                 FulltextQuery("another long query string".to_string()),
             ],
             terms: vec![],
+            likes: vec![],
         };
         predicates1.insert(1, request1);
         let key1 = PredicateKey::new_fulltext(Arc::new(predicates1));
@@ -357,6 +363,7 @@ mod tests {
                 FulltextQuery("another long query string".to_string()),
             ],
             terms: vec![],
+            likes: vec![],
         };
         predicates2.insert(1, request2);
         let key2 = PredicateKey::new_fulltext(Arc::new(predicates2));
@@ -427,6 +434,7 @@ mod tests {
                 col_lowered: false,
                 term: "test term".to_string(),
             }],
+            likes: vec![],
         };
         predicates1.insert(1, request1);
         let key1 = PredicateKey::new_fulltext(Arc::new(predicates1));

src/mito2/src/engine/basic_test.rs

@@ -1262,13 +1262,13 @@ async fn test_all_region_infos() {
 async fn test_all_index_metas_list_all_types() {
     test_all_index_metas_list_all_types_with_format(false, r#"
 PuffinIndexMetaEntry { table_dir: "test/", index_file_path: "test/11_0000000001/index/<file_id>.puffin", region_id: 47244640257(11, 1), table_id: 11, region_number: 1, region_group: 0, region_sequence: 1, file_id: "<file_id>", index_file_size: Some(6000), index_type: "bloom_filter", target_type: "column", target_key: "1", target_json: "{\"column\":1}", blob_size: 751, meta_json: Some("{\"bloom\":{\"bloom_filter_size\":640,\"row_count\":20,\"rows_per_segment\":2,\"segment_count\":10}}"), node_id: None }
-PuffinIndexMetaEntry { table_dir: "test/", index_file_path: "test/11_0000000001/index/<file_id>.puffin", region_id: 47244640257(11, 1), table_id: 11, region_number: 1, region_group: 0, region_sequence: 1, file_id: "<file_id>", index_file_size: Some(6000), index_type: "fulltext_bloom", target_type: "column", target_key: "4", target_json: "{\"column\":4}", blob_size: 89, meta_json: Some("{\"bloom\":{\"bloom_filter_size\":64,\"row_count\":20,\"rows_per_segment\":4,\"segment_count\":5},\"fulltext\":{\"analyzer\":\"English\",\"case_sensitive\":false}}"), node_id: None }
+PuffinIndexMetaEntry { table_dir: "test/", index_file_path: "test/11_0000000001/index/<file_id>.puffin", region_id: 47244640257(11, 1), table_id: 11, region_number: 1, region_group: 0, region_sequence: 1, file_id: "<file_id>", index_file_size: Some(6000), index_type: "fulltext_bloom", target_type: "column", target_key: "4", target_json: "{\"column\":4}", blob_size: 89, meta_json: Some("{\"bloom\":{\"bloom_filter_size\":64,\"row_count\":20,\"rows_per_segment\":4,\"segment_count\":5},\"fulltext\":{\"analyzer\":\"English\",\"bloom_version\":2,\"case_sensitive\":false}}"), node_id: None }
 PuffinIndexMetaEntry { table_dir: "test/", index_file_path: "test/11_0000000001/index/<file_id>.puffin", region_id: 47244640257(11, 1), table_id: 11, region_number: 1, region_group: 0, region_sequence: 1, file_id: "<file_id>", index_file_size: Some(6000), index_type: "fulltext_tantivy", target_type: "column", target_key: "5", target_json: "{\"column\":5}", blob_size: 1000, meta_json: Some("{\"fulltext\":{\"analyzer\":\"Chinese\",\"case_sensitive\":true}}"), node_id: None }
 PuffinIndexMetaEntry { table_dir: "test/", index_file_path: "test/11_0000000001/index/<file_id>.puffin", region_id: 47244640257(11, 1), table_id: 11, region_number: 1, region_group: 0, region_sequence: 1, file_id: "<file_id>", index_file_size: Some(6000), index_type: "inverted", target_type: "column", target_key: "1", target_json: "{\"column\":1}", blob_size: 518, meta_json: Some("{\"inverted\":{\"base_offset\":0,\"bitmap_type\":\"Roaring\",\"fst_size\":150,\"inverted_index_size\":518,\"null_bitmap_size\":8,\"relative_fst_offset\":368,\"relative_null_bitmap_offset\":0,\"segment_row_count\":1024,\"total_row_count\":20}}"), node_id: None }
 PuffinIndexMetaEntry { table_dir: "test/", index_file_path: "test/11_0000000001/index/<file_id>.puffin", region_id: 47244640257(11, 1), table_id: 11, region_number: 1, region_group: 0, region_sequence: 1, file_id: "<file_id>", index_file_size: Some(6000), index_type: "inverted", target_type: "column", target_key: "2", target_json: "{\"column\":2}", blob_size: 515, meta_json: Some("{\"inverted\":{\"base_offset\":0,\"bitmap_type\":\"Roaring\",\"fst_size\":147,\"inverted_index_size\":515,\"null_bitmap_size\":8,\"relative_fst_offset\":368,\"relative_null_bitmap_offset\":0,\"segment_row_count\":1024,\"total_row_count\":20}}"), node_id: None }"#).await;
     test_all_index_metas_list_all_types_with_format(true, r#"
 PuffinIndexMetaEntry { table_dir: "test/", index_file_path: "test/11_0000000001/index/<file_id>.puffin", region_id: 47244640257(11, 1), table_id: 11, region_number: 1, region_group: 0, region_sequence: 1, file_id: "<file_id>", index_file_size: Some(6000), index_type: "bloom_filter", target_type: "column", target_key: "1", target_json: "{\"column\":1}", blob_size: 751, meta_json: Some("{\"bloom\":{\"bloom_filter_size\":640,\"row_count\":20,\"rows_per_segment\":2,\"segment_count\":10}}"), node_id: None }
-PuffinIndexMetaEntry { table_dir: "test/", index_file_path: "test/11_0000000001/index/<file_id>.puffin", region_id: 47244640257(11, 1), table_id: 11, region_number: 1, region_group: 0, region_sequence: 1, file_id: "<file_id>", index_file_size: Some(6000), index_type: "fulltext_bloom", target_type: "column", target_key: "4", target_json: "{\"column\":4}", blob_size: 89, meta_json: Some("{\"bloom\":{\"bloom_filter_size\":64,\"row_count\":20,\"rows_per_segment\":4,\"segment_count\":5},\"fulltext\":{\"analyzer\":\"English\",\"case_sensitive\":false}}"), node_id: None }
+PuffinIndexMetaEntry { table_dir: "test/", index_file_path: "test/11_0000000001/index/<file_id>.puffin", region_id: 47244640257(11, 1), table_id: 11, region_number: 1, region_group: 0, region_sequence: 1, file_id: "<file_id>", index_file_size: Some(6000), index_type: "fulltext_bloom", target_type: "column", target_key: "4", target_json: "{\"column\":4}", blob_size: 89, meta_json: Some("{\"bloom\":{\"bloom_filter_size\":64,\"row_count\":20,\"rows_per_segment\":4,\"segment_count\":5},\"fulltext\":{\"analyzer\":\"English\",\"bloom_version\":2,\"case_sensitive\":false}}"), node_id: None }
 PuffinIndexMetaEntry { table_dir: "test/", index_file_path: "test/11_0000000001/index/<file_id>.puffin", region_id: 47244640257(11, 1), table_id: 11, region_number: 1, region_group: 0, region_sequence: 1, file_id: "<file_id>", index_file_size: Some(6000), index_type: "fulltext_tantivy", target_type: "column", target_key: "5", target_json: "{\"column\":5}", blob_size: 1000, meta_json: Some("{\"fulltext\":{\"analyzer\":\"Chinese\",\"case_sensitive\":true}}"), node_id: None }
 PuffinIndexMetaEntry { table_dir: "test/", index_file_path: "test/11_0000000001/index/<file_id>.puffin", region_id: 47244640257(11, 1), table_id: 11, region_number: 1, region_group: 0, region_sequence: 1, file_id: "<file_id>", index_file_size: Some(6000), index_type: "inverted", target_type: "column", target_key: "1", target_json: "{\"column\":1}", blob_size: 518, meta_json: Some("{\"inverted\":{\"base_offset\":0,\"bitmap_type\":\"Roaring\",\"fst_size\":150,\"inverted_index_size\":518,\"null_bitmap_size\":8,\"relative_fst_offset\":368,\"relative_null_bitmap_offset\":0,\"segment_row_count\":1024,\"total_row_count\":20}}"), node_id: None }
 PuffinIndexMetaEntry { table_dir: "test/", index_file_path: "test/11_0000000001/index/<file_id>.puffin", region_id: 47244640257(11, 1), table_id: 11, region_number: 1, region_group: 0, region_sequence: 1, file_id: "<file_id>", index_file_size: Some(6000), index_type: "inverted", target_type: "column", target_key: "2", target_json: "{\"column\":2}", blob_size: 515, meta_json: Some("{\"inverted\":{\"base_offset\":0,\"bitmap_type\":\"Roaring\",\"fst_size\":147,\"inverted_index_size\":515,\"null_bitmap_size\":8,\"relative_fst_offset\":368,\"relative_null_bitmap_offset\":0,\"segment_row_count\":1024,\"total_row_count\":20}}"), node_id: None }"#).await;

src/mito2/src/engine/puffin_index.rs

@@ -39,7 +39,8 @@ use crate::cache::index::inverted_index::{CachedInvertedIndexBlobReader, Inverte
 use crate::sst::file::RegionIndexId;
 use crate::sst::index::bloom_filter::INDEX_BLOB_TYPE as BLOOM_BLOB_TYPE;
 use crate::sst::index::fulltext_index::{
-    INDEX_BLOB_TYPE_BLOOM as FULLTEXT_BLOOM_BLOB_TYPE,
+    FULLTEXT_BLOOM_VERSION, INDEX_BLOB_TYPE_BLOOM as FULLTEXT_BLOOM_BLOB_TYPE,
+    INDEX_BLOB_TYPE_BLOOM_V1 as FULLTEXT_BLOOM_V1_BLOB_TYPE,
     INDEX_BLOB_TYPE_TANTIVY as FULLTEXT_TANTIVY_BLOB_TYPE,
 };
 use crate::sst::index::inverted_index::INDEX_BLOB_TYPE as INVERTED_BLOB_TYPE;
@@ -126,7 +127,7 @@ pub(crate) async fn collect_index_entries_from_puffin(
                 );
                 entries.push(entry);
             }
-            Some(BlobIndexTypeTargetKey::FulltextBloom(target_key)) => {
+            Some(BlobIndexTypeTargetKey::FulltextBloom(target_key, bloom_version)) => {
                 let bloom_meta = try_read_bloom_meta(
                     &reader,
                     region_index_id,
@@ -139,7 +140,7 @@ pub(crate) async fn collect_index_entries_from_puffin(
                 .await;
 
                 let bloom_value = bloom_meta.as_deref().map(bloom_meta_value);
-                let fulltext_value = Some(fulltext_meta_value(blob));
+                let fulltext_value = Some(fulltext_meta_value(blob, Some(bloom_version)));
                 let (target_type, target_json) = decode_target_info(target_key);
                 let meta_json = build_meta_json(bloom_value, fulltext_value, None);
                 let entry = build_index_entry(
@@ -154,7 +155,7 @@ pub(crate) async fn collect_index_entries_from_puffin(
                 entries.push(entry);
             }
             Some(BlobIndexTypeTargetKey::FulltextTantivy(target_key)) => {
-                let fulltext_value = Some(fulltext_meta_value(blob));
+                let fulltext_value = Some(fulltext_meta_value(blob, None));
                 let (target_type, target_json) = decode_target_info(target_key);
                 let meta_json = build_meta_json(None, fulltext_value, None);
                 let entry = build_index_entry(
@@ -392,12 +393,16 @@ fn bloom_meta_value(meta: &BloomFilterMeta) -> Value {
     })
 }
 
-fn fulltext_meta_value(blob: &BlobMetadata) -> Value {
+fn fulltext_meta_value(blob: &BlobMetadata, bloom_version: Option<u8>) -> Value {
     let config = FulltextConfig::from_blob_metadata(blob).unwrap_or_default();
-    json!({
+    let mut value = json!({
         "analyzer": config.analyzer.to_str(),
         "case_sensitive": config.case_sensitive,
-    })
+    });
+    if let Some(version) = bloom_version {
+        value["bloom_version"] = json!(version);
+    }
+    value
 }
 
 fn inverted_meta_value(meta: &InvertedIndexMeta, metas: &InvertedIndexMetas) -> Value {
@@ -441,7 +446,8 @@ fn build_meta_json(
 
 enum BlobIndexTypeTargetKey<'a> {
     BloomFilter(&'a str),
-    FulltextBloom(&'a str),
+    /// Target key and version of a fulltext bloom blob.
+    FulltextBloom(&'a str, u8),
     FulltextTantivy(&'a str),
     Inverted,
 }
@@ -451,9 +457,17 @@ impl<'a> BlobIndexTypeTargetKey<'a> {
         if let Some(target_key) = Self::target_key_from_blob(blob_type, BLOOM_BLOB_TYPE) {
             Some(BlobIndexTypeTargetKey::BloomFilter(target_key))
         } else if let Some(target_key) =
+            // v2 goes first: the v1 type is a prefix of it.
             Self::target_key_from_blob(blob_type, FULLTEXT_BLOOM_BLOB_TYPE)
         {
-            Some(BlobIndexTypeTargetKey::FulltextBloom(target_key))
+            Some(BlobIndexTypeTargetKey::FulltextBloom(
+                target_key,
+                FULLTEXT_BLOOM_VERSION,
+            ))
+        } else if let Some(target_key) =
+            Self::target_key_from_blob(blob_type, FULLTEXT_BLOOM_V1_BLOB_TYPE)
+        {
+            Some(BlobIndexTypeTargetKey::FulltextBloom(target_key, 1))
         } else if let Some(target_key) =
             Self::target_key_from_blob(blob_type, FULLTEXT_TANTIVY_BLOB_TYPE)
         {

src/mito2/src/sst/file.rs

@@ -26,6 +26,7 @@ use bytes::Bytes;
 use common_base::readable_size::ReadableSize;
 use common_telemetry::{debug, error, warn};
 use common_time::Timestamp;
+use datatypes::schema::FulltextBackend;
 use partition::expr::PartitionExpr;
 use serde::{Deserialize, Serialize};
 use smallvec::SmallVec;
@@ -37,6 +38,7 @@ use crate::access_layer::AccessLayerRef;
 use crate::cache::CacheManagerRef;
 use crate::cache::file_cache::{FileType, IndexKey};
 use crate::sst::file_purger::FilePurgerRef;
+use crate::sst::index::fulltext_index::FULLTEXT_BLOOM_VERSION;
 use crate::sst::location;
 use crate::sst::parquet::SstInfo;
 use crate::sst::primary_key::PrimaryKeyRangeMapper;
@@ -397,6 +399,15 @@ pub struct ColumnIndexMetadata {
     pub column_id: ColumnId,
     /// List of index types that have been successfully created for this column.
     pub created_indexes: IndexTypes,
+    /// Version of the column's fulltext bloom index. `0` means not recorded: the column
+    /// has no bloom index, or its index is the legacy format, which `ssts_index_meta`
+    /// reports as version 1.
+    #[serde(skip_serializing_if = "is_zero")]
+    pub fulltext_bloom_version: u8,
+}
+
+fn is_zero(version: &u8) -> bool {
+    *version == 0
 }
 
 impl FileMeta {
@@ -450,13 +461,14 @@ impl FileMeta {
         let id_to_indexes = self
             .indexes
             .iter()
-            .map(|index| (index.column_id, index.created_indexes.clone()))
+            .map(|index| (index.column_id, index))
             .collect::<std::collections::HashMap<_, _>>();
         for column in metadata {
             if !column.column_schema.is_indexed() {
                 continue;
             }
-            if let Some(indexes) = id_to_indexes.get(&column.column_id) {
+            if let Some(index) = id_to_indexes.get(&column.column_id) {
+                let indexes = &index.created_indexes;
                 if column.column_schema.is_inverted_indexed()
                     && !indexes.contains(&IndexType::InvertedIndex)
                 {
@@ -467,6 +479,26 @@ impl FileMeta {
                 {
                     return false;
                 }
+                if column
+                    .column_schema
+                    .fulltext_options()
+                    .ok()
+                    .flatten()
+                    .is_some_and(|options| {
+                        options.enable && options.backend == FulltextBackend::Bloom
+                    })
+                    && index.fulltext_bloom_version < FULLTEXT_BLOOM_VERSION
+                {
+                    debug!(
+                        "Fulltext bloom index is outdated, region_id: {}, file_id: {}, column_id: {}, version: {}, current: {}",
+                        self.region_id,
+                        self.file_id,
+                        column.column_id,
+                        index.fulltext_bloom_version,
+                        FULLTEXT_BLOOM_VERSION
+                    );
+                    return false;
+                }
                 if column.column_schema.is_skipping_indexed()
                     && !indexes.contains(&IndexType::BloomFilterIndex)
                 {
@@ -1038,6 +1070,7 @@ mod tests {
             indexes: vec![ColumnIndexMetadata {
                 column_id: 0,
                 created_indexes: SmallVec::from_iter([IndexType::InvertedIndex]),
+                ..Default::default()
             }],
             index_file_size: 0,
             index_version: 0,
@@ -1116,6 +1149,7 @@ mod tests {
             indexes: vec![ColumnIndexMetadata {
                 column_id: 0,
                 created_indexes: SmallVec::from_iter([IndexType::InvertedIndex]),
+                ..Default::default()
             }],
             index_file_size: 0,
             index_version: 0,
@@ -1340,6 +1374,7 @@ mod tests {
             indexes: vec![ColumnIndexMetadata {
                 column_id: 1,
                 created_indexes: SmallVec::from_iter([IndexType::InvertedIndex]),
+                ..Default::default()
             }],
             ..Default::default()
         };
@@ -1353,6 +1388,7 @@ mod tests {
                 IndexType::InvertedIndex,
                 IndexType::BloomFilterIndex,
             ]),
+            ..Default::default()
         }];
         let region_meta = vec![new_column_meta(1, "tag1", true, false, false)];
         assert!(file_meta.is_index_consistent_with_region(&region_meta));
@@ -1361,6 +1397,7 @@ mod tests {
         file_meta.indexes = vec![ColumnIndexMetadata {
             column_id: 1,
             created_indexes: SmallVec::from_iter([IndexType::InvertedIndex]),
+            ..Default::default()
         }];
         let region_meta = vec![new_column_meta(1, "tag1", true, true, false)]; // Requires fulltext too
         assert!(!file_meta.is_index_consistent_with_region(&region_meta));
@@ -1369,6 +1406,7 @@ mod tests {
         file_meta.indexes = vec![ColumnIndexMetadata {
             column_id: 2, // File only has index for column 2
             created_indexes: SmallVec::from_iter([IndexType::InvertedIndex]),
+            ..Default::default()
         }];
         let region_meta = vec![new_column_meta(1, "tag1", true, false, false)]; // Requires index on column 1
         assert!(!file_meta.is_index_consistent_with_region(&region_meta));
@@ -1377,6 +1415,7 @@ mod tests {
         file_meta.indexes = vec![ColumnIndexMetadata {
             column_id: 1,
             created_indexes: SmallVec::from_iter([IndexType::InvertedIndex]),
+            ..Default::default()
         }];
         let region_meta = vec![new_column_meta(1, "tag1", false, false, false)]; // No index required
         assert!(file_meta.is_index_consistent_with_region(&region_meta));
@@ -1391,10 +1430,12 @@ mod tests {
             ColumnIndexMetadata {
                 column_id: 1,
                 created_indexes: SmallVec::from_iter([IndexType::InvertedIndex]),
+                ..Default::default()
             },
             ColumnIndexMetadata {
                 column_id: 2, // Column 2 is missing the required BloomFilterIndex
                 created_indexes: SmallVec::from_iter([IndexType::FulltextIndex]),
+                ..Default::default()
             },
         ];
         let region_meta = vec![
@@ -1403,4 +1444,41 @@ mod tests {
         ];
         assert!(!file_meta.is_index_consistent_with_region(&region_meta));
     }
+
+    #[test]
+    fn test_index_consistency_requires_current_fulltext_bloom_version() {
+        let fulltext_column = |backend| {
+            let column_schema = ColumnSchema::new("msg", ConcreteDataType::string_datatype(), true)
+                .with_fulltext_options(FulltextOptions::new_unchecked(
+                    true,
+                    FulltextAnalyzer::English,
+                    false,
+                    backend,
+                    1000,
+                    0.01,
+                ))
+                .unwrap();
+            vec![ColumnMetadata {
+                column_schema,
+                semantic_type: api::v1::SemanticType::Field,
+                column_id: 1,
+            }]
+        };
+        let file_meta = |fulltext_bloom_version| FileMeta {
+            indexes: vec![ColumnIndexMetadata {
+                column_id: 1,
+                created_indexes: SmallVec::from_iter([IndexType::FulltextIndex]),
+                fulltext_bloom_version,
+            }],
+            ..Default::default()
+        };
+
+        let bloom = fulltext_column(FulltextBackend::Bloom);
+        // Written before the version was recorded, i.e. with the old bloom blob.
+        assert!(!file_meta(0).is_index_consistent_with_region(&bloom));
+        assert!(file_meta(FULLTEXT_BLOOM_VERSION).is_index_consistent_with_region(&bloom));
+        // Tantivy indexes carry no bloom version.
+        let tantivy = fulltext_column(FulltextBackend::Tantivy);
+        assert!(file_meta(0).is_index_consistent_with_region(&tantivy));
+    }
 }

src/mito2/src/sst/file_purger.rs

@@ -434,6 +434,7 @@ mod tests {
                     indexes: vec![ColumnIndexMetadata {
                         column_id: 0,
                         created_indexes: SmallVec::from_iter([IndexType::InvertedIndex]),
+                        ..Default::default()
                     }],
                     index_file_size: 4096,
                     index_version: 0,

src/mito2/src/sst/file_ref.rs

@@ -273,6 +273,7 @@ mod tests {
             indexes: vec![ColumnIndexMetadata {
                 column_id: 0,
                 created_indexes: SmallVec::from_iter([IndexType::InvertedIndex]),
+                ..Default::default()
             }],
             index_file_size: 4096,
             index_version: 0,

src/mito2/src/sst/index.rs

@@ -74,6 +74,7 @@ use crate::sst::file::{
     ColumnIndexMetadata, FileHandle, IndexType, IndexTypes, RegionFileId, RegionIndexId,
 };
 use crate::sst::file_purger::FilePurgerRef;
+use crate::sst::index::fulltext_index::FULLTEXT_BLOOM_VERSION;
 use crate::sst::index::fulltext_index::creator::FulltextIndexer;
 use crate::sst::index::intermediate::IntermediateManager;
 use crate::sst::index::inverted_index::creator::InvertedIndexer;
@@ -160,6 +161,8 @@ pub struct IndexOutput {
     pub inverted_index: InvertedIndexOutput,
     /// Fulltext index output.
     pub fulltext_index: FulltextIndexOutput,
+    /// Columns of `fulltext_index` that use the bloom backend.
+    pub fulltext_bloom_columns: Vec<ColumnId>,
     /// Bloom filter output.
     pub bloom_filter: BloomFilterOutput,
 }
@@ -203,6 +206,13 @@ impl IndexOutput {
         map.into_iter()
             .map(|(column_id, created_indexes)| ColumnIndexMetadata {
                 column_id,
+                fulltext_bloom_version: if self.fulltext_index.is_available()
+                    && self.fulltext_bloom_columns.contains(&column_id)
+                {
+                    FULLTEXT_BLOOM_VERSION
+                } else {
+                    0
+                },
                 created_indexes,
             })
             .collect::<Vec<_>>()

src/mito2/src/sst/index/fulltext_index.rs

@@ -16,4 +16,13 @@ pub(crate) mod applier;
 pub(crate) mod creator;
 
 pub(crate) const INDEX_BLOB_TYPE_TANTIVY: &str = "greptime-fulltext-index-v1";
-pub(crate) const INDEX_BLOB_TYPE_BLOOM: &str = "greptime-fulltext-index-bloom";
+/// Bloom blobs tokenized by `ScriptTokenizer`. A new key instead of a config field so that
+/// older versions, which would probe these tokens with their own rules and drop rows,
+/// don't find the blob and scan without pruning.
+pub(crate) const INDEX_BLOB_TYPE_BLOOM: &str = "greptime-fulltext-index-bloom-v2";
+/// Version of the bloom blobs stored under [`INDEX_BLOB_TYPE_BLOOM`], recorded in the
+/// manifest so that `ADMIN BUILD_INDEX` can find SSTs with older blobs.
+pub(crate) const FULLTEXT_BLOOM_VERSION: u8 = 2;
+/// Bloom blobs from older versions. Their tokens don't follow `matches_term` boundaries,
+/// so queries never probe them; they are kept only for index metadata listing.
+pub(crate) const INDEX_BLOB_TYPE_BLOOM_V1: &str = "greptime-fulltext-index-bloom";

src/mito2/src/sst/index/fulltext_index/applier.rs

@@ -19,12 +19,12 @@ use std::sync::Arc;
 use std::time::Instant;
 
 use common_base::range_read::RangeReader;
+use common_base::term_token::{extract_like_probes, extract_term_probes};
 use common_telemetry::{tracing, warn};
 use index::bloom_filter::applier::{BloomFilterApplier, InListPredicate};
 use index::bloom_filter::reader::{BloomFilterReadMetrics, BloomFilterReaderImpl};
+use index::fulltext_index::Config;
 use index::fulltext_index::search::{FulltextIndexSearcher, RowId, TantivyFulltextIndexSearcher};
-use index::fulltext_index::tokenizer::{ChineseTokenizer, EnglishTokenizer, Tokenizer};
-use index::fulltext_index::{Analyzer, Config};
 use index::target::IndexTarget;
 use object_store::ObjectStore;
 use puffin::puffin_manager::cache::PuffinMetadataCacheRef;
@@ -45,7 +45,7 @@ use crate::error::{
 };
 use crate::metrics::INDEX_APPLY_ELAPSED;
 use crate::sst::file::RegionIndexId;
-use crate::sst::index::fulltext_index::applier::builder::{FulltextRequest, FulltextTerm};
+use crate::sst::index::fulltext_index::applier::builder::FulltextRequest;
 use crate::sst::index::fulltext_index::{INDEX_BLOB_TYPE_BLOOM, INDEX_BLOB_TYPE_TANTIVY};
 use crate::sst::index::puffin_manager::{
     PuffinManagerFactory, SstPuffinBlob, SstPuffinDir, SstPuffinReader,
@@ -233,7 +233,10 @@ impl FulltextIndexApplier {
 
         let mut row_ids: Option<BTreeSet<RowId>> = None;
         for (column_id, request) in self.requests.iter() {
-            if request.queries.is_empty() && request.terms.is_empty() {
+            // Tantivy tokens don't follow `matches_term` boundaries, e.g. `SimpleTokenizer`
+            // keeps "错误error日志" as one token, so terms and patterns are left to the
+            // bloom backend.
+            if request.queries.is_empty() {
                 continue;
             }
 
@@ -307,7 +310,6 @@ impl FulltextIndexApplier {
             TantivyFulltextIndexSearcher::new(path, config).context(ApplyFulltextIndexSnafu)?;
         let mut row_ids: Option<BTreeSet<RowId>> = None;
 
-        // 1. Apply queries
         for query in &request.queries {
             let result = searcher
                 .search(&query.0)
@@ -327,21 +329,6 @@ impl FulltextIndexApplier {
             }
         }
 
-        // 2. Apply terms
-        let query = request.terms_as_query(config.case_sensitive);
-        if !query.0.is_empty() {
-            let result = searcher
-                .search(&query.0)
-                .await
-                .context(ApplyFulltextIndexSnafu)?;
-
-            if let Some(ids) = row_ids.as_mut() {
-                ids.retain(|id| result.contains(id));
-            } else {
-                row_ids = Some(result);
-            }
-        }
-
         Ok(row_ids)
     }
 }
@@ -376,8 +363,7 @@ impl FulltextIndexApplier {
         let mut applied = false;
 
         for (column_id, request) in self.requests.iter() {
-            if request.terms.is_empty() {
-                // only apply terms
+            if request.terms.is_empty() && request.likes.is_empty() {
                 continue;
             }
 
@@ -386,7 +372,7 @@ impl FulltextIndexApplier {
                     file_id,
                     file_size_hint,
                     *column_id,
-                    &request.terms,
+                    request,
                     &mut output,
                     metrics.as_deref_mut(),
                 )
@@ -417,7 +403,7 @@ impl FulltextIndexApplier {
         file_id: RegionIndexId,
         file_size_hint: Option<u64>,
         column_id: ColumnId,
-        terms: &[FulltextTerm],
+        request: &FulltextRequest,
         output: &mut [(usize, Vec<Range<usize>>)],
         mut metrics: Option<&mut FulltextIndexApplyMetrics>,
     ) -> Result<bool> {
@@ -435,7 +421,7 @@ impl FulltextIndexApplier {
         let config =
             Config::from_blob_metadata(reader.metadata()).context(ApplyFulltextIndexSnafu)?;
 
-        let predicates = Self::terms_to_predicates(terms, &config);
+        let predicates = Self::request_to_predicates(request, &config);
         if predicates.is_empty() {
             return Ok(false);
         }
@@ -524,43 +510,39 @@ impl FulltextIndexApplier {
         }
     }
 
-    /// Converts terms to predicates.
-    ///
-    /// Split terms by non-alphanumeric characters and convert them to lowercase if case-insensitive.
-    /// Multiple terms are combined with AND semantics.
-    fn terms_to_predicates(terms: &[FulltextTerm], config: &Config) -> Vec<InListPredicate> {
-        let mut probes = HashSet::new();
-        for term in terms {
-            if config.case_sensitive && term.col_lowered {
-                // lowercased terms are not indexed
-                continue;
+    /// Converts terms and `LIKE` patterns to predicates, one per token that a matching
+    /// row must contain. Multiple predicates are combined with AND semantics.
+    fn request_to_predicates(request: &FulltextRequest, config: &Config) -> Vec<InListPredicate> {
+        // Probes of a lowercased column come from lowercased text and are indexed as is;
+        // a case-sensitive index has no tokens of the lowercased text.
+        let probes = |col_lowered: bool, probes: Vec<String>| -> Vec<Vec<u8>> {
+            match (col_lowered, config.case_sensitive) {
+                (true, true) => vec![],
+                (false, false) => probes
+                    .iter()
+                    .map(|p| p.to_lowercase().into_bytes())
+                    .collect(),
+                _ => probes.into_iter().map(String::into_bytes).collect(),
             }
-            probes.extend(Self::term_to_probes(&term.term, config));
-        }
+        };
+        let term_probes = request
+            .terms
+            .iter()
+            .flat_map(|term| probes(term.col_lowered, extract_term_probes(&term.term)));
+        let like_probes = request
+            .likes
+            .iter()
+            .flat_map(|like| probes(like.col_lowered, extract_like_probes(&like.pattern)));
 
-        probes
+        term_probes
+            .chain(like_probes)
+            .collect::<HashSet<_>>()
             .into_iter()
             .map(|p| InListPredicate {
                 list: iter::once(p).collect(),
             })
             .collect::<Vec<_>>()
     }
-
-    fn term_to_probes<'a>(term: &'a str, config: &'a Config) -> impl Iterator<Item = Vec<u8>> + 'a {
-        let tokens = match config.analyzer {
-            Analyzer::English => EnglishTokenizer {}.tokenize(term),
-            Analyzer::Chinese => ChineseTokenizer {}.tokenize(term),
-        };
-
-        tokens.into_iter().map(|t| {
-            if !config.case_sensitive {
-                t.to_lowercase()
-            } else {
-                t.to_string()
-            }
-            .into_bytes()
-        })
-    }
 }
 
 /// The source of the index.
@@ -774,3 +756,247 @@ impl IndexSource {
         Ok(reader)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use common_function::scalars::matches_term::MatchesTermFinder;
+    use datatypes::arrow::array::{Scalar, StringArray};
+    use datatypes::arrow::compute::kernels::comparison::like;
+    use index::fulltext_index::Analyzer;
+    use index::fulltext_index::tokenizer::{self, ScriptTokenizer};
+    use rand::rngs::StdRng;
+    use rand::{Rng, SeedableRng};
+
+    use super::*;
+    use crate::sst::index::fulltext_index::applier::builder::{FulltextLike, FulltextTerm};
+
+    /// A `matches_term` or `LIKE` predicate on `col`, or on `lower(col)` when `lowered`.
+    enum Predicate<'a> {
+        Term(&'a str),
+        Like(&'a str),
+        LowerTerm(&'a str),
+        LowerLike(&'a str),
+    }
+
+    impl Predicate<'_> {
+        fn eval(&self, text: &str) -> bool {
+            let like_matches = |text: &str, pattern: &str| {
+                like(
+                    &StringArray::from(vec![text]),
+                    &Scalar::new(StringArray::from(vec![pattern])),
+                )
+                .unwrap()
+                .value(0)
+            };
+            match self {
+                Predicate::Term(term) => MatchesTermFinder::new(term).find(text),
+                Predicate::Like(pattern) => like_matches(text, pattern),
+                Predicate::LowerTerm(term) => {
+                    MatchesTermFinder::new(term).find(&text.to_lowercase())
+                }
+                Predicate::LowerLike(pattern) => like_matches(&text.to_lowercase(), pattern),
+            }
+        }
+
+        fn request(&self) -> FulltextRequest {
+            let mut request = FulltextRequest::default();
+            let (term, like, col_lowered) = match *self {
+                Predicate::Term(term) => (Some(term), None, false),
+                Predicate::Like(pattern) => (None, Some(pattern), false),
+                Predicate::LowerTerm(term) => (Some(term), None, true),
+                Predicate::LowerLike(pattern) => (None, Some(pattern), true),
+            };
+            request.terms.extend(term.map(|term| FulltextTerm {
+                col_lowered,
+                term: term.to_string(),
+            }));
+            request.likes.extend(like.map(|pattern| FulltextLike {
+                col_lowered,
+                pattern: pattern.to_string(),
+            }));
+            request
+        }
+
+        /// Asserts that every probe is among `text`'s indexed tokens, i.e. the bloom
+        /// filter keeps the row, and returns the number of probes.
+        fn assert_probes_indexed(&self, text: &str) -> usize {
+            let mut num_probes = 0;
+            for case_sensitive in [true, false] {
+                let config = Config {
+                    analyzer: Analyzer::English,
+                    case_sensitive,
+                };
+                let tokens = tokenizer::Analyzer::new(Box::new(ScriptTokenizer), case_sensitive)
+                    .analyze_text(text)
+                    .unwrap()
+                    .into_iter()
+                    .collect::<HashSet<_>>();
+                let predicates =
+                    FulltextIndexApplier::request_to_predicates(&self.request(), &config);
+                for predicate in &predicates {
+                    assert!(
+                        predicate.list.iter().any(|probe| tokens.contains(probe)),
+                        "probe {:?} not indexed, text: {text:?}, case_sensitive: {case_sensitive}",
+                        predicate
+                            .list
+                            .iter()
+                            .map(|p| String::from_utf8_lossy(p))
+                            .collect::<Vec<_>>(),
+                    );
+                }
+                num_probes = predicates.len();
+            }
+            num_probes
+        }
+    }
+
+    #[test]
+    fn test_probes_keep_matching_rows() {
+        // (text, predicate, whether the predicate yields probes)
+        let cases = [
+            ("hello_world", Predicate::Term("world"), true),
+            ("trace_id=abc", Predicate::Term("id"), true),
+            ("错误error日志", Predicate::Term("error"), true),
+            (
+                "登录手机号18888888888的动态key",
+                Predicate::Term("手机号"),
+                true,
+            ),
+            (
+                "登录手机号18888888888的动态key",
+                Predicate::Term("机号"),
+                true,
+            ),
+            (
+                "登录手机号18888888888的动态key",
+                Predicate::Term("18888888888"),
+                true,
+            ),
+            // The ASCII edge of a Han term may continue in the text.
+            (
+                "登录手机号18888888888的动态key",
+                Predicate::Term("机号1888"),
+                true,
+            ),
+            ("中国农业银行", Predicate::Term("农业"), true),
+            ("中国农业银行", Predicate::Term("农"), false),
+            ("连接timeout.5次", Predicate::Term("timeout"), true),
+            ("用户user-123登录", Predicate::Term("user"), true),
+            (
+                "CRITICAL error: disk",
+                Predicate::Term("CRITICAL error"),
+                true,
+            ),
+            ("café>", Predicate::Term("café"), true),
+            (
+                "connection timeout after 5s",
+                Predicate::Like("%timeout%"),
+                false,
+            ),
+            (
+                "connection timeout after 5s",
+                Predicate::Like("% timeout %"),
+                true,
+            ),
+            (
+                "connection timeout after 5s",
+                Predicate::Like("connection%"),
+                false,
+            ),
+            (
+                "connection timeout after 5s",
+                Predicate::Like("connection %"),
+                true,
+            ),
+            (
+                "connection timeout after 5s",
+                Predicate::Like("%after 5s"),
+                true,
+            ),
+            ("trace_id=abc", Predicate::Like(r"trace\_id=%"), true),
+            ("trace_id=abc", Predicate::Like("trace_id=%"), false),
+            ("100% done", Predicate::Like(r"100\% done"), true),
+            ("错误error日志", Predicate::Like("%错误error日%"), true),
+            ("错误error日志", Predicate::Like("%r日志"), true),
+        ];
+        let lowered_cases = [
+            (
+                "Connection TIMEOUT after 5s",
+                Predicate::LowerTerm("timeout"),
+                true,
+            ),
+            (
+                "Connection TIMEOUT after 5s",
+                Predicate::LowerLike("% timeout %"),
+                true,
+            ),
+            // Lowercasing changes the class of U+212A KELVIN SIGN, splits U+0130 into
+            // two characters, and lowercases `Σ` depending on what follows it.
+            ("a\u{212A}b", Predicate::LowerTerm("akb"), true),
+            ("\u{130}stanbul", Predicate::LowerTerm("stanbul"), true),
+            ("ΟΔΟΣ.x", Predicate::LowerTerm("οδοσ"), true),
+        ];
+        for (text, predicate, has_probes) in cases.into_iter().chain(lowered_cases) {
+            assert!(predicate.eval(text), "case doesn't match: {text:?}");
+            assert_eq!(
+                predicate.assert_probes_indexed(text) > 0,
+                has_probes,
+                "text: {text:?}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_probes_keep_matching_rows_random() {
+        // Word characters of every class, separators, LIKE metacharacters, and
+        // characters whose lowercase changes length or class.
+        const ALPHABET: &[char] = &[
+            'a', 'b', 'A', '1', '_', '-', ' ', '.', '%', '\\', '错', '误', 'é', 'Д', '\u{212A}',
+            '\u{130}', 'Σ',
+        ];
+        let mut rng = StdRng::seed_from_u64(42);
+        let random_char = |rng: &mut StdRng| ALPHABET[rng.random_range(0..ALPHABET.len())];
+
+        for _ in 0..20000 {
+            let text = (0..rng.random_range(0..12))
+                .map(|_| random_char(&mut rng))
+                .collect::<String>();
+            let chars = text.chars().collect::<Vec<_>>();
+            let start = rng.random_range(0..=chars.len());
+            let end = rng.random_range(start..=chars.len());
+
+            let term = chars[start..end].iter().collect::<String>();
+            let lowered = term.to_lowercase();
+            for predicate in [Predicate::Term(&term), Predicate::LowerTerm(&lowered)] {
+                if predicate.eval(&text) {
+                    predicate.assert_probes_indexed(&text);
+                }
+            }
+
+            let mut pattern = String::new();
+            if start > 0 && rng.random_bool(0.5) {
+                pattern.push('%');
+            }
+            for &c in &chars[start..end] {
+                match rng.random_range(0..10) {
+                    0 => pattern.push('%'),
+                    1 => pattern.push('_'),
+                    _ if matches!(c, '%' | '_' | '\\') => {
+                        pattern.push('\\');
+                        pattern.push(c);
+                    }
+                    _ => pattern.push(c),
+                }
+            }
+            if end < chars.len() && rng.random_bool(0.5) {
+                pattern.push('%');
+            }
+            let lowered = pattern.to_lowercase();
+            for predicate in [Predicate::Like(&pattern), Predicate::LowerLike(&lowered)] {
+                if predicate.eval(&text) {
+                    predicate.assert_probes_indexed(&text);
+                }
+            }
+        }
+    }
+}

src/mito2/src/sst/index/fulltext_index/applier/builder.rs

@@ -14,9 +14,11 @@
 
 use std::collections::BTreeMap;
 
+use common_base::term_token::extract_like_probes;
 use datafusion_common::ScalarValue;
-use datafusion_expr::expr::ScalarFunction;
+use datafusion_expr::expr::{Like, ScalarFunction};
 use datafusion_expr::{BinaryExpr, Expr, Operator};
+use datatypes::schema::FulltextBackend;
 use object_store::ObjectStore;
 use puffin::puffin_manager::cache::PuffinMetadataCacheRef;
 use store_api::metadata::RegionMetadata;
@@ -36,31 +38,7 @@ use crate::sst::index::puffin_manager::PuffinManagerFactory;
 pub struct FulltextRequest {
     pub queries: Vec<FulltextQuery>,
     pub terms: Vec<FulltextTerm>,
-}
-
-impl FulltextRequest {
-    /// Convert terms to a query string.
-    ///
-    /// For example, if the terms are ["foo", "bar"], the query string will be `r#"+"foo" +"bar""#`.
-    /// Need to escape the `"` in the term.
-    ///
-    /// `skip_lowercased` is used for the situation that lowercased terms are not indexed.
-    pub fn terms_as_query(&self, skip_lowercased: bool) -> FulltextQuery {
-        let mut query = String::new();
-        for term in &self.terms {
-            if skip_lowercased && term.col_lowered {
-                continue;
-            }
-            // Escape the `"` in the term.
-            let escaped_term = term.term.replace("\"", "\\\"");
-            if query.is_empty() {
-                query = format!("+\"{escaped_term}\"");
-            } else {
-                query.push_str(&format!(" +\"{escaped_term}\""));
-            }
-        }
-        FulltextQuery(query)
-    }
+    pub likes: Vec<FulltextLike>,
 }
 
 /// A query to be matched in fulltext index.
@@ -79,6 +57,16 @@ pub struct FulltextTerm {
     pub term: String,
 }
 
+/// A case-sensitive `LIKE` pattern to be matched in fulltext index.
+///
+/// `pattern` is e.g. "%foo bar%" in `SELECT * FROM t WHERE text LIKE '%foo bar%'`.
+/// `col_lowered` indicates whether the column is lowercased, e.g. `lower(text) LIKE '%foo bar%'`.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct FulltextLike {
+    pub col_lowered: bool,
+    pub pattern: String,
+}
+
 /// `FulltextIndexApplierBuilder` is a builder for `FulltextIndexApplier`.
 pub struct FulltextIndexApplierBuilder<'a> {
     table_dir: String,
@@ -144,9 +132,9 @@ impl<'a> FulltextIndexApplierBuilder<'a> {
         }
 
         // Check if any requests have queries or terms
-        let has_requests = requests
-            .iter()
-            .any(|(_, request)| !request.queries.is_empty() || !request.terms.is_empty());
+        let has_requests = requests.iter().any(|(_, request)| {
+            !request.queries.is_empty() || !request.terms.is_empty() || !request.likes.is_empty()
+        });
 
         Ok(has_requests.then(|| {
             FulltextIndexApplier::new(
@@ -176,6 +164,11 @@ impl<'a> FulltextIndexApplierBuilder<'a> {
                 Self::extract_requests(left, metadata, requests);
                 Self::extract_requests(right, metadata, requests);
             }
+            Expr::Like(like) => {
+                if let Some((column_id, like)) = Self::expr_to_like(metadata, like) {
+                    requests.entry(column_id).or_default().likes.push(like);
+                }
+            }
             Expr::ScalarFunction(func) => {
                 if let Some((column_id, query)) = Self::expr_to_query(metadata, func) {
                     requests.entry(column_id).or_default().queries.push(query);
@@ -261,6 +254,58 @@ impl<'a> FulltextIndexApplierBuilder<'a> {
         ))
     }
 
+    fn expr_to_like(metadata: &RegionMetadata, like: &Like) -> Option<(ColumnId, FulltextLike)> {
+        // `ILIKE` uses Unicode case folding, which disagrees with the index's
+        // `to_lowercase` on characters like 'ſ', so its probes could miss rows.
+        if like.negated || like.case_insensitive {
+            return None;
+        }
+        // Probes assume arrow's default `\` escape.
+        if !matches!(like.escape_char, None | Some('\\')) {
+            return None;
+        }
+
+        let (c, col_lowered) = match like.expr.as_ref() {
+            Expr::Column(c) => (c, false),
+            Expr::ScalarFunction(f) => match Self::extract_lower_arg(f)? {
+                Expr::Column(c) => (c, true),
+                _ => return None,
+            },
+            _ => return None,
+        };
+        let column = metadata.column_by_name(&c.name)?;
+        if column.column_schema.data_type != ConcreteDataType::string_datatype() {
+            return None;
+        }
+        // `LIKE` is common on unindexed columns and with patterns like '%word%' that have
+        // no probe; skip them instead of opening every SST's index for nothing.
+        let options = column.column_schema.fulltext_options().ok()??;
+        if !options.enable || options.backend != FulltextBackend::Bloom {
+            return None;
+        }
+
+        let Expr::Literal(
+            ScalarValue::Utf8(Some(pattern))
+            | ScalarValue::LargeUtf8(Some(pattern))
+            | ScalarValue::Utf8View(Some(pattern)),
+            _,
+        ) = like.pattern.as_ref()
+        else {
+            return None;
+        };
+        if extract_like_probes(pattern).is_empty() {
+            return None;
+        }
+
+        Some((
+            column.column_id,
+            FulltextLike {
+                col_lowered,
+                pattern: pattern.clone(),
+            },
+        ))
+    }
+
     fn extract_lower_arg(lower_func: &ScalarFunction) -> Option<&Expr> {
         if lower_func.args.len() != 1 {
             return None;
@@ -289,9 +334,9 @@ mod tests {
     use common_function::scalars::matches_term::MatchesTermFunction;
     use datafusion::functions::string::lower;
     use datafusion_common::Column;
-    use datafusion_expr::expr::ScalarFunction;
+    use datafusion_expr::expr::{Like, ScalarFunction};
     use datafusion_expr::{Literal, ScalarUDF};
-    use datatypes::schema::ColumnSchema;
+    use datatypes::schema::{ColumnSchema, FulltextAnalyzer, FulltextOptions};
     use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder};
     use store_api::storage::RegionId;
 
@@ -512,6 +557,107 @@ mod tests {
         assert_eq!(request.queries[0], FulltextQuery("foo".to_string()));
     }
 
+    fn mock_metadata_with_fulltext(backend: FulltextBackend) -> RegionMetadata {
+        let mut builder = RegionMetadataBuilder::new(RegionId::new(1, 2));
+        builder
+            .push_column_metadata(ColumnMetadata {
+                column_schema: ColumnSchema::new("text", ConcreteDataType::string_datatype(), true)
+                    .with_fulltext_options(FulltextOptions::new_unchecked(
+                        true,
+                        FulltextAnalyzer::English,
+                        false,
+                        backend,
+                        10240,
+                        0.01,
+                    ))
+                    .unwrap(),
+                semantic_type: SemanticType::Field,
+                column_id: 1,
+            })
+            .push_column_metadata(ColumnMetadata {
+                column_schema: ColumnSchema::new(
+                    "ts",
+                    ConcreteDataType::timestamp_millisecond_datatype(),
+                    false,
+                ),
+                semantic_type: SemanticType::Timestamp,
+                column_id: 2,
+            });
+
+        builder.build().unwrap()
+    }
+
+    #[test]
+    fn test_expr_to_like() {
+        let like = |negated, case_insensitive, escape_char, pattern: &str| Like {
+            negated,
+            expr: Box::new(Expr::Column(Column::from_name("text"))),
+            pattern: Box::new(pattern.lit()),
+            escape_char,
+            case_insensitive,
+        };
+        let bloom = mock_metadata_with_fulltext(FulltextBackend::Bloom);
+
+        for escape_char in [None, Some('\\')] {
+            assert_eq!(
+                FulltextIndexApplierBuilder::expr_to_like(
+                    &bloom,
+                    &like(false, false, escape_char, "%foo bar baz%")
+                ),
+                Some((
+                    1,
+                    FulltextLike {
+                        col_lowered: false,
+                        pattern: "%foo bar baz%".to_string(),
+                    }
+                ))
+            );
+        }
+        let lowered = Like {
+            expr: Box::new(Expr::ScalarFunction(ScalarFunction {
+                args: vec![Expr::Column(Column::from_name("text"))],
+                func: lower(),
+            })),
+            ..like(false, false, None, "%foo bar baz%")
+        };
+        assert_eq!(
+            FulltextIndexApplierBuilder::expr_to_like(&bloom, &lowered),
+            Some((
+                1,
+                FulltextLike {
+                    col_lowered: true,
+                    pattern: "%foo bar baz%".to_string(),
+                }
+            ))
+        );
+        // NOT LIKE, ILIKE and other escape characters can't use the probes, and
+        // '%foo%' has none.
+        for like in [
+            like(true, false, None, "%foo bar baz%"),
+            like(false, true, None, "%foo bar baz%"),
+            like(false, false, Some('!'), "%foo bar baz%"),
+            like(false, false, None, "%foo%"),
+        ] {
+            assert_eq!(
+                FulltextIndexApplierBuilder::expr_to_like(&bloom, &like),
+                None
+            );
+        }
+        // Only columns with a bloom fulltext index.
+        for metadata in [
+            mock_metadata(),
+            mock_metadata_with_fulltext(FulltextBackend::Tantivy),
+        ] {
+            assert_eq!(
+                FulltextIndexApplierBuilder::expr_to_like(
+                    &metadata,
+                    &like(false, false, None, "%foo bar baz%")
+                ),
+                None
+            );
+        }
+    }
+
     #[test]
     fn test_extract_multiple_requests() {
         let metadata = mock_metadata();
@@ -551,92 +697,4 @@ mod tests {
             }
         );
     }
-
-    #[test]
-    fn test_terms_as_query() {
-        // Test with empty terms
-        let request = FulltextRequest::default();
-        assert_eq!(request.terms_as_query(false), FulltextQuery(String::new()));
-        assert_eq!(request.terms_as_query(true), FulltextQuery(String::new()));
-
-        // Test with a single term (not lowercased)
-        let mut request = FulltextRequest::default();
-        request.terms.push(FulltextTerm {
-            col_lowered: false,
-            term: "foo".to_string(),
-        });
-        assert_eq!(
-            request.terms_as_query(false),
-            FulltextQuery("+\"foo\"".to_string())
-        );
-        assert_eq!(
-            request.terms_as_query(true),
-            FulltextQuery("+\"foo\"".to_string())
-        );
-
-        // Test with a single lowercased term and skip_lowercased=true
-        let mut request = FulltextRequest::default();
-        request.terms.push(FulltextTerm {
-            col_lowered: true,
-            term: "foo".to_string(),
-        });
-        assert_eq!(
-            request.terms_as_query(false),
-            FulltextQuery("+\"foo\"".to_string())
-        );
-        assert_eq!(request.terms_as_query(true), FulltextQuery(String::new())); // Should skip lowercased term
-
-        // Test with multiple terms, mix of lowercased and not
-        let mut request = FulltextRequest::default();
-        request.terms.push(FulltextTerm {
-            col_lowered: false,
-            term: "foo".to_string(),
-        });
-        request.terms.push(FulltextTerm {
-            col_lowered: true,
-            term: "bar".to_string(),
-        });
-        assert_eq!(
-            request.terms_as_query(false),
-            FulltextQuery("+\"foo\" +\"bar\"".to_string())
-        );
-        assert_eq!(
-            request.terms_as_query(true),
-            FulltextQuery("+\"foo\"".to_string()) // Only the non-lowercased term
-        );
-
-        // Test with term containing quotes that need escaping
-        let mut request = FulltextRequest::default();
-        request.terms.push(FulltextTerm {
-            col_lowered: false,
-            term: "foo\"bar".to_string(),
-        });
-        assert_eq!(
-            request.terms_as_query(false),
-            FulltextQuery("+\"foo\\\"bar\"".to_string())
-        );
-
-        // Test with a complex mix of terms
-        let mut request = FulltextRequest::default();
-        request.terms.push(FulltextTerm {
-            col_lowered: false,
-            term: "foo".to_string(),
-        });
-        request.terms.push(FulltextTerm {
-            col_lowered: true,
-            term: "bar\"quoted\"".to_string(),
-        });
-        request.terms.push(FulltextTerm {
-            col_lowered: false,
-            term: "baz\\escape".to_string(),
-        });
-        assert_eq!(
-            request.terms_as_query(false),
-            FulltextQuery("+\"foo\" +\"bar\\\"quoted\\\"\" +\"baz\\escape\"".to_string())
-        );
-        assert_eq!(
-            request.terms_as_query(true),
-            FulltextQuery("+\"foo\" +\"baz\\escape\"".to_string()) // Skips the lowercased term
-        );
-    }
 }

src/mito2/src/sst/index/fulltext_index/creator.rs

@@ -229,6 +229,14 @@ impl FulltextIndexer {
     pub fn column_ids(&self) -> impl Iterator<Item = ColumnId> + '_ {
         self.creators.keys().copied()
     }
+
+    /// Returns IDs of columns indexed with the bloom backend.
+    pub fn bloom_column_ids(&self) -> impl Iterator<Item = ColumnId> + '_ {
+        self.creators
+            .values()
+            .filter(|c| matches!(c.inner, AltFulltextCreator::Bloom(_)))
+            .map(|c| c.column_id)
+    }
 }
 
 impl FulltextIndexer {
@@ -778,28 +786,8 @@ mod tests {
         let row_ids = applier_factory(vec![(1, "World")], vec![], None).await;
         assert_eq!(row_ids, Some(rows([3])));
 
+        // Terms are left to the bloom backend.
         let row_ids = applier_factory(vec![], vec![(1, vec![(false, "hello")])], None).await;
-        assert_eq!(row_ids, Some(rows([0])));
-
-        let row_ids = applier_factory(vec![], vec![(1, vec![(true, "hello")])], None).await;
-        assert_eq!(row_ids, None);
-
-        let row_ids = applier_factory(vec![], vec![(1, vec![(false, "world")])], None).await;
-        assert_eq!(row_ids, Some(rows([1])));
-
-        let row_ids = applier_factory(vec![], vec![(1, vec![(true, "world")])], None).await;
-        assert_eq!(row_ids, None);
-
-        let row_ids = applier_factory(vec![], vec![(1, vec![(false, "Hello")])], None).await;
-        assert_eq!(row_ids, Some(rows([3])));
-
-        let row_ids = applier_factory(vec![], vec![(1, vec![(true, "Hello")])], None).await;
-        assert_eq!(row_ids, None);
-
-        let row_ids = applier_factory(vec![], vec![(1, vec![(false, "Hello, World")])], None).await;
-        assert_eq!(row_ids, Some(rows([3])));
-
-        let row_ids = applier_factory(vec![], vec![(1, vec![(true, "Hello, World")])], None).await;
         assert_eq!(row_ids, None);
     }
 
@@ -939,30 +927,6 @@ mod tests {
 
         let row_ids = applier_factory(vec![(2, "World")], vec![], None).await;
         assert_eq!(row_ids, Some(rows([2, 3])));
-
-        let row_ids = applier_factory(vec![], vec![(2, vec![(false, "hello")])], None).await;
-        assert_eq!(row_ids, Some(rows([0, 3])));
-
-        let row_ids = applier_factory(vec![], vec![(2, vec![(true, "hello")])], None).await;
-        assert_eq!(row_ids, Some(rows([0, 3])));
-
-        let row_ids = applier_factory(vec![], vec![(2, vec![(false, "world")])], None).await;
-        assert_eq!(row_ids, Some(rows([2, 3])));
-
-        let row_ids = applier_factory(vec![], vec![(2, vec![(true, "world")])], None).await;
-        assert_eq!(row_ids, Some(rows([2, 3])));
-
-        let row_ids = applier_factory(vec![], vec![(2, vec![(false, "Hello")])], None).await;
-        assert_eq!(row_ids, Some(rows([0, 3])));
-
-        let row_ids = applier_factory(vec![], vec![(2, vec![(true, "Hello")])], None).await;
-        assert_eq!(row_ids, Some(rows([0, 3])));
-
-        let row_ids = applier_factory(vec![], vec![(2, vec![(false, "World")])], None).await;
-        assert_eq!(row_ids, Some(rows([2, 3])));
-
-        let row_ids = applier_factory(vec![], vec![(2, vec![(true, "World")])], None).await;
-        assert_eq!(row_ids, Some(rows([2, 3])));
     }
 
     #[tokio::test]
@@ -995,22 +959,6 @@ mod tests {
         .await;
         assert_eq!(row_ids, Some(rows([3])));
 
-        let row_ids = applier_factory(
-            vec![],
-            vec![(2, vec![(true, "hello")])],
-            Some(BitVec::from_slice(&[0b1111])),
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([0, 3])));
-
-        let row_ids = applier_factory(
-            vec![],
-            vec![(2, vec![(true, "hello")])],
-            Some(BitVec::from_slice(&[0b1110])), // row 0 is filtered out
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([3])));
-
         let row_ids = applier_factory(
             vec![],
             vec![(2, vec![(false, "world")])],
@@ -1027,22 +975,6 @@ mod tests {
         .await;
         assert_eq!(row_ids, Some(rows([3])));
 
-        let row_ids = applier_factory(
-            vec![],
-            vec![(2, vec![(true, "world")])],
-            Some(BitVec::from_slice(&[0b1111])),
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([2, 3])));
-
-        let row_ids = applier_factory(
-            vec![],
-            vec![(2, vec![(true, "world")])],
-            Some(BitVec::from_slice(&[0b1011])), // row 2 is filtered out
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([3])));
-
         let row_ids = applier_factory(
             vec![],
             vec![(2, vec![(false, "Hello")])],
@@ -1059,22 +991,6 @@ mod tests {
         .await;
         assert_eq!(row_ids, Some(rows([0])));
 
-        let row_ids = applier_factory(
-            vec![],
-            vec![(2, vec![(true, "Hello")])],
-            Some(BitVec::from_slice(&[0b1111])),
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([0, 3])));
-
-        let row_ids = applier_factory(
-            vec![],
-            vec![(2, vec![(true, "Hello")])],
-            Some(BitVec::from_slice(&[0b1110])), // row 0 is filtered out
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([3])));
-
         let row_ids = applier_factory(
             vec![],
             vec![(2, vec![(false, "World")])],
@@ -1093,19 +1009,11 @@ mod tests {
 
         let row_ids = applier_factory(
             vec![],
-            vec![(2, vec![(true, "World")])],
+            vec![(2, vec![(true, "hello")])],
             Some(BitVec::from_slice(&[0b1111])),
         )
         .await;
-        assert_eq!(row_ids, Some(rows([2, 3])));
-
-        let row_ids = applier_factory(
-            vec![],
-            vec![(2, vec![(true, "World")])],
-            Some(BitVec::from_slice(&[0b1011])), // row 2 is filtered out
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([3])));
+        assert_eq!(row_ids, Some(rows([0, 3])));
     }
 
     #[tokio::test]
@@ -1127,12 +1035,6 @@ mod tests {
 
         let row_ids = applier_factory(vec![(3, "世界")], vec![], None).await;
         assert_eq!(row_ids, Some(rows([2, 3])));
-
-        let row_ids = applier_factory(vec![], vec![(3, vec![(false, "你好")])], None).await;
-        assert_eq!(row_ids, Some(rows([0, 3])));
-
-        let row_ids = applier_factory(vec![], vec![(3, vec![(false, "世界")])], None).await;
-        assert_eq!(row_ids, Some(rows([2, 3])));
     }
 
     #[tokio::test]
@@ -1182,61 +1084,6 @@ mod tests {
         assert_eq!(row_ids, Some(rows([3])));
     }
 
-    #[tokio::test]
-    async fn test_fulltext_index_multi_terms_case_sensitive_tantivy() {
-        let applier_factory = build_fulltext_applier_factory(
-            "test_fulltext_index_multi_terms_case_sensitive_tantivy_",
-            FulltextBackend::Tantivy,
-            &[
-                (Some("Hello"), None, None),
-                (Some("World"), None, None),
-                (None, None, None),
-                (Some("Hello, World"), None, None),
-            ],
-        )
-        .await;
-
-        let row_ids = applier_factory(
-            vec![],
-            vec![(1, vec![(false, "hello"), (false, "world")])],
-            None,
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([])));
-
-        let row_ids = applier_factory(
-            vec![],
-            vec![(1, vec![(false, "Hello"), (false, "World")])],
-            None,
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([3])));
-
-        let row_ids = applier_factory(
-            vec![],
-            vec![(1, vec![(true, "Hello"), (false, "World")])],
-            None,
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([1, 3])));
-
-        let row_ids = applier_factory(
-            vec![],
-            vec![(1, vec![(false, "Hello"), (true, "World")])],
-            None,
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([0, 3])));
-
-        let row_ids = applier_factory(
-            vec![],
-            vec![(1, vec![(true, "Hello"), (true, "World")])],
-            None,
-        )
-        .await;
-        assert_eq!(row_ids, None);
-    }
-
     #[tokio::test]
     async fn test_fulltext_index_multi_terms_case_sensitive_bloom() {
         let applier_factory = build_fulltext_applier_factory(
@@ -1292,53 +1139,6 @@ mod tests {
         assert_eq!(row_ids, None);
     }
 
-    #[tokio::test]
-    async fn test_fulltext_index_multi_terms_case_insensitive_tantivy() {
-        let applier_factory = build_fulltext_applier_factory(
-            "test_fulltext_index_multi_terms_case_insensitive_tantivy_",
-            FulltextBackend::Tantivy,
-            &[
-                (None, Some("hello"), None),
-                (None, None, None),
-                (None, Some("world"), None),
-                (None, Some("Hello, World"), None),
-            ],
-        )
-        .await;
-
-        let row_ids = applier_factory(
-            vec![],
-            vec![(2, vec![(false, "hello"), (false, "world")])],
-            None,
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([3])));
-
-        let row_ids = applier_factory(
-            vec![],
-            vec![(2, vec![(true, "hello"), (false, "world")])],
-            None,
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([3])));
-
-        let row_ids = applier_factory(
-            vec![],
-            vec![(2, vec![(false, "hello"), (true, "world")])],
-            None,
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([3])));
-
-        let row_ids = applier_factory(
-            vec![],
-            vec![(2, vec![(true, "hello"), (true, "world")])],
-            None,
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([3])));
-    }
-
     #[tokio::test]
     async fn test_fulltext_index_multi_terms_case_insensitive_bloom() {
         let applier_factory = build_fulltext_applier_factory(
@@ -1404,16 +1204,10 @@ mod tests {
         )
         .await;
 
-        let row_ids = applier_factory(
-            vec![(1, "Hello"), (3, "你好")],
-            vec![(2, vec![(false, "world")])],
-            None,
-        )
-        .await;
-        assert_eq!(row_ids, Some(rows([3])));
+        let row_ids = applier_factory(vec![(1, "Hello"), (3, "你好")], vec![], None).await;
+        assert_eq!(row_ids, Some(rows([0, 3])));
 
-        let row_ids =
-            applier_factory(vec![(2, "World")], vec![(1, vec![(false, "World")])], None).await;
+        let row_ids = applier_factory(vec![(1, "World"), (2, "World")], vec![], None).await;
         assert_eq!(row_ids, Some(rows([1, 3])));
     }
 

src/mito2/src/sst/index/indexer/finish.rs

@@ -160,6 +160,7 @@ impl Indexer {
         };
 
         let column_ids = indexer.column_ids().collect();
+        index_output.fulltext_bloom_columns = indexer.bloom_column_ids().collect();
         let err = match indexer.finish(puffin_writer).await {
             Ok((row_count, byte_count)) => {
                 self.fill_fulltext_index_output(

src/query/src/optimizer/constant_term.rs

@@ -18,6 +18,7 @@ use std::hash::{Hash, Hasher};
 use std::sync::Arc;
 
 use arrow::array::BooleanArray;
+use common_base::term_token::extract_term_probes;
 use common_function::scalars::matches_term::MatchesTermFinder;
 use datafusion::config::ConfigOptions;
 use datafusion::error::Result as DfResult;
@@ -179,12 +180,8 @@ impl PhysicalOptimizerRule for MatchesConstantTermOptimizer {
                             {
                                 let finder = MatchesTermFinder::new(term);
 
-                                // For debugging purpose. Not really precise but enough for most cases.
-                                let probes = term
-                                    .split(|c: char| !c.is_alphanumeric() && c != '_')
-                                    .filter(|s| !s.is_empty())
-                                    .map(|s| s.to_string())
-                                    .collect();
+                                // Shown in plans to explain fulltext bloom pruning.
+                                let probes = extract_term_probes(term);
 
                                 let expr = PreCompiledMatchesTermExpr {
                                     text: args[0].clone(),
@@ -495,7 +492,7 @@ mod tests {
         let plan_str = get_plan_string(&physical_plan).join("\n");
         assert!(plan_str.contains("MatchesConstTerm(text@0, term: \"foo\", probes: [\"foo\"]"));
         assert!(plan_str.contains(
-            "MatchesConstTerm(text@0, term: \"hello wo_rld\", probes: [\"hello\", \"wo_rld\"]"
+            "MatchesConstTerm(text@0, term: \"hello wo_rld\", probes: [\"hello\", \"wo\", \"rld\"]"
         ));
         assert!(plan_str.contains("MatchesConstTerm(text@0, term: \"world\", probes: [\"world\"]"));
         assert!(

tests/cases/standalone/common/function/admin/build_index_table_restart.result

@@ -147,7 +147,7 @@ WHERE table_id = (
 +------------+
 | index_size |
 +------------+
-| 318        |
+| 321        |
 +------------+
 
 -- SQLNESS ARG restart=true

tests/cases/standalone/common/function/fulltext_index_pruning.sql

@@ -0,0 +1,296 @@
+-- Fulltext index pruning must never drop rows that `matches_term` or `LIKE` accepts.
+-- Each table holds the same rows in a single flushed SST. The unfiltered
+-- projections are the full-scan answers; every filtered query below must return
+-- exactly the rows marked true there.
+
+CREATE TABLE ft_en_bloom (
+    ts TIMESTAMP TIME INDEX,
+    msg STRING FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false')
+);
+
+INSERT INTO ft_en_bloom VALUES
+    (1, 'hello_world'),
+    (2, 'trace_id=abc'),
+    (3, '错误error日志'),
+    (4, '登录手机号18888888888的动态key'),
+    (5, '中国农业银行'),
+    (6, '连接timeout.5次'),
+    (7, '用户user-123登录'),
+    (8, 'Disk QUOTA Exceeded on VOL_7');
+
+ADMIN flush_table('ft_en_bloom');
+
+SELECT ts, msg @@ 'world' AS "world", msg @@ 'id' AS "id", msg @@ 'error' AS "error", msg @@ '手机号' AS "手机号", msg @@ '手机' AS "手机", msg @@ '机号' AS "机号", msg @@ '18888888888' AS "18888888888", msg @@ '农业' AS "农业", msg @@ 'timeout' AS "timeout", msg @@ 'user' AS "user" FROM ft_en_bloom ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg @@ 'world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg @@ 'id' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg @@ 'error' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg @@ '手机号' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg @@ '手机' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg @@ '机号' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg @@ '18888888888' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg @@ '农业' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg @@ 'timeout' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg @@ 'user' ORDER BY ts;
+
+SELECT ts, msg LIKE '%world' AS "%world", msg LIKE 'hello\_world' AS "hello\_world", msg LIKE 'hello_world' AS "hello_world", msg LIKE '%id=%' AS "%id=%", msg LIKE 'trace%abc' AS "trace%abc", msg LIKE '%error%' AS "%error%", msg LIKE '%手机号1888%' AS "%手机号1888%", msg LIKE '%农业%' AS "%农业%", msg LIKE '连接timeout.%' AS "连接timeout.%", msg LIKE '%user-123%' AS "%user-123%" FROM ft_en_bloom ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE '%world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE 'hello\_world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE 'hello_world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE '%id=%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE 'trace%abc' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE '%error%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE '%手机号1888%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE '%农业%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE '连接timeout.%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE '%user-123%' ORDER BY ts;
+
+SELECT ts, matches_term(lower(msg), 'quota') AS "matches_term(lower(msg), 'quota')", matches_term(lower(msg), 'vol') AS "matches_term(lower(msg), 'vol')", lower(msg) LIKE '%disk quota exceeded%' AS "lower(msg) LIKE '%disk quota exceeded%'", lower(msg) LIKE 'disk quota%' AS "lower(msg) LIKE 'disk quota%'" FROM ft_en_bloom ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE matches_term(lower(msg), 'quota') ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE matches_term(lower(msg), 'vol') ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE lower(msg) LIKE '%disk quota exceeded%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_bloom WHERE lower(msg) LIKE 'disk quota%' ORDER BY ts;
+
+DROP TABLE ft_en_bloom;
+
+CREATE TABLE ft_zh_bloom (
+    ts TIMESTAMP TIME INDEX,
+    msg STRING FULLTEXT INDEX WITH(analyzer = 'Chinese', backend = 'bloom', case_sensitive = 'false')
+);
+
+INSERT INTO ft_zh_bloom VALUES
+    (1, 'hello_world'),
+    (2, 'trace_id=abc'),
+    (3, '错误error日志'),
+    (4, '登录手机号18888888888的动态key'),
+    (5, '中国农业银行'),
+    (6, '连接timeout.5次'),
+    (7, '用户user-123登录'),
+    (8, 'Disk QUOTA Exceeded on VOL_7');
+
+ADMIN flush_table('ft_zh_bloom');
+
+SELECT ts, msg @@ 'world' AS "world", msg @@ 'id' AS "id", msg @@ 'error' AS "error", msg @@ '手机号' AS "手机号", msg @@ '手机' AS "手机", msg @@ '机号' AS "机号", msg @@ '18888888888' AS "18888888888", msg @@ '农业' AS "农业", msg @@ 'timeout' AS "timeout", msg @@ 'user' AS "user" FROM ft_zh_bloom ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ 'world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ 'id' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ 'error' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ '手机号' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ '手机' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ '机号' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ '18888888888' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ '农业' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ 'timeout' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ 'user' ORDER BY ts;
+
+SELECT ts, msg LIKE '%world' AS "%world", msg LIKE 'hello\_world' AS "hello\_world", msg LIKE 'hello_world' AS "hello_world", msg LIKE '%id=%' AS "%id=%", msg LIKE 'trace%abc' AS "trace%abc", msg LIKE '%error%' AS "%error%", msg LIKE '%手机号1888%' AS "%手机号1888%", msg LIKE '%农业%' AS "%农业%", msg LIKE '连接timeout.%' AS "连接timeout.%", msg LIKE '%user-123%' AS "%user-123%" FROM ft_zh_bloom ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE '%world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE 'hello\_world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE 'hello_world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE '%id=%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE 'trace%abc' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE '%error%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE '%手机号1888%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE '%农业%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE '连接timeout.%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE '%user-123%' ORDER BY ts;
+
+SELECT ts, matches_term(lower(msg), 'quota') AS "matches_term(lower(msg), 'quota')", matches_term(lower(msg), 'vol') AS "matches_term(lower(msg), 'vol')", lower(msg) LIKE '%disk quota exceeded%' AS "lower(msg) LIKE '%disk quota exceeded%'", lower(msg) LIKE 'disk quota%' AS "lower(msg) LIKE 'disk quota%'" FROM ft_zh_bloom ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE matches_term(lower(msg), 'quota') ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE matches_term(lower(msg), 'vol') ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE lower(msg) LIKE '%disk quota exceeded%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_bloom WHERE lower(msg) LIKE 'disk quota%' ORDER BY ts;
+
+DROP TABLE ft_zh_bloom;
+
+CREATE TABLE ft_en_tantivy (
+    ts TIMESTAMP TIME INDEX,
+    msg STRING FULLTEXT INDEX WITH(analyzer = 'English', backend = 'tantivy', case_sensitive = 'false')
+);
+
+INSERT INTO ft_en_tantivy VALUES
+    (1, 'hello_world'),
+    (2, 'trace_id=abc'),
+    (3, '错误error日志'),
+    (4, '登录手机号18888888888的动态key'),
+    (5, '中国农业银行'),
+    (6, '连接timeout.5次'),
+    (7, '用户user-123登录'),
+    (8, 'Disk QUOTA Exceeded on VOL_7');
+
+ADMIN flush_table('ft_en_tantivy');
+
+SELECT ts, msg @@ 'world' AS "world", msg @@ 'id' AS "id", msg @@ 'error' AS "error", msg @@ '手机号' AS "手机号", msg @@ '手机' AS "手机", msg @@ '机号' AS "机号", msg @@ '18888888888' AS "18888888888", msg @@ '农业' AS "农业", msg @@ 'timeout' AS "timeout", msg @@ 'user' AS "user" FROM ft_en_tantivy ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ 'world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ 'id' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ 'error' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ '手机号' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ '手机' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ '机号' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ '18888888888' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ '农业' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ 'timeout' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ 'user' ORDER BY ts;
+
+SELECT ts, msg LIKE '%world' AS "%world", msg LIKE 'hello\_world' AS "hello\_world", msg LIKE 'hello_world' AS "hello_world", msg LIKE '%id=%' AS "%id=%", msg LIKE 'trace%abc' AS "trace%abc", msg LIKE '%error%' AS "%error%", msg LIKE '%手机号1888%' AS "%手机号1888%", msg LIKE '%农业%' AS "%农业%", msg LIKE '连接timeout.%' AS "连接timeout.%", msg LIKE '%user-123%' AS "%user-123%" FROM ft_en_tantivy ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE '%world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE 'hello\_world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE 'hello_world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE '%id=%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE 'trace%abc' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE '%error%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE '%手机号1888%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE '%农业%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE '连接timeout.%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE '%user-123%' ORDER BY ts;
+
+SELECT ts, matches_term(lower(msg), 'quota') AS "matches_term(lower(msg), 'quota')", matches_term(lower(msg), 'vol') AS "matches_term(lower(msg), 'vol')", lower(msg) LIKE '%disk quota exceeded%' AS "lower(msg) LIKE '%disk quota exceeded%'", lower(msg) LIKE 'disk quota%' AS "lower(msg) LIKE 'disk quota%'" FROM ft_en_tantivy ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE matches_term(lower(msg), 'quota') ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE matches_term(lower(msg), 'vol') ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE lower(msg) LIKE '%disk quota exceeded%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_en_tantivy WHERE lower(msg) LIKE 'disk quota%' ORDER BY ts;
+
+DROP TABLE ft_en_tantivy;
+
+CREATE TABLE ft_zh_tantivy (
+    ts TIMESTAMP TIME INDEX,
+    msg STRING FULLTEXT INDEX WITH(analyzer = 'Chinese', backend = 'tantivy', case_sensitive = 'false')
+);
+
+INSERT INTO ft_zh_tantivy VALUES
+    (1, 'hello_world'),
+    (2, 'trace_id=abc'),
+    (3, '错误error日志'),
+    (4, '登录手机号18888888888的动态key'),
+    (5, '中国农业银行'),
+    (6, '连接timeout.5次'),
+    (7, '用户user-123登录'),
+    (8, 'Disk QUOTA Exceeded on VOL_7');
+
+ADMIN flush_table('ft_zh_tantivy');
+
+SELECT ts, msg @@ 'world' AS "world", msg @@ 'id' AS "id", msg @@ 'error' AS "error", msg @@ '手机号' AS "手机号", msg @@ '手机' AS "手机", msg @@ '机号' AS "机号", msg @@ '18888888888' AS "18888888888", msg @@ '农业' AS "农业", msg @@ 'timeout' AS "timeout", msg @@ 'user' AS "user" FROM ft_zh_tantivy ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ 'world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ 'id' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ 'error' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ '手机号' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ '手机' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ '机号' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ '18888888888' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ '农业' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ 'timeout' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ 'user' ORDER BY ts;
+
+SELECT ts, msg LIKE '%world' AS "%world", msg LIKE 'hello\_world' AS "hello\_world", msg LIKE 'hello_world' AS "hello_world", msg LIKE '%id=%' AS "%id=%", msg LIKE 'trace%abc' AS "trace%abc", msg LIKE '%error%' AS "%error%", msg LIKE '%手机号1888%' AS "%手机号1888%", msg LIKE '%农业%' AS "%农业%", msg LIKE '连接timeout.%' AS "连接timeout.%", msg LIKE '%user-123%' AS "%user-123%" FROM ft_zh_tantivy ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE '%world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE 'hello\_world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE 'hello_world' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE '%id=%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE 'trace%abc' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE '%error%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE '%手机号1888%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE '%农业%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE '连接timeout.%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE '%user-123%' ORDER BY ts;
+
+SELECT ts, matches_term(lower(msg), 'quota') AS "matches_term(lower(msg), 'quota')", matches_term(lower(msg), 'vol') AS "matches_term(lower(msg), 'vol')", lower(msg) LIKE '%disk quota exceeded%' AS "lower(msg) LIKE '%disk quota exceeded%'", lower(msg) LIKE 'disk quota%' AS "lower(msg) LIKE 'disk quota%'" FROM ft_zh_tantivy ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE matches_term(lower(msg), 'quota') ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE matches_term(lower(msg), 'vol') ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE lower(msg) LIKE '%disk quota exceeded%' ORDER BY ts;
+
+SELECT ts, msg FROM ft_zh_tantivy WHERE lower(msg) LIKE 'disk quota%' ORDER BY ts;
+
+DROP TABLE ft_zh_tantivy;

tests/cases/standalone/common/information_schema/ssts.result

@@ -120,9 +120,9 @@ SELECT * FROM information_schema.ssts_index_meta ORDER BY meta_json;
 +----------------------------+---------------------------------------------------------------------------------------------+---------------+----------+---------------+--------------+-----------------+--------------------------------------+-----------------+----------------+-------------+------------+--------------+-----------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+
 | table_dir                  | index_file_path                                                                             | region_id     | table_id | region_number | region_group | region_sequence | file_id                              | index_file_size | index_type     | target_type | target_key | target_json  | blob_size | meta_json                                                                                                                                                                                                                | node_id |
 +----------------------------+---------------------------------------------------------------------------------------------+---------------+----------+---------------+--------------+-----------------+--------------------------------------+-----------------+----------------+-------------+------------+--------------+-----------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+
-| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","case_sensitive":false}}                                                                     |<NUM>|
-| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","case_sensitive":false}}                                                                     |<NUM>|
-| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","case_sensitive":false}}                                                                     |<NUM>|
+| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","bloom_version":2,"case_sensitive":false}}                                                   |<NUM>|
+| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","bloom_version":2,"case_sensitive":false}}                                                   |<NUM>|
+| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","bloom_version":2,"case_sensitive":false}}                                                   |<NUM>|
 | data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| bloom_filter   | column      |<NUM>| {"column":1} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1}}                                                                                                                              |<NUM>|
 | data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| bloom_filter   | column      |<NUM>| {"column":1} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1}}                                                                                                                              |<NUM>|
 | data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| bloom_filter   | column      |<NUM>| {"column":1} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1}}                                                                                                                              |<NUM>|
@@ -191,15 +191,15 @@ SELECT * FROM information_schema.ssts_index_meta ORDER BY meta_json;
 +----------------------------+---------------------------------------------------------------------------------------------+---------------+----------+---------------+--------------+-----------------+--------------------------------------+-----------------+----------------+-------------+------------+--------------+-----------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+
 | table_dir                  | index_file_path                                                                             | region_id     | table_id | region_number | region_group | region_sequence | file_id                              | index_file_size | index_type     | target_type | target_key | target_json  | blob_size | meta_json                                                                                                                                                                                                                 | node_id |
 +----------------------------+---------------------------------------------------------------------------------------------+---------------+----------+---------------+--------------+-----------------+--------------------------------------+-----------------+----------------+-------------+------------+--------------+-----------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+
-| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","case_sensitive":false}}                                                                      |<NUM>|
-| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","case_sensitive":false}}                                                                      |<NUM>|
-| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","case_sensitive":false}}                                                                      |<NUM>|
-| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","case_sensitive":false}}                                                                      |<NUM>|
+| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","bloom_version":2,"case_sensitive":false}}                                                    |<NUM>|
+| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","bloom_version":2,"case_sensitive":false}}                                                    |<NUM>|
+| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","bloom_version":2,"case_sensitive":false}}                                                    |<NUM>|
+| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","bloom_version":2,"case_sensitive":false}}                                                    |<NUM>|
 | data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| bloom_filter   | column      |<NUM>| {"column":1} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1}}                                                                                                                               |<NUM>|
 | data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| bloom_filter   | column      |<NUM>| {"column":1} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1}}                                                                                                                               |<NUM>|
 | data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| bloom_filter   | column      |<NUM>| {"column":1} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1}}                                                                                                                               |<NUM>|
 | data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| bloom_filter   | column      |<NUM>| {"column":1} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":1,"rows_per_segment":10240,"segment_count":1}}                                                                                                                               |<NUM>|
-| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":2,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","case_sensitive":false}}                                                                      |<NUM>|
+| data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| fulltext_bloom | column      |<NUM>| {"column":2} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":2,"rows_per_segment":10240,"segment_count":1},"fulltext":{"analyzer":"English","bloom_version":2,"case_sensitive":false}}                                                    |<NUM>|
 | data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| bloom_filter   | column      |<NUM>| {"column":1} |<NUM>| {"bloom":{"bloom_filter_size":64,"row_count":2,"rows_per_segment":10240,"segment_count":1}}                                                                                                                               |<NUM>|
 | data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| inverted       | column      |<NUM>| {"column":0} |<NUM>| {"inverted":{"base_offset":0,"bitmap_type":"Roaring","fst_size":55,"inverted_index_size":81,"null_bitmap_size":8,"relative_fst_offset":26,"relative_null_bitmap_offset":0,"segment_row_count":1024,"total_row_count":1}}  |<NUM>|
 | data/greptime/public/<TABLE_ID>/ | data/greptime/public/<TABLE_ID>/<REGION_ID>_<REGION_NUMBER>/index/<UUID>.puffin |<NUM>|<NUM>|<NUM>|<NUM>|<NUM>| <UUID> |<NUM>| inverted       | column      |<NUM>| {"column":0} |<NUM>| {"inverted":{"base_offset":0,"bitmap_type":"Roaring","fst_size":55,"inverted_index_size":81,"null_bitmap_size":8,"relative_fst_offset":26,"relative_null_bitmap_offset":0,"segment_row_count":1024,"total_row_count":1}}  |<NUM>|

tests/compatibility/cases/downgrade_compatibility/setup.sql

@@ -47,3 +47,14 @@ INSERT INTO t_sst_float_field_encoding_downgrade VALUES
 ('2024-02-10 00:02:00+0000', 'host_c', NULL, NULL);
 
 ADMIN FLUSH_TABLE('t_sst_float_field_encoding_downgrade');
+
+CREATE TABLE t_fulltext_bloom_v2_downgrade (
+    ts TIMESTAMP TIME INDEX,
+    msg STRING FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false')
+);
+
+INSERT INTO t_fulltext_bloom_v2_downgrade VALUES
+    (1, 'hello_world'),
+    (2, '错误error日志');
+
+ADMIN FLUSH_TABLE('t_fulltext_bloom_v2_downgrade');

tests/compatibility/cases/downgrade_compatibility/verify.result

@@ -29,3 +29,21 @@ ORDER BY ts, host;
 | 2024-02-10T00:01:00 | host_b | -2.5 | 20.25 |
 | 2024-02-10T00:02:00 | host_c |      |       |
 +---------------------+--------+------+-------+
+
+-- Older binaries don't recognize the bloom blob written by the current one and must
+-- scan without pruning instead of probing it with their own tokens.
+SELECT ts, msg FROM t_fulltext_bloom_v2_downgrade WHERE msg @@ 'hello_world' ORDER BY ts;
+
++-------------------------+-------------+
+| ts                      | msg         |
++-------------------------+-------------+
+| 1970-01-01T00:00:00.001 | hello_world |
++-------------------------+-------------+
+
+SELECT ts, msg FROM t_fulltext_bloom_v2_downgrade WHERE msg @@ '错误error日志' ORDER BY ts;
+
++-------------------------+---------------+
+| ts                      | msg           |
++-------------------------+---------------+
+| 1970-01-01T00:00:00.002 | 错误error日志 |
++-------------------------+---------------+

tests/compatibility/cases/downgrade_compatibility/verify.sql

@@ -5,3 +5,9 @@ SELECT ts, host, val FROM t_preserve_sequence_downgrade ORDER BY ts, host;
 SELECT ts, host, f, d
 FROM t_sst_float_field_encoding_downgrade
 ORDER BY ts, host;
+
+-- Older binaries don't recognize the bloom blob written by the current one and must
+-- scan without pruning instead of probing it with their own tokens.
+SELECT ts, msg FROM t_fulltext_bloom_v2_downgrade WHERE msg @@ 'hello_world' ORDER BY ts;
+
+SELECT ts, msg FROM t_fulltext_bloom_v2_downgrade WHERE msg @@ '错误error日志' ORDER BY ts;

tests/compatibility/cases/fulltext_bloom_tokens/case.toml

@@ -0,0 +1,8 @@
+name = "fulltext_bloom_tokens"
+reason = "Verify matches_term and LIKE return every matching row from fulltext bloom SSTs written by older binaries, whose tokens don't follow matches_term boundaries, alone and mixed with SSTs written by the current binary."
+introduced_by = "fix: fulltext bloom index drops rows matched by matches_term"
+topologies = ["distributed", "standalone"]
+from_range = [">=v1.0.0"]
+to_range = ["*"]
+features = ["table", "sst", "fulltext"]
+owner = "storage"

tests/compatibility/cases/fulltext_bloom_tokens/setup.sql

@@ -0,0 +1,28 @@
+CREATE TABLE t_fulltext_bloom_tokens (
+    ts TIMESTAMP TIME INDEX,
+    msg STRING FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false')
+);
+
+INSERT INTO t_fulltext_bloom_tokens VALUES
+    (1, 'hello_world'),
+    (2, '错误error日志'),
+    (3, '中国农业银行');
+
+ADMIN FLUSH_TABLE('t_fulltext_bloom_tokens');
+
+-- Two SSTs with old bloom blobs, left for compaction to rewrite.
+CREATE TABLE t_fulltext_bloom_compact (
+    ts TIMESTAMP TIME INDEX,
+    msg STRING FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false')
+);
+
+INSERT INTO t_fulltext_bloom_compact VALUES
+    (1, 'hello_world'),
+    (2, '错误error日志');
+
+ADMIN FLUSH_TABLE('t_fulltext_bloom_compact');
+
+INSERT INTO t_fulltext_bloom_compact VALUES
+    (3, '中国农业银行');
+
+ADMIN FLUSH_TABLE('t_fulltext_bloom_compact');

tests/compatibility/cases/fulltext_bloom_tokens/verify.result

@@ -0,0 +1,197 @@
+-- Only the SST written by the old binary.
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg @@ 'world' ORDER BY ts;
+
++-------------------------+-------------+
+| ts                      | msg         |
++-------------------------+-------------+
+| 1970-01-01T00:00:00.001 | hello_world |
++-------------------------+-------------+
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg @@ 'error' ORDER BY ts;
+
++-------------------------+---------------+
+| ts                      | msg           |
++-------------------------+---------------+
+| 1970-01-01T00:00:00.002 | 错误error日志 |
++-------------------------+---------------+
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg @@ '农业' ORDER BY ts;
+
++-------------------------+--------------+
+| ts                      | msg          |
++-------------------------+--------------+
+| 1970-01-01T00:00:00.003 | 中国农业银行 |
++-------------------------+--------------+
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg LIKE 'hello\_%' ORDER BY ts;
+
++-------------------------+-------------+
+| ts                      | msg         |
++-------------------------+-------------+
+| 1970-01-01T00:00:00.001 | hello_world |
++-------------------------+-------------+
+
+INSERT INTO t_fulltext_bloom_tokens VALUES
+    (4, 'hello_world again'),
+    (5, '农业 error');
+
+Affected Rows: 2
+
+ADMIN FLUSH_TABLE('t_fulltext_bloom_tokens');
+
++----------------------------------------------+
+| ADMIN FLUSH_TABLE('t_fulltext_bloom_tokens') |
++----------------------------------------------+
+| 0                                            |
++----------------------------------------------+
+
+-- Old and new SSTs: one bloom blob of each version.
+SELECT meta_json LIKE '%"bloom_version":2%' AS is_v2, count(*) AS ssts
+FROM information_schema.ssts_index_meta
+WHERE table_id = (SELECT table_id FROM information_schema.tables WHERE table_name = 't_fulltext_bloom_tokens')
+  AND index_type = 'fulltext_bloom'
+GROUP BY is_v2
+ORDER BY is_v2;
+
++-------+------+
+| is_v2 | ssts |
++-------+------+
+| false | 1    |
+| true  | 1    |
++-------+------+
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg @@ 'world' ORDER BY ts;
+
++-------------------------+-------------------+
+| ts                      | msg               |
++-------------------------+-------------------+
+| 1970-01-01T00:00:00.001 | hello_world       |
+| 1970-01-01T00:00:00.004 | hello_world again |
++-------------------------+-------------------+
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg @@ 'error' ORDER BY ts;
+
++-------------------------+---------------+
+| ts                      | msg           |
++-------------------------+---------------+
+| 1970-01-01T00:00:00.002 | 错误error日志 |
+| 1970-01-01T00:00:00.005 | 农业 error    |
++-------------------------+---------------+
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg @@ '农业' ORDER BY ts;
+
++-------------------------+--------------+
+| ts                      | msg          |
++-------------------------+--------------+
+| 1970-01-01T00:00:00.003 | 中国农业银行 |
+| 1970-01-01T00:00:00.005 | 农业 error   |
++-------------------------+--------------+
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg LIKE 'hello\_%' ORDER BY ts;
+
++-------------------------+-------------------+
+| ts                      | msg               |
++-------------------------+-------------------+
+| 1970-01-01T00:00:00.001 | hello_world       |
+| 1970-01-01T00:00:00.004 | hello_world again |
++-------------------------+-------------------+
+
+-- Rebuilds the old SST's index with the current bloom version.
+ADMIN BUILD_INDEX('t_fulltext_bloom_tokens');
+
++----------------------------------------------+
+| ADMIN BUILD_INDEX('t_fulltext_bloom_tokens') |
++----------------------------------------------+
+| 0                                            |
++----------------------------------------------+
+
+-- SQLNESS SLEEP 2s
+SELECT meta_json LIKE '%"bloom_version":2%' AS is_v2, count(*) AS ssts
+FROM information_schema.ssts_index_meta
+WHERE table_id = (SELECT table_id FROM information_schema.tables WHERE table_name = 't_fulltext_bloom_tokens')
+  AND index_type = 'fulltext_bloom'
+GROUP BY is_v2
+ORDER BY is_v2;
+
++-------+------+
+| is_v2 | ssts |
++-------+------+
+| true  | 2    |
++-------+------+
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg @@ 'world' ORDER BY ts;
+
++-------------------------+-------------------+
+| ts                      | msg               |
++-------------------------+-------------------+
+| 1970-01-01T00:00:00.001 | hello_world       |
+| 1970-01-01T00:00:00.004 | hello_world again |
++-------------------------+-------------------+
+
+DROP TABLE t_fulltext_bloom_tokens;
+
+Affected Rows: 0
+
+-- Two SSTs written by the old binary.
+SELECT meta_json LIKE '%"bloom_version":2%' AS is_v2, count(*) AS ssts
+FROM information_schema.ssts_index_meta
+WHERE table_id = (SELECT table_id FROM information_schema.tables WHERE table_name = 't_fulltext_bloom_compact')
+  AND index_type = 'fulltext_bloom'
+GROUP BY is_v2
+ORDER BY is_v2;
+
++-------+------+
+| is_v2 | ssts |
++-------+------+
+| false | 2    |
++-------+------+
+
+SELECT ts, msg FROM t_fulltext_bloom_compact WHERE msg @@ 'world' ORDER BY ts;
+
++-------------------------+-------------+
+| ts                      | msg         |
++-------------------------+-------------+
+| 1970-01-01T00:00:00.001 | hello_world |
++-------------------------+-------------+
+
+-- Merges both SSTs into one with a current bloom blob.
+ADMIN COMPACT_TABLE('t_fulltext_bloom_compact', 'strict_window', 'window=3600');
+
++---------------------------------------------------------------------------------+
+| ADMIN COMPACT_TABLE('t_fulltext_bloom_compact', 'strict_window', 'window=3600') |
++---------------------------------------------------------------------------------+
+| 0                                                                               |
++---------------------------------------------------------------------------------+
+
+SELECT meta_json LIKE '%"bloom_version":2%' AS is_v2, count(*) AS ssts
+FROM information_schema.ssts_index_meta
+WHERE table_id = (SELECT table_id FROM information_schema.tables WHERE table_name = 't_fulltext_bloom_compact')
+  AND index_type = 'fulltext_bloom'
+GROUP BY is_v2
+ORDER BY is_v2;
+
++-------+------+
+| is_v2 | ssts |
++-------+------+
+| true  | 1    |
++-------+------+
+
+SELECT ts, msg FROM t_fulltext_bloom_compact WHERE msg @@ 'world' ORDER BY ts;
+
++-------------------------+-------------+
+| ts                      | msg         |
++-------------------------+-------------+
+| 1970-01-01T00:00:00.001 | hello_world |
++-------------------------+-------------+
+
+SELECT ts, msg FROM t_fulltext_bloom_compact WHERE msg @@ '农业' ORDER BY ts;
+
++-------------------------+--------------+
+| ts                      | msg          |
++-------------------------+--------------+
+| 1970-01-01T00:00:00.003 | 中国农业银行 |
++-------------------------+--------------+
+
+DROP TABLE t_fulltext_bloom_compact;
+
+Affected Rows: 0

tests/compatibility/cases/fulltext_bloom_tokens/verify.sql

@@ -0,0 +1,71 @@
+-- Only the SST written by the old binary.
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg @@ 'world' ORDER BY ts;
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg @@ 'error' ORDER BY ts;
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg @@ '农业' ORDER BY ts;
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg LIKE 'hello\_%' ORDER BY ts;
+
+INSERT INTO t_fulltext_bloom_tokens VALUES
+    (4, 'hello_world again'),
+    (5, '农业 error');
+
+ADMIN FLUSH_TABLE('t_fulltext_bloom_tokens');
+
+-- Old and new SSTs: one bloom blob of each version.
+SELECT meta_json LIKE '%"bloom_version":2%' AS is_v2, count(*) AS ssts
+FROM information_schema.ssts_index_meta
+WHERE table_id = (SELECT table_id FROM information_schema.tables WHERE table_name = 't_fulltext_bloom_tokens')
+  AND index_type = 'fulltext_bloom'
+GROUP BY is_v2
+ORDER BY is_v2;
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg @@ 'world' ORDER BY ts;
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg @@ 'error' ORDER BY ts;
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg @@ '农业' ORDER BY ts;
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg LIKE 'hello\_%' ORDER BY ts;
+
+-- Rebuilds the old SST's index with the current bloom version.
+ADMIN BUILD_INDEX('t_fulltext_bloom_tokens');
+
+-- SQLNESS SLEEP 2s
+SELECT meta_json LIKE '%"bloom_version":2%' AS is_v2, count(*) AS ssts
+FROM information_schema.ssts_index_meta
+WHERE table_id = (SELECT table_id FROM information_schema.tables WHERE table_name = 't_fulltext_bloom_tokens')
+  AND index_type = 'fulltext_bloom'
+GROUP BY is_v2
+ORDER BY is_v2;
+
+SELECT ts, msg FROM t_fulltext_bloom_tokens WHERE msg @@ 'world' ORDER BY ts;
+
+DROP TABLE t_fulltext_bloom_tokens;
+
+-- Two SSTs written by the old binary.
+SELECT meta_json LIKE '%"bloom_version":2%' AS is_v2, count(*) AS ssts
+FROM information_schema.ssts_index_meta
+WHERE table_id = (SELECT table_id FROM information_schema.tables WHERE table_name = 't_fulltext_bloom_compact')
+  AND index_type = 'fulltext_bloom'
+GROUP BY is_v2
+ORDER BY is_v2;
+
+SELECT ts, msg FROM t_fulltext_bloom_compact WHERE msg @@ 'world' ORDER BY ts;
+
+-- Merges both SSTs into one with a current bloom blob.
+ADMIN COMPACT_TABLE('t_fulltext_bloom_compact', 'strict_window', 'window=3600');
+
+SELECT meta_json LIKE '%"bloom_version":2%' AS is_v2, count(*) AS ssts
+FROM information_schema.ssts_index_meta
+WHERE table_id = (SELECT table_id FROM information_schema.tables WHERE table_name = 't_fulltext_bloom_compact')
+  AND index_type = 'fulltext_bloom'
+GROUP BY is_v2
+ORDER BY is_v2;
+
+SELECT ts, msg FROM t_fulltext_bloom_compact WHERE msg @@ 'world' ORDER BY ts;
+
+SELECT ts, msg FROM t_fulltext_bloom_compact WHERE msg @@ '农业' ORDER BY ts;
+
+DROP TABLE t_fulltext_bloom_compact;