Files
greptimedb/tests/cases/standalone/common/function/fulltext_index_pruning.sql
T
dennis zhuang 5c4d9eb4f8 fix: align fulltext bloom tokens with matches_term and use them for LIKE (#9470)
* fix: align fulltext bloom tokens with matches_term and use them for LIKE

The bloom fulltext index stored tokens that did not follow
matches_term's word boundaries (`_`, ASCII next to Han, Han substrings),
so pruning dropped rows matches_term accepts.

Tokens and query probes now share the character classes of
matches_term, Han runs are indexed as bigrams, and LIKE patterns use the
same probes. The bloom blob is written under a new key; blobs from older
versions are no longer probed. Tantivy no longer serves matches_term.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* chore: drop unused lazy_static from index and avoid typo-flagged test tokens

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* docs: link like_probes to tokenize instead of the downstream ScriptTokenizer

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* refactor: name probe extraction with verbs and document CharClass

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* feat: rebuild legacy fulltext bloom indexes and probe lower(col)

The manifest records the fulltext bloom version per column, so ADMIN
BUILD_INDEX now rebuilds SSTs whose bloom index predates the current
tokens, and ssts_index_meta shows the version.

Case-insensitive bloom indexes also store the tokens of the whole
lowercased text for rows where it can differ (U+0130, U+212A, sigma),
so matches_term(lower(col), ..) and lower(col) LIKE .. use the index.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* test: cover legacy bloom index rewrite by compaction in the compat case

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* chore: log outdated fulltext bloom indexes in the consistency check

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* docs: explain the unrecorded fulltext bloom version

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

---------

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
2026-10-14 08:07:24 +00:00

297 lines
14 KiB
SQL

-- Fulltext index pruning must never drop rows that `matches_term` or `LIKE` accepts.
-- Each table holds the same rows in a single flushed SST. The unfiltered
-- projections are the full-scan answers; every filtered query below must return
-- exactly the rows marked true there.
CREATE TABLE ft_en_bloom (
ts TIMESTAMP TIME INDEX,
msg STRING FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false')
);
INSERT INTO ft_en_bloom VALUES
(1, 'hello_world'),
(2, 'trace_id=abc'),
(3, '错误error日志'),
(4, '登录手机号18888888888的动态key'),
(5, '中国农业银行'),
(6, '连接timeout.5次'),
(7, '用户user-123登录'),
(8, 'Disk QUOTA Exceeded on VOL_7');
ADMIN flush_table('ft_en_bloom');
SELECT ts, msg @@ 'world' AS "world", msg @@ 'id' AS "id", msg @@ 'error' AS "error", msg @@ '手机号' AS "手机号", msg @@ '手机' AS "手机", msg @@ '机号' AS "机号", msg @@ '18888888888' AS "18888888888", msg @@ '农业' AS "农业", msg @@ 'timeout' AS "timeout", msg @@ 'user' AS "user" FROM ft_en_bloom ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg @@ 'world' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg @@ 'id' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg @@ 'error' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg @@ '手机号' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg @@ '手机' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg @@ '机号' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg @@ '18888888888' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg @@ '农业' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg @@ 'timeout' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg @@ 'user' ORDER BY ts;
SELECT ts, msg LIKE '%world' AS "%world", msg LIKE 'hello\_world' AS "hello\_world", msg LIKE 'hello_world' AS "hello_world", msg LIKE '%id=%' AS "%id=%", msg LIKE 'trace%abc' AS "trace%abc", msg LIKE '%error%' AS "%error%", msg LIKE '%手机号1888%' AS "%手机号1888%", msg LIKE '%农业%' AS "%农业%", msg LIKE '连接timeout.%' AS "连接timeout.%", msg LIKE '%user-123%' AS "%user-123%" FROM ft_en_bloom ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE '%world' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE 'hello\_world' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE 'hello_world' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE '%id=%' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE 'trace%abc' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE '%error%' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE '%手机号1888%' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE '%农业%' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE '连接timeout.%' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE msg LIKE '%user-123%' ORDER BY ts;
SELECT ts, matches_term(lower(msg), 'quota') AS "matches_term(lower(msg), 'quota')", matches_term(lower(msg), 'vol') AS "matches_term(lower(msg), 'vol')", lower(msg) LIKE '%disk quota exceeded%' AS "lower(msg) LIKE '%disk quota exceeded%'", lower(msg) LIKE 'disk quota%' AS "lower(msg) LIKE 'disk quota%'" FROM ft_en_bloom ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE matches_term(lower(msg), 'quota') ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE matches_term(lower(msg), 'vol') ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE lower(msg) LIKE '%disk quota exceeded%' ORDER BY ts;
SELECT ts, msg FROM ft_en_bloom WHERE lower(msg) LIKE 'disk quota%' ORDER BY ts;
DROP TABLE ft_en_bloom;
CREATE TABLE ft_zh_bloom (
ts TIMESTAMP TIME INDEX,
msg STRING FULLTEXT INDEX WITH(analyzer = 'Chinese', backend = 'bloom', case_sensitive = 'false')
);
INSERT INTO ft_zh_bloom VALUES
(1, 'hello_world'),
(2, 'trace_id=abc'),
(3, '错误error日志'),
(4, '登录手机号18888888888的动态key'),
(5, '中国农业银行'),
(6, '连接timeout.5次'),
(7, '用户user-123登录'),
(8, 'Disk QUOTA Exceeded on VOL_7');
ADMIN flush_table('ft_zh_bloom');
SELECT ts, msg @@ 'world' AS "world", msg @@ 'id' AS "id", msg @@ 'error' AS "error", msg @@ '手机号' AS "手机号", msg @@ '手机' AS "手机", msg @@ '机号' AS "机号", msg @@ '18888888888' AS "18888888888", msg @@ '农业' AS "农业", msg @@ 'timeout' AS "timeout", msg @@ 'user' AS "user" FROM ft_zh_bloom ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ 'world' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ 'id' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ 'error' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ '手机号' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ '手机' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ '机号' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ '18888888888' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ '农业' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ 'timeout' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg @@ 'user' ORDER BY ts;
SELECT ts, msg LIKE '%world' AS "%world", msg LIKE 'hello\_world' AS "hello\_world", msg LIKE 'hello_world' AS "hello_world", msg LIKE '%id=%' AS "%id=%", msg LIKE 'trace%abc' AS "trace%abc", msg LIKE '%error%' AS "%error%", msg LIKE '%手机号1888%' AS "%手机号1888%", msg LIKE '%农业%' AS "%农业%", msg LIKE '连接timeout.%' AS "连接timeout.%", msg LIKE '%user-123%' AS "%user-123%" FROM ft_zh_bloom ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE '%world' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE 'hello\_world' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE 'hello_world' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE '%id=%' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE 'trace%abc' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE '%error%' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE '%手机号1888%' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE '%农业%' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE '连接timeout.%' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE msg LIKE '%user-123%' ORDER BY ts;
SELECT ts, matches_term(lower(msg), 'quota') AS "matches_term(lower(msg), 'quota')", matches_term(lower(msg), 'vol') AS "matches_term(lower(msg), 'vol')", lower(msg) LIKE '%disk quota exceeded%' AS "lower(msg) LIKE '%disk quota exceeded%'", lower(msg) LIKE 'disk quota%' AS "lower(msg) LIKE 'disk quota%'" FROM ft_zh_bloom ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE matches_term(lower(msg), 'quota') ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE matches_term(lower(msg), 'vol') ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE lower(msg) LIKE '%disk quota exceeded%' ORDER BY ts;
SELECT ts, msg FROM ft_zh_bloom WHERE lower(msg) LIKE 'disk quota%' ORDER BY ts;
DROP TABLE ft_zh_bloom;
CREATE TABLE ft_en_tantivy (
ts TIMESTAMP TIME INDEX,
msg STRING FULLTEXT INDEX WITH(analyzer = 'English', backend = 'tantivy', case_sensitive = 'false')
);
INSERT INTO ft_en_tantivy VALUES
(1, 'hello_world'),
(2, 'trace_id=abc'),
(3, '错误error日志'),
(4, '登录手机号18888888888的动态key'),
(5, '中国农业银行'),
(6, '连接timeout.5次'),
(7, '用户user-123登录'),
(8, 'Disk QUOTA Exceeded on VOL_7');
ADMIN flush_table('ft_en_tantivy');
SELECT ts, msg @@ 'world' AS "world", msg @@ 'id' AS "id", msg @@ 'error' AS "error", msg @@ '手机号' AS "手机号", msg @@ '手机' AS "手机", msg @@ '机号' AS "机号", msg @@ '18888888888' AS "18888888888", msg @@ '农业' AS "农业", msg @@ 'timeout' AS "timeout", msg @@ 'user' AS "user" FROM ft_en_tantivy ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ 'world' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ 'id' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ 'error' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ '手机号' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ '手机' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ '机号' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ '18888888888' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ '农业' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ 'timeout' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg @@ 'user' ORDER BY ts;
SELECT ts, msg LIKE '%world' AS "%world", msg LIKE 'hello\_world' AS "hello\_world", msg LIKE 'hello_world' AS "hello_world", msg LIKE '%id=%' AS "%id=%", msg LIKE 'trace%abc' AS "trace%abc", msg LIKE '%error%' AS "%error%", msg LIKE '%手机号1888%' AS "%手机号1888%", msg LIKE '%农业%' AS "%农业%", msg LIKE '连接timeout.%' AS "连接timeout.%", msg LIKE '%user-123%' AS "%user-123%" FROM ft_en_tantivy ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE '%world' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE 'hello\_world' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE 'hello_world' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE '%id=%' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE 'trace%abc' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE '%error%' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE '%手机号1888%' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE '%农业%' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE '连接timeout.%' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE msg LIKE '%user-123%' ORDER BY ts;
SELECT ts, matches_term(lower(msg), 'quota') AS "matches_term(lower(msg), 'quota')", matches_term(lower(msg), 'vol') AS "matches_term(lower(msg), 'vol')", lower(msg) LIKE '%disk quota exceeded%' AS "lower(msg) LIKE '%disk quota exceeded%'", lower(msg) LIKE 'disk quota%' AS "lower(msg) LIKE 'disk quota%'" FROM ft_en_tantivy ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE matches_term(lower(msg), 'quota') ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE matches_term(lower(msg), 'vol') ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE lower(msg) LIKE '%disk quota exceeded%' ORDER BY ts;
SELECT ts, msg FROM ft_en_tantivy WHERE lower(msg) LIKE 'disk quota%' ORDER BY ts;
DROP TABLE ft_en_tantivy;
CREATE TABLE ft_zh_tantivy (
ts TIMESTAMP TIME INDEX,
msg STRING FULLTEXT INDEX WITH(analyzer = 'Chinese', backend = 'tantivy', case_sensitive = 'false')
);
INSERT INTO ft_zh_tantivy VALUES
(1, 'hello_world'),
(2, 'trace_id=abc'),
(3, '错误error日志'),
(4, '登录手机号18888888888的动态key'),
(5, '中国农业银行'),
(6, '连接timeout.5次'),
(7, '用户user-123登录'),
(8, 'Disk QUOTA Exceeded on VOL_7');
ADMIN flush_table('ft_zh_tantivy');
SELECT ts, msg @@ 'world' AS "world", msg @@ 'id' AS "id", msg @@ 'error' AS "error", msg @@ '手机号' AS "手机号", msg @@ '手机' AS "手机", msg @@ '机号' AS "机号", msg @@ '18888888888' AS "18888888888", msg @@ '农业' AS "农业", msg @@ 'timeout' AS "timeout", msg @@ 'user' AS "user" FROM ft_zh_tantivy ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ 'world' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ 'id' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ 'error' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ '手机号' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ '手机' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ '机号' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ '18888888888' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ '农业' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ 'timeout' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg @@ 'user' ORDER BY ts;
SELECT ts, msg LIKE '%world' AS "%world", msg LIKE 'hello\_world' AS "hello\_world", msg LIKE 'hello_world' AS "hello_world", msg LIKE '%id=%' AS "%id=%", msg LIKE 'trace%abc' AS "trace%abc", msg LIKE '%error%' AS "%error%", msg LIKE '%手机号1888%' AS "%手机号1888%", msg LIKE '%农业%' AS "%农业%", msg LIKE '连接timeout.%' AS "连接timeout.%", msg LIKE '%user-123%' AS "%user-123%" FROM ft_zh_tantivy ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE '%world' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE 'hello\_world' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE 'hello_world' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE '%id=%' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE 'trace%abc' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE '%error%' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE '%手机号1888%' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE '%农业%' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE '连接timeout.%' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE msg LIKE '%user-123%' ORDER BY ts;
SELECT ts, matches_term(lower(msg), 'quota') AS "matches_term(lower(msg), 'quota')", matches_term(lower(msg), 'vol') AS "matches_term(lower(msg), 'vol')", lower(msg) LIKE '%disk quota exceeded%' AS "lower(msg) LIKE '%disk quota exceeded%'", lower(msg) LIKE 'disk quota%' AS "lower(msg) LIKE 'disk quota%'" FROM ft_zh_tantivy ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE matches_term(lower(msg), 'quota') ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE matches_term(lower(msg), 'vol') ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE lower(msg) LIKE '%disk quota exceeded%' ORDER BY ts;
SELECT ts, msg FROM ft_zh_tantivy WHERE lower(msg) LIKE 'disk quota%' ORDER BY ts;
DROP TABLE ft_zh_tantivy;