Files
greptimedb/tests/compatibility/cases/downgrade_compatibility/setup.sql
T
dennis zhuang 5c4d9eb4f8 fix: align fulltext bloom tokens with matches_term and use them for LIKE (#9470)
* fix: align fulltext bloom tokens with matches_term and use them for LIKE

The bloom fulltext index stored tokens that did not follow
matches_term's word boundaries (`_`, ASCII next to Han, Han substrings),
so pruning dropped rows matches_term accepts.

Tokens and query probes now share the character classes of
matches_term, Han runs are indexed as bigrams, and LIKE patterns use the
same probes. The bloom blob is written under a new key; blobs from older
versions are no longer probed. Tantivy no longer serves matches_term.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* chore: drop unused lazy_static from index and avoid typo-flagged test tokens

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* docs: link like_probes to tokenize instead of the downstream ScriptTokenizer

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* refactor: name probe extraction with verbs and document CharClass

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* feat: rebuild legacy fulltext bloom indexes and probe lower(col)

The manifest records the fulltext bloom version per column, so ADMIN
BUILD_INDEX now rebuilds SSTs whose bloom index predates the current
tokens, and ssts_index_meta shows the version.

Case-insensitive bloom indexes also store the tokens of the whole
lowercased text for rows where it can differ (U+0130, U+212A, sigma),
so matches_term(lower(col), ..) and lower(col) LIKE .. use the index.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* test: cover legacy bloom index rewrite by compaction in the compat case

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* chore: log outdated fulltext bloom indexes in the consistency check

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* docs: explain the unrecorded fulltext bloom version

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

---------

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
2026-10-14 08:07:24 +00:00

61 lines
1.7 KiB
SQL

CREATE TABLE t_downgrade_compatibility(
ts TIMESTAMP TIME INDEX,
host STRING PRIMARY KEY,
val INT
);
INSERT INTO t_downgrade_compatibility VALUES
('2024-02-09 00:00:00+0000', 'host_a', 1),
('2024-02-09 00:01:00+0000', 'host_b', 2);
ADMIN FLUSH_TABLE('t_downgrade_compatibility');
CREATE TABLE t_preserve_sequence_downgrade(
ts TIMESTAMP TIME INDEX,
host STRING,
val INT,
PRIMARY KEY(host)
)
ENGINE=mito
WITH(append_mode='true', preserve_row_sequence='true');
INSERT INTO t_preserve_sequence_downgrade VALUES
('2024-02-09 00:00:00+0000', 'host_a', 1),
('2024-02-09 00:01:00+0000', 'host_b', 2);
ADMIN FLUSH_TABLE('t_preserve_sequence_downgrade');
INSERT INTO t_preserve_sequence_downgrade VALUES
('2024-02-09 00:02:00+0000', 'host_a', 3),
('2024-02-09 00:03:00+0000', 'host_c', 4);
ADMIN FLUSH_TABLE('t_preserve_sequence_downgrade');
ADMIN COMPACT_TABLE('t_preserve_sequence_downgrade');
CREATE TABLE t_sst_float_field_encoding_downgrade(
ts TIMESTAMP TIME INDEX,
host STRING PRIMARY KEY,
f FLOAT,
d DOUBLE
) ENGINE=mito
WITH('experimental_sst_float_field_encoding'='byte_stream_split');
INSERT INTO t_sst_float_field_encoding_downgrade VALUES
('2024-02-10 00:00:00+0000', 'host_a', 1.25, 10.5),
('2024-02-10 00:01:00+0000', 'host_b', -2.5, 20.25),
('2024-02-10 00:02:00+0000', 'host_c', NULL, NULL);
ADMIN FLUSH_TABLE('t_sst_float_field_encoding_downgrade');
CREATE TABLE t_fulltext_bloom_v2_downgrade (
ts TIMESTAMP TIME INDEX,
msg STRING FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false')
);
INSERT INTO t_fulltext_bloom_v2_downgrade VALUES
(1, 'hello_world'),
(2, '错误error日志');
ADMIN FLUSH_TABLE('t_fulltext_bloom_v2_downgrade');