mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-10-15 16:32:34 +00:00
* fix: align fulltext bloom tokens with matches_term and use them for LIKE The bloom fulltext index stored tokens that did not follow matches_term's word boundaries (`_`, ASCII next to Han, Han substrings), so pruning dropped rows matches_term accepts. Tokens and query probes now share the character classes of matches_term, Han runs are indexed as bigrams, and LIKE patterns use the same probes. The bloom blob is written under a new key; blobs from older versions are no longer probed. Tantivy no longer serves matches_term. Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * chore: drop unused lazy_static from index and avoid typo-flagged test tokens Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * docs: link like_probes to tokenize instead of the downstream ScriptTokenizer Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * refactor: name probe extraction with verbs and document CharClass Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * feat: rebuild legacy fulltext bloom indexes and probe lower(col) The manifest records the fulltext bloom version per column, so ADMIN BUILD_INDEX now rebuilds SSTs whose bloom index predates the current tokens, and ssts_index_meta shows the version. Case-insensitive bloom indexes also store the tokens of the whole lowercased text for rows where it can differ (U+0130, U+212A, sigma), so matches_term(lower(col), ..) and lower(col) LIKE .. use the index. Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * test: cover legacy bloom index rewrite by compaction in the compat case Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * chore: log outdated fulltext bloom indexes in the consistency check Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * docs: explain the unrecorded fulltext bloom version Signed-off-by: Dennis Zhuang <killme2008@gmail.com> --------- Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
61 lines
1.7 KiB
SQL
61 lines
1.7 KiB
SQL
CREATE TABLE t_downgrade_compatibility(
|
|
ts TIMESTAMP TIME INDEX,
|
|
host STRING PRIMARY KEY,
|
|
val INT
|
|
);
|
|
|
|
INSERT INTO t_downgrade_compatibility VALUES
|
|
('2024-02-09 00:00:00+0000', 'host_a', 1),
|
|
('2024-02-09 00:01:00+0000', 'host_b', 2);
|
|
|
|
ADMIN FLUSH_TABLE('t_downgrade_compatibility');
|
|
|
|
CREATE TABLE t_preserve_sequence_downgrade(
|
|
ts TIMESTAMP TIME INDEX,
|
|
host STRING,
|
|
val INT,
|
|
PRIMARY KEY(host)
|
|
)
|
|
ENGINE=mito
|
|
WITH(append_mode='true', preserve_row_sequence='true');
|
|
|
|
INSERT INTO t_preserve_sequence_downgrade VALUES
|
|
('2024-02-09 00:00:00+0000', 'host_a', 1),
|
|
('2024-02-09 00:01:00+0000', 'host_b', 2);
|
|
|
|
ADMIN FLUSH_TABLE('t_preserve_sequence_downgrade');
|
|
|
|
INSERT INTO t_preserve_sequence_downgrade VALUES
|
|
('2024-02-09 00:02:00+0000', 'host_a', 3),
|
|
('2024-02-09 00:03:00+0000', 'host_c', 4);
|
|
|
|
ADMIN FLUSH_TABLE('t_preserve_sequence_downgrade');
|
|
|
|
ADMIN COMPACT_TABLE('t_preserve_sequence_downgrade');
|
|
|
|
CREATE TABLE t_sst_float_field_encoding_downgrade(
|
|
ts TIMESTAMP TIME INDEX,
|
|
host STRING PRIMARY KEY,
|
|
f FLOAT,
|
|
d DOUBLE
|
|
) ENGINE=mito
|
|
WITH('experimental_sst_float_field_encoding'='byte_stream_split');
|
|
|
|
INSERT INTO t_sst_float_field_encoding_downgrade VALUES
|
|
('2024-02-10 00:00:00+0000', 'host_a', 1.25, 10.5),
|
|
('2024-02-10 00:01:00+0000', 'host_b', -2.5, 20.25),
|
|
('2024-02-10 00:02:00+0000', 'host_c', NULL, NULL);
|
|
|
|
ADMIN FLUSH_TABLE('t_sst_float_field_encoding_downgrade');
|
|
|
|
CREATE TABLE t_fulltext_bloom_v2_downgrade (
|
|
ts TIMESTAMP TIME INDEX,
|
|
msg STRING FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false')
|
|
);
|
|
|
|
INSERT INTO t_fulltext_bloom_v2_downgrade VALUES
|
|
(1, 'hello_world'),
|
|
(2, '错误error日志');
|
|
|
|
ADMIN FLUSH_TABLE('t_fulltext_bloom_v2_downgrade');
|