diff --git a/src/mito2/src/sst/index/inverted_index/applier.rs b/src/mito2/src/sst/index/inverted_index/applier.rs index 0f665eec549..8f0c309a003 100644 --- a/src/mito2/src/sst/index/inverted_index/applier.rs +++ b/src/mito2/src/sst/index/inverted_index/applier.rs @@ -221,8 +221,9 @@ impl InvertedIndexApplier { let start = Instant::now(); let context = SearchContext { - // Encountering a non-existing column indicates that it doesn't match predicates. - index_not_found_strategy: IndexNotFoundStrategy::ReturnEmpty, + // Older SSTs may lack an index enabled by ALTER, or a column with a default value. + // Missing indexes cannot rule out matching rows. + index_not_found_strategy: IndexNotFoundStrategy::Ignore, }; let mut cache_miss = 0; diff --git a/src/mito2/src/sst/index/inverted_index/creator.rs b/src/mito2/src/sst/index/inverted_index/creator.rs index d1dee226515..9c1ba5cbca0 100644 --- a/src/mito2/src/sst/index/inverted_index/creator.rs +++ b/src/mito2/src/sst/index/inverted_index/creator.rs @@ -556,6 +556,7 @@ mod tests { async fn build_applier_factory( prefix: &str, rows: BTreeSet<(&'static str, i32, [u64; 2])>, + sst_indexed_column_ids: HashSet, ) -> impl Fn(DfExpr) -> BoxFuture<'static, Vec> { let (d, factory) = PuffinManagerFactory::new_for_test_async(prefix).await; let table_dir = "table0".to_string(); @@ -573,7 +574,7 @@ mod tests { intm_mgr, memory_threshold, NonZeroUsize::new(segment_row_count).unwrap(), - indexed_column_ids.clone(), + sst_indexed_column_ids, ); for (str_tag, i32_tag, u64_field) in &rows { @@ -624,6 +625,42 @@ mod tests { } } + #[tokio::test] + async fn test_create_and_query_missing_index() { + let rows = BTreeSet::from_iter([("aaa", 1, [1, 2]), ("bbb", 2, [3, 4])]); + // The SST predates enabling indexes on tag_i32 and field_u64. + let applier_factory = build_applier_factory( + "test_create_and_query_missing_index_", + rows, + HashSet::from([1]), + ) + .await; + + let cases = [ + (col("tag_i32").eq(lit(1)), vec![0, 1]), + ( + col("tag_i32") + .eq(lit(1)) + .and(col("field_u64").eq(lit(1u64))), + vec![0, 1], + ), + ( + col("tag_str").eq(lit("aaa")).and(col("tag_i32").eq(lit(1))), + vec![0], + ), + ( + col("tag_str") + .eq(lit("missing")) + .and(col("tag_i32").eq(lit(1))), + vec![], + ), + ]; + for (expr, expected) in cases { + let actual = applier_factory(expr.clone()).await; + assert_eq!(actual, expected, "{expr}"); + } + } + #[tokio::test] async fn test_create_and_query_get_key() { let rows = BTreeSet::from_iter([ @@ -638,7 +675,12 @@ mod tests { ("abc", 3, [9, 10]), ]); - let applier_factory = build_applier_factory("test_create_and_query_get_key_", rows).await; + let applier_factory = build_applier_factory( + "test_create_and_query_get_key_", + rows, + HashSet::from([1, 2, 4]), + ) + .await; let expr = col("tag_str").eq(lit("aaa")); let res = applier_factory(expr).await; @@ -681,7 +723,12 @@ mod tests { ("abc", 3, [9, 10]), ]); - let applier_factory = build_applier_factory("test_create_and_query_range_", rows).await; + let applier_factory = build_applier_factory( + "test_create_and_query_range_", + rows, + HashSet::from([1, 2, 4]), + ) + .await; let expr = col("tag_str").between(lit("aaa"), lit("aab")); let res = applier_factory(expr).await; @@ -718,8 +765,12 @@ mod tests { ("abc", 3, [9, 10]), ]); - let applier_factory = - build_applier_factory("test_create_and_query_comparison_", rows).await; + let applier_factory = build_applier_factory( + "test_create_and_query_comparison_", + rows, + HashSet::from([1, 2, 4]), + ) + .await; let expr = col("tag_str").lt(lit("aab")); let res = applier_factory(expr).await; @@ -800,7 +851,12 @@ mod tests { ("abc", 3, [9, 10]), ]); - let applier_factory = build_applier_factory("test_create_and_query_regex_", rows).await; + let applier_factory = build_applier_factory( + "test_create_and_query_regex_", + rows, + HashSet::from([1, 2, 4]), + ) + .await; let expr = binary_expr(col("tag_str"), Operator::RegexMatch, lit(".*")); let res = applier_factory(expr).await; diff --git a/tests/cases/standalone/common/alter/change_col_inverted_index.result b/tests/cases/standalone/common/alter/change_col_inverted_index.result index df684fec436..c1265f2958e 100644 --- a/tests/cases/standalone/common/alter/change_col_inverted_index.result +++ b/tests/cases/standalone/common/alter/change_col_inverted_index.result @@ -180,3 +180,151 @@ DROP TABLE test_pk; Affected Rows: 0 +-- The old SST has an inverted index, but not on host. +CREATE TABLE missing_inverted_index ( + ts TIMESTAMP TIME INDEX, + indexed_col STRING INVERTED INDEX, + host STRING +) WITH ('append_mode' = 'true'); + +Affected Rows: 0 + +INSERT INTO missing_inverted_index VALUES + (1, 'a', 'server1'), + (2, 'b', 'server2'); + +Affected Rows: 2 + +ADMIN FLUSH_TABLE('missing_inverted_index'); + ++---------------------------------------------+ +| ADMIN FLUSH_TABLE('missing_inverted_index') | ++---------------------------------------------+ +| 0 | ++---------------------------------------------+ + +SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts; + ++-------------------------+-------------+---------+ +| ts | indexed_col | host | ++-------------------------+-------------+---------+ +| 1970-01-01T00:00:00.001 | a | server1 | ++-------------------------+-------------+---------+ + +ALTER TABLE missing_inverted_index MODIFY COLUMN host SET INVERTED INDEX; + +Affected Rows: 0 + +SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts; + ++-------------------------+-------------+---------+ +| ts | indexed_col | host | ++-------------------------+-------------+---------+ +| 1970-01-01T00:00:00.001 | a | server1 | ++-------------------------+-------------+---------+ + +SELECT * FROM missing_inverted_index WHERE host = 'missing' ORDER BY ts; + ++----+-------------+------+ +| ts | indexed_col | host | ++----+-------------+------+ ++----+-------------+------+ + +SELECT * FROM missing_inverted_index WHERE indexed_col = 'a' AND host = 'server1' ORDER BY ts; + ++-------------------------+-------------+---------+ +| ts | indexed_col | host | ++-------------------------+-------------+---------+ +| 1970-01-01T00:00:00.001 | a | server1 | ++-------------------------+-------------+---------+ + +SELECT * FROM missing_inverted_index WHERE indexed_col = 'b' AND host = 'server1' ORDER BY ts; + ++----+-------------+------+ +| ts | indexed_col | host | ++----+-------------+------+ ++----+-------------+------+ + +-- New SSTs have both indexes; queries must also retain rows from the old SST. +INSERT INTO missing_inverted_index VALUES + (3, 'a', 'server1'), + (4, 'b', 'server2'); + +Affected Rows: 2 + +ADMIN FLUSH_TABLE('missing_inverted_index'); + ++---------------------------------------------+ +| ADMIN FLUSH_TABLE('missing_inverted_index') | ++---------------------------------------------+ +| 0 | ++---------------------------------------------+ + +SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts; + ++-------------------------+-------------+---------+ +| ts | indexed_col | host | ++-------------------------+-------------+---------+ +| 1970-01-01T00:00:00.001 | a | server1 | +| 1970-01-01T00:00:00.003 | a | server1 | ++-------------------------+-------------+---------+ + +SELECT * FROM missing_inverted_index WHERE indexed_col = 'a' AND host = 'server1' ORDER BY ts; + ++-------------------------+-------------+---------+ +| ts | indexed_col | host | ++-------------------------+-------------+---------+ +| 1970-01-01T00:00:00.001 | a | server1 | +| 1970-01-01T00:00:00.003 | a | server1 | ++-------------------------+-------------+---------+ + +-- SQLNESS ARG restart=true +SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts; + ++-------------------------+-------------+---------+ +| ts | indexed_col | host | ++-------------------------+-------------+---------+ +| 1970-01-01T00:00:00.001 | a | server1 | +| 1970-01-01T00:00:00.003 | a | server1 | ++-------------------------+-------------+---------+ + +-- A column absent from old SSTs can still match its default value. +ALTER TABLE missing_inverted_index ADD COLUMN source STRING DEFAULT 'legacy'; + +Affected Rows: 0 + +ALTER TABLE missing_inverted_index MODIFY COLUMN source SET INVERTED INDEX; + +Affected Rows: 0 + +SELECT * FROM missing_inverted_index WHERE source = 'legacy' ORDER BY ts; + ++-------------------------+-------------+---------+--------+ +| ts | indexed_col | host | source | ++-------------------------+-------------+---------+--------+ +| 1970-01-01T00:00:00.001 | a | server1 | legacy | +| 1970-01-01T00:00:00.002 | b | server2 | legacy | +| 1970-01-01T00:00:00.003 | a | server1 | legacy | +| 1970-01-01T00:00:00.004 | b | server2 | legacy | ++-------------------------+-------------+---------+--------+ + +SELECT * FROM missing_inverted_index WHERE source = 'missing' ORDER BY ts; + ++----+-------------+------+--------+ +| ts | indexed_col | host | source | ++----+-------------+------+--------+ ++----+-------------+------+--------+ + +SELECT * FROM missing_inverted_index WHERE indexed_col = 'a' AND source = 'legacy' ORDER BY ts; + ++-------------------------+-------------+---------+--------+ +| ts | indexed_col | host | source | ++-------------------------+-------------+---------+--------+ +| 1970-01-01T00:00:00.001 | a | server1 | legacy | +| 1970-01-01T00:00:00.003 | a | server1 | legacy | ++-------------------------+-------------+---------+--------+ + +DROP TABLE missing_inverted_index; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/alter/change_col_inverted_index.sql b/tests/cases/standalone/common/alter/change_col_inverted_index.sql index aadc463d6fc..857bbb0aaf2 100644 --- a/tests/cases/standalone/common/alter/change_col_inverted_index.sql +++ b/tests/cases/standalone/common/alter/change_col_inverted_index.sql @@ -52,3 +52,48 @@ ALTER TABLE test_pk MODIFY COLUMN foo SET INVERTED INDEX; SHOW INDEX FROM test_pk; DROP TABLE test_pk; + +-- The old SST has an inverted index, but not on host. +CREATE TABLE missing_inverted_index ( + ts TIMESTAMP TIME INDEX, + indexed_col STRING INVERTED INDEX, + host STRING +) WITH ('append_mode' = 'true'); + +INSERT INTO missing_inverted_index VALUES + (1, 'a', 'server1'), + (2, 'b', 'server2'); + +ADMIN FLUSH_TABLE('missing_inverted_index'); + +SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts; + +ALTER TABLE missing_inverted_index MODIFY COLUMN host SET INVERTED INDEX; + +SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts; +SELECT * FROM missing_inverted_index WHERE host = 'missing' ORDER BY ts; +SELECT * FROM missing_inverted_index WHERE indexed_col = 'a' AND host = 'server1' ORDER BY ts; +SELECT * FROM missing_inverted_index WHERE indexed_col = 'b' AND host = 'server1' ORDER BY ts; + +-- New SSTs have both indexes; queries must also retain rows from the old SST. +INSERT INTO missing_inverted_index VALUES + (3, 'a', 'server1'), + (4, 'b', 'server2'); + +ADMIN FLUSH_TABLE('missing_inverted_index'); + +SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts; +SELECT * FROM missing_inverted_index WHERE indexed_col = 'a' AND host = 'server1' ORDER BY ts; + +-- SQLNESS ARG restart=true +SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts; + +-- A column absent from old SSTs can still match its default value. +ALTER TABLE missing_inverted_index ADD COLUMN source STRING DEFAULT 'legacy'; +ALTER TABLE missing_inverted_index MODIFY COLUMN source SET INVERTED INDEX; + +SELECT * FROM missing_inverted_index WHERE source = 'legacy' ORDER BY ts; +SELECT * FROM missing_inverted_index WHERE source = 'missing' ORDER BY ts; +SELECT * FROM missing_inverted_index WHERE indexed_col = 'a' AND source = 'legacy' ORDER BY ts; + +DROP TABLE missing_inverted_index;