fix(mito2): ignore missing inverted indexes in old SSTs (#9545)

Signed-off-by: WenyXu <wenymedia@gmail.com>

src/mito2/src/sst/index/inverted_index/applier.rs

@@ -221,8 +221,9 @@ impl InvertedIndexApplier {
         let start = Instant::now();
 
         let context = SearchContext {
-            // Encountering a non-existing column indicates that it doesn't match predicates.
-            index_not_found_strategy: IndexNotFoundStrategy::ReturnEmpty,
+            // Older SSTs may lack an index enabled by ALTER, or a column with a default value.
+            // Missing indexes cannot rule out matching rows.
+            index_not_found_strategy: IndexNotFoundStrategy::Ignore,
         };
 
         let mut cache_miss = 0;

src/mito2/src/sst/index/inverted_index/creator.rs

@@ -556,6 +556,7 @@ mod tests {
     async fn build_applier_factory(
         prefix: &str,
         rows: BTreeSet<(&'static str, i32, [u64; 2])>,
+        sst_indexed_column_ids: HashSet<ColumnId>,
     ) -> impl Fn(DfExpr) -> BoxFuture<'static, Vec<usize>> {
         let (d, factory) = PuffinManagerFactory::new_for_test_async(prefix).await;
         let table_dir = "table0".to_string();
@@ -573,7 +574,7 @@ mod tests {
             intm_mgr,
             memory_threshold,
             NonZeroUsize::new(segment_row_count).unwrap(),
-            indexed_column_ids.clone(),
+            sst_indexed_column_ids,
         );
 
         for (str_tag, i32_tag, u64_field) in &rows {
@@ -624,6 +625,42 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_create_and_query_missing_index() {
+        let rows = BTreeSet::from_iter([("aaa", 1, [1, 2]), ("bbb", 2, [3, 4])]);
+        // The SST predates enabling indexes on tag_i32 and field_u64.
+        let applier_factory = build_applier_factory(
+            "test_create_and_query_missing_index_",
+            rows,
+            HashSet::from([1]),
+        )
+        .await;
+
+        let cases = [
+            (col("tag_i32").eq(lit(1)), vec![0, 1]),
+            (
+                col("tag_i32")
+                    .eq(lit(1))
+                    .and(col("field_u64").eq(lit(1u64))),
+                vec![0, 1],
+            ),
+            (
+                col("tag_str").eq(lit("aaa")).and(col("tag_i32").eq(lit(1))),
+                vec![0],
+            ),
+            (
+                col("tag_str")
+                    .eq(lit("missing"))
+                    .and(col("tag_i32").eq(lit(1))),
+                vec![],
+            ),
+        ];
+        for (expr, expected) in cases {
+            let actual = applier_factory(expr.clone()).await;
+            assert_eq!(actual, expected, "{expr}");
+        }
+    }
+
     #[tokio::test]
     async fn test_create_and_query_get_key() {
         let rows = BTreeSet::from_iter([
@@ -638,7 +675,12 @@ mod tests {
             ("abc", 3, [9, 10]),
         ]);
 
-        let applier_factory = build_applier_factory("test_create_and_query_get_key_", rows).await;
+        let applier_factory = build_applier_factory(
+            "test_create_and_query_get_key_",
+            rows,
+            HashSet::from([1, 2, 4]),
+        )
+        .await;
 
         let expr = col("tag_str").eq(lit("aaa"));
         let res = applier_factory(expr).await;
@@ -681,7 +723,12 @@ mod tests {
             ("abc", 3, [9, 10]),
         ]);
 
-        let applier_factory = build_applier_factory("test_create_and_query_range_", rows).await;
+        let applier_factory = build_applier_factory(
+            "test_create_and_query_range_",
+            rows,
+            HashSet::from([1, 2, 4]),
+        )
+        .await;
 
         let expr = col("tag_str").between(lit("aaa"), lit("aab"));
         let res = applier_factory(expr).await;
@@ -718,8 +765,12 @@ mod tests {
             ("abc", 3, [9, 10]),
         ]);
 
-        let applier_factory =
-            build_applier_factory("test_create_and_query_comparison_", rows).await;
+        let applier_factory = build_applier_factory(
+            "test_create_and_query_comparison_",
+            rows,
+            HashSet::from([1, 2, 4]),
+        )
+        .await;
 
         let expr = col("tag_str").lt(lit("aab"));
         let res = applier_factory(expr).await;
@@ -800,7 +851,12 @@ mod tests {
             ("abc", 3, [9, 10]),
         ]);
 
-        let applier_factory = build_applier_factory("test_create_and_query_regex_", rows).await;
+        let applier_factory = build_applier_factory(
+            "test_create_and_query_regex_",
+            rows,
+            HashSet::from([1, 2, 4]),
+        )
+        .await;
 
         let expr = binary_expr(col("tag_str"), Operator::RegexMatch, lit(".*"));
         let res = applier_factory(expr).await;

tests/cases/standalone/common/alter/change_col_inverted_index.result

@@ -180,3 +180,151 @@ DROP TABLE test_pk;
 
 Affected Rows: 0
 
+-- The old SST has an inverted index, but not on host.
+CREATE TABLE missing_inverted_index (
+    ts TIMESTAMP TIME INDEX,
+    indexed_col STRING INVERTED INDEX,
+    host STRING
+) WITH ('append_mode' = 'true');
+
+Affected Rows: 0
+
+INSERT INTO missing_inverted_index VALUES
+    (1, 'a', 'server1'),
+    (2, 'b', 'server2');
+
+Affected Rows: 2
+
+ADMIN FLUSH_TABLE('missing_inverted_index');
+
++---------------------------------------------+
+| ADMIN FLUSH_TABLE('missing_inverted_index') |
++---------------------------------------------+
+| 0                                           |
++---------------------------------------------+
+
+SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts;
+
++-------------------------+-------------+---------+
+| ts                      | indexed_col | host    |
++-------------------------+-------------+---------+
+| 1970-01-01T00:00:00.001 | a           | server1 |
++-------------------------+-------------+---------+
+
+ALTER TABLE missing_inverted_index MODIFY COLUMN host SET INVERTED INDEX;
+
+Affected Rows: 0
+
+SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts;
+
++-------------------------+-------------+---------+
+| ts                      | indexed_col | host    |
++-------------------------+-------------+---------+
+| 1970-01-01T00:00:00.001 | a           | server1 |
++-------------------------+-------------+---------+
+
+SELECT * FROM missing_inverted_index WHERE host = 'missing' ORDER BY ts;
+
++----+-------------+------+
+| ts | indexed_col | host |
++----+-------------+------+
++----+-------------+------+
+
+SELECT * FROM missing_inverted_index WHERE indexed_col = 'a' AND host = 'server1' ORDER BY ts;
+
++-------------------------+-------------+---------+
+| ts                      | indexed_col | host    |
++-------------------------+-------------+---------+
+| 1970-01-01T00:00:00.001 | a           | server1 |
++-------------------------+-------------+---------+
+
+SELECT * FROM missing_inverted_index WHERE indexed_col = 'b' AND host = 'server1' ORDER BY ts;
+
++----+-------------+------+
+| ts | indexed_col | host |
++----+-------------+------+
++----+-------------+------+
+
+-- New SSTs have both indexes; queries must also retain rows from the old SST.
+INSERT INTO missing_inverted_index VALUES
+    (3, 'a', 'server1'),
+    (4, 'b', 'server2');
+
+Affected Rows: 2
+
+ADMIN FLUSH_TABLE('missing_inverted_index');
+
++---------------------------------------------+
+| ADMIN FLUSH_TABLE('missing_inverted_index') |
++---------------------------------------------+
+| 0                                           |
++---------------------------------------------+
+
+SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts;
+
++-------------------------+-------------+---------+
+| ts                      | indexed_col | host    |
++-------------------------+-------------+---------+
+| 1970-01-01T00:00:00.001 | a           | server1 |
+| 1970-01-01T00:00:00.003 | a           | server1 |
++-------------------------+-------------+---------+
+
+SELECT * FROM missing_inverted_index WHERE indexed_col = 'a' AND host = 'server1' ORDER BY ts;
+
++-------------------------+-------------+---------+
+| ts                      | indexed_col | host    |
++-------------------------+-------------+---------+
+| 1970-01-01T00:00:00.001 | a           | server1 |
+| 1970-01-01T00:00:00.003 | a           | server1 |
++-------------------------+-------------+---------+
+
+-- SQLNESS ARG restart=true
+SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts;
+
++-------------------------+-------------+---------+
+| ts                      | indexed_col | host    |
++-------------------------+-------------+---------+
+| 1970-01-01T00:00:00.001 | a           | server1 |
+| 1970-01-01T00:00:00.003 | a           | server1 |
++-------------------------+-------------+---------+
+
+-- A column absent from old SSTs can still match its default value.
+ALTER TABLE missing_inverted_index ADD COLUMN source STRING DEFAULT 'legacy';
+
+Affected Rows: 0
+
+ALTER TABLE missing_inverted_index MODIFY COLUMN source SET INVERTED INDEX;
+
+Affected Rows: 0
+
+SELECT * FROM missing_inverted_index WHERE source = 'legacy' ORDER BY ts;
+
++-------------------------+-------------+---------+--------+
+| ts                      | indexed_col | host    | source |
++-------------------------+-------------+---------+--------+
+| 1970-01-01T00:00:00.001 | a           | server1 | legacy |
+| 1970-01-01T00:00:00.002 | b           | server2 | legacy |
+| 1970-01-01T00:00:00.003 | a           | server1 | legacy |
+| 1970-01-01T00:00:00.004 | b           | server2 | legacy |
++-------------------------+-------------+---------+--------+
+
+SELECT * FROM missing_inverted_index WHERE source = 'missing' ORDER BY ts;
+
++----+-------------+------+--------+
+| ts | indexed_col | host | source |
++----+-------------+------+--------+
++----+-------------+------+--------+
+
+SELECT * FROM missing_inverted_index WHERE indexed_col = 'a' AND source = 'legacy' ORDER BY ts;
+
++-------------------------+-------------+---------+--------+
+| ts                      | indexed_col | host    | source |
++-------------------------+-------------+---------+--------+
+| 1970-01-01T00:00:00.001 | a           | server1 | legacy |
+| 1970-01-01T00:00:00.003 | a           | server1 | legacy |
++-------------------------+-------------+---------+--------+
+
+DROP TABLE missing_inverted_index;
+
+Affected Rows: 0
+

tests/cases/standalone/common/alter/change_col_inverted_index.sql

@@ -52,3 +52,48 @@ ALTER TABLE test_pk MODIFY COLUMN foo SET INVERTED INDEX;
 SHOW INDEX FROM test_pk;
 
 DROP TABLE test_pk;
+
+-- The old SST has an inverted index, but not on host.
+CREATE TABLE missing_inverted_index (
+    ts TIMESTAMP TIME INDEX,
+    indexed_col STRING INVERTED INDEX,
+    host STRING
+) WITH ('append_mode' = 'true');
+
+INSERT INTO missing_inverted_index VALUES
+    (1, 'a', 'server1'),
+    (2, 'b', 'server2');
+
+ADMIN FLUSH_TABLE('missing_inverted_index');
+
+SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts;
+
+ALTER TABLE missing_inverted_index MODIFY COLUMN host SET INVERTED INDEX;
+
+SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts;
+SELECT * FROM missing_inverted_index WHERE host = 'missing' ORDER BY ts;
+SELECT * FROM missing_inverted_index WHERE indexed_col = 'a' AND host = 'server1' ORDER BY ts;
+SELECT * FROM missing_inverted_index WHERE indexed_col = 'b' AND host = 'server1' ORDER BY ts;
+
+-- New SSTs have both indexes; queries must also retain rows from the old SST.
+INSERT INTO missing_inverted_index VALUES
+    (3, 'a', 'server1'),
+    (4, 'b', 'server2');
+
+ADMIN FLUSH_TABLE('missing_inverted_index');
+
+SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts;
+SELECT * FROM missing_inverted_index WHERE indexed_col = 'a' AND host = 'server1' ORDER BY ts;
+
+-- SQLNESS ARG restart=true
+SELECT * FROM missing_inverted_index WHERE host = 'server1' ORDER BY ts;
+
+-- A column absent from old SSTs can still match its default value.
+ALTER TABLE missing_inverted_index ADD COLUMN source STRING DEFAULT 'legacy';
+ALTER TABLE missing_inverted_index MODIFY COLUMN source SET INVERTED INDEX;
+
+SELECT * FROM missing_inverted_index WHERE source = 'legacy' ORDER BY ts;
+SELECT * FROM missing_inverted_index WHERE source = 'missing' ORDER BY ts;
+SELECT * FROM missing_inverted_index WHERE indexed_col = 'a' AND source = 'legacy' ORDER BY ts;
+
+DROP TABLE missing_inverted_index;