fix: rewrite match_all_raw fns to match_all (#8399)

### **User description**
This adds a sql re-writer which re-writes `match_all_raw` and
`match_all_raw_ignore_case` to just `match_all` for backward
compatibility.


___

### **PR Type**
Bug fix, Tests


___

### **Description**
- Add SQL rewriter for `match_all_raw`

- Map ignore-case variant to `match_all`

- Integrate visitor into SQL pipeline

- Add unit tests for rewrites


___

### Diagram Walkthrough


```mermaid
flowchart LR
  parser["Parse SQL statement"] --> removeDash["Remove DASHBOARD_ALL placeholders"]
  removeDash --> matchAllRaw["Rewrite match_all_raw* -> match_all"]
  matchAllRaw --> columns["Collect columns/limits"]
  columns --> matchAll["Detect match_all usage"]
  matchAll --> schema["Build used schema"]
  schema --> rest["Index/optimizations pipeline"]
```



<details> <summary><h3> File Walkthrough</h3></summary>

<table><thead><tr><th></th><th align="left">Relevant
files</th></tr></thead><tbody><tr><td><strong>Enhancement</strong></td><td><table>
<tr>
  <td>
    <details>
<summary><strong>mod.rs</strong><dd><code>Integrate match_all_raw
rewriter into SQL flow</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></summary>
<hr>

src/service/search/sql/mod.rs

<ul><li>Import <code>MatchAllRawVisitor</code><br> <li> Visit AST to
rewrite <code>match_all_raw*</code><br> <li> Renumber subsequent
processing step comments</ul>


</details>


  </td>
<td><a
href="https://github.com/openobserve/openobserve/pull/8399/files#diff-dc0ab53d0ff551cf3cd12027c73d0da5e66680c0bf2d1830778639a33bac3464">+18/-13</a>&nbsp;
</td>

</tr>

<tr>
  <td>
    <details>
<summary><strong>mod.rs</strong><dd><code>Register match_all_raw
rewriter module</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; </dd></summary>
<hr>

src/service/search/sql/rewriter/mod.rs

- Export new `match_all_raw` module


</details>


  </td>
<td><a
href="https://github.com/openobserve/openobserve/pull/8399/files#diff-5e0199e85e9e2501ae1c5038ca5afb773dac8501a42f08b360fbaf9ab77c0440">+1/-0</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>
</table></td></tr><tr><td><strong>Bug fix</strong></td><td><table>
<tr>
  <td>
    <details>
<summary><strong>match_all_raw.rs</strong><dd><code>New visitor to
rewrite match_all_raw variants</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </dd></summary>
<hr>

src/service/search/sql/rewriter/match_all_raw.rs

<ul><li>Add <code>MatchAllRawVisitor</code> implementing
<code>VisitorMut</code><br> <li> Rewrite <code>match_all_raw*</code>
function names to <code>match_all</code><br> <li> Include comprehensive
unit tests</ul>


</details>


  </td>
<td><a
href="https://github.com/openobserve/openobserve/pull/8399/files#diff-933586a29412bf099e28da7c412feeb69d8eead2d69f3ebda1ba08abb2e6a2dc">+114/-0</a>&nbsp;
</td>

</tr>
</table></td></tr></tr></tbody></table>

</details>

___

---------

Signed-off-by: Yashodhan Joshi <yjdoc2@gmail.com>
Co-authored-by: Yashodhan Joshi <yjdoc2@gmail.com>

src/service/search/sql/mod.rs

@@ -45,7 +45,7 @@ use crate::service::search::sql::{
     rewriter::{
         add_o2_id::AddO2IdVisitor, add_timestamp::AddTimestampVisitor,
         approx_percentile::ReplaceApproxPercentiletVisitor, index::IndexVisitor,
-        remove_dashboard_placeholder::RemoveDashboardAllVisitor,
+        match_all_raw::MatchAllRawVisitor, remove_dashboard_placeholder::RemoveDashboardAllVisitor,
         track_total_hits::TrackTotalHitsVisitor,
     },
     schema::{generate_schema_fields, generate_select_star_schema, has_original_column},
@@ -143,9 +143,14 @@ impl Sql {
         // 3. rewrite all filter that include DASHBOARD_ALL with true
         let mut remove_dashboard_all_visitor = RemoveDashboardAllVisitor::new();
         let _ = statement.visit(&mut remove_dashboard_all_visitor);
+
+        // 4. rewrite match_all_raw and match_all_raw_ignore_case to match_all
+        let mut match_all_raw_visitor = MatchAllRawVisitor::new();
+        let _ = statement.visit(&mut match_all_raw_visitor);
+
         //********************Change the sql end*********************************//
 
-        // 4. get column name, alias, group by, order by
+        // 5. get column name, alias, group by, order by
         let mut column_visitor = ColumnVisitor::new(&total_schemas);
         let _ = statement.visit(&mut column_visitor);
 
@@ -179,11 +184,11 @@ impl Sql {
             limit = n;
         }
 
-        // 5. get match_all() value
+        // 6. get match_all() value
         let mut match_visitor = MatchVisitor::new();
         let _ = statement.visit(&mut match_visitor);
 
-        // 6. check if have full text search filed in stream
+        // 7. check if have full text search filed in stream
         if match_visitor.has_match_all {
             if match_visitor.is_multi_stream {
                 return Err(Error::ErrorCode(ErrorCodes::SearchSQLNotValid(
@@ -205,7 +210,7 @@ impl Sql {
             }
         }
 
-        // 7. generate used schema
+        // 8. generate used schema
         let mut used_schemas = HashMap::with_capacity(total_schemas.len());
         if column_visitor.is_wildcard {
             let has_original_column = has_original_column(&column_visitor.columns);
@@ -227,11 +232,11 @@ impl Sql {
             }
         }
 
-        // 8. get partition column value
+        // 9. get partition column value
         let mut partition_column_visitor = PartitionColumnVisitor::new(&used_schemas);
         let _ = statement.visit(&mut partition_column_visitor);
 
-        // 9. pick up histogram interval
+        // 10. pick up histogram interval
         let mut histogram_interval_visitor =
             HistogramIntervalVisitor::new(Some((query.start_time, query.end_time)));
         let _ = statement.visit(&mut histogram_interval_visitor);
@@ -245,11 +250,11 @@ impl Sql {
         };
 
         //********************Change the sql start*********************************//
-        // 10. replace approx_percentile_cont to new format
+        // 11. replace approx_percentile_cont to new format
         let mut replace_approx_percentilet_visitor = ReplaceApproxPercentiletVisitor::new();
         let _ = statement.visit(&mut replace_approx_percentilet_visitor);
 
-        // 11. add _timestamp and _o2_id if need
+        // 12. add _timestamp and _o2_id if need
         if !is_complex_query(&mut statement) {
             let mut add_timestamp_visitor = AddTimestampVisitor::new();
             let _ = statement.visit(&mut add_timestamp_visitor);
@@ -259,7 +264,7 @@ impl Sql {
             }
         }
 
-        // 12. generate tantivy query
+        // 13. generate tantivy query
         // TODO: merge IndexVisitor and IndexOptimizeModeVisitor
         let mut index_condition = None;
         let mut can_optimize = false;
@@ -282,7 +287,7 @@ impl Sql {
             });
         }
 
-        // 13. check `select * from table where match_all()` optimizer
+        // 14. check `select * from table where match_all()` optimizer
         let mut index_optimize_mode = None;
         if !is_complex_query(&mut statement)
             && order_by.len() == 1
@@ -295,7 +300,7 @@ impl Sql {
             ));
         }
 
-        // 14. check other inverted index optimize modes
+        // 15. check other inverted index optimize modes
         // `select count(*) from table where match_all` -> SimpleCount
         // or `select histogram(..), count(*) from table where match_all` -> SimpleHistogram
         // or `select id, count(*) from t group by id order by cnt desc limit 10` -> SimpleTopN
@@ -326,7 +331,7 @@ impl Sql {
             }
         }
 
-        // 15. replace the Utf8 to Utf8View type
+        // 16. replace the Utf8 to Utf8View type
         let final_schemas = if cfg.common.utf8_view_enabled {
             let mut final_schemas = HashMap::with_capacity(used_schemas.len());
             for (stream, schema) in used_schemas.iter() {

src/service/search/sql/rewriter/match_all_raw.rs

@@ -0,0 +1,114 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::ops::ControlFlow;
+
+use sqlparser::ast::{Expr, Ident, ObjectName, ObjectNamePart, VisitorMut};
+
+pub struct MatchAllRawVisitor {}
+
+impl MatchAllRawVisitor {
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+impl VisitorMut for MatchAllRawVisitor {
+    type Break = ();
+
+    fn pre_visit_expr(&mut self, expr: &mut Expr) -> ControlFlow<Self::Break> {
+        if let Expr::Function(f) = expr {
+            let fname = f.name.to_string().to_lowercase();
+            // for backward compatibility, we re-write these functions as match all
+            if fname == "match_all_raw" || fname == "match_all_raw_ignore_case" {
+                f.name = ObjectName(vec![ObjectNamePart::Identifier(Ident::new("match_all"))])
+            }
+        }
+        ControlFlow::Continue(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use sqlparser::{ast::VisitMut, dialect::GenericDialect};
+
+    use super::*;
+
+    #[test]
+    fn test_match_all_raw1() {
+        let sql = "SELECT * FROM t WHERE match_all_raw('test')";
+        let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql)
+            .unwrap()
+            .pop()
+            .unwrap();
+        let mut visitor = MatchAllRawVisitor::new();
+        let _ = statement.visit(&mut visitor);
+        let expected_sql = "SELECT * FROM t WHERE match_all('test')";
+        assert_eq!(statement.to_string(), expected_sql);
+    }
+
+    #[test]
+    fn test_match_all_raw2() {
+        let sql =
+            "SELECT * FROM t WHERE match_all_raw_ignore_case('test') group by name order by name";
+        let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql)
+            .unwrap()
+            .pop()
+            .unwrap();
+        let mut visitor = MatchAllRawVisitor::new();
+        let _ = statement.visit(&mut visitor);
+        let expected_sql = "SELECT * FROM t WHERE match_all('test') GROUP BY name ORDER BY name";
+        assert_eq!(statement.to_string(), expected_sql);
+    }
+
+    #[test]
+    fn test_match_all_raw3() {
+        let sql = "SELECT t1.name, t2.name from t1 join t2 on t1.name = t2.name where match_all_raw('capture') group by t1.name, t2.name order by t1.name, t2.name";
+        let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql)
+            .unwrap()
+            .pop()
+            .unwrap();
+        let mut visitor = MatchAllRawVisitor::new();
+        let _ = statement.visit(&mut visitor);
+        let expected_sql = "SELECT t1.name, t2.name FROM t1 JOIN t2 ON t1.name = t2.name WHERE match_all('capture') GROUP BY t1.name, t2.name ORDER BY t1.name, t2.name";
+        assert_eq!(statement.to_string(), expected_sql);
+    }
+
+    #[test]
+    fn test_match_all_raw4() {
+        let sql = "SELECT match_all_raw_ignore_case('fine') from t1";
+        let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql)
+            .unwrap()
+            .pop()
+            .unwrap();
+        let mut visitor = MatchAllRawVisitor::new();
+        let _ = statement.visit(&mut visitor);
+        let expected_sql = "SELECT match_all('fine') FROM t1";
+        assert_eq!(statement.to_string(), expected_sql);
+    }
+
+    #[test]
+    fn test_match_all_raw5() {
+        let sql = "SELECT match_all_raw('fine') from t1";
+        let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql)
+            .unwrap()
+            .pop()
+            .unwrap();
+        let mut visitor = MatchAllRawVisitor::new();
+        let _ = statement.visit(&mut visitor);
+        let expected_sql = "SELECT match_all('fine') FROM t1";
+        assert_eq!(statement.to_string(), expected_sql);
+    }
+}

src/service/search/sql/rewriter/mod.rs

@@ -19,5 +19,6 @@ pub mod add_ordering_term;
 pub mod add_timestamp;
 pub mod approx_percentile;
 pub mod index;
+pub mod match_all_raw;
 pub mod remove_dashboard_placeholder;
 pub mod track_total_hits;