From 2bfde6bc0b265147bd0aeb77285dfe7f0589b574 Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Fri, 12 Sep 2025 13:09:58 +0530 Subject: [PATCH] fix: rewrite match_all_raw fns to match_all (#8399) ### **User description** This adds a sql re-writer which re-writes `match_all_raw` and `match_all_raw_ignore_case` to just `match_all` for backward compatibility. ___ ### **PR Type** Bug fix, Tests ___ ### **Description** - Add SQL rewriter for `match_all_raw` - Map ignore-case variant to `match_all` - Integrate visitor into SQL pipeline - Add unit tests for rewrites ___ ### Diagram Walkthrough ```mermaid flowchart LR parser["Parse SQL statement"] --> removeDash["Remove DASHBOARD_ALL placeholders"] removeDash --> matchAllRaw["Rewrite match_all_raw* -> match_all"] matchAllRaw --> columns["Collect columns/limits"] columns --> matchAll["Detect match_all usage"] matchAll --> schema["Build used schema"] schema --> rest["Index/optimizations pipeline"] ```

File Walkthrough
Relevant files
Enhancement
mod.rs
Integrate match_all_raw rewriter into SQL flow                     
src/service/search/sql/mod.rs
  • Import MatchAllRawVisitor
  • Visit AST to rewrite match_all_raw*
  • Renumber subsequent processing step comments
 +18/-13 
mod.rs
Register match_all_raw rewriter module                                     
src/service/search/sql/rewriter/mod.rs - Export new `match_all_raw` module
+1/-0     
Bug fix
match_all_raw.rs
New visitor to rewrite match_all_raw variants                       
src/service/search/sql/rewriter/match_all_raw.rs
  • Add MatchAllRawVisitor implementing VisitorMut
  • Rewrite match_all_raw* function names to match_all
  • Include comprehensive unit tests
 +114/-0 
___ --------- Signed-off-by: Yashodhan Joshi Co-authored-by: Yashodhan Joshi --- src/service/search/sql/mod.rs | 31 +++-- .../search/sql/rewriter/match_all_raw.rs | 114 ++++++++++++++++++ src/service/search/sql/rewriter/mod.rs | 1 + 3 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 src/service/search/sql/rewriter/match_all_raw.rs diff --git a/src/service/search/sql/mod.rs b/src/service/search/sql/mod.rs index 496ff18bdd..570c58b58b 100644 --- a/src/service/search/sql/mod.rs +++ b/src/service/search/sql/mod.rs @@ -45,7 +45,7 @@ use crate::service::search::sql::{ rewriter::{ add_o2_id::AddO2IdVisitor, add_timestamp::AddTimestampVisitor, approx_percentile::ReplaceApproxPercentiletVisitor, index::IndexVisitor, - remove_dashboard_placeholder::RemoveDashboardAllVisitor, + match_all_raw::MatchAllRawVisitor, remove_dashboard_placeholder::RemoveDashboardAllVisitor, track_total_hits::TrackTotalHitsVisitor, }, schema::{generate_schema_fields, generate_select_star_schema, has_original_column}, @@ -143,9 +143,14 @@ impl Sql { // 3. rewrite all filter that include DASHBOARD_ALL with true let mut remove_dashboard_all_visitor = RemoveDashboardAllVisitor::new(); let _ = statement.visit(&mut remove_dashboard_all_visitor); + + // 4. rewrite match_all_raw and match_all_raw_ignore_case to match_all + let mut match_all_raw_visitor = MatchAllRawVisitor::new(); + let _ = statement.visit(&mut match_all_raw_visitor); + //********************Change the sql end*********************************// - // 4. get column name, alias, group by, order by + // 5. get column name, alias, group by, order by let mut column_visitor = ColumnVisitor::new(&total_schemas); let _ = statement.visit(&mut column_visitor); @@ -179,11 +184,11 @@ impl Sql { limit = n; } - // 5. get match_all() value + // 6. get match_all() value let mut match_visitor = MatchVisitor::new(); let _ = statement.visit(&mut match_visitor); - // 6. check if have full text search filed in stream + // 7. check if have full text search filed in stream if match_visitor.has_match_all { if match_visitor.is_multi_stream { return Err(Error::ErrorCode(ErrorCodes::SearchSQLNotValid( @@ -205,7 +210,7 @@ impl Sql { } } - // 7. generate used schema + // 8. generate used schema let mut used_schemas = HashMap::with_capacity(total_schemas.len()); if column_visitor.is_wildcard { let has_original_column = has_original_column(&column_visitor.columns); @@ -227,11 +232,11 @@ impl Sql { } } - // 8. get partition column value + // 9. get partition column value let mut partition_column_visitor = PartitionColumnVisitor::new(&used_schemas); let _ = statement.visit(&mut partition_column_visitor); - // 9. pick up histogram interval + // 10. pick up histogram interval let mut histogram_interval_visitor = HistogramIntervalVisitor::new(Some((query.start_time, query.end_time))); let _ = statement.visit(&mut histogram_interval_visitor); @@ -245,11 +250,11 @@ impl Sql { }; //********************Change the sql start*********************************// - // 10. replace approx_percentile_cont to new format + // 11. replace approx_percentile_cont to new format let mut replace_approx_percentilet_visitor = ReplaceApproxPercentiletVisitor::new(); let _ = statement.visit(&mut replace_approx_percentilet_visitor); - // 11. add _timestamp and _o2_id if need + // 12. add _timestamp and _o2_id if need if !is_complex_query(&mut statement) { let mut add_timestamp_visitor = AddTimestampVisitor::new(); let _ = statement.visit(&mut add_timestamp_visitor); @@ -259,7 +264,7 @@ impl Sql { } } - // 12. generate tantivy query + // 13. generate tantivy query // TODO: merge IndexVisitor and IndexOptimizeModeVisitor let mut index_condition = None; let mut can_optimize = false; @@ -282,7 +287,7 @@ impl Sql { }); } - // 13. check `select * from table where match_all()` optimizer + // 14. check `select * from table where match_all()` optimizer let mut index_optimize_mode = None; if !is_complex_query(&mut statement) && order_by.len() == 1 @@ -295,7 +300,7 @@ impl Sql { )); } - // 14. check other inverted index optimize modes + // 15. check other inverted index optimize modes // `select count(*) from table where match_all` -> SimpleCount // or `select histogram(..), count(*) from table where match_all` -> SimpleHistogram // or `select id, count(*) from t group by id order by cnt desc limit 10` -> SimpleTopN @@ -326,7 +331,7 @@ impl Sql { } } - // 15. replace the Utf8 to Utf8View type + // 16. replace the Utf8 to Utf8View type let final_schemas = if cfg.common.utf8_view_enabled { let mut final_schemas = HashMap::with_capacity(used_schemas.len()); for (stream, schema) in used_schemas.iter() { diff --git a/src/service/search/sql/rewriter/match_all_raw.rs b/src/service/search/sql/rewriter/match_all_raw.rs new file mode 100644 index 0000000000..d53d7f5c87 --- /dev/null +++ b/src/service/search/sql/rewriter/match_all_raw.rs @@ -0,0 +1,114 @@ +// Copyright 2025 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::ops::ControlFlow; + +use sqlparser::ast::{Expr, Ident, ObjectName, ObjectNamePart, VisitorMut}; + +pub struct MatchAllRawVisitor {} + +impl MatchAllRawVisitor { + pub fn new() -> Self { + Self {} + } +} + +impl VisitorMut for MatchAllRawVisitor { + type Break = (); + + fn pre_visit_expr(&mut self, expr: &mut Expr) -> ControlFlow { + if let Expr::Function(f) = expr { + let fname = f.name.to_string().to_lowercase(); + // for backward compatibility, we re-write these functions as match all + if fname == "match_all_raw" || fname == "match_all_raw_ignore_case" { + f.name = ObjectName(vec![ObjectNamePart::Identifier(Ident::new("match_all"))]) + } + } + ControlFlow::Continue(()) + } +} + +#[cfg(test)] +mod tests { + use sqlparser::{ast::VisitMut, dialect::GenericDialect}; + + use super::*; + + #[test] + fn test_match_all_raw1() { + let sql = "SELECT * FROM t WHERE match_all_raw('test')"; + let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql) + .unwrap() + .pop() + .unwrap(); + let mut visitor = MatchAllRawVisitor::new(); + let _ = statement.visit(&mut visitor); + let expected_sql = "SELECT * FROM t WHERE match_all('test')"; + assert_eq!(statement.to_string(), expected_sql); + } + + #[test] + fn test_match_all_raw2() { + let sql = + "SELECT * FROM t WHERE match_all_raw_ignore_case('test') group by name order by name"; + let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql) + .unwrap() + .pop() + .unwrap(); + let mut visitor = MatchAllRawVisitor::new(); + let _ = statement.visit(&mut visitor); + let expected_sql = "SELECT * FROM t WHERE match_all('test') GROUP BY name ORDER BY name"; + assert_eq!(statement.to_string(), expected_sql); + } + + #[test] + fn test_match_all_raw3() { + let sql = "SELECT t1.name, t2.name from t1 join t2 on t1.name = t2.name where match_all_raw('capture') group by t1.name, t2.name order by t1.name, t2.name"; + let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql) + .unwrap() + .pop() + .unwrap(); + let mut visitor = MatchAllRawVisitor::new(); + let _ = statement.visit(&mut visitor); + let expected_sql = "SELECT t1.name, t2.name FROM t1 JOIN t2 ON t1.name = t2.name WHERE match_all('capture') GROUP BY t1.name, t2.name ORDER BY t1.name, t2.name"; + assert_eq!(statement.to_string(), expected_sql); + } + + #[test] + fn test_match_all_raw4() { + let sql = "SELECT match_all_raw_ignore_case('fine') from t1"; + let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql) + .unwrap() + .pop() + .unwrap(); + let mut visitor = MatchAllRawVisitor::new(); + let _ = statement.visit(&mut visitor); + let expected_sql = "SELECT match_all('fine') FROM t1"; + assert_eq!(statement.to_string(), expected_sql); + } + + #[test] + fn test_match_all_raw5() { + let sql = "SELECT match_all_raw('fine') from t1"; + let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql) + .unwrap() + .pop() + .unwrap(); + let mut visitor = MatchAllRawVisitor::new(); + let _ = statement.visit(&mut visitor); + let expected_sql = "SELECT match_all('fine') FROM t1"; + assert_eq!(statement.to_string(), expected_sql); + } +} diff --git a/src/service/search/sql/rewriter/mod.rs b/src/service/search/sql/rewriter/mod.rs index 47f9ce221c..16f085b92e 100644 --- a/src/service/search/sql/rewriter/mod.rs +++ b/src/service/search/sql/rewriter/mod.rs @@ -19,5 +19,6 @@ pub mod add_ordering_term; pub mod add_timestamp; pub mod approx_percentile; pub mod index; +pub mod match_all_raw; pub mod remove_dashboard_placeholder; pub mod track_total_hits;