diff --git a/src/service/search/sql/mod.rs b/src/service/search/sql/mod.rs
index 496ff18bdd..570c58b58b 100644
--- a/src/service/search/sql/mod.rs
+++ b/src/service/search/sql/mod.rs
@@ -45,7 +45,7 @@ use crate::service::search::sql::{
rewriter::{
add_o2_id::AddO2IdVisitor, add_timestamp::AddTimestampVisitor,
approx_percentile::ReplaceApproxPercentiletVisitor, index::IndexVisitor,
- remove_dashboard_placeholder::RemoveDashboardAllVisitor,
+ match_all_raw::MatchAllRawVisitor, remove_dashboard_placeholder::RemoveDashboardAllVisitor,
track_total_hits::TrackTotalHitsVisitor,
},
schema::{generate_schema_fields, generate_select_star_schema, has_original_column},
@@ -143,9 +143,14 @@ impl Sql {
// 3. rewrite all filter that include DASHBOARD_ALL with true
let mut remove_dashboard_all_visitor = RemoveDashboardAllVisitor::new();
let _ = statement.visit(&mut remove_dashboard_all_visitor);
+
+ // 4. rewrite match_all_raw and match_all_raw_ignore_case to match_all
+ let mut match_all_raw_visitor = MatchAllRawVisitor::new();
+ let _ = statement.visit(&mut match_all_raw_visitor);
+
//********************Change the sql end*********************************//
- // 4. get column name, alias, group by, order by
+ // 5. get column name, alias, group by, order by
let mut column_visitor = ColumnVisitor::new(&total_schemas);
let _ = statement.visit(&mut column_visitor);
@@ -179,11 +184,11 @@ impl Sql {
limit = n;
}
- // 5. get match_all() value
+ // 6. get match_all() value
let mut match_visitor = MatchVisitor::new();
let _ = statement.visit(&mut match_visitor);
- // 6. check if have full text search filed in stream
+ // 7. check if have full text search filed in stream
if match_visitor.has_match_all {
if match_visitor.is_multi_stream {
return Err(Error::ErrorCode(ErrorCodes::SearchSQLNotValid(
@@ -205,7 +210,7 @@ impl Sql {
}
}
- // 7. generate used schema
+ // 8. generate used schema
let mut used_schemas = HashMap::with_capacity(total_schemas.len());
if column_visitor.is_wildcard {
let has_original_column = has_original_column(&column_visitor.columns);
@@ -227,11 +232,11 @@ impl Sql {
}
}
- // 8. get partition column value
+ // 9. get partition column value
let mut partition_column_visitor = PartitionColumnVisitor::new(&used_schemas);
let _ = statement.visit(&mut partition_column_visitor);
- // 9. pick up histogram interval
+ // 10. pick up histogram interval
let mut histogram_interval_visitor =
HistogramIntervalVisitor::new(Some((query.start_time, query.end_time)));
let _ = statement.visit(&mut histogram_interval_visitor);
@@ -245,11 +250,11 @@ impl Sql {
};
//********************Change the sql start*********************************//
- // 10. replace approx_percentile_cont to new format
+ // 11. replace approx_percentile_cont to new format
let mut replace_approx_percentilet_visitor = ReplaceApproxPercentiletVisitor::new();
let _ = statement.visit(&mut replace_approx_percentilet_visitor);
- // 11. add _timestamp and _o2_id if need
+ // 12. add _timestamp and _o2_id if need
if !is_complex_query(&mut statement) {
let mut add_timestamp_visitor = AddTimestampVisitor::new();
let _ = statement.visit(&mut add_timestamp_visitor);
@@ -259,7 +264,7 @@ impl Sql {
}
}
- // 12. generate tantivy query
+ // 13. generate tantivy query
// TODO: merge IndexVisitor and IndexOptimizeModeVisitor
let mut index_condition = None;
let mut can_optimize = false;
@@ -282,7 +287,7 @@ impl Sql {
});
}
- // 13. check `select * from table where match_all()` optimizer
+ // 14. check `select * from table where match_all()` optimizer
let mut index_optimize_mode = None;
if !is_complex_query(&mut statement)
&& order_by.len() == 1
@@ -295,7 +300,7 @@ impl Sql {
));
}
- // 14. check other inverted index optimize modes
+ // 15. check other inverted index optimize modes
// `select count(*) from table where match_all` -> SimpleCount
// or `select histogram(..), count(*) from table where match_all` -> SimpleHistogram
// or `select id, count(*) from t group by id order by cnt desc limit 10` -> SimpleTopN
@@ -326,7 +331,7 @@ impl Sql {
}
}
- // 15. replace the Utf8 to Utf8View type
+ // 16. replace the Utf8 to Utf8View type
let final_schemas = if cfg.common.utf8_view_enabled {
let mut final_schemas = HashMap::with_capacity(used_schemas.len());
for (stream, schema) in used_schemas.iter() {
diff --git a/src/service/search/sql/rewriter/match_all_raw.rs b/src/service/search/sql/rewriter/match_all_raw.rs
new file mode 100644
index 0000000000..d53d7f5c87
--- /dev/null
+++ b/src/service/search/sql/rewriter/match_all_raw.rs
@@ -0,0 +1,114 @@
+// Copyright 2025 OpenObserve Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+use std::ops::ControlFlow;
+
+use sqlparser::ast::{Expr, Ident, ObjectName, ObjectNamePart, VisitorMut};
+
+pub struct MatchAllRawVisitor {}
+
+impl MatchAllRawVisitor {
+ pub fn new() -> Self {
+ Self {}
+ }
+}
+
+impl VisitorMut for MatchAllRawVisitor {
+ type Break = ();
+
+ fn pre_visit_expr(&mut self, expr: &mut Expr) -> ControlFlow {
+ if let Expr::Function(f) = expr {
+ let fname = f.name.to_string().to_lowercase();
+ // for backward compatibility, we re-write these functions as match all
+ if fname == "match_all_raw" || fname == "match_all_raw_ignore_case" {
+ f.name = ObjectName(vec![ObjectNamePart::Identifier(Ident::new("match_all"))])
+ }
+ }
+ ControlFlow::Continue(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use sqlparser::{ast::VisitMut, dialect::GenericDialect};
+
+ use super::*;
+
+ #[test]
+ fn test_match_all_raw1() {
+ let sql = "SELECT * FROM t WHERE match_all_raw('test')";
+ let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql)
+ .unwrap()
+ .pop()
+ .unwrap();
+ let mut visitor = MatchAllRawVisitor::new();
+ let _ = statement.visit(&mut visitor);
+ let expected_sql = "SELECT * FROM t WHERE match_all('test')";
+ assert_eq!(statement.to_string(), expected_sql);
+ }
+
+ #[test]
+ fn test_match_all_raw2() {
+ let sql =
+ "SELECT * FROM t WHERE match_all_raw_ignore_case('test') group by name order by name";
+ let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql)
+ .unwrap()
+ .pop()
+ .unwrap();
+ let mut visitor = MatchAllRawVisitor::new();
+ let _ = statement.visit(&mut visitor);
+ let expected_sql = "SELECT * FROM t WHERE match_all('test') GROUP BY name ORDER BY name";
+ assert_eq!(statement.to_string(), expected_sql);
+ }
+
+ #[test]
+ fn test_match_all_raw3() {
+ let sql = "SELECT t1.name, t2.name from t1 join t2 on t1.name = t2.name where match_all_raw('capture') group by t1.name, t2.name order by t1.name, t2.name";
+ let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql)
+ .unwrap()
+ .pop()
+ .unwrap();
+ let mut visitor = MatchAllRawVisitor::new();
+ let _ = statement.visit(&mut visitor);
+ let expected_sql = "SELECT t1.name, t2.name FROM t1 JOIN t2 ON t1.name = t2.name WHERE match_all('capture') GROUP BY t1.name, t2.name ORDER BY t1.name, t2.name";
+ assert_eq!(statement.to_string(), expected_sql);
+ }
+
+ #[test]
+ fn test_match_all_raw4() {
+ let sql = "SELECT match_all_raw_ignore_case('fine') from t1";
+ let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql)
+ .unwrap()
+ .pop()
+ .unwrap();
+ let mut visitor = MatchAllRawVisitor::new();
+ let _ = statement.visit(&mut visitor);
+ let expected_sql = "SELECT match_all('fine') FROM t1";
+ assert_eq!(statement.to_string(), expected_sql);
+ }
+
+ #[test]
+ fn test_match_all_raw5() {
+ let sql = "SELECT match_all_raw('fine') from t1";
+ let mut statement = sqlparser::parser::Parser::parse_sql(&GenericDialect {}, sql)
+ .unwrap()
+ .pop()
+ .unwrap();
+ let mut visitor = MatchAllRawVisitor::new();
+ let _ = statement.visit(&mut visitor);
+ let expected_sql = "SELECT match_all('fine') FROM t1";
+ assert_eq!(statement.to_string(), expected_sql);
+ }
+}
diff --git a/src/service/search/sql/rewriter/mod.rs b/src/service/search/sql/rewriter/mod.rs
index 47f9ce221c..16f085b92e 100644
--- a/src/service/search/sql/rewriter/mod.rs
+++ b/src/service/search/sql/rewriter/mod.rs
@@ -19,5 +19,6 @@ pub mod add_ordering_term;
pub mod add_timestamp;
pub mod approx_percentile;
pub mod index;
+pub mod match_all_raw;
pub mod remove_dashboard_placeholder;
pub mod track_total_hits;