common_function/scalars/
matches_term.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::fmt;
16use std::sync::Arc;
17
18use common_base::term_token::{CharClass, classify_char};
19use datafusion_common::arrow::array::{Array, AsArray, BooleanArray, BooleanBuilder};
20use datafusion_common::arrow::compute;
21use datafusion_common::arrow::datatypes::DataType;
22use datafusion_common::{DataFusionError, ScalarValue};
23use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, Volatility};
24use memchr::memmem;
25
26use crate::function::Function;
27use crate::function_registry::FunctionRegistry;
28
29/// Exact term/phrase matching function for text columns.
30///
31/// This function uses script-aware matching rules:
32/// - ASCII-only terms keep whole-word style boundary matching, like Whole-word matching (e.g. "cat" in "cat!" but not in "category")
33/// - Phrase matching (e.g. "hello world" in "note:hello world!")
34/// - Terms containing Han characters match as contiguous substrings
35/// - Mixed-script identifiers and numeric terms remain searchable in Chinese text
36///
37/// # Signature
38/// `matches_term(text: String, term: String) -> Boolean`
39///
40/// # Arguments
41/// * `text` - String column to search
42/// * `term` - Search term/phrase
43///
44/// # Returns
45/// BooleanVector where each element indicates if the corresponding text
46/// contains an exact match of the term, following these rules:
47/// 1. Exact substring match found (case-sensitive)
48/// 2. For ASCII-only terms, adjacent ASCII word characters block the match
49/// 3. For Han-containing terms, contiguous substring match is sufficient
50///
51/// # Examples
52/// ```
53/// -- SQL examples --
54/// -- Match phrase with space --
55/// SELECT matches_term(column, 'hello world') FROM table;
56/// -- Text: "warning:hello world!" => true
57/// -- Text: "hello-world"          => false (hyphen instead of space)
58/// -- Text: "hello world2023"      => false (ending with numbers)
59///
60/// -- Match multiple words with boundaries --
61/// SELECT matches_term(column, 'critical error') FROM logs;
62/// -- Match in: "ERROR:critical error!"
63/// -- No match: "critical_errors"
64/// -- Chinese substring examples --
65/// SELECT matches_term(column, '手机') FROM table;
66/// -- Text: "登录手机号18888888888的动态key" => true
67///
68/// -- Empty string handling --
69/// SELECT matches_term(column, '') FROM table;
70/// -- Text: "" => true
71/// -- Text: "any" => false
72///
73/// -- Case sensitivity --
74/// SELECT matches_term(column, 'Cat') FROM table;
75/// -- Text: "Cat" => true
76/// -- Text: "cat" => false
77/// ```
78pub struct MatchesTermFunction {
79    signature: Signature,
80}
81
82impl MatchesTermFunction {
83    pub fn register(registry: &FunctionRegistry) {
84        registry.register_scalar(MatchesTermFunction::default());
85    }
86}
87
88impl Default for MatchesTermFunction {
89    fn default() -> Self {
90        Self {
91            signature: Signature::string(2, Volatility::Immutable),
92        }
93    }
94}
95
96impl fmt::Display for MatchesTermFunction {
97    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
98        write!(f, "MATCHES_TERM")
99    }
100}
101
102impl Function for MatchesTermFunction {
103    fn name(&self) -> &str {
104        "matches_term"
105    }
106
107    fn return_type(&self, _: &[DataType]) -> datafusion_common::Result<DataType> {
108        Ok(DataType::Boolean)
109    }
110
111    fn signature(&self) -> &Signature {
112        &self.signature
113    }
114
115    fn invoke_with_args(
116        &self,
117        args: ScalarFunctionArgs,
118    ) -> datafusion_common::Result<ColumnarValue> {
119        let [arg0, arg1] = datafusion_common::utils::take_function_args(self.name(), &args.args)?;
120
121        fn as_str(v: &ScalarValue) -> Option<&str> {
122            match v {
123                ScalarValue::Utf8View(Some(x))
124                | ScalarValue::Utf8(Some(x))
125                | ScalarValue::LargeUtf8(Some(x)) => Some(x.as_str()),
126                _ => None,
127            }
128        }
129
130        if let (ColumnarValue::Scalar(text), ColumnarValue::Scalar(term)) = (arg0, arg1) {
131            let text = as_str(text);
132            let term = as_str(term);
133            let result = match (text, term) {
134                (Some(text), Some(term)) => Some(MatchesTermFinder::new(term).find(text)),
135                _ => None,
136            };
137            return Ok(ColumnarValue::Scalar(ScalarValue::Boolean(result)));
138        };
139
140        let v = match (arg0, arg1) {
141            (ColumnarValue::Scalar(_), ColumnarValue::Scalar(_)) => {
142                // Unreachable because we have checked this case above and returned if matched.
143                unreachable!()
144            }
145            (ColumnarValue::Scalar(text), ColumnarValue::Array(terms)) => {
146                let text = as_str(text);
147                if let Some(text) = text {
148                    let terms = compute::cast(terms, &DataType::Utf8View)?;
149                    let terms = terms.as_string_view();
150
151                    let mut builder = BooleanBuilder::with_capacity(terms.len());
152                    terms.iter().for_each(|term| {
153                        builder.append_option(term.map(|x| MatchesTermFinder::new(x).find(text)))
154                    });
155                    ColumnarValue::Array(Arc::new(builder.finish()))
156                } else {
157                    ColumnarValue::Array(Arc::new(BooleanArray::new_null(terms.len())))
158                }
159            }
160            (ColumnarValue::Array(texts), ColumnarValue::Scalar(term)) => {
161                let term = as_str(term);
162                if let Some(term) = term {
163                    let finder = MatchesTermFinder::new(term);
164
165                    let texts = compute::cast(texts, &DataType::Utf8View)?;
166                    let texts = texts.as_string_view();
167
168                    let mut builder = BooleanBuilder::with_capacity(texts.len());
169                    texts
170                        .iter()
171                        .for_each(|text| builder.append_option(text.map(|x| finder.find(x))));
172                    ColumnarValue::Array(Arc::new(builder.finish()))
173                } else {
174                    ColumnarValue::Array(Arc::new(BooleanArray::new_null(texts.len())))
175                }
176            }
177            (ColumnarValue::Array(texts), ColumnarValue::Array(terms)) => {
178                let terms = compute::cast(terms, &DataType::Utf8View)?;
179                let terms = terms.as_string_view();
180                let texts = compute::cast(texts, &DataType::Utf8View)?;
181                let texts = texts.as_string_view();
182
183                let len = texts.len();
184                if terms.len() != len {
185                    return Err(DataFusionError::Internal(format!(
186                        "input arrays have different lengths: {len}, {}",
187                        terms.len()
188                    )));
189                }
190
191                let mut builder = BooleanBuilder::with_capacity(len);
192                for (text, term) in texts.iter().zip(terms.iter()) {
193                    let result = match (text, term) {
194                        (Some(text), Some(term)) => Some(MatchesTermFinder::new(term).find(text)),
195                        _ => None,
196                    };
197                    builder.append_option(result);
198                }
199                ColumnarValue::Array(Arc::new(builder.finish()))
200            }
201        };
202        Ok(v)
203    }
204}
205
206/// A compiled finder for `matches_term` function that holds the compiled term
207/// and its metadata for efficient matching.
208///
209/// A term is considered matched when:
210/// 1. The exact sequence appears in the text
211/// 2. ASCII-only terms are not adjacent to ASCII word characters
212/// 3. Han-containing terms match as contiguous substrings
213///
214/// # Examples
215/// ```
216/// let finder = MatchesTermFinder::new("cat");
217/// assert!(finder.find("cat!"));      // Term at end with punctuation
218/// assert!(finder.find("dog,cat"));   // Term preceded by comma
219/// assert!(!finder.find("category")); // Partial match rejected
220///
221/// let finder = MatchesTermFinder::new("手机");
222/// assert!(finder.find("登录手机号18888888888的动态key"));
223/// ```
224#[derive(Clone, Debug)]
225pub struct MatchesTermFinder {
226    finder: memmem::Finder<'static>,
227    term: String,
228    term_kind: TermKind,
229    starts_with_other: bool,
230    ends_with_other: bool,
231}
232
233#[derive(Clone, Copy, Debug, PartialEq, Eq)]
234enum TermKind {
235    AsciiLike,
236    UnicodeWord,
237    HanContaining,
238}
239
240fn classify_term(term: &str) -> TermKind {
241    let mut has_han = false;
242    let mut has_unicode_word = false;
243    for c in term.chars() {
244        match classify_char(c) {
245            CharClass::AsciiWord => {}
246            CharClass::Han => has_han = true,
247            CharClass::UnicodeWord => has_unicode_word = true,
248            CharClass::Other => {}
249        }
250    }
251
252    if has_han {
253        TermKind::HanContaining
254    } else if has_unicode_word {
255        TermKind::UnicodeWord
256    } else {
257        TermKind::AsciiLike
258    }
259}
260
261fn boundary_ok(term_kind: TermKind, neighbor: Option<char>, term_has_other_boundary: bool) -> bool {
262    if term_has_other_boundary {
263        return true;
264    }
265
266    match term_kind {
267        TermKind::AsciiLike => !matches!(neighbor.map(classify_char), Some(CharClass::AsciiWord)),
268        TermKind::UnicodeWord => !matches!(
269            neighbor.map(classify_char),
270            Some(CharClass::AsciiWord | CharClass::UnicodeWord | CharClass::Han)
271        ),
272        TermKind::HanContaining => true,
273    }
274}
275
276impl MatchesTermFinder {
277    /// Create a new `MatchesTermFinder` for the given term.
278    pub fn new(term: &str) -> Self {
279        let starts_with_other = term
280            .chars()
281            .next()
282            .is_some_and(|c| classify_char(c) == CharClass::Other);
283        let ends_with_other = term
284            .chars()
285            .last()
286            .is_some_and(|c| classify_char(c) == CharClass::Other);
287        Self {
288            finder: memmem::Finder::new(term).into_owned(),
289            term: term.to_string(),
290            term_kind: classify_term(term),
291            starts_with_other,
292            ends_with_other,
293        }
294    }
295
296    /// Find the term in the text.
297    pub fn find(&self, text: &str) -> bool {
298        if self.term.is_empty() {
299            return text.is_empty();
300        }
301
302        if text.len() < self.term.len() {
303            return false;
304        }
305
306        let mut pos = 0;
307        while let Some(found_pos) = self.finder.find(&text.as_bytes()[pos..]) {
308            let actual_pos = pos + found_pos;
309
310            let prev = text[..actual_pos].chars().last();
311            let prev_ok = self.starts_with_other || boundary_ok(self.term_kind, prev, false);
312
313            if prev_ok {
314                if self.term_kind == TermKind::HanContaining {
315                    return true;
316                }
317
318                let next_pos = actual_pos + self.finder.needle().len();
319                let next = text[next_pos..].chars().next();
320                let next_ok = self.ends_with_other || boundary_ok(self.term_kind, next, false);
321
322                if next_ok {
323                    return true;
324                }
325            }
326
327            if let Some(next_char) = text[actual_pos..].chars().next() {
328                pos = actual_pos + next_char.len_utf8();
329            } else {
330                break;
331            }
332        }
333
334        false
335    }
336}
337
338#[cfg(test)]
339mod tests {
340    use super::*;
341
342    #[test]
343    fn matches_term_example() {
344        let finder = MatchesTermFinder::new("hello world");
345        assert!(finder.find("warning:hello world!"));
346        assert!(!finder.find("hello-world"));
347        assert!(!finder.find("hello world2023"));
348
349        let finder = MatchesTermFinder::new("critical error");
350        assert!(finder.find("ERROR:critical error!"));
351        assert!(!finder.find("critical_errors"));
352
353        let finder = MatchesTermFinder::new("");
354        assert!(finder.find(""));
355        assert!(!finder.find("any"));
356
357        let finder = MatchesTermFinder::new("Cat");
358        assert!(finder.find("Cat"));
359        assert!(!finder.find("cat"));
360    }
361
362    #[test]
363    fn matches_term_with_punctuation() {
364        assert!(MatchesTermFinder::new("cat").find("cat!"));
365        assert!(MatchesTermFinder::new("dog").find("!dog"));
366    }
367
368    #[test]
369    fn matches_phrase_with_boundaries() {
370        assert!(MatchesTermFinder::new("hello-world").find("hello-world"));
371        assert!(MatchesTermFinder::new("'foo bar'").find("test: 'foo bar'"));
372    }
373
374    #[test]
375    fn matches_at_text_boundaries() {
376        assert!(MatchesTermFinder::new("start").find("start..."));
377        assert!(MatchesTermFinder::new("end").find("...end"));
378    }
379
380    // Negative cases
381    #[test]
382    fn rejects_partial_matches() {
383        assert!(!MatchesTermFinder::new("cat").find("category"));
384        assert!(!MatchesTermFinder::new("boot").find("rebooted"));
385    }
386
387    #[test]
388    fn rejects_missing_term() {
389        assert!(!MatchesTermFinder::new("foo").find("hello world"));
390    }
391
392    // Edge cases
393    #[test]
394    fn handles_empty_inputs() {
395        assert!(!MatchesTermFinder::new("test").find(""));
396        assert!(!MatchesTermFinder::new("").find("text"));
397    }
398
399    #[test]
400    fn different_unicode_boundaries() {
401        assert!(MatchesTermFinder::new("café").find("café>"));
402        assert!(!MatchesTermFinder::new("café").find("口café>"));
403        assert!(!MatchesTermFinder::new("café").find("café口"));
404        assert!(!MatchesTermFinder::new("café").find("cafémore"));
405        assert!(MatchesTermFinder::new("русский").find("русский!"));
406        assert!(MatchesTermFinder::new("русский").find("русский！"));
407    }
408
409    #[test]
410    fn case_sensitive_matching() {
411        assert!(!MatchesTermFinder::new("cat").find("Cat"));
412        assert!(MatchesTermFinder::new("CaT").find("CaT"));
413    }
414
415    #[test]
416    fn numbers_in_term() {
417        assert!(MatchesTermFinder::new("v1.0").find("v1.0!"));
418        assert!(!MatchesTermFinder::new("v1.0").find("v1.0a"));
419    }
420
421    #[test]
422    fn mixed_script_terms_match_inside_chinese_context() {
423        let text = "登录手机号18888888888的动态key";
424        assert!(MatchesTermFinder::new("手机号").find(text));
425        assert!(MatchesTermFinder::new("18888888888").find(text));
426        assert!(MatchesTermFinder::new("手机").find(text));
427        assert!(MatchesTermFinder::new("机号").find(text));
428        assert!(MatchesTermFinder::new("机号1888").find(text));
429        assert!(MatchesTermFinder::new("农业").find("中国农业银行"));
430        assert!(MatchesTermFinder::new("error").find("错误error日志"));
431    }
432
433    #[test]
434    fn underscore_still_counts_as_boundary_for_ascii_terms() {
435        assert!(MatchesTermFinder::new("world").find("hello_world"));
436        assert!(MatchesTermFinder::new("id").find("trace_id=abc"));
437        assert!(!MatchesTermFinder::new("error").find("criticalerrors"));
438    }
439
440    #[test]
441    fn adjacent_alphanumeric_fails() {
442        assert!(!MatchesTermFinder::new("cat").find("cat5"));
443        assert!(!MatchesTermFinder::new("dog").find("dogcat"));
444    }
445
446    #[test]
447    fn empty_term_text() {
448        assert!(!MatchesTermFinder::new("").find("text"));
449        assert!(MatchesTermFinder::new("").find(""));
450        assert!(!MatchesTermFinder::new("text").find(""));
451    }
452
453    #[test]
454    fn leading_non_alphanumeric() {
455        assert!(MatchesTermFinder::new("/cat").find("dog/cat"));
456        assert!(MatchesTermFinder::new("dog/").find("dog/cat"));
457        assert!(MatchesTermFinder::new("dog/cat").find("dog/cat"));
458    }
459
460    #[test]
461    fn continues_searching_after_boundary_mismatch() {
462        assert!(!MatchesTermFinder::new("log").find("bloglog!"));
463        assert!(MatchesTermFinder::new("log").find("bloglog log"));
464        assert!(MatchesTermFinder::new("log").find("alogblog_log!"));
465
466        assert!(MatchesTermFinder::new("error").find("errorlog_error_case"));
467        assert!(MatchesTermFinder::new("test").find("atestbtestc_test_end"));
468        assert!(MatchesTermFinder::new("data").find("database_data_store"));
469        assert!(!MatchesTermFinder::new("data").find("database_datastore"));
470        assert!(MatchesTermFinder::new("log.txt").find("catalog.txt_log.txt!"));
471        assert!(!MatchesTermFinder::new("log.txt").find("catalog.txtlog.txt!"));
472        assert!(MatchesTermFinder::new("data-set").find("bigdata-set_data-set!"));
473
474        assert!(MatchesTermFinder::new("中文").find("这是中文测试，中文！"));
475        assert!(MatchesTermFinder::new("error").find("错误errorerror日志_error!"));
476    }
477
478    #[test]
479    fn han_terms_match_as_contiguous_substrings() {
480        assert!(MatchesTermFinder::new("行账号").find("中国农业银行账号"));
481        assert!(MatchesTermFinder::new("登录").find("登录手机号18888888888的动态key"));
482    }
483}
common_function/scalars/matches_term.rs

common_function/scalars/
matches_term.rs
