feat: fuzzy match (#168)

core/expression/Cargo.toml

@@ -25,6 +25,7 @@ nohash-hasher = "0.2.0"
 thiserror = { workspace = true }
 rust_decimal = { workspace = true, features = ["maths-nopanic"] }
 rust_decimal_macros = { workspace = true }
+strsim = "0.11.1"
 
 [dev-dependencies]
 anyhow = { workspace = true }

core/expression/src/compiler/compiler.rs

@@ -329,6 +329,11 @@ impl<'arena, 'bytecode_ref> CompilerInner<'arena, 'bytecode_ref> {
                     self.compile_argument(kind, arguments, 1)?;
                     Ok(self.emit(Opcode::Matches))
                 }
+                BuiltInFunction::FuzzyMatch => {
+                    self.compile_argument(kind, arguments, 0)?;
+                    self.compile_argument(kind, arguments, 1)?;
+                    Ok(self.emit(Opcode::FuzzyMatch))
+                }
                 BuiltInFunction::Extract => {
                     self.compile_argument(kind, arguments, 0)?;
                     self.compile_argument(kind, arguments, 1)?;

core/expression/src/compiler/opcode.rs

@@ -53,6 +53,7 @@ pub enum Opcode<'a> {
     StartsWith,
     EndsWith,
     Matches,
+    FuzzyMatch,
     Extract,
     Slice,
     Array,

core/expression/src/parser/builtin.rs

@@ -22,6 +22,7 @@ pub enum BuiltInFunction {
     EndsWith,
     Matches,
     Extract,
+    FuzzyMatch,
 
     // Math
     Abs,
@@ -88,6 +89,7 @@ impl BuiltInFunction {
             BuiltInFunction::EndsWith => Arity::Dual,
             BuiltInFunction::Matches => Arity::Dual,
             BuiltInFunction::Extract => Arity::Dual,
+            BuiltInFunction::FuzzyMatch => Arity::Dual,
 
             // Math
             BuiltInFunction::Abs => Arity::Single,

core/expression/src/parser/unary.rs

@@ -303,6 +303,7 @@ impl From<&Node<'_>> for UnaryNodeBehaviour {
                 BuiltInFunction::StartsWith => AsBoolean,
                 BuiltInFunction::EndsWith => AsBoolean,
                 BuiltInFunction::Matches => AsBoolean,
+                BuiltInFunction::FuzzyMatch => CompareWithReference(Equal),
                 BuiltInFunction::IsNumeric => AsBoolean,
                 BuiltInFunction::Keys => CompareWithReference(In),
                 BuiltInFunction::All => AsBoolean,

core/expression/src/vm/vm.rs

@@ -10,7 +10,7 @@ use chrono::{Datelike, Timelike};
 use regex::Regex;
 #[cfg(feature = "regex-lite")]
 use regex_lite::Regex;
-use rust_decimal::prelude::ToPrimitive;
+use rust_decimal::prelude::{FromPrimitive, ToPrimitive};
 use rust_decimal::{Decimal, MathematicalOps};
 use rust_decimal_macros::dec;
 
@@ -989,6 +989,49 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'arena, 'parent_ref, 'bytecode_
 
                     self.push(Bool(regex.is_match(a)));
                 }
+                Opcode::FuzzyMatch => {
+                    let b = self.pop()?;
+                    let a = self.pop()?;
+
+                    let String(b) = b else {
+                        return Err(OpcodeErr {
+                            opcode: "FuzzyMatch".into(),
+                            message: "Unsupported type".into(),
+                        });
+                    };
+
+                    match a {
+                        String(a) => {
+                            let sim = strsim::normalized_damerau_levenshtein(a, b);
+                            // This is okay, as NDL will return [0, 1]
+                            self.push(Number(Decimal::from_f64(sim).unwrap_or(dec!(0))));
+                        }
+                        Array(a) => {
+                            let mut sims = BumpVec::with_capacity_in(a.len(), &self.bump);
+                            for v in a.iter() {
+                                let String(s) = v else {
+                                    return Err(OpcodeErr {
+                                        opcode: "FuzzyMatch".into(),
+                                        message: "Unsupported type".into(),
+                                    });
+                                };
+
+                                let sim =
+                                    Decimal::from_f64(strsim::normalized_damerau_levenshtein(s, b))
+                                        .unwrap_or(dec!(0));
+                                sims.push(&*self.bump.alloc(Number(sim)));
+                            }
+
+                            self.push(Array(sims.into_bump_slice()))
+                        }
+                        _ => {
+                            return Err(OpcodeErr {
+                                opcode: "FuzzyMatch".into(),
+                                message: "Unsupported type".into(),
+                            })
+                        }
+                    }
+                }
                 Opcode::Extract => {
                     let b = self.pop()?;
                     let a = self.pop()?;

core/expression/tests/data/standard.csv

@@ -120,6 +120,10 @@ contains("Hello, World!", "foo");; false
 matches("Hello, World!", "H[a-z]+, W[a-z]+!");; true
 matches("Hello, World!", "[0-9]+");; false
 extract("2022-09-18", "(\d{4})-(\d{2})-(\d{2})");; ["2022-09-18", "2022", "09", "18"]
+fuzzyMatch("hello", "hello");; 1
+fuzzyMatch("world", "hello");; 0.2
+fuzzyMatch(["hello", "world"], "hello");;[1, 0.2]
+
 
 # String Slice
 string[0:5];{string: 'sample_string'};'sample'