From 7ce6ceb9e448e499673ddffd7e0828ceda1a69cc Mon Sep 17 00:00:00 2001 From: stefan-gorules <127550877+stefan-gorules@users.noreply.github.com> Date: Thu, 23 May 2024 12:00:58 +0200 Subject: [PATCH] feat: fuzzy match (#168) --- core/expression/Cargo.toml | 1 + core/expression/src/compiler/compiler.rs | 5 +++ core/expression/src/compiler/opcode.rs | 1 + core/expression/src/parser/builtin.rs | 2 ++ core/expression/src/parser/unary.rs | 1 + core/expression/src/vm/vm.rs | 45 +++++++++++++++++++++++- core/expression/tests/data/standard.csv | 4 +++ 7 files changed, 58 insertions(+), 1 deletion(-) diff --git a/core/expression/Cargo.toml b/core/expression/Cargo.toml index 521fa09f..09a3c6cf 100644 --- a/core/expression/Cargo.toml +++ b/core/expression/Cargo.toml @@ -25,6 +25,7 @@ nohash-hasher = "0.2.0" thiserror = { workspace = true } rust_decimal = { workspace = true, features = ["maths-nopanic"] } rust_decimal_macros = { workspace = true } +strsim = "0.11.1" [dev-dependencies] anyhow = { workspace = true } diff --git a/core/expression/src/compiler/compiler.rs b/core/expression/src/compiler/compiler.rs index adf339a7..b6d89e0b 100644 --- a/core/expression/src/compiler/compiler.rs +++ b/core/expression/src/compiler/compiler.rs @@ -329,6 +329,11 @@ impl<'arena, 'bytecode_ref> CompilerInner<'arena, 'bytecode_ref> { self.compile_argument(kind, arguments, 1)?; Ok(self.emit(Opcode::Matches)) } + BuiltInFunction::FuzzyMatch => { + self.compile_argument(kind, arguments, 0)?; + self.compile_argument(kind, arguments, 1)?; + Ok(self.emit(Opcode::FuzzyMatch)) + } BuiltInFunction::Extract => { self.compile_argument(kind, arguments, 0)?; self.compile_argument(kind, arguments, 1)?; diff --git a/core/expression/src/compiler/opcode.rs b/core/expression/src/compiler/opcode.rs index 35a6dc7e..0f3f3cc9 100644 --- a/core/expression/src/compiler/opcode.rs +++ b/core/expression/src/compiler/opcode.rs @@ -53,6 +53,7 @@ pub enum Opcode<'a> { StartsWith, EndsWith, Matches, + FuzzyMatch, Extract, Slice, Array, diff --git a/core/expression/src/parser/builtin.rs b/core/expression/src/parser/builtin.rs index 6d57bdff..04a1b7ea 100644 --- a/core/expression/src/parser/builtin.rs +++ b/core/expression/src/parser/builtin.rs @@ -22,6 +22,7 @@ pub enum BuiltInFunction { EndsWith, Matches, Extract, + FuzzyMatch, // Math Abs, @@ -88,6 +89,7 @@ impl BuiltInFunction { BuiltInFunction::EndsWith => Arity::Dual, BuiltInFunction::Matches => Arity::Dual, BuiltInFunction::Extract => Arity::Dual, + BuiltInFunction::FuzzyMatch => Arity::Dual, // Math BuiltInFunction::Abs => Arity::Single, diff --git a/core/expression/src/parser/unary.rs b/core/expression/src/parser/unary.rs index b531b7ae..1dba0bf3 100644 --- a/core/expression/src/parser/unary.rs +++ b/core/expression/src/parser/unary.rs @@ -303,6 +303,7 @@ impl From<&Node<'_>> for UnaryNodeBehaviour { BuiltInFunction::StartsWith => AsBoolean, BuiltInFunction::EndsWith => AsBoolean, BuiltInFunction::Matches => AsBoolean, + BuiltInFunction::FuzzyMatch => CompareWithReference(Equal), BuiltInFunction::IsNumeric => AsBoolean, BuiltInFunction::Keys => CompareWithReference(In), BuiltInFunction::All => AsBoolean, diff --git a/core/expression/src/vm/vm.rs b/core/expression/src/vm/vm.rs index 7f84eec0..0f3b3ce2 100644 --- a/core/expression/src/vm/vm.rs +++ b/core/expression/src/vm/vm.rs @@ -10,7 +10,7 @@ use chrono::{Datelike, Timelike}; use regex::Regex; #[cfg(feature = "regex-lite")] use regex_lite::Regex; -use rust_decimal::prelude::ToPrimitive; +use rust_decimal::prelude::{FromPrimitive, ToPrimitive}; use rust_decimal::{Decimal, MathematicalOps}; use rust_decimal_macros::dec; @@ -989,6 +989,49 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'arena, 'parent_ref, 'bytecode_ self.push(Bool(regex.is_match(a))); } + Opcode::FuzzyMatch => { + let b = self.pop()?; + let a = self.pop()?; + + let String(b) = b else { + return Err(OpcodeErr { + opcode: "FuzzyMatch".into(), + message: "Unsupported type".into(), + }); + }; + + match a { + String(a) => { + let sim = strsim::normalized_damerau_levenshtein(a, b); + // This is okay, as NDL will return [0, 1] + self.push(Number(Decimal::from_f64(sim).unwrap_or(dec!(0)))); + } + Array(a) => { + let mut sims = BumpVec::with_capacity_in(a.len(), &self.bump); + for v in a.iter() { + let String(s) = v else { + return Err(OpcodeErr { + opcode: "FuzzyMatch".into(), + message: "Unsupported type".into(), + }); + }; + + let sim = + Decimal::from_f64(strsim::normalized_damerau_levenshtein(s, b)) + .unwrap_or(dec!(0)); + sims.push(&*self.bump.alloc(Number(sim))); + } + + self.push(Array(sims.into_bump_slice())) + } + _ => { + return Err(OpcodeErr { + opcode: "FuzzyMatch".into(), + message: "Unsupported type".into(), + }) + } + } + } Opcode::Extract => { let b = self.pop()?; let a = self.pop()?; diff --git a/core/expression/tests/data/standard.csv b/core/expression/tests/data/standard.csv index ff66ccf1..4ed19794 100644 --- a/core/expression/tests/data/standard.csv +++ b/core/expression/tests/data/standard.csv @@ -120,6 +120,10 @@ contains("Hello, World!", "foo");; false matches("Hello, World!", "H[a-z]+, W[a-z]+!");; true matches("Hello, World!", "[0-9]+");; false extract("2022-09-18", "(\d{4})-(\d{2})-(\d{2})");; ["2022-09-18", "2022", "09", "18"] +fuzzyMatch("hello", "hello");; 1 +fuzzyMatch("world", "hello");; 0.2 +fuzzyMatch(["hello", "world"], "hello");;[1, 0.2] + # String Slice string[0:5];{string: 'sample_string'};'sample'