From 5f423b7910feb62d28c84c159705dc8db296d469 Mon Sep 17 00:00:00 2001 From: stefan-gorules <127550877+stefan-gorules@users.noreply.github.com> Date: Tue, 12 Dec 2023 08:50:25 +0100 Subject: [PATCH] feat: zen expression rewrite (#107) * temp: zen-expression rewrite * rewrite standard parser * stabilise parser, upgrade dependencies * implement unary parser * align builtin behavior * fix tests and minor bugs * refactor exports, update docs * fix expression repl * cleanup * cleanup vm * replace expression with expression-rewrite * update d.ts file * fix failing test --- bindings/nodejs/index.d.ts | 2 +- bindings/nodejs/src/expression.rs | 28 +- bindings/python/src/expression.rs | 39 +- core/engine/Cargo.toml | 6 +- core/engine/src/engine.rs | 2 +- core/engine/src/handler/expression/mod.rs | 10 +- core/engine/src/handler/table/zen.rs | 24 +- core/engine/src/handler/traversal.rs | 11 +- core/engine/src/loader/closure.rs | 6 +- core/engine/tests/engine.rs | 15 +- core/expression/Cargo.toml | 11 +- core/expression/README.md | 71 --- core/expression/benches/isolate.rs | 20 +- core/expression/benches/lexer.rs | 2 +- core/expression/benches/standard.rs | 10 +- core/expression/benches/unary.rs | 9 +- .../expression/src/{ => compiler}/compiler.rs | 402 ++++++------- core/expression/src/compiler/error.rs | 15 + core/expression/src/compiler/mod.rs | 10 + core/expression/src/compiler/opcode.rs | 85 +++ core/expression/src/function.rs | 21 + core/expression/src/isolate.rs | 365 ++++++------ core/expression/src/lexer/codes.rs | 13 +- core/expression/src/lexer/cursor.rs | 4 +- core/expression/src/lexer/error.rs | 13 +- core/expression/src/lexer/lexer.rs | 128 +++-- core/expression/src/lexer/mod.rs | 13 +- core/expression/src/lexer/token.rs | 166 +++++- core/expression/src/lib.rs | 64 ++- core/expression/src/{ => parser}/ast.rs | 9 +- core/expression/src/parser/builtin.rs | 136 +++++ core/expression/src/parser/constants.rs | 71 +++ core/expression/src/parser/definitions.rs | 42 -- core/expression/src/parser/error.rs | 4 +- core/expression/src/parser/iter.rs | 188 ------- core/expression/src/parser/mod.rs | 26 +- core/expression/src/parser/parser.rs | 527 ++++++++++++++++++ core/expression/src/parser/standard.rs | 107 ++++ .../src/parser/standard/constants.rs | 99 ---- core/expression/src/parser/standard/mod.rs | 399 ------------- core/expression/src/parser/unary.rs | 313 +++++++++++ core/expression/src/parser/unary/constants.rs | 61 -- core/expression/src/parser/unary/mod.rs | 269 --------- core/expression/src/vm/error.rs | 21 + core/expression/src/{ => vm}/helpers.rs | 11 +- core/expression/src/vm/mod.rs | 11 + .../src/{opcodes.rs => vm/variable.rs} | 139 ++--- core/expression/src/{ => vm}/vm.rs | 271 ++++----- core/expression/tests/data/unary.csv | 98 +++- core/expression/tests/isolate.rs | 70 ++- core/expression/tests/lexer.rs | 104 ++-- core/expression/tests/standard.rs | 74 +-- core/expression/tests/unary.rs | 67 ++- core/expression_repl/Cargo.toml | 3 +- core/expression_repl/src/main.rs | 4 +- 55 files changed, 2570 insertions(+), 2119 deletions(-) delete mode 100644 core/expression/README.md rename core/expression/src/{ => compiler}/compiler.rs (52%) create mode 100644 core/expression/src/compiler/error.rs create mode 100644 core/expression/src/compiler/mod.rs create mode 100644 core/expression/src/compiler/opcode.rs create mode 100644 core/expression/src/function.rs rename core/expression/src/{ => parser}/ast.rs (84%) create mode 100644 core/expression/src/parser/builtin.rs create mode 100644 core/expression/src/parser/constants.rs delete mode 100644 core/expression/src/parser/definitions.rs delete mode 100644 core/expression/src/parser/iter.rs create mode 100644 core/expression/src/parser/parser.rs create mode 100644 core/expression/src/parser/standard.rs delete mode 100644 core/expression/src/parser/standard/constants.rs delete mode 100644 core/expression/src/parser/standard/mod.rs create mode 100644 core/expression/src/parser/unary.rs delete mode 100644 core/expression/src/parser/unary/constants.rs delete mode 100644 core/expression/src/parser/unary/mod.rs create mode 100644 core/expression/src/vm/error.rs rename core/expression/src/{ => vm}/helpers.rs (95%) create mode 100644 core/expression/src/vm/mod.rs rename core/expression/src/{opcodes.rs => vm/variable.rs} (69%) rename core/expression/src/{ => vm}/vm.rs (87%) diff --git a/bindings/nodejs/index.d.ts b/bindings/nodejs/index.d.ts index bd30c05d..0c744e8d 100644 --- a/bindings/nodejs/index.d.ts +++ b/bindings/nodejs/index.d.ts @@ -11,7 +11,7 @@ export interface ZenEngineOptions { loader?: (key: string) => Promise } export function evaluateExpression(expression: string, context?: any | undefined | null): Promise -export function evaluateUnaryExpression(expression: string, context: any): Promise +export function evaluateUnaryExpression(expression: string, context: any): Promise export class ZenDecision { constructor() evaluate(context: any, opts?: ZenEvaluateOptions | undefined | null): Promise diff --git a/bindings/nodejs/src/expression.rs b/bindings/nodejs/src/expression.rs index 582bfe32..08e8bd87 100644 --- a/bindings/nodejs/src/expression.rs +++ b/bindings/nodejs/src/expression.rs @@ -2,21 +2,16 @@ use napi::anyhow::anyhow; use napi_derive::napi; use serde_json::Value; -use zen_expression::isolate::Isolate; - #[allow(dead_code)] #[napi] pub async fn evaluate_expression( expression: String, context: Option, ) -> napi::Result { - let result: Value = napi::tokio::spawn(async move { - let isolate = Isolate::default(); - if let Some(ctx) = context { - isolate.inject_env(&ctx); - } + let ctx = context.unwrap_or(Value::Null); - isolate.run_standard(expression.as_str()) + let result: Value = napi::tokio::spawn(async move { + zen_expression::evaluate_expression(expression.as_str(), &ctx) }) .await .map_err(|_| anyhow!("Hook timed out"))? @@ -27,20 +22,9 @@ pub async fn evaluate_expression( #[allow(dead_code)] #[napi] -pub async fn evaluate_unary_expression(expression: String, context: Value) -> napi::Result { - let Some(context_object) = context.as_object() else { - return Err(anyhow!("Context must be an object").into()); - }; - - if !context_object.contains_key("$") { - return Err(anyhow!("Context must contain '$' reference.").into()); - } - - let result: Value = napi::tokio::spawn(async move { - let isolate = Isolate::default(); - isolate.inject_env(&context); - - isolate.run_unary(expression.as_str()) +pub async fn evaluate_unary_expression(expression: String, context: Value) -> napi::Result { + let result: bool = napi::tokio::spawn(async move { + zen_expression::evaluate_unary_expression(expression.as_str(), &context) }) .await .map_err(|_| anyhow!("Hook timed out"))? diff --git a/bindings/python/src/expression.rs b/bindings/python/src/expression.rs index d455d7b1..c93d0b77 100644 --- a/bindings/python/src/expression.rs +++ b/bindings/python/src/expression.rs @@ -4,8 +4,6 @@ use pyo3::{pyfunction, PyObject, PyResult, Python, ToPyObject}; use pythonize::depythonize; use serde_json::Value; -use zen_expression::isolate::Isolate; - use crate::value::PyValue; #[pyfunction] @@ -14,45 +12,24 @@ pub fn evaluate_expression( expression: String, ctx: Option<&PyDict>, ) -> PyResult { - let isolate = Isolate::default(); - let ctx = ctx + let context = ctx .map(|ctx| depythonize(ctx)) .transpose() - .context("Failed to convert context")?; + .context("Failed to convert context")? + .unwrap_or(Value::Null); - if let Some(env_value) = ctx { - isolate.inject_env(&env_value); - } - - let result = isolate - .run_standard(expression.as_str()) + let result = zen_expression::evaluate_expression(expression.as_str(), &context) .map_err(|e| anyhow!(serde_json::to_string(&e).unwrap_or_else(|_| e.to_string())))?; Ok(PyValue(result).to_object(py)) } #[pyfunction] -pub fn evaluate_unary_expression( - py: Python, - expression: String, - ctx: &PyDict, -) -> PyResult { - let env_value: Value = depythonize(ctx).context("Failed to convert context")?; +pub fn evaluate_unary_expression(expression: String, ctx: &PyDict) -> PyResult { + let context: Value = depythonize(ctx).context("Failed to convert context")?; - let Some(env_object) = env_value.as_object() else { - return Err(anyhow!("Context must be an object").into()); - }; - - if !env_object.contains_key("$") { - return Err(anyhow!("Context must contain '$' reference.").into()); - } - - let isolate = Isolate::default(); - isolate.inject_env(&env_value); - - let result = isolate - .run_unary(expression.as_str()) + let result = zen_expression::evaluate_unary_expression(expression.as_str(), &context) .map_err(|e| anyhow!(serde_json::to_string(&e).unwrap_or_else(|_| e.to_string())))?; - Ok(PyValue(result).to_object(py)) + Ok(result) } diff --git a/core/engine/Cargo.toml b/core/engine/Cargo.toml index be609882..04ba0333 100644 --- a/core/engine/Cargo.toml +++ b/core/engine/Cargo.toml @@ -19,12 +19,12 @@ bincode = { workspace = true, optional = true } petgraph = { workspace = true } serde_json = { workspace = true, features = ["arbitrary_precision"] } serde = { workspace = true, features = ["derive"] } -serde_v8 = { version = "0.141.0" } +serde_v8 = { version = "0.144.0" } once_cell = { version = "1.18.0" } fixedbitset = "0.4.2" futures = "0.3.29" -v8 = { version = "0.81.0" } -zen-expression = { path = "../expression", version = "0.16.1" } +v8 = { version = "0.82.0" } +zen-expression = { path = "../expression" } [dev-dependencies] tokio = { version = "1.34.0", features = ["rt", "macros"] } diff --git a/core/engine/src/engine.rs b/core/engine/src/engine.rs index 8bdfd155..90e30b78 100644 --- a/core/engine/src/engine.rs +++ b/core/engine/src/engine.rs @@ -34,7 +34,7 @@ impl Default for DecisionEngine { impl DecisionEngine> where - F: Fn(&str) -> O + Sync + Send, + F: Fn(String) -> O + Sync + Send, O: Future + Send, { pub fn async_loader(loader: F) -> Self { diff --git a/core/engine/src/handler/expression/mod.rs b/core/engine/src/handler/expression/mod.rs index ee3f174a..58ad711c 100644 --- a/core/engine/src/handler/expression/mod.rs +++ b/core/engine/src/handler/expression/mod.rs @@ -6,7 +6,7 @@ use crate::util::json_map::FlatJsonMap; use anyhow::{anyhow, Context}; use serde::Serialize; use serde_json::Value; -use zen_expression::isolate::Isolate; +use zen_expression::Isolate; pub struct ExpressionHandler<'a> { trace: bool, @@ -22,11 +22,11 @@ impl<'a> ExpressionHandler<'a> { pub fn new(trace: bool) -> Self { Self { trace, - isolate: Default::default(), + isolate: Isolate::new(), } } - pub async fn handle(&self, request: &'a NodeRequest<'_>) -> NodeResult { + pub async fn handle(&mut self, request: &'a NodeRequest<'_>) -> NodeResult { let content = match &request.node.kind { DecisionNodeKind::ExpressionNode { content } => Ok(content), _ => Err(anyhow!("Unexpected node type")), @@ -35,7 +35,7 @@ impl<'a> ExpressionHandler<'a> { let mut result = FlatJsonMap::with_capacity(content.expressions.len()); let mut trace_map = self.trace.then(|| HashMap::<&str, ExpressionTrace>::new()); - self.isolate.inject_env(&request.input); + self.isolate.set_environment(&request.input); for expression in &content.expressions { let value = self.evaluate_expression(&expression.value)?; if let Some(tmap) = &mut trace_map { @@ -60,7 +60,7 @@ impl<'a> ExpressionHandler<'a> { }) } - fn evaluate_expression(&self, expression: &'a str) -> anyhow::Result { + fn evaluate_expression(&mut self, expression: &'a str) -> anyhow::Result { self.isolate .run_standard(expression) .with_context(|| format!(r#"Failed to evaluate expression: "{expression}""#)) diff --git a/core/engine/src/handler/table/zen.rs b/core/engine/src/handler/table/zen.rs index fd59dbd0..fb49b7bb 100644 --- a/core/engine/src/handler/table/zen.rs +++ b/core/engine/src/handler/table/zen.rs @@ -3,11 +3,11 @@ use std::collections::HashMap; use serde::Serialize; use serde_json::Value; +use zen_expression::Isolate; use crate::handler::node::{NodeRequest, NodeResponse, NodeResult}; use crate::handler::table::{RowOutput, RowOutputKind}; use crate::model::{DecisionNodeKind, DecisionTableContent, DecisionTableHitPolicy}; -use zen_expression::isolate::Isolate; #[derive(Debug, Serialize)] struct RowResult { @@ -18,7 +18,7 @@ struct RowResult { output: RowOutput, } -#[derive(Debug, Default)] +#[derive(Debug)] pub struct DecisionTableHandler<'a> { isolate: Isolate<'a>, trace: bool, @@ -27,18 +27,18 @@ pub struct DecisionTableHandler<'a> { impl<'a> DecisionTableHandler<'a> { pub fn new(trace: bool) -> Self { Self { - isolate: Default::default(), + isolate: Isolate::new(), trace, } } - pub async fn handle(&self, request: &'a NodeRequest<'_>) -> NodeResult { + pub async fn handle(&mut self, request: &'a NodeRequest<'_>) -> NodeResult { let content = match &request.node.kind { DecisionNodeKind::DecisionTableNode { content } => Ok(content), _ => Err(anyhow!("Unexpected node type")), }?; - self.isolate.inject_env(&request.input); + self.isolate.set_environment(&request.input); match &content.hit_policy { DecisionTableHitPolicy::First => self.handle_first_hit(&content).await, @@ -46,7 +46,7 @@ impl<'a> DecisionTableHandler<'a> { } } - async fn handle_first_hit(&self, content: &'a DecisionTableContent) -> NodeResult { + async fn handle_first_hit(&mut self, content: &'a DecisionTableContent) -> NodeResult { for i in 0..content.rules.len() { if let Some(result) = self.evaluate_row(&content, i) { return Ok(NodeResponse { @@ -67,7 +67,7 @@ impl<'a> DecisionTableHandler<'a> { }) } - async fn handle_collect(&self, content: &'a DecisionTableContent) -> NodeResult { + async fn handle_collect(&mut self, content: &'a DecisionTableContent) -> NodeResult { let mut results = Vec::new(); for i in 0..content.rules.len() { if let Some(result) = self.evaluate_row(&content, i) { @@ -89,7 +89,11 @@ impl<'a> DecisionTableHandler<'a> { }) } - fn evaluate_row(&self, content: &'a DecisionTableContent, index: usize) -> Option { + fn evaluate_row( + &mut self, + content: &'a DecisionTableContent, + index: usize, + ) -> Option { let rule = content.rules.get(index)?; for input in &content.inputs { let rule_value = rule.get(input.id.as_str())?; @@ -108,9 +112,7 @@ impl<'a> DecisionTableHandler<'a> { }; self.isolate.set_reference(input_field.as_str()).ok()?; - let result = self.isolate.run_unary(rule_value.as_str()).ok()?; - - let is_ok = result.as_bool().unwrap_or(false); + let is_ok = self.isolate.run_unary(rule_value.as_str()).ok()?; if !is_ok { return None; } diff --git a/core/engine/src/handler/traversal.rs b/core/engine/src/handler/traversal.rs index 8db83925..791a0175 100644 --- a/core/engine/src/handler/traversal.rs +++ b/core/engine/src/handler/traversal.rs @@ -8,7 +8,7 @@ use petgraph::visit::{EdgeRef, IntoNeighbors, IntoNodeIdentifiers, Reversed, Vis use petgraph::{Incoming, Outgoing}; use serde_json::{json, Map, Value}; -use zen_expression::isolate::Isolate; +use zen_expression::Isolate; use crate::model::{ DecisionEdge, DecisionNode, DecisionNodeKind, SwitchStatement, SwitchStatementHitPolicy, @@ -102,17 +102,16 @@ impl GraphWalker { let input_context = json!({ "$": &input_data }); merge_json(&mut input_data, &input_context, true); - let isolate = Isolate::default(); - isolate.inject_env(&input_data); + let mut isolate = Isolate::with_environment(&input_data); let mut statement_iter = content.statements.iter(); let valid_statements: Vec<&SwitchStatement> = match content.hit_policy { SwitchStatementHitPolicy::First => statement_iter - .find(|&s| switch_statement_evaluate(&isolate, &s)) + .find(|&s| switch_statement_evaluate(&mut isolate, &s)) .into_iter() .collect(), SwitchStatementHitPolicy::Collect => statement_iter - .filter(|&s| switch_statement_evaluate(&isolate, &s)) + .filter(|&s| switch_statement_evaluate(&mut isolate, &s)) .collect(), }; @@ -163,7 +162,7 @@ impl GraphWalker { } fn switch_statement_evaluate<'a>( - isolate: &Isolate<'a>, + isolate: &mut Isolate<'a>, switch_statement: &'a SwitchStatement, ) -> bool { if switch_statement.condition.is_empty() { diff --git a/core/engine/src/loader/closure.rs b/core/engine/src/loader/closure.rs index 7872b9f2..c22f33b0 100644 --- a/core/engine/src/loader/closure.rs +++ b/core/engine/src/loader/closure.rs @@ -13,7 +13,7 @@ where impl ClosureLoader where - F: Fn(&str) -> O + Sync + Send, + F: Fn(String) -> O + Sync + Send, O: Future + Send, { pub fn new(closure: F) -> Self { @@ -24,11 +24,11 @@ where #[async_trait] impl DecisionLoader for ClosureLoader where - F: Fn(&str) -> O + Sync + Send, + F: Fn(String) -> O + Sync + Send, O: Future + Send, { async fn load(&self, key: &str) -> LoaderResponse { let closure = &self.closure; - closure(key).await + closure(key.to_string()).await } } diff --git a/core/engine/tests/engine.rs b/core/engine/tests/engine.rs index 8f94e519..ccce6a35 100644 --- a/core/engine/tests/engine.rs +++ b/core/engine/tests/engine.rs @@ -52,16 +52,11 @@ async fn engine_filesystem_loader() { #[tokio::test] #[cfg_attr(miri, ignore)] async fn engine_closure_loader() { - let engine = DecisionEngine::async_loader(|key| { - // TODO: Improve once async closures become stable in Rust - let mv_key = key.to_string(); - - async move { - match mv_key.as_str() { - "function" => Ok(Arc::new(load_test_data("function.json"))), - "table" => Ok(Arc::new(load_test_data("table.json"))), - _ => Err(LoaderError::NotFound(mv_key).into()), - } + let engine = DecisionEngine::async_loader(|key| async { + match key.as_str() { + "function" => Ok(Arc::new(load_test_data("function.json"))), + "table" => Ok(Arc::new(load_test_data("table.json"))), + _ => Err(LoaderError::NotFound(key).into()), } }); diff --git a/core/expression/Cargo.toml b/core/expression/Cargo.toml index 59d76e03..e186a9de 100644 --- a/core/expression/Cargo.toml +++ b/core/expression/Cargo.toml @@ -6,20 +6,23 @@ license = "MIT" version = "0.16.1" edition = "2021" repository = "https://github.com/gorules/zen.git" +publish = false [dependencies] ahash = { workspace = true } -bumpalo = { workspace = true } +bumpalo = { workspace = true, features = ["collections"] } chrono = { workspace = true } hashbrown = { workspace = true, features = ["bumpalo"] } humantime = { workspace = true } fastrand = { workspace = true } once_cell = { workspace = true } -phf = { workspace = true, features = ["macros"] } regex = { workspace = true, optional = true } regex-lite = { workspace = true, optional = true } serde = { workspace = true } serde_json = { workspace = true, features = ["arbitrary_precision"] } +strum = { version = "0.25.0" } +strum_macros = "0.25.3" +nohash-hasher = "0.2.0" thiserror = { workspace = true } rust_decimal = { workspace = true, features = ["maths-nopanic"] } rust_decimal_macros = { workspace = true } @@ -37,7 +40,7 @@ regex-lite = ["dep:regex-lite"] [[bench]] harness = false -name = "unary" +name = "lexer" [[bench]] harness = false @@ -45,7 +48,7 @@ name = "standard" [[bench]] harness = false -name = "lexer" +name = "unary" [[bench]] harness = false diff --git a/core/expression/README.md b/core/expression/README.md deleted file mode 100644 index f5531b14..00000000 --- a/core/expression/README.md +++ /dev/null @@ -1,71 +0,0 @@ -# ZEN Expression - -Zen Expression is business-first expression language used in -Zen Business rules engine by GoRules. The language is designed -to follow these principles: - -- Side-effect free -- Dynamic types -- Simple syntax for broad audiences - -It's primary objective is to bridge the gap between business analysts and engineers, -while providing outstanding performance and readability. - -## Resources - -[Documentation](https://gorules.io/docs/) - -[Zen Language Playground](https://gorules.io/docs/rules-engine/expression-language#playground) - -[Online Rules Editor](https://editor.gorules.io/) - -## Unary tests -Unary test is a comma-separated list of simple expressions which -evaluate to a boolean value. Each comma separation is treated as -or operator. Inside unary expressions, a special symbol is available -$ which refers to a current column. - -Some examples: -```js -// Given: $ = 1 -1, 2, 3 // true -1 // true ->= 1 // true -< 1 // false -[0..10] // true, (internally this is $ >= 0 and $ <= 10) -> 0 and < 10 // true - -// Given: $ = 'USD' -'GBP', 'USD' // true -'EUR' // false -startsWith($, "US") // true - defaults to expression mode, comma is unavailable -endsWith($, "US") // false - defaults to expression mode -lower($) == "usd" // true - defaults to expression mode -``` - -## Standard tests - -Expressions feature full capability syntax of ZEN language. -They give you access to all functions, and are most useful when -defining columns or outputs. Full syntax is also available in unary -expressions when $ is used (as it forces the expression mode). - -```js -100 + 100 // 200 -10 * 5 // 50 -10 ^ 2 // 100 -1 in [1, 2, 3] // true -5 in (5..10] // false -sum([1, 2, 3]) // 6 -max([1, 2, 3]) // 3 - -"hello" + " " + "world" // "hello world" -len("world") // 5 -weekdayString(date("2022-11-08")) // "Tue" -contains("hello world", "hello") // true -upper('john') // "JOHN" - -some(['admin', 'user'], # == "admin") // true -not all([100, 200, 400, 800], # in (100..800)) // false -filter([100, 200, 400, 800], # >= 200) // [200, 400, 800] -``` \ No newline at end of file diff --git a/core/expression/benches/isolate.rs b/core/expression/benches/isolate.rs index e117c5a0..1a8a0ec5 100644 --- a/core/expression/benches/isolate.rs +++ b/core/expression/benches/isolate.rs @@ -1,23 +1,33 @@ use criterion::{criterion_group, criterion_main, Bencher, Criterion}; use serde_json::Value; - -use zen_expression::isolate::Isolate; +use zen_expression::Isolate; fn bench_source(b: &mut Bencher, source: &'static str) { let s: Value = serde_json::from_str(r#"{ "$": "ru" }"#).unwrap(); - let isolate = Isolate::default(); - isolate.inject_env(&s); - + let mut isolate = Isolate::with_environment(&s); b.iter(|| { criterion::black_box(isolate.run_unary(source).unwrap()); }) } +fn bench_standard(b: &mut Bencher, source: &'static str) { + let s: Value = serde_json::from_str(r#"{ "$": "ru" }"#).unwrap(); + + let mut isolate = Isolate::with_environment(&s); + b.iter(|| { + criterion::black_box(isolate.run_standard(source).unwrap()); + }) +} + fn bench_functions(c: &mut Criterion) { c.bench_function("isolate/simple", |b| { bench_source(b, "'ru', 'se'"); }); + + c.bench_function("isolate/standard", |b| { + bench_standard(b, "contains(['ru', 'se'], $)"); + }); } criterion_group!(benches, bench_functions); diff --git a/core/expression/benches/lexer.rs b/core/expression/benches/lexer.rs index e3b60a5f..1931081d 100644 --- a/core/expression/benches/lexer.rs +++ b/core/expression/benches/lexer.rs @@ -3,7 +3,7 @@ use criterion::{criterion_group, criterion_main, Bencher, Criterion}; use zen_expression::lexer::Lexer; fn bench_source(b: &mut Bencher, source: &'static str) { - let lexer = Lexer::new(); + let mut lexer = Lexer::new(); b.iter(|| { criterion::black_box(lexer.tokenize(source).unwrap()); diff --git a/core/expression/benches/standard.rs b/core/expression/benches/standard.rs index 942df51b..90d67bf1 100644 --- a/core/expression/benches/standard.rs +++ b/core/expression/benches/standard.rs @@ -2,16 +2,16 @@ use bumpalo::Bump; use criterion::{criterion_group, criterion_main, Bencher, Criterion}; use zen_expression::lexer::Lexer; -use zen_expression::parser::StandardParser; +use zen_expression::parser::Parser; fn bench_source(b: &mut Bencher, src: &'static str) { - let lexer = Lexer::new(); + let mut lexer = Lexer::new(); let mut bump = Bump::new(); - let t_res = lexer.tokenize(src).unwrap(); - let tokens = t_res.borrow(); + + let tokens = lexer.tokenize(src).unwrap(); b.iter(|| { - let std_parser = StandardParser::try_new(tokens.as_ref(), &bump).unwrap(); + let std_parser = Parser::try_new(tokens, &bump).unwrap().standard(); criterion::black_box(std_parser.parse().unwrap()); bump.reset(); diff --git a/core/expression/benches/unary.rs b/core/expression/benches/unary.rs index b6f8621b..12443d3c 100644 --- a/core/expression/benches/unary.rs +++ b/core/expression/benches/unary.rs @@ -2,16 +2,15 @@ use bumpalo::Bump; use criterion::{criterion_group, criterion_main, Bencher, Criterion}; use zen_expression::lexer::Lexer; -use zen_expression::parser::UnaryParser; +use zen_expression::parser::Parser; fn bench_source(b: &mut Bencher, src: &'static str) { - let lexer = Lexer::new(); + let mut lexer = Lexer::new(); let mut bump = Bump::new(); - let t_res = lexer.tokenize(src).unwrap(); - let tokens = t_res.borrow(); + let tokens = lexer.tokenize(src).unwrap(); b.iter(|| { - let unary_parser = UnaryParser::try_new(tokens.as_ref(), &bump).unwrap(); + let unary_parser = Parser::try_new(tokens, &bump).unwrap().unary(); criterion::black_box(unary_parser.parse().unwrap()); bump.reset(); diff --git a/core/expression/src/compiler.rs b/core/expression/src/compiler/compiler.rs similarity index 52% rename from core/expression/src/compiler.rs rename to core/expression/src/compiler/compiler.rs index a0acdbeb..f4e021a1 100644 --- a/core/expression/src/compiler.rs +++ b/core/expression/src/compiler/compiler.rs @@ -1,69 +1,64 @@ -use std::cell::UnsafeCell; -use std::rc::Rc; - -use bumpalo::Bump; use rust_decimal::Decimal; use rust_decimal_macros::dec; -use thiserror::Error; -use crate::ast::Node; +use crate::compiler::error::{CompilerError, CompilerResult}; +use crate::compiler::{Opcode, TypeCheckKind, TypeConversionKind}; +use crate::lexer::{ArithmeticOperator, ComparisonOperator, LogicalOperator, Operator}; +use crate::parser::{BuiltInFunction, Node}; +use crate::vm::Variable; -use crate::compiler::CompilerError::{ - ArgumentNotFound, UnknownBinaryOperator, UnknownBuiltIn, UnknownUnaryOperator, -}; -use crate::opcodes::{Opcode, TypeCheckKind, TypeConversionKind, Variable}; - -type Bytecode<'a> = Rc>>>; - -#[derive(Debug, Error)] -pub enum CompilerError { - #[error("Unknown unary operator: {operator}")] - UnknownUnaryOperator { operator: String }, - - #[error("Unknown binary operator: {operator}")] - UnknownBinaryOperator { operator: String }, - - #[error("Unknown builtin: {builtin}")] - UnknownBuiltIn { builtin: String }, - - #[error("Argument not found for builtin {builtin} at index {index}")] - ArgumentNotFound { builtin: String, index: usize }, +#[derive(Debug)] +pub struct Compiler<'arena> { + bytecode: Vec>, } -pub struct Compiler<'a> { - root: &'a Node<'a>, - bytecode: Bytecode<'a>, - bump: &'a Bump, -} - -impl<'a> Compiler<'a> { - pub fn new(root: &'a Node<'a>, bytecode: Bytecode<'a>, bump: &'a Bump) -> Self { +impl<'arena> Compiler<'arena> { + pub fn new() -> Self { Self { - root, - bytecode, - bump, + bytecode: Default::default(), } } - pub fn compile(&self) -> Result<(), CompilerError> { + pub fn compile(&mut self, root: &'arena Node<'arena>) -> CompilerResult<&[Opcode<'arena>]> { + self.bytecode.clear(); + + CompilerInner::new(&mut self.bytecode, root).compile()?; + Ok(self.bytecode.as_slice()) + } +} + +#[derive(Debug)] +struct CompilerInner<'arena, 'bytecode_ref> { + root: &'arena Node<'arena>, + bytecode: &'bytecode_ref mut Vec>, +} + +impl<'arena, 'bytecode_ref> CompilerInner<'arena, 'bytecode_ref> { + pub fn new( + bytecode: &'bytecode_ref mut Vec>, + root: &'arena Node<'arena>, + ) -> Self { + Self { root, bytecode } + } + + pub fn compile(&mut self) -> CompilerResult<()> { self.compile_node(self.root)?; Ok(()) } - fn emit(&self, op: Opcode<'a>) -> usize { - let bc = unsafe { &mut *self.bytecode.get() }; - bc.push(self.bump.alloc(op)); - bc.len() + fn emit(&mut self, op: Opcode<'arena>) -> usize { + self.bytecode.push(op); + self.bytecode.len() } - fn emit_loop(&self, mut body: F) -> Result<(), CompilerError> + fn emit_loop(&mut self, body: F) -> CompilerResult<()> where - F: FnMut() -> Result<(), CompilerError>, + F: FnOnce(&mut Self) -> CompilerResult<()>, { - let begin = unsafe { (*self.bytecode.get()).len() }; + let begin = self.bytecode.len(); let end = self.emit(Opcode::JumpIfEnd(0)); - body()?; + body(self)?; self.emit(Opcode::IncrementIt); let e = self.emit(Opcode::JumpBackward(self.calc_backward_jump(begin))); @@ -71,14 +66,14 @@ impl<'a> Compiler<'a> { Ok(()) } - fn emit_cond(&self, mut body: F) + fn emit_cond(&mut self, mut body: F) where - F: FnMut(), + F: FnMut(&mut Self), { let noop = self.emit(Opcode::JumpIfFalse(0)); self.emit(Opcode::Pop); - body(); + body(self); let jmp = self.emit(Opcode::Jump(0)); self.replace(noop, Opcode::JumpIfFalse(jmp - noop)); @@ -86,30 +81,31 @@ impl<'a> Compiler<'a> { self.replace(jmp, Opcode::Jump(e - jmp)); } - fn replace(&self, at: usize, op: Opcode<'a>) { - let bytecode = unsafe { &mut *self.bytecode.get() }; - let _ = std::mem::replace(&mut bytecode[at - 1], self.bump.alloc(op)); + fn replace(&mut self, at: usize, op: Opcode<'arena>) { + let _ = std::mem::replace(&mut self.bytecode[at - 1], op); } fn calc_backward_jump(&self, to: usize) -> usize { - unsafe { (*self.bytecode.get()).len() + 1 - to } + self.bytecode.len() + 1 - to } fn compile_argument( - &self, - name: &str, - arguments: &&[&'a Node<'a>], + &mut self, + builtin: &BuiltInFunction, + arguments: &[&'arena Node<'arena>], index: usize, - ) -> Result { - let arg = arguments.get(index).ok_or_else(|| ArgumentNotFound { - index, - builtin: name.to_string(), - })?; + ) -> CompilerResult { + let arg = arguments + .get(index) + .ok_or_else(|| CompilerError::ArgumentNotFound { + index, + builtin: builtin.to_string(), + })?; self.compile_node(arg) } - fn compile_node(&self, node: &'a Node<'a>) -> Result { + fn compile_node(&mut self, node: &'arena Node<'arena>) -> CompilerResult { match node { Node::Null => Ok(self.emit(Opcode::Push(Variable::Null))), Node::Bool(v) => Ok(self.emit(Opcode::Push(Variable::Bool(*v)))), @@ -182,10 +178,12 @@ impl<'a> Compiler<'a> { Node::Unary { node, operator } => { let curr = self.compile_node(node)?; match *operator { - "+" => Ok(curr), - "!" | "not" => Ok(self.emit(Opcode::Not)), - "-" => Ok(self.emit(Opcode::Negate)), - _ => Err(UnknownUnaryOperator { + Operator::Arithmetic(ArithmeticOperator::Add) => Ok(curr), + Operator::Arithmetic(ArithmeticOperator::Subtract) => { + Ok(self.emit(Opcode::Negate)) + } + Operator::Logical(LogicalOperator::Not) => Ok(self.emit(Opcode::Not)), + _ => Err(CompilerError::UnknownUnaryOperator { operator: operator.to_string(), }), } @@ -195,20 +193,20 @@ impl<'a> Compiler<'a> { right, operator, } => match *operator { - "==" => { + Operator::Comparison(ComparisonOperator::Equal) => { self.compile_node(left)?; self.compile_node(right)?; Ok(self.emit(Opcode::Equal)) } - "!=" => { + Operator::Comparison(ComparisonOperator::NotEqual) => { self.compile_node(left)?; self.compile_node(right)?; self.emit(Opcode::Equal); Ok(self.emit(Opcode::Not)) } - "or" => { + Operator::Logical(LogicalOperator::Or) => { self.compile_node(left)?; let end = self.emit(Opcode::JumpIfTrue(0)); self.emit(Opcode::Pop); @@ -217,7 +215,7 @@ impl<'a> Compiler<'a> { Ok(r) } - "and" => { + Operator::Logical(LogicalOperator::And) => { self.compile_node(left)?; let end = self.emit(Opcode::JumpIfFalse(0)); self.emit(Opcode::Pop); @@ -226,243 +224,254 @@ impl<'a> Compiler<'a> { Ok(r) } - "in" => { + Operator::Comparison(ComparisonOperator::In) => { self.compile_node(left)?; self.compile_node(right)?; Ok(self.emit(Opcode::In)) } - "not in" => { + Operator::Comparison(ComparisonOperator::NotIn) => { self.compile_node(left)?; self.compile_node(right)?; self.emit(Opcode::In); Ok(self.emit(Opcode::Not)) } - "<" => { + Operator::Comparison(ComparisonOperator::LessThan) => { self.compile_node(left)?; self.compile_node(right)?; Ok(self.emit(Opcode::Less)) } - "<=" => { + Operator::Comparison(ComparisonOperator::LessThanOrEqual) => { self.compile_node(left)?; self.compile_node(right)?; Ok(self.emit(Opcode::LessOrEqual)) } - ">" => { + Operator::Comparison(ComparisonOperator::GreaterThan) => { self.compile_node(left)?; self.compile_node(right)?; Ok(self.emit(Opcode::More)) } - ">=" => { + Operator::Comparison(ComparisonOperator::GreaterThanOrEqual) => { self.compile_node(left)?; self.compile_node(right)?; Ok(self.emit(Opcode::MoreOrEqual)) } - "+" => { + Operator::Arithmetic(ArithmeticOperator::Add) => { self.compile_node(left)?; self.compile_node(right)?; Ok(self.emit(Opcode::Add)) } - "-" => { + Operator::Arithmetic(ArithmeticOperator::Subtract) => { self.compile_node(left)?; self.compile_node(right)?; Ok(self.emit(Opcode::Subtract)) } - "*" => { + Operator::Arithmetic(ArithmeticOperator::Multiply) => { self.compile_node(left)?; self.compile_node(right)?; Ok(self.emit(Opcode::Multiply)) } - "/" => { + Operator::Arithmetic(ArithmeticOperator::Divide) => { self.compile_node(left)?; self.compile_node(right)?; Ok(self.emit(Opcode::Divide)) } - "%" => { + Operator::Arithmetic(ArithmeticOperator::Modulus) => { self.compile_node(left)?; self.compile_node(right)?; Ok(self.emit(Opcode::Modulo)) } - "^" => { + Operator::Arithmetic(ArithmeticOperator::Power) => { self.compile_node(left)?; self.compile_node(right)?; Ok(self.emit(Opcode::Exponent)) } - _ => Err(UnknownBinaryOperator { + _ => Err(CompilerError::UnknownBinaryOperator { operator: operator.to_string(), }), }, - Node::BuiltIn { name, arguments } => match *name { - "len" => { - self.compile_argument(name, arguments, 0)?; + Node::BuiltIn { kind, arguments } => match kind { + BuiltInFunction::Len => { + self.compile_argument(kind, arguments, 0)?; self.emit(Opcode::Len); self.emit(Opcode::Rot); Ok(self.emit(Opcode::Pop)) } - "date" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Date => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::ParseDateTime)) } - "time" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Time => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::ParseTime)) } - "duration" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Duration => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::ParseDuration)) } - "startsWith" => { - self.compile_argument(name, arguments, 0)?; - self.compile_argument(name, arguments, 1)?; + BuiltInFunction::StartsWith => { + self.compile_argument(kind, arguments, 0)?; + self.compile_argument(kind, arguments, 1)?; Ok(self.emit(Opcode::StartsWith)) } - "endsWith" => { - self.compile_argument(name, arguments, 0)?; - self.compile_argument(name, arguments, 1)?; + BuiltInFunction::EndsWith => { + self.compile_argument(kind, arguments, 0)?; + self.compile_argument(kind, arguments, 1)?; Ok(self.emit(Opcode::EndsWith)) } - "contains" => { - self.compile_argument(name, arguments, 0)?; - self.compile_argument(name, arguments, 1)?; + BuiltInFunction::Contains => { + self.compile_argument(kind, arguments, 0)?; + self.compile_argument(kind, arguments, 1)?; Ok(self.emit(Opcode::Contains)) } - "matches" => { - self.compile_argument(name, arguments, 0)?; - self.compile_argument(name, arguments, 1)?; + BuiltInFunction::Matches => { + self.compile_argument(kind, arguments, 0)?; + self.compile_argument(kind, arguments, 1)?; Ok(self.emit(Opcode::Matches)) } - "extract" => { - self.compile_argument(name, arguments, 0)?; - self.compile_argument(name, arguments, 1)?; + BuiltInFunction::Extract => { + self.compile_argument(kind, arguments, 0)?; + self.compile_argument(kind, arguments, 1)?; Ok(self.emit(Opcode::Extract)) } - "flatten" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Flatten => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::Flatten)) } - "upper" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Upper => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::Uppercase)) } - "lower" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Lower => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::Lowercase)) } - "abs" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Abs => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::Abs)) } - "avg" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Avg => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::Average)) } - "median" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Median => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::Median)) } - "mode" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Mode => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::Mode)) } - "max" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Max => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::Max)) } - "min" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Min => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::Min)) } - "sum" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Sum => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::Sum)) } - "floor" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Floor => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::Floor)) } - "ceil" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Ceil => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::Ceil)) } - "round" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Round => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::Round)) } - "rand" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Rand => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::Random)) } - "string" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::String => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::TypeConversion(TypeConversionKind::String))) } - "number" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Number => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::TypeConversion(TypeConversionKind::Number))) } - "isNumeric" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Bool => { + self.compile_argument(kind, arguments, 0)?; + Ok(self.emit(Opcode::TypeConversion(TypeConversionKind::Bool))) + } + BuiltInFunction::IsNumeric => { + self.compile_argument(kind, arguments, 0)?; Ok(self.emit(Opcode::TypeCheck(TypeCheckKind::Numeric))) } - "startOf" | "endOf" => { - self.compile_argument(name, arguments, 0)?; - self.compile_argument(name, arguments, 1)?; - Ok(self.emit(Opcode::DateFunction(name))) + BuiltInFunction::StartOf | BuiltInFunction::EndOf => { + self.compile_argument(kind, arguments, 0)?; + self.compile_argument(kind, arguments, 1)?; + Ok(self.emit(Opcode::DateFunction(kind.into()))) } - "dayOfWeek" | "dayOfMonth" | "dayOfYear" | "weekOfYear" | "monthOfYear" - | "monthString" | "weekdayString" | "year" | "dateString" => { - self.compile_argument(name, arguments, 0)?; - Ok(self.emit(Opcode::DateManipulation(name))) + BuiltInFunction::DayOfWeek + | BuiltInFunction::DayOfMonth + | BuiltInFunction::DayOfYear + | BuiltInFunction::WeekOfYear + | BuiltInFunction::MonthOfYear + | BuiltInFunction::MonthString + | BuiltInFunction::WeekdayString + | BuiltInFunction::Year + | BuiltInFunction::DateString => { + self.compile_argument(kind, arguments, 0)?; + Ok(self.emit(Opcode::DateManipulation(kind.into()))) } - "all" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::All => { + self.compile_argument(kind, arguments, 0)?; self.emit(Opcode::Begin); let mut loop_break: usize = 0; - self.emit_loop(|| { - self.compile_argument(name, arguments, 1)?; - loop_break = self.emit(Opcode::JumpIfFalse(0)); - self.emit(Opcode::Pop); + self.emit_loop(|c| { + c.compile_argument(kind, arguments, 1)?; + loop_break = c.emit(Opcode::JumpIfFalse(0)); + c.emit(Opcode::Pop); Ok(()) })?; let e = self.emit(Opcode::Push(Variable::Bool(true))); self.replace(loop_break, Opcode::JumpIfFalse(e - loop_break)); Ok(self.emit(Opcode::End)) } - "none" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::None => { + self.compile_argument(kind, arguments, 0)?; self.emit(Opcode::Begin); let mut loop_break: usize = 0; - self.emit_loop(|| { - self.compile_argument(name, arguments, 1)?; - self.emit(Opcode::Not); - loop_break = self.emit(Opcode::JumpIfFalse(0)); - self.emit(Opcode::Pop); + self.emit_loop(|c| { + c.compile_argument(kind, arguments, 1)?; + c.emit(Opcode::Not); + loop_break = c.emit(Opcode::JumpIfFalse(0)); + c.emit(Opcode::Pop); Ok(()) })?; let e = self.emit(Opcode::Push(Variable::Bool(true))); self.replace(loop_break, Opcode::JumpIfFalse(e - loop_break)); Ok(self.emit(Opcode::End)) } - "some" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Some => { + self.compile_argument(kind, arguments, 0)?; self.emit(Opcode::Begin); let mut loop_break: usize = 0; - self.emit_loop(|| { - self.compile_argument(name, arguments, 1)?; - loop_break = self.emit(Opcode::JumpIfTrue(0)); - self.emit(Opcode::Pop); + self.emit_loop(|c| { + c.compile_argument(kind, arguments, 1)?; + loop_break = c.emit(Opcode::JumpIfTrue(0)); + c.emit(Opcode::Pop); Ok(()) })?; let e = self.emit(Opcode::Push(Variable::Bool(false))); self.replace(loop_break, Opcode::JumpIfTrue(e - loop_break)); Ok(self.emit(Opcode::End)) } - "one" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::One => { + self.compile_argument(kind, arguments, 0)?; self.emit(Opcode::Begin); - self.emit_loop(|| { - self.compile_argument(name, arguments, 1)?; - self.emit_cond(|| { - self.emit(Opcode::IncrementCount); + self.emit_loop(|c| { + c.compile_argument(kind, arguments, 1)?; + c.emit_cond(|c| { + c.emit(Opcode::IncrementCount); }); Ok(()) })?; @@ -471,14 +480,14 @@ impl<'a> Compiler<'a> { self.emit(Opcode::Equal); Ok(self.emit(Opcode::End)) } - "filter" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Filter => { + self.compile_argument(kind, arguments, 0)?; self.emit(Opcode::Begin); - self.emit_loop(|| { - self.compile_argument(name, arguments, 1)?; - self.emit_cond(|| { - self.emit(Opcode::IncrementCount); - self.emit(Opcode::Pointer); + self.emit_loop(|c| { + c.compile_argument(kind, arguments, 1)?; + c.emit_cond(|c| { + c.emit(Opcode::IncrementCount); + c.emit(Opcode::Pointer); }); Ok(()) })?; @@ -486,22 +495,22 @@ impl<'a> Compiler<'a> { self.emit(Opcode::End); Ok(self.emit(Opcode::Array)) } - "map" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Map => { + self.compile_argument(kind, arguments, 0)?; self.emit(Opcode::Begin); - self.emit_loop(|| { - self.compile_argument(name, arguments, 1)?; + self.emit_loop(|c| { + c.compile_argument(kind, arguments, 1)?; Ok(()) })?; self.emit(Opcode::GetLen); self.emit(Opcode::End); Ok(self.emit(Opcode::Array)) } - "flatMap" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::FlatMap => { + self.compile_argument(kind, arguments, 0)?; self.emit(Opcode::Begin); - self.emit_loop(|| { - self.compile_argument(name, arguments, 1)?; + self.emit_loop(|c| { + c.compile_argument(kind, arguments, 1)?; Ok(()) })?; self.emit(Opcode::GetLen); @@ -509,22 +518,19 @@ impl<'a> Compiler<'a> { self.emit(Opcode::Array); Ok(self.emit(Opcode::Flatten)) } - "count" => { - self.compile_argument(name, arguments, 0)?; + BuiltInFunction::Count => { + self.compile_argument(kind, arguments, 0)?; self.emit(Opcode::Begin); - self.emit_loop(|| { - self.compile_argument(name, arguments, 1)?; - self.emit_cond(|| { - self.emit(Opcode::IncrementCount); + self.emit_loop(|c| { + c.compile_argument(kind, arguments, 1)?; + c.emit_cond(|c| { + c.emit(Opcode::IncrementCount); }); Ok(()) })?; self.emit(Opcode::GetCount); Ok(self.emit(Opcode::End)) } - _ => Err(UnknownBuiltIn { - builtin: name.to_string(), - }), }, } } diff --git a/core/expression/src/compiler/error.rs b/core/expression/src/compiler/error.rs new file mode 100644 index 00000000..fffe938f --- /dev/null +++ b/core/expression/src/compiler/error.rs @@ -0,0 +1,15 @@ +use thiserror::Error; + +#[derive(Debug, PartialEq, Eq, Clone, Error)] +pub enum CompilerError { + #[error("Unknown unary operator: {operator}")] + UnknownUnaryOperator { operator: String }, + + #[error("Unknown binary operator: {operator}")] + UnknownBinaryOperator { operator: String }, + + #[error("Argument not found for builtin {builtin} at index {index}")] + ArgumentNotFound { builtin: String, index: usize }, +} + +pub(crate) type CompilerResult = Result; diff --git a/core/expression/src/compiler/mod.rs b/core/expression/src/compiler/mod.rs new file mode 100644 index 00000000..88d07316 --- /dev/null +++ b/core/expression/src/compiler/mod.rs @@ -0,0 +1,10 @@ +//! Compilation from AST into Opcodes +//! +//! The Compiler module transforms an Abstract Syntax Tree (AST) representation of source code into machine-readable opcodes. +mod compiler; +mod error; +mod opcode; + +pub use compiler::Compiler; +pub use error::CompilerError; +pub use opcode::{Opcode, TypeCheckKind, TypeConversionKind}; diff --git a/core/expression/src/compiler/opcode.rs b/core/expression/src/compiler/opcode.rs new file mode 100644 index 00000000..ba33343e --- /dev/null +++ b/core/expression/src/compiler/opcode.rs @@ -0,0 +1,85 @@ +use crate::vm::Variable; +use strum_macros::Display; + +/// Machine code interpreted by VM +#[derive(Debug, PartialEq, Eq, Clone, Display)] +pub enum Opcode<'a> { + Push(Variable<'a>), + Pop, + Rot, + Fetch, + FetchEnv(&'a str), + Negate, + Not, + Equal, + Jump(usize), + JumpIfTrue(usize), + JumpIfFalse(usize), + JumpIfEnd(usize), + JumpBackward(usize), + In, + Less, + More, + LessOrEqual, + MoreOrEqual, + Abs, + Average, + Median, + Mode, + Min, + Max, + Round, + Floor, + Ceil, + Sum, + Random, + Add, + Subtract, + Multiply, + Divide, + Modulo, + Exponent, + Interval { + left_bracket: &'a str, + right_bracket: &'a str, + }, + Contains, + DateFunction(&'a str), + DateManipulation(&'a str), + Uppercase, + Lowercase, + StartsWith, + EndsWith, + Matches, + Extract, + Slice, + Array, + Len, + ParseDateTime, + ParseTime, + ParseDuration, + IncrementIt, + IncrementCount, + GetCount, + GetLen, + Pointer, + Begin, + End, + Flatten, + TypeConversion(TypeConversionKind), + TypeCheck(TypeCheckKind), +} + +/// Metadata for TypeConversion Opcode +#[derive(Debug, PartialEq, Eq, Clone, Copy, Display)] +pub enum TypeConversionKind { + Number, + String, + Bool, +} + +/// Metadata for TypeCheck Opcode +#[derive(Debug, PartialEq, Eq, Clone, Copy, Display)] +pub enum TypeCheckKind { + Numeric, +} diff --git a/core/expression/src/function.rs b/core/expression/src/function.rs new file mode 100644 index 00000000..dba5c98d --- /dev/null +++ b/core/expression/src/function.rs @@ -0,0 +1,21 @@ +use serde_json::Value; + +use crate::{Isolate, IsolateError}; + +/// Evaluates a standard expression +pub fn evaluate_expression(expression: &str, context: &Value) -> Result { + Isolate::with_environment(context).run_standard(expression) +} + +/// Evaluates a unary expression; Required: context must be an object with "$" key. +pub fn evaluate_unary_expression(expression: &str, context: &Value) -> Result { + let Some(context_object) = context.as_object() else { + return Err(IsolateError::MissingContextReference); + }; + + if !context_object.contains_key("$") { + return Err(IsolateError::MissingContextReference); + } + + Isolate::with_environment(context).run_unary(expression) +} diff --git a/core/expression/src/isolate.rs b/core/expression/src/isolate.rs index f49f18aa..e59607e8 100644 --- a/core/expression/src/isolate.rs +++ b/core/expression/src/isolate.rs @@ -1,9 +1,5 @@ -use std::cell::{RefCell, UnsafeCell}; use std::collections::HashMap; use std::hash::BuildHasherDefault; -use std::mem::ManuallyDrop; -use std::ops::DerefMut; -use std::rc::Rc; use ahash::AHasher; use bumpalo::Bump; @@ -12,18 +8,170 @@ use serde::{Serialize, Serializer}; use serde_json::Value; use thiserror::Error; -use crate::lexer::error::LexerError; -use crate::lexer::Lexer; -use crate::parser::error::ParserError; -use crate::parser::{StandardParser, UnaryParser}; - use crate::compiler::{Compiler, CompilerError}; -use crate::lexer::token::TokenKind; -use crate::opcodes::{Opcode, Variable}; -use crate::vm::{Scope, VMError, VM}; +use crate::lexer::{Lexer, LexerError}; +use crate::parser::{Parser, ParserError}; +use crate::vm::{VMError, Variable, VM}; type ADefHasher = BuildHasherDefault; +/// Isolate is a component that encapsulates an isolated environment for executing expressions. +/// +/// Rerunning the Isolate allows for efficient memory reuse through an arena allocator. +/// The arena allocator optimizes memory management by reusing memory blocks for subsequent evaluations, +/// contributing to improved performance and resource utilization in scenarios where the Isolate is reused multiple times. +#[derive(Debug)] +pub struct Isolate<'arena> { + lexer: Lexer<'arena>, + compiler: Compiler<'arena>, + vm: VM<'arena>, + + bump: Bump, + reference_bump: Bump, + + environment: Option<&'arena mut Variable<'arena>>, + references: HashMap<&'arena str, &'arena Variable<'arena>, ADefHasher>, +} + +impl<'a> Isolate<'a> { + pub fn new() -> Self { + Self { + lexer: Lexer::new(), + compiler: Compiler::new(), + vm: VM::new(), + + bump: Default::default(), + reference_bump: Default::default(), + + environment: None, + references: Default::default(), + } + } + + pub fn with_environment(value: &Value) -> Self { + let mut isolate = Isolate::new(); + isolate.set_environment(value); + + isolate + } + + pub fn set_environment(&mut self, value: &Value) { + let bump = self.get_reference_bump(); + let new_environment = Variable::from_serde(value, self.get_reference_bump()); + + self.environment.replace(bump.alloc(new_environment)); + } + + pub fn set_reference(&mut self, reference: &'a str) -> Result<(), IsolateError> { + let bump = self.get_reference_bump(); + + let reference_value = match self.references.get(reference) { + Some(value) => value, + None => { + let result = self.run_standard(reference)?; + let value = &*bump.alloc(Variable::from_serde(&result, bump)); + self.references.insert(reference, value); + value + } + }; + + let environment = self.environment.as_deref().unwrap_or(&Variable::Null); + if !matches!(environment, Variable::Object(_)) { + let new_environment = bump.alloc(Variable::empty_object_in(bump)); + self.environment.replace(new_environment); + } + + let Some(Variable::Object(environment_object)) = self.environment else { + return Err(IsolateError::ReferenceError); + }; + + environment_object.insert("$", reference_value); + Ok(()) + } + + pub fn get_reference(&self, reference: &str) -> Option { + let reference_variable = self.references.get(reference)?; + (*reference_variable).try_into().ok() + } + + pub fn run_standard(&mut self, source: &'a str) -> Result { + self.bump.reset(); + let bump = self.get_bump(); + + let tokens = self + .lexer + .tokenize(source) + .map_err(|source| IsolateError::LexerError { source })?; + + let parser = Parser::try_new(tokens, bump) + .map_err(|source| IsolateError::ParserError { source })? + .standard(); + + let ast = parser + .parse() + .map_err(|source| IsolateError::ParserError { source })?; + + let bytecode = self + .compiler + .compile(ast) + .map_err(|source| IsolateError::CompilerError { source })?; + + let result = self + .vm + .run( + bytecode, + bump, + self.environment.as_deref().unwrap_or(&Variable::Null), + ) + .map_err(|source| IsolateError::VMError { source })?; + + result.try_into().map_err(|_| IsolateError::ValueCastError) + } + + pub fn run_unary(&mut self, source: &'a str) -> Result { + self.bump.reset(); + let bump = self.get_bump(); + + let tokens = self + .lexer + .tokenize(source) + .map_err(|source| IsolateError::LexerError { source })?; + + let parser = Parser::try_new(tokens, bump) + .map_err(|source| IsolateError::ParserError { source })? + .unary(); + + let ast = parser + .parse() + .map_err(|source| IsolateError::ParserError { source })?; + + let bytecode = self + .compiler + .compile(ast) + .map_err(|source| IsolateError::CompilerError { source })?; + + let result = self + .vm + .run( + bytecode, + bump, + self.environment.as_deref().unwrap_or(&Variable::Null), + ) + .map_err(|source| IsolateError::VMError { source })?; + + result.as_bool().ok_or_else(|| IsolateError::ValueCastError) + } + + fn get_bump(&self) -> &'a Bump { + unsafe { std::mem::transmute::<&Bump, &'a Bump>(&self.bump) } + } + + fn get_reference_bump(&self) -> &'a Bump { + unsafe { std::mem::transmute::<&Bump, &'a Bump>(&self.reference_bump) } + } +} + +/// Errors which happen within isolate or during evaluation #[derive(Debug, Error)] pub enum IsolateError { #[error("Lexer error")] @@ -43,6 +191,9 @@ pub enum IsolateError { #[error("Failed to compute reference")] ReferenceError, + + #[error("Missing context reference")] + MissingContextReference, } impl Serialize for IsolateError { @@ -56,6 +207,9 @@ impl Serialize for IsolateError { IsolateError::ReferenceError => { map.serialize_entry("type", "referenceError")?; } + IsolateError::MissingContextReference => { + map.serialize_entry("type", "missingContextReference")?; + } IsolateError::ValueCastError => { map.serialize_entry("type", "valueCastError")?; } @@ -80,190 +234,3 @@ impl Serialize for IsolateError { map.end() } } - -#[derive(Debug)] -pub struct Isolate<'a> { - lexer: Lexer<'a>, - bump: RefCell, - reference_bump: Bump, - bytecode: Rc>>>, - stack: UnsafeCell>>, - scopes: UnsafeCell>>, - environment: UnsafeCell>>, - references: RefCell, ADefHasher>>, -} - -impl<'a> Default for Isolate<'a> { - fn default() -> Self { - Self { - lexer: Lexer::new(), - bump: Default::default(), - reference_bump: Default::default(), - bytecode: Default::default(), - stack: Default::default(), - scopes: Default::default(), - environment: UnsafeCell::new(ManuallyDrop::new(Variable::Null)), - references: Default::default(), - } - } -} - -impl<'a> Isolate<'a> { - pub fn inject_env(&self, value: &Value) { - let new_env = Variable::from_serde(value, self.get_reference_bump()); - let env = unsafe { &mut (*self.environment.get()) }; - *env = ManuallyDrop::new(new_env); - } - - fn get_bump(&self) -> &'a Bump { - unsafe { std::mem::transmute::<&Bump, &'a Bump>(&self.bump.borrow()) } - } - - fn get_reference_bump(&self) -> &'a Bump { - unsafe { std::mem::transmute(&self.reference_bump) } - } - - pub fn set_reference(&self, reference: &'a str) -> Result<(), IsolateError> { - let mut references = self.references.borrow_mut(); - let bump = self.get_reference_bump(); - - if !references.contains_key(reference) { - let result = self.run_standard(reference)?; - let value = bump.alloc(Variable::from_serde(&result, bump)); - - references.insert(reference, value); - } - - let value = references - .get(reference) - .ok_or(IsolateError::ReferenceError)?; - - let env_w = unsafe { &mut (*self.environment.get()) }; - let env = env_w.deref_mut(); - match env { - Variable::Object(..) => { - // - } - _ => { - *env = Variable::empty_object_in(bump); - } - } - - if let Variable::Object(obj) = env { - obj.insert("$", value); - } - - Ok(()) - } - - pub fn get_reference(&self, reference: &str) -> Option { - let refs = self.references.borrow(); - let var = refs.get(reference)?; - - (*var).try_into().ok() - } - - pub fn run_standard(&self, source: &'a str) -> Result { - self.clear(); - - let tokens = self - .lexer - .tokenize(source) - .map_err(|source| IsolateError::LexerError { source })?; - - let tkn = tokens.borrow(); - let bump = self.get_bump(); - - let parser = StandardParser::try_new(tkn.as_ref(), bump) - .map_err(|source| IsolateError::ParserError { source })?; - - let ast = parser - .parse() - .map_err(|source| IsolateError::ParserError { source })?; - - let compiler = Compiler::new(ast, self.bytecode.clone(), bump); - compiler - .compile() - .map_err(|source| IsolateError::CompilerError { source })?; - - let mut vm = unsafe { - VM::new( - &*self.bytecode.get(), - &mut *self.stack.get(), - &mut *self.scopes.get(), - bump, - ) - }; - - let res = vm - .run(unsafe { &*self.environment.get() }) - .map_err(|source| IsolateError::VMError { source })?; - - res.try_into().map_err(|_| IsolateError::ValueCastError) - } - - /// Runs unary test - /// If reference identifier is present ($) it will use standard parser - pub fn run_unary(&self, source: &'a str) -> Result { - self.clear(); - - let tokens = self - .lexer - .tokenize(source) - .map_err(|source| IsolateError::LexerError { source })?; - - let tkn = tokens.borrow(); - let unary_disallowed = tkn - .iter() - .any(|token| token.kind == TokenKind::Identifier && token.value == "$"); - - let bump = self.get_bump(); - let ast = match unary_disallowed { - true => { - let parser = StandardParser::try_new(tkn.as_ref(), bump) - .map_err(|source| IsolateError::ParserError { source })?; - - parser - .parse() - .map_err(|source| IsolateError::ParserError { source })? - } - false => { - let parser = UnaryParser::try_new(tkn.as_ref(), bump) - .map_err(|source| IsolateError::ParserError { source })?; - - parser - .parse() - .map_err(|source| IsolateError::ParserError { source })? - } - }; - - let compiler = Compiler::new(ast, self.bytecode.clone(), bump); - compiler - .compile() - .map_err(|source| IsolateError::CompilerError { source })?; - - let mut vm = unsafe { - VM::new( - &*self.bytecode.get(), - &mut *self.stack.get(), - &mut *self.scopes.get(), - bump, - ) - }; - - let res = vm - .run(unsafe { &*self.environment.get() }) - .map_err(|source| IsolateError::VMError { source })?; - - res.try_into().map_err(|_| IsolateError::ValueCastError) - } - - fn clear(&self) { - self.bump.borrow_mut().reset(); - unsafe { - (*self.bytecode.get()).clear(); - (*self.stack.get()).clear(); - (*self.scopes.get()).clear(); - } - } -} diff --git a/core/expression/src/lexer/codes.rs b/core/expression/src/lexer/codes.rs index 8e126e97..79b23a3c 100644 --- a/core/expression/src/lexer/codes.rs +++ b/core/expression/src/lexer/codes.rs @@ -1,18 +1,19 @@ -#[macro_export] macro_rules! token_type { ("space") => { ' ' | '\n' | '\t' }; ("quote") => { '\'' | '"' }; ("digit") => { '0'..='9' }; ("bracket") => { '(' | ')' | '[' | ']' }; ("cmp_operator") => { '>' | '<' | '!' | '=' }; - ("operator") => { '#' | ',' | '?' | ':' | '+' | '-' | '/' | '*' | '^' | '%' }; - ("alpha") => { 'A'..='Z' | 'a'..='z' | '$' | '_' }; - ("alphanumeric") => { 'A'..='Z' | 'a'..='z' | '0'..='9' | '$' | '_' }; + ("operator") => { ',' | '?' | ':' | '+' | '-' | '/' | '*' | '^' | '%' }; + ("alpha") => { 'A'..='Z' | 'a'..='z' | '$' | '_' | '#' }; + ("alphanumeric") => { 'A'..='Z' | 'a'..='z' | '0'..='9' | '$' | '_' | '#' }; } -#[macro_export] macro_rules! is_token_type { ($str: expr, $t: tt) => { - matches!($str, crate::token_type!($t)) + matches!($str, crate::lexer::codes::token_type!($t)) }; } + +pub(crate) use is_token_type; +pub(crate) use token_type; diff --git a/core/expression/src/lexer/cursor.rs b/core/expression/src/lexer/cursor.rs index d8705c45..08775326 100644 --- a/core/expression/src/lexer/cursor.rs +++ b/core/expression/src/lexer/cursor.rs @@ -1,12 +1,12 @@ use std::cell::Cell; #[derive(Debug)] -pub(crate) struct Cursor<'a> { +pub(super) struct Cursor<'a> { chars: &'a [u8], current: Cell, } -pub(crate) type CursorItem = (usize, char); +pub(super) type CursorItem = (usize, char); impl<'a> From<&'a str> for Cursor<'a> { fn from(source: &'a str) -> Self { diff --git a/core/expression/src/lexer/error.rs b/core/expression/src/lexer/error.rs index da8bbcc0..a8f8abfd 100644 --- a/core/expression/src/lexer/error.rs +++ b/core/expression/src/lexer/error.rs @@ -1,6 +1,7 @@ +use strum::ParseError; use thiserror::Error; -#[derive(Debug, Error)] +#[derive(Debug, PartialEq, Eq, Clone, Error)] pub enum LexerError { #[error("Unexpected symbol: {symbol}")] UnexpectedSymbol { symbol: String }, @@ -11,3 +12,13 @@ pub enum LexerError { #[error("Unexpected end of file: {symbol} at position {position}")] UnexpectedEof { symbol: char, position: usize }, } + +impl From for LexerError { + fn from(value: ParseError) -> Self { + Self::UnexpectedSymbol { + symbol: value.to_string(), + } + } +} + +pub(crate) type LexerResult = Result; diff --git a/core/expression/src/lexer/lexer.rs b/core/expression/src/lexer/lexer.rs index 9495bf39..eabf8fd4 100644 --- a/core/expression/src/lexer/lexer.rs +++ b/core/expression/src/lexer/lexer.rs @@ -1,49 +1,37 @@ -use std::cell::RefCell; -use std::rc::Rc; - +use crate::lexer::codes::{is_token_type, token_type}; use crate::lexer::cursor::{Cursor, CursorItem}; -use crate::lexer::error::LexerError; use crate::lexer::error::LexerError::{UnexpectedEof, UnmatchedSymbol}; -use crate::lexer::token::{Token, TokenKind}; -use crate::{is_token_type, token_type}; +use crate::lexer::error::LexerResult; +use crate::lexer::token::{ + Bracket, ComparisonOperator, Identifier, LogicalOperator, Operator, Token, TokenKind, +}; -type TokenSlice<'a> = Rc>>>; - -type VoidResult = Result<(), LexerError>; - -#[derive(Debug)] -pub struct Lexer<'a> { - tokens: TokenSlice<'a>, +#[derive(Debug, Default)] +pub struct Lexer<'arena> { + tokens: Vec>, } -impl<'a> Default for Lexer<'a> { - fn default() -> Self { - Lexer::new() - } -} - -impl<'a> Lexer<'a> { +impl<'arena> Lexer<'arena> { pub fn new() -> Self { - Self { - tokens: Rc::new(RefCell::new(Vec::new())), - } + Self::default() } - pub fn tokenize(&self, source: &'a str) -> Result, LexerError> { - self.tokens.borrow_mut().clear(); - Scanner::new(source, self.tokens.clone()).scan()?; - Ok(self.tokens.clone()) + pub fn tokenize(&mut self, source: &'arena str) -> LexerResult<&[Token<'arena>]> { + self.tokens.clear(); + + Scanner::new(source, &mut self.tokens).scan()?; + Ok(&self.tokens) } } -struct Scanner<'a> { - cursor: Cursor<'a>, - tokens: TokenSlice<'a>, - source: &'a str, +struct Scanner<'arena, 'self_ref> { + cursor: Cursor<'arena>, + tokens: &'self_ref mut Vec>, + source: &'arena str, } -impl<'a> Scanner<'a> { - pub fn new(source: &'a str, tokens: TokenSlice<'a>) -> Self { +impl<'arena, 'self_ref> Scanner<'arena, 'self_ref> { + pub fn new(source: &'arena str, tokens: &'self_ref mut Vec>) -> Self { Self { cursor: Cursor::from(source), source, @@ -51,7 +39,7 @@ impl<'a> Scanner<'a> { } } - pub fn scan(&self) -> VoidResult { + pub fn scan(&mut self) -> LexerResult<()> { while let Some((i, s)) = self.cursor.peek() { match s { token_type!("space") => { @@ -76,7 +64,7 @@ impl<'a> Scanner<'a> { Ok(()) } - fn next(&self) -> Result { + fn next(&self) -> LexerResult { self.cursor.next().ok_or_else(|| { let (a, b) = self.cursor.peek_back().unwrap_or((0, ' ')); @@ -87,11 +75,11 @@ impl<'a> Scanner<'a> { }) } - fn push(&self, token: Token<'a>) { - self.tokens.borrow_mut().push(token); + fn push(&mut self, token: Token<'arena>) { + self.tokens.push(token); } - fn string(&self) -> VoidResult { + fn string(&mut self) -> LexerResult<()> { let (start, opener) = self.next()?; let end: usize; @@ -112,7 +100,7 @@ impl<'a> Scanner<'a> { Ok(()) } - fn number(&self) -> VoidResult { + fn number(&mut self) -> LexerResult<()> { let (start, _) = self.next()?; let mut end = start; let mut fractal = false; @@ -149,19 +137,20 @@ impl<'a> Scanner<'a> { Ok(()) } - fn bracket(&self) -> VoidResult { + fn bracket(&mut self) -> LexerResult<()> { let (start, _) = self.next()?; + let value = &self.source[start..=start]; self.push(Token { - kind: TokenKind::Bracket, + kind: TokenKind::Bracket(Bracket::try_from(value)?), span: (start, start + 1), - value: &self.source[start..=start], + value, }); Ok(()) } - fn dot(&self) -> VoidResult { + fn dot(&mut self) -> LexerResult<()> { let (start, _) = self.next()?; let mut end = start; @@ -169,16 +158,17 @@ impl<'a> Scanner<'a> { end += 1; } + let value = &self.source[start..=end]; self.push(Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::try_from(value)?), span: (start, end + 1), - value: &self.source[start..=end], + value, }); Ok(()) } - fn cmp_operator(&self) -> VoidResult { + fn cmp_operator(&mut self) -> LexerResult<()> { let (start, _) = self.next()?; let mut end = start; @@ -186,33 +176,35 @@ impl<'a> Scanner<'a> { end += 1; } + let value = &self.source[start..=end]; self.push(Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::try_from(value)?), span: (start, end + 1), - value: &self.source[start..=end], + value, }); Ok(()) } - fn operator(&self) -> VoidResult { + fn operator(&mut self) -> LexerResult<()> { let (start, _) = self.next()?; + let value = &self.source[start..=start]; self.push(Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::try_from(value)?), span: (start, start + 1), - value: &self.source[start..=start], + value, }); Ok(()) } - fn not(&self, start: usize) -> VoidResult { + fn not(&mut self, start: usize) -> LexerResult<()> { if self.cursor.next_if_is(" in ") { let end = self.cursor.position(); self.push(Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Comparison(ComparisonOperator::NotIn)), span: (start, end - 1), value: "not in", }) @@ -220,7 +212,7 @@ impl<'a> Scanner<'a> { let end = self.cursor.position(); self.push(Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Logical(LogicalOperator::Not)), span: (start, end), value: "not", }) @@ -229,7 +221,7 @@ impl<'a> Scanner<'a> { Ok(()) } - fn identifier(&self) -> VoidResult { + fn identifier(&mut self) -> LexerResult<()> { let (start, _) = self.next()?; let mut end = start; @@ -239,14 +231,34 @@ impl<'a> Scanner<'a> { let value = &self.source[start..=end]; match value { - "and" | "or" | "in" => self.push(Token { - kind: TokenKind::Operator, + "and" => self.push(Token { + kind: TokenKind::Operator(Operator::Logical(LogicalOperator::And)), + span: (start, end + 1), + value, + }), + "or" => self.push(Token { + kind: TokenKind::Operator(Operator::Logical(LogicalOperator::Or)), + span: (start, end + 1), + value, + }), + "in" => self.push(Token { + kind: TokenKind::Operator(Operator::Comparison(ComparisonOperator::In)), + span: (start, end + 1), + value, + }), + "true" => self.push(Token { + kind: TokenKind::Boolean(true), + span: (start, end + 1), + value, + }), + "false" => self.push(Token { + kind: TokenKind::Boolean(false), span: (start, end + 1), value, }), "not" => self.not(start)?, _ => self.push(Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::from(value)), span: (start, end + 1), value, }), diff --git a/core/expression/src/lexer/mod.rs b/core/expression/src/lexer/mod.rs index 4ddef8ce..f54af4fb 100644 --- a/core/expression/src/lexer/mod.rs +++ b/core/expression/src/lexer/mod.rs @@ -1,8 +1,13 @@ -pub use lexer::Lexer; - -pub mod error; -pub mod token; +//! Performs lexical analysis on string inputs +//! +//! The Lexer module transforms strings into tokens using Strum. +mod error; +mod token; mod codes; mod cursor; mod lexer; + +pub use error::LexerError; +pub use lexer::Lexer; +pub use token::*; diff --git a/core/expression/src/lexer/token.rs b/core/expression/src/lexer/token.rs index 18b5cd80..f8cf5636 100644 --- a/core/expression/src/lexer/token.rs +++ b/core/expression/src/lexer/token.rs @@ -1,3 +1,9 @@ +use std::hash::{Hash, Hasher}; + +use nohash_hasher::IsEnabled; +use strum_macros::{Display, EnumString}; + +/// Contains information from lexical analysis #[derive(Debug, PartialEq, Eq, Clone)] pub struct Token<'a> { pub span: (usize, usize), @@ -5,11 +11,163 @@ pub struct Token<'a> { pub value: &'a str, } -#[derive(Debug, PartialEq, Eq, Clone)] +/// Classification of tokens +#[derive(Debug, PartialEq, Eq, Clone, Copy, Display)] pub enum TokenKind { - Identifier, + Identifier(Identifier), + Boolean(bool), Number, String, - Operator, - Bracket, + Operator(Operator), + Bracket(Bracket), } + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Display)] +pub enum Identifier { + ContextReference, // $ + CallbackReference, // # + Null, // null + Variable, +} + +impl From<&str> for Identifier { + fn from(value: &str) -> Self { + match value { + "$" => Identifier::ContextReference, + "#" => Identifier::CallbackReference, + "null" => Identifier::Null, + _ => Identifier::Variable, + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Display)] +pub enum Operator { + Arithmetic(ArithmeticOperator), + Logical(LogicalOperator), + Comparison(ComparisonOperator), + Range, // .. + Comma, // , + Slice, // : + Dot, // . + QuestionMark, // ? +} + +impl TryFrom<&str> for Operator { + type Error = strum::ParseError; + + fn try_from(operator: &str) -> Result { + match operator { + ".." => Ok(Operator::Range), + "," => Ok(Operator::Comma), + ":" => Ok(Operator::Slice), + "." => Ok(Operator::Dot), + "?" => Ok(Operator::QuestionMark), + _ => ArithmeticOperator::try_from(operator) + .map(Operator::Arithmetic) + .or_else(|_| LogicalOperator::try_from(operator).map(Operator::Logical)) + .or_else(|_| ComparisonOperator::try_from(operator).map(Operator::Comparison)), + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Display, EnumString)] +pub enum ArithmeticOperator { + #[strum(serialize = "+")] + Add, + #[strum(serialize = "-")] + Subtract, + #[strum(serialize = "*")] + Multiply, + #[strum(serialize = "/")] + Divide, + #[strum(serialize = "%")] + Modulus, + #[strum(serialize = "^")] + Power, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Display, EnumString)] +pub enum LogicalOperator { + #[strum(serialize = "and")] + And, + #[strum(serialize = "or")] + Or, + #[strum(serialize = "not", serialize = "!")] + Not, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Display, EnumString)] +pub enum ComparisonOperator { + #[strum(serialize = "==")] + Equal, + #[strum(serialize = "!=")] + NotEqual, + #[strum(serialize = "<")] + LessThan, + #[strum(serialize = ">")] + GreaterThan, + #[strum(serialize = "<=")] + LessThanOrEqual, + #[strum(serialize = ">=")] + GreaterThanOrEqual, + #[strum(serialize = "in")] + In, + #[strum(serialize = "not in")] + NotIn, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Display, EnumString)] +pub enum Bracket { + #[strum(serialize = "(")] + LeftParenthesis, + #[strum(serialize = ")")] + RightParenthesis, + #[strum(serialize = "[")] + LeftSquareBracket, + #[strum(serialize = "]")] + RightSquareBracket, +} + +impl Operator { + pub fn variant(&self) -> u8 { + match &self { + Operator::Arithmetic(a) => match a { + ArithmeticOperator::Add => 1, + ArithmeticOperator::Subtract => 2, + ArithmeticOperator::Multiply => 3, + ArithmeticOperator::Divide => 4, + ArithmeticOperator::Modulus => 5, + ArithmeticOperator::Power => 6, + }, + Operator::Logical(l) => match l { + LogicalOperator::And => 7, + LogicalOperator::Or => 8, + LogicalOperator::Not => 9, + }, + Operator::Comparison(c) => match c { + ComparisonOperator::Equal => 10, + ComparisonOperator::NotEqual => 11, + ComparisonOperator::LessThan => 12, + ComparisonOperator::GreaterThan => 13, + ComparisonOperator::LessThanOrEqual => 14, + ComparisonOperator::GreaterThanOrEqual => 15, + ComparisonOperator::In => 16, + ComparisonOperator::NotIn => 17, + }, + Operator::Range => 18, + Operator::Comma => 19, + Operator::Slice => 20, + Operator::Dot => 21, + Operator::QuestionMark => 22, + } + } +} + +impl Hash for Operator { + fn hash(&self, state: &mut H) { + state.write_u8(self.variant()); + } +} + +impl IsEnabled for Operator {} diff --git a/core/expression/src/lib.rs b/core/expression/src/lib.rs index 91b069e7..f3457a5b 100644 --- a/core/expression/src/lib.rs +++ b/core/expression/src/lib.rs @@ -1,14 +1,62 @@ -#![deny(clippy::unwrap_used)] -#![deny(clippy::expect_used)] +//! A lightweight expression language designed for the evaluation of expressions in various contexts. +//! +//! Zen Expression is a versatile single-threaded expression language designed for simplicity +//! high-performance. It's primarily used for evaluating and processing JSON data offers key components that empower developers in creating responsive and +//! non-blocking I/O applications +//! Out of the box, it comes with amazing benefits: +//! - 🚀 Blazingly fast - Perform millions of evaluations per second +//! - 🧠 Intuitive syntax - Minimalistic and expressive syntax +//! - 💼 Portable - Can be compiled for all standard architectures including WASM +//! +//! For a full list of language references, visit [documentation](https://gorules.io/docs/rules-engine/expression-language/). +//! +//! # Example +//! Evaluate expression using isolate: +//! ``` +//! use zen_expression::{evaluate_expression, json}; +//! +//! fn main() { +//! let context = json!({ "tax": { "percentage": 10 } }); +//! let tax_amount = evaluate_expression("50 * tax.percentage / 100", &context).unwrap(); +//! +//! assert_eq!(tax_amount, json!(5)); +//! } +//! ``` +//! +//! ## High Performance +//! When evaluating a lot of expressions at once, you can use Isolate directly. Under the hood, Isolate +//! will re-use allocated memory from previous evaluations, drastically improving performance. +//! +//! ``` +//! use zen_expression::{Isolate, json}; +//! +//! fn main() { +//! let context = json!({ "tax": { "percentage": 10 } }); +//! let mut isolate = Isolate::with_environment(&context); +//! +//! // Fast 🚀 +//! for _ in 0..1_000 { +//! let tax_amount = isolate.run_standard("50 * tax.percentage / 100").unwrap(); +//! assert_eq!(tax_amount, json!(5)); +//! } +//! } +//! ``` +//! +//! # Feature flags +//! +//! Name | Description | Default? +//! ---|---|--- +//! `regex-deprecated` | Uses standard `regex` crate | Yes +//! `regex-lite` | Opts for usage of lightweight `regex-lite` crate. Useful for reducing build size, especially in WASM. | No -extern crate core; +mod isolate; -mod helpers; - -pub mod ast; pub mod compiler; -pub mod isolate; +mod function; pub mod lexer; -pub mod opcodes; pub mod parser; pub mod vm; + +pub use function::{evaluate_expression, evaluate_unary_expression}; +pub use isolate::{Isolate, IsolateError}; +pub use serde_json::json; diff --git a/core/expression/src/ast.rs b/core/expression/src/parser/ast.rs similarity index 84% rename from core/expression/src/ast.rs rename to core/expression/src/parser/ast.rs index ed024a63..0720dc21 100644 --- a/core/expression/src/ast.rs +++ b/core/expression/src/parser/ast.rs @@ -1,5 +1,8 @@ use rust_decimal::Decimal; +use crate::lexer::Operator; +use crate::parser::builtin::BuiltInFunction; + #[derive(Debug, PartialEq, Clone)] pub enum Node<'a> { Null, @@ -32,15 +35,15 @@ pub enum Node<'a> { }, Unary { node: &'a Node<'a>, - operator: &'a str, + operator: Operator, }, Binary { left: &'a Node<'a>, - operator: &'a str, + operator: Operator, right: &'a Node<'a>, }, BuiltIn { - name: &'a str, + kind: BuiltInFunction, arguments: &'a [&'a Node<'a>], }, } diff --git a/core/expression/src/parser/builtin.rs b/core/expression/src/parser/builtin.rs new file mode 100644 index 00000000..8d688713 --- /dev/null +++ b/core/expression/src/parser/builtin.rs @@ -0,0 +1,136 @@ +use strum_macros::{Display, EnumString, IntoStaticStr}; + +#[derive(Debug, PartialEq)] +pub(crate) enum Arity { + Single, + Closure, + Dual, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, EnumString, Display, IntoStaticStr)] +#[strum(serialize_all = "camelCase")] +pub enum BuiltInFunction { + // General + Len, + Contains, + Flatten, + + // String + Upper, + Lower, + StartsWith, + EndsWith, + Matches, + Extract, + + // Math + Abs, + Sum, + Avg, + Min, + Max, + Rand, + Median, + Mode, + Floor, + Ceil, + Round, + + // Type + IsNumeric, + String, + Number, + Bool, + // Json, + + // Date + time + Date, + Time, + Duration, + Year, + DayOfWeek, + DayOfMonth, + DayOfYear, + WeekOfYear, + MonthOfYear, + MonthString, + DateString, + WeekdayString, + StartOf, + EndOf, + + // Closures + All, + Some, + None, + Filter, + Map, + Count, + One, + FlatMap, +} + +impl BuiltInFunction { + pub(crate) fn arity(&self) -> Arity { + match &self { + // General + BuiltInFunction::Len => Arity::Single, + BuiltInFunction::Contains => Arity::Dual, + BuiltInFunction::Flatten => Arity::Single, + + // String + BuiltInFunction::Upper => Arity::Single, + BuiltInFunction::Lower => Arity::Single, + BuiltInFunction::StartsWith => Arity::Dual, + BuiltInFunction::EndsWith => Arity::Dual, + BuiltInFunction::Matches => Arity::Dual, + BuiltInFunction::Extract => Arity::Dual, + + // Math + BuiltInFunction::Abs => Arity::Single, + BuiltInFunction::Sum => Arity::Single, + BuiltInFunction::Avg => Arity::Single, + BuiltInFunction::Min => Arity::Single, + BuiltInFunction::Max => Arity::Single, + BuiltInFunction::Rand => Arity::Single, + BuiltInFunction::Median => Arity::Single, + BuiltInFunction::Mode => Arity::Single, + BuiltInFunction::Floor => Arity::Single, + BuiltInFunction::Ceil => Arity::Single, + BuiltInFunction::Round => Arity::Single, + + // Date + time + BuiltInFunction::Date => Arity::Single, + BuiltInFunction::Time => Arity::Single, + BuiltInFunction::Duration => Arity::Single, + BuiltInFunction::Year => Arity::Single, + BuiltInFunction::DayOfWeek => Arity::Single, + BuiltInFunction::DayOfMonth => Arity::Single, + BuiltInFunction::DayOfYear => Arity::Single, + BuiltInFunction::WeekOfYear => Arity::Single, + BuiltInFunction::MonthOfYear => Arity::Single, + BuiltInFunction::MonthString => Arity::Single, + BuiltInFunction::DateString => Arity::Single, + BuiltInFunction::WeekdayString => Arity::Single, + BuiltInFunction::StartOf => Arity::Dual, + BuiltInFunction::EndOf => Arity::Dual, + + // Type + BuiltInFunction::String => Arity::Single, + BuiltInFunction::Number => Arity::Single, + BuiltInFunction::Bool => Arity::Single, + // BuiltInFunction::Json => Arity::Single, + BuiltInFunction::IsNumeric => Arity::Single, + + // Closure + BuiltInFunction::All => Arity::Closure, + BuiltInFunction::Some => Arity::Closure, + BuiltInFunction::None => Arity::Closure, + BuiltInFunction::Filter => Arity::Closure, + BuiltInFunction::Map => Arity::Closure, + BuiltInFunction::Count => Arity::Closure, + BuiltInFunction::One => Arity::Closure, + BuiltInFunction::FlatMap => Arity::Closure, + } + } +} diff --git a/core/expression/src/parser/constants.rs b/core/expression/src/parser/constants.rs new file mode 100644 index 00000000..e9a170b9 --- /dev/null +++ b/core/expression/src/parser/constants.rs @@ -0,0 +1,71 @@ +use nohash_hasher::BuildNoHashHasher; +use once_cell::sync::Lazy; +use std::collections::HashMap; + +use crate::lexer::{ArithmeticOperator, ComparisonOperator, LogicalOperator, Operator}; +use Associativity::{Left, Right}; + +type NoHasher = BuildNoHashHasher; + +#[derive(Debug, PartialEq)] +pub(crate) enum Associativity { + Left, + Right, +} + +#[derive(Debug, PartialEq)] +pub(crate) struct ParserOperator { + pub precedence: u8, + pub associativity: Associativity, +} + +macro_rules! hashmap { + (@single $($x:tt)*) => (()); + (@count $($rest:expr),*) => (<[()]>::len(&[$(hashmap!(@single $rest)),*])); + + ($($key:expr => $value:expr,)+) => { hashmap!($($key => $value),+) }; + ($($key:expr => $value:expr),*) => { + { + let _cap = hashmap!(@count $($key),*); + let mut _map = ::std::collections::HashMap::default(); + _map.reserve(_cap); + $( + let _ = _map.insert($key, $value); + )* + _map + } + }; +} + +pub(crate) static BINARY_OPERATORS: Lazy> = Lazy::new( + || { + hashmap! { + Operator::Logical(LogicalOperator::Or) => ParserOperator { precedence: 10, associativity: Left }, + Operator::Logical(LogicalOperator::And) => ParserOperator { precedence: 15, associativity: Left }, + Operator::Comparison(ComparisonOperator::Equal) => ParserOperator { precedence: 20, associativity: Left }, + Operator::Comparison(ComparisonOperator::NotEqual) => ParserOperator { precedence: 20, associativity: Left }, + Operator::Comparison(ComparisonOperator::LessThan) => ParserOperator { precedence: 20, associativity: Left }, + Operator::Comparison(ComparisonOperator::GreaterThan) => ParserOperator { precedence: 20, associativity: Left }, + Operator::Comparison(ComparisonOperator::LessThanOrEqual) => ParserOperator { precedence: 20, associativity: Left }, + Operator::Comparison(ComparisonOperator::GreaterThanOrEqual) => ParserOperator { precedence: 20, associativity: Left }, + Operator::Comparison(ComparisonOperator::NotIn) => ParserOperator { precedence: 20, associativity: Left }, + Operator::Comparison(ComparisonOperator::In) => ParserOperator { precedence: 20, associativity: Left }, + Operator::Arithmetic(ArithmeticOperator::Add) => ParserOperator { precedence: 30, associativity: Left }, + Operator::Arithmetic(ArithmeticOperator::Subtract) => ParserOperator { precedence: 30, associativity: Left }, + Operator::Arithmetic(ArithmeticOperator::Multiply) => ParserOperator { precedence: 60, associativity: Left }, + Operator::Arithmetic(ArithmeticOperator::Divide) => ParserOperator { precedence: 60, associativity: Left }, + Operator::Arithmetic(ArithmeticOperator::Modulus) => ParserOperator { precedence: 60, associativity: Left }, + Operator::Arithmetic(ArithmeticOperator::Power) => ParserOperator { precedence: 70, associativity: Right }, + } + }, +); + +pub(crate) static UNARY_OPERATORS: Lazy> = Lazy::new( + || { + hashmap! { + Operator::Logical(LogicalOperator::Not) => ParserOperator { precedence: 50, associativity: Left }, + Operator::Arithmetic(ArithmeticOperator::Add) => ParserOperator { precedence: 200, associativity: Left }, + Operator::Arithmetic(ArithmeticOperator::Subtract) => ParserOperator { precedence: 200, associativity: Left }, + } + }, +); diff --git a/core/expression/src/parser/definitions.rs b/core/expression/src/parser/definitions.rs deleted file mode 100644 index 700861e9..00000000 --- a/core/expression/src/parser/definitions.rs +++ /dev/null @@ -1,42 +0,0 @@ -#[derive(Debug, PartialEq)] -pub(crate) enum Associativity { - Left, - Right, -} - -#[derive(Debug, PartialEq)] -pub(crate) enum Arity { - Single, - Closure, - Dual, -} - -#[derive(Debug, PartialEq)] -pub(crate) struct Operator { - pub precedence: u8, - pub associativity: Associativity, -} - -#[derive(Debug, PartialEq)] -pub(crate) struct BuiltIn { - pub arity: Arity, -} - -#[macro_export] -macro_rules! hashmap { - (@single $($x:tt)*) => (()); - (@count $($rest:expr),*) => (<[()]>::len(&[$(hashmap!(@single $rest)),*])); - - ($($key:expr => $value:expr,)+) => { hashmap!($($key => $value),+) }; - ($($key:expr => $value:expr),*) => { - { - let _cap = hashmap!(@count $($key),*); - let mut _map = ::std::collections::HashMap::default(); - _map.reserve(_cap); - $( - let _ = _map.insert($key, $value); - )* - _map - } - }; -} diff --git a/core/expression/src/parser/error.rs b/core/expression/src/parser/error.rs index e2fe3706..38202ebd 100644 --- a/core/expression/src/parser/error.rs +++ b/core/expression/src/parser/error.rs @@ -1,6 +1,6 @@ use thiserror::Error; -#[derive(Debug, PartialEq, Eq, Error)] +#[derive(Debug, PartialEq, Eq, Clone, Error)] pub enum ParserError { #[error("Token out of bounds")] TokenOutOfBounds, @@ -21,4 +21,4 @@ pub enum ParserError { UnsupportedBuiltIn { token: String }, } -pub type ParserResult = Result; +pub(crate) type ParserResult = Result; diff --git a/core/expression/src/parser/iter.rs b/core/expression/src/parser/iter.rs deleted file mode 100644 index fa969ce5..00000000 --- a/core/expression/src/parser/iter.rs +++ /dev/null @@ -1,188 +0,0 @@ -use std::cell::Cell; -use std::fmt::Debug; - -use bumpalo::Bump; -use rust_decimal::Decimal; - -use crate::ast::Node; -use crate::lexer::token::{Token, TokenKind}; -use crate::parser::error::{ParserError, ParserResult}; - -type TokenValues<'a> = Option<&'a [&'a str]>; - -#[derive(Debug)] -pub(crate) struct ParserIterator<'a, 'b> { - tokens: &'a Vec>, - current: Cell<&'a Token<'a>>, - position: Cell, - bump: &'b Bump, - is_done: Cell, - has_interval: bool, -} - -impl<'a, 'b> ParserIterator<'a, 'b> { - pub fn try_new(tokens: &'a Vec>, bump: &'b Bump) -> Result { - let current = tokens.get(0).ok_or(ParserError::TokenOutOfBounds)?; - let has_interval = tokens - .iter() - .any(|t| t.kind == TokenKind::Operator && t.value == ".."); - - Ok(Self { - tokens, - bump, - has_interval, - current: Cell::new(current), - position: Cell::new(0), - is_done: Cell::new(false), - }) - } - - pub fn has_interval(&self) -> bool { - self.has_interval - } - - pub fn current(&self) -> &'a Token<'a> { - self.current.get() - } - - pub fn position(&self) -> usize { - self.position.get() - } - - pub fn set_position(&self, position: usize) -> ParserResult<()> { - let Some(token) = self.tokens.get(position) else { - return Err(ParserError::TokenOutOfBounds); - }; - - self.position.set(position); - self.current.set(token); - Ok(()) - } - - pub fn is_done(&self) -> bool { - self.is_done.get() - } - - pub fn next(&self) -> ParserResult<()> { - self.position.set(self.position.get() + 1); - - if let Some(token) = self.tokens.get(self.position.get()) { - self.current.set(token); - Ok(()) - } else { - if self.is_done.get() { - return Err(ParserError::TokenOutOfBounds); - } - - self.is_done.set(true); - Ok(()) - } - } - - pub fn str_value(&self, str: &str) -> &'b str { - self.bump.alloc_str(str) - } - - pub fn expect(&self, kind: TokenKind, values: TokenValues<'a>) -> Result<(), ParserError> { - self.token_cmp(kind, values)?; - self.next()?; - Ok(()) - } - - #[allow(dead_code)] - pub fn lookup(&self, dx: usize, kind: TokenKind, values: TokenValues<'a>) -> bool { - self.token_cmp_at_bool(self.position.get() + dx, kind, values) - } - - pub fn lookup_back(&self, dx: usize, kind: TokenKind, values: TokenValues<'a>) -> bool { - if self.position.get() < dx { - return false; - } - - self.token_cmp_at_bool(self.position.get() - dx, kind, values) - } - - pub fn number(&self, token: &'a Token) -> Result<&'b Node<'b>, ParserError> { - self.next()?; - - let decimal = - Decimal::from_str_exact(token.value).map_err(|_| ParserError::FailedToParse { - message: format!("unknown float value: {:?}", token.value), - })?; - - self.node(Node::Number(decimal)) - } - - pub fn string(&self, token: &'a Token) -> Result<&'b Node<'b>, ParserError> { - self.next()?; - self.node(Node::String(self.bump.alloc_str(token.value))) - } - - pub fn bool(&self, token: &'a Token) -> Result<&'b Node<'b>, ParserError> { - match token.value { - "true" => self.node(Node::Bool(true)), - "false" => self.node(Node::Bool(false)), - _ => Err(ParserError::FailedToParse { - message: format!("unknown bool value: {:?}", token.value), - }), - } - } - - pub fn null(&self, _token: &'a Token) -> Result<&'b Node<'b>, ParserError> { - self.node(Node::Null) - } - - pub fn node(&self, node: Node<'b>) -> Result<&'b Node<'b>, ParserError> { - Ok(self.bump.alloc(node)) - } - - fn token_cmp_at_bool(&self, index: usize, kind: TokenKind, values: TokenValues<'a>) -> bool { - return if let Some(token) = self.tokens.get(index) { - if token.kind != kind { - return false; - } - - if let Some(vals) = values { - return vals.iter().any(|&c| c == token.value); - } - - true - } else { - false - }; - } - - fn token_cmp_at( - &self, - index: usize, - kind: TokenKind, - values: TokenValues<'a>, - ) -> Result<(), ParserError> { - let token: &'a Token = self - .tokens - .get(index) - .ok_or(ParserError::TokenOutOfBounds)?; - - if token.kind != kind { - return Err(ParserError::UnexpectedToken { - expected: format!("{kind:?} {values:?}"), - received: format!("{token:?}"), - }); - } - - if let Some(vals) = values { - if !vals.iter().any(|&c| c == token.value) { - return Err(ParserError::UnexpectedToken { - expected: format!("{kind:?} {values:?}"), - received: format!("{token:?}"), - }); - } - } - - Ok(()) - } - - fn token_cmp(&self, kind: TokenKind, values: TokenValues<'a>) -> Result<(), ParserError> { - self.token_cmp_at(self.position.get(), kind, values) - } -} diff --git a/core/expression/src/parser/mod.rs b/core/expression/src/parser/mod.rs index 0144b0b3..1d5b3afe 100644 --- a/core/expression/src/parser/mod.rs +++ b/core/expression/src/parser/mod.rs @@ -1,9 +1,21 @@ -pub use standard::StandardParser; -pub use unary::UnaryParser; - -pub mod error; - -mod definitions; -mod iter; +//! Parses Tokens into AST +//! +//! The Parser module processes tokens from the Lexer, constructing an Abstract Syntax Tree (AST). +//! +//! It's available in two specialized variants: +//! - Standard, designed for comprehensive expression evaluation yielding any result +//! - Unary, specifically created for truthy tests with exclusive boolean outcomes +mod ast; +mod builtin; +mod constants; +mod error; +mod parser; mod standard; mod unary; + +pub use ast::Node; +pub use builtin::BuiltInFunction; +pub use error::ParserError; +pub use parser::Parser; +pub use standard::Standard; +pub use unary::Unary; diff --git a/core/expression/src/parser/parser.rs b/core/expression/src/parser/parser.rs new file mode 100644 index 00000000..3306284f --- /dev/null +++ b/core/expression/src/parser/parser.rs @@ -0,0 +1,527 @@ +use std::cell::Cell; +use std::fmt::Debug; +use std::marker::PhantomData; + +use bumpalo::collections::Vec as BumpVec; +use bumpalo::Bump; +use rust_decimal::Decimal; + +use crate::lexer::{Bracket, ComparisonOperator, Identifier, Operator, Token, TokenKind}; +use crate::parser::ast::Node; +use crate::parser::builtin::{Arity, BuiltInFunction}; +use crate::parser::error::{ParserError, ParserResult}; +use crate::parser::standard::Standard; +use crate::parser::unary::Unary; + +#[derive(Debug)] +pub struct BaseParser; + +#[derive(Debug)] +pub struct Parser<'arena, 'token_ref, Flavor> { + tokens: &'token_ref [Token<'arena>], + current: Cell<&'token_ref Token<'arena>>, + pub(crate) bump: &'arena Bump, + is_done: Cell, + position: Cell, + depth: Cell, + marker_flavor: PhantomData, + has_range_operator: bool, +} + +impl<'arena, 'token_ref> Parser<'arena, 'token_ref, BaseParser> { + pub fn try_new( + tokens: &'token_ref [Token<'arena>], + bump: &'arena Bump, + ) -> Result { + let current = tokens.get(0).ok_or(ParserError::TokenOutOfBounds)?; + let has_range_operator = tokens + .iter() + .any(|t| t.kind == TokenKind::Operator(Operator::Range)); + + Ok(Self { + tokens, + bump, + current: Cell::new(current), + depth: Cell::new(0), + position: Cell::new(0), + is_done: Cell::new(false), + has_range_operator, + marker_flavor: PhantomData, + }) + } + + pub fn standard(self) -> Parser<'arena, 'token_ref, Standard> { + Parser { + tokens: self.tokens, + bump: self.bump, + current: self.current, + depth: self.depth, + position: self.position, + is_done: self.is_done, + has_range_operator: self.has_range_operator, + marker_flavor: PhantomData, + } + } + + pub fn unary(self) -> Parser<'arena, 'token_ref, Unary> { + Parser { + tokens: self.tokens, + bump: self.bump, + current: self.current, + depth: self.depth, + position: self.position, + is_done: self.is_done, + has_range_operator: self.has_range_operator, + marker_flavor: PhantomData, + } + } +} + +impl<'arena, 'token_ref, Flavor> Parser<'arena, 'token_ref, Flavor> { + pub(crate) fn current(&self) -> &Token<'arena> { + self.current.get() + } + + fn position(&self) -> usize { + self.position.get() + } + + fn set_position(&self, position: usize) -> ParserResult<()> { + let Some(token) = self.tokens.get(position) else { + return Err(ParserError::TokenOutOfBounds); + }; + + self.position.set(position); + self.current.set(token); + Ok(()) + } + + pub(crate) fn depth(&self) -> u8 { + self.depth.get() + } + + pub(crate) fn is_done(&self) -> bool { + self.is_done.get() + } + + pub(crate) fn next(&self) -> ParserResult<()> { + self.position.set(self.position.get() + 1); + + if let Some(token) = self.tokens.get(self.position.get()) { + self.current.set(token); + Ok(()) + } else { + if self.is_done.get() { + return Err(ParserError::TokenOutOfBounds); + } + + self.is_done.set(true); + Ok(()) + } + } + + pub(crate) fn expect(&self, kind: TokenKind) -> Result<(), ParserError> { + let token = self.current(); + if token.kind != kind { + return Err(ParserError::UnexpectedToken { + expected: format!("{kind:?}"), + received: format!("{token:?}"), + }); + } + + self.next() + } + + pub(crate) fn number(&self) -> ParserResult>> { + let Ok(decimal) = Decimal::from_str_exact(self.current().value) else { + return Ok(None); + }; + + self.next()?; + Ok(Some(self.node(Node::Number(decimal)))) + } + + pub(crate) fn string(&self) -> ParserResult>> { + let current_token = self.current(); + if current_token.kind != TokenKind::String { + return Ok(None); + } + + self.next()?; + Ok(Some(self.node(Node::String(current_token.value)))) + } + + pub(crate) fn bool(&self) -> ParserResult>> { + let current_token = self.current(); + let TokenKind::Boolean(boolean) = current_token.kind else { + return Ok(None); + }; + + self.next()?; + Ok(Some(self.node(Node::Bool(boolean)))) + } + + pub(crate) fn null(&self) -> ParserResult>> { + let current_token = self.current(); + if current_token.kind != TokenKind::Identifier(Identifier::Null) { + return Ok(None); + } + + self.next()?; + Ok(Some(self.node(Node::Null))) + } + + pub(crate) fn node(&self, node: Node<'arena>) -> &'arena Node<'arena> { + self.bump.alloc(node) + } + + // Higher level constructs + + pub(crate) fn with_postfix( + &self, + node: &'arena Node<'arena>, + expression_parser: F, + ) -> ParserResult<&'arena Node<'arena>> + where + F: Fn() -> ParserResult<&'arena Node<'arena>>, + { + let postfix_token = self.current(); + let postfix_kind = PostfixKind::from(postfix_token); + + let processed_token = match postfix_kind { + PostfixKind::Other => return Ok(node), + PostfixKind::MemberAccess => { + self.next()?; + let property_token = self.current(); + self.next()?; + + if !is_valid_property(property_token) { + return Err(ParserError::UnexpectedToken { + expected: "member identifier token".to_string(), + received: format!("{postfix_token:?}"), + }); + } + + let property = self.node(Node::String(property_token.value)); + Ok(self.node(Node::Member { node, property })) + } + PostfixKind::PropertyAccess => { + self.next()?; + let mut from: Option<&'arena Node<'arena>> = None; + let mut to: Option<&'arena Node<'arena>> = None; + + let mut c = self.current(); + if c.kind == TokenKind::Operator(Operator::Slice) { + self.next()?; + c = self.current(); + + if c.kind != TokenKind::Bracket(Bracket::RightSquareBracket) { + to = Some(expression_parser()?); + } + + self.expect(TokenKind::Bracket(Bracket::RightSquareBracket))?; + Ok(self.node(Node::Slice { node, to, from })) + } else { + from = Some(expression_parser()?); + c = self.current(); + + if c.kind == TokenKind::Operator(Operator::Slice) { + self.next()?; + c = self.current(); + + if c.kind != TokenKind::Bracket(Bracket::RightSquareBracket) { + to = Some(expression_parser()?); + } + + self.expect(TokenKind::Bracket(Bracket::RightSquareBracket))?; + Ok(self.node(Node::Slice { node, from, to })) + } else { + // Slice operator [:] was not found, + // it should be just an index node. + self.expect(TokenKind::Bracket(Bracket::RightSquareBracket))?; + Ok(self.node(Node::Member { + node, + property: from.ok_or(ParserError::MemoryFailure)?, + })) + } + } + } + }?; + + self.with_postfix(processed_token, expression_parser) + } + + /// Closure + pub(crate) fn closure(&self, expression_parser: F) -> ParserResult<&'arena Node<'arena>> + where + F: Fn() -> ParserResult<&'arena Node<'arena>>, + { + self.depth.set(self.depth.get() + 1); + let node = expression_parser()?; + self.depth.set(self.depth.get() - 1); + + Ok(self.node(Node::Closure(node))) + } + + /// Identifier expression + /// Either or + pub(crate) fn identifier( + &self, + expression_parser: F, + ) -> ParserResult>> + where + F: Fn() -> ParserResult<&'arena Node<'arena>>, + { + let TokenKind::Identifier(_) = &self.current().kind else { + return Ok(None); + }; + + let identifier_token = self.current(); + self.next()?; + let current_token = self.current(); + if current_token.kind != TokenKind::Bracket(Bracket::LeftParenthesis) { + let identifier_node = self.node(Node::Identifier(identifier_token.value)); + return self + .with_postfix(identifier_node, expression_parser) + .map(Some); + } + + // Potentially it might be a built in expression + let builtin = BuiltInFunction::try_from(identifier_token.value).map_err(|_| { + ParserError::UnknownBuiltIn { + token: identifier_token.value.to_string(), + } + })?; + + self.next()?; + let builtin_node = match builtin.arity() { + Arity::Single => { + let arg = expression_parser()?; + self.expect(TokenKind::Bracket(Bracket::RightParenthesis))?; + + Node::BuiltIn { + kind: builtin, + arguments: self.bump.alloc_slice_copy(&[arg]), + } + } + Arity::Dual => { + let arg1 = expression_parser()?; + self.expect(TokenKind::Operator(Operator::Comma))?; + let arg2 = expression_parser()?; + self.expect(TokenKind::Bracket(Bracket::RightParenthesis))?; + + Node::BuiltIn { + kind: builtin, + arguments: self.bump.alloc_slice_copy(&[arg1, arg2]), + } + } + Arity::Closure => { + let arg1 = expression_parser()?; + self.expect(TokenKind::Operator(Operator::Comma))?; + let arg2 = self.closure(&expression_parser)?; + self.expect(TokenKind::Bracket(Bracket::RightParenthesis))?; + + Node::BuiltIn { + kind: builtin, + arguments: self.bump.alloc_slice_copy(&[arg1, arg2]), + } + } + }; + + self.with_postfix(self.node(builtin_node), expression_parser) + .map(Some) + } + + /// Interval node + pub(crate) fn interval( + &self, + expression_parser: F, + ) -> ParserResult>> + where + F: Fn() -> ParserResult<&'arena Node<'arena>>, + { + // Performance optimisation: skip if expression does not contain an interval for faster evaluation + if !self.has_range_operator { + return Ok(None); + } + + let TokenKind::Bracket(_) = &self.current().kind else { + return Ok(None); + }; + + let initial_position = self.position(); + let left_bracket = self.current().value; + + let TokenKind::Bracket(_) = &self.current().kind else { + self.set_position(initial_position)?; + return Ok(None); + }; + self.next()?; + + let Ok(left) = expression_parser() else { + self.set_position(initial_position)?; + return Ok(None); + }; + + if let Err(_) = self.expect(TokenKind::Operator(Operator::Range)) { + self.set_position(initial_position)?; + return Ok(None); + }; + + let Ok(right) = expression_parser() else { + self.set_position(initial_position)?; + return Ok(None); + }; + + let right_bracket = self.current().value; + + let TokenKind::Bracket(_) = &self.current().kind else { + self.set_position(initial_position)?; + return Ok(None); + }; + self.next()?; + + let interval_node = self.node(Node::Interval { + left_bracket, + left, + right, + right_bracket, + }); + + self.with_postfix(interval_node, expression_parser) + .map(Some) + } + + /// Array nodes + pub(crate) fn array( + &self, + expression_parser: F, + ) -> ParserResult>> + where + F: Fn() -> ParserResult<&'arena Node<'arena>>, + { + let current_token = self.current(); + if current_token.kind != TokenKind::Bracket(Bracket::LeftSquareBracket) { + return Ok(None); + } + + self.next()?; + let mut nodes = BumpVec::new_in(self.bump); + while !(self.current().kind == TokenKind::Bracket(Bracket::RightSquareBracket)) { + if !nodes.is_empty() { + self.expect(TokenKind::Operator(Operator::Comma))?; + if self.current().kind == TokenKind::Bracket(Bracket::RightSquareBracket) { + break; + } + } + + nodes.push(expression_parser()?); + } + + self.expect(TokenKind::Bracket(Bracket::RightSquareBracket))?; + let node = Node::Array(nodes.into_bump_slice()); + + self.with_postfix(self.node(node), expression_parser) + .map(Some) + } + + /// Conditional + /// condition_node ? on_true : on_false + pub(crate) fn conditional( + &self, + condition: &'arena Node<'arena>, + expression_parser: F, + ) -> ParserResult>> + where + F: Fn() -> ParserResult<&'arena Node<'arena>>, + { + let current_token = self.current(); + if current_token.kind != TokenKind::Operator(Operator::QuestionMark) { + return Ok(None); + } + + self.next()?; + + let on_true = expression_parser()?; + self.expect(TokenKind::Operator(Operator::Slice))?; + let on_false = expression_parser()?; + + let conditional_node = Node::Conditional { + condition, + on_true, + on_false, + }; + + Ok(Some(self.node(conditional_node))) + } + + /// Literal - number, string, array etc. + pub(crate) fn literal(&self, expression_parser: F) -> ParserResult<&'arena Node<'arena>> + where + F: Fn() -> ParserResult<&'arena Node<'arena>>, + { + let current_token = self.current(); + match ¤t_token.kind { + TokenKind::Identifier(identifier) => match identifier { + Identifier::Null => self.null()?.ok_or_else(|| ParserError::FailedToParse { + message: format!("failed to parse null identifier: {:?}", current_token), + }), + _ => { + self.identifier(&expression_parser)? + .ok_or_else(|| ParserError::FailedToParse { + message: format!("failed to parse identifier: {:?}", current_token), + }) + } + }, + TokenKind::Boolean(_) => self.bool()?.ok_or_else(|| ParserError::FailedToParse { + message: format!("failed to parse boolean: {:?}", current_token), + }), + TokenKind::Number => self.number()?.ok_or_else(|| ParserError::FailedToParse { + message: format!("failed to parse number: {:?}", current_token), + }), + TokenKind::String => self.string()?.ok_or_else(|| ParserError::FailedToParse { + message: format!("failed to parse string: {:?}", current_token), + }), + TokenKind::Bracket(_) => self + .interval(&expression_parser) + .transpose() + .or_else(|| self.array(&expression_parser).transpose()) + .transpose()? + .ok_or_else(|| ParserError::FailedToParse { + message: format!("unexpected bracket: {:?}", current_token), + }), + TokenKind::Operator(_) => Err(ParserError::FailedToParse { + message: format!("unexpected literal token: {:?}", current_token), + }), + } + } +} + +fn is_valid_property(token: &Token) -> bool { + match &token.kind { + TokenKind::Identifier(_) => true, + TokenKind::Operator(operator) => match operator { + Operator::Logical(_) => true, + Operator::Comparison(comparison) => matches!(comparison, ComparisonOperator::In), + _ => false, + }, + _ => false, + } +} + +#[derive(Debug)] +enum PostfixKind { + MemberAccess, + PropertyAccess, + Other, +} + +impl From<&Token<'_>> for PostfixKind { + fn from(token: &Token) -> Self { + match &token.kind { + TokenKind::Bracket(Bracket::LeftSquareBracket) => Self::PropertyAccess, + TokenKind::Operator(Operator::Dot) => Self::MemberAccess, + _ => Self::Other, + } + } +} diff --git a/core/expression/src/parser/standard.rs b/core/expression/src/parser/standard.rs new file mode 100644 index 00000000..d8f908d4 --- /dev/null +++ b/core/expression/src/parser/standard.rs @@ -0,0 +1,107 @@ +use crate::lexer::{Bracket, Identifier, TokenKind}; +use crate::parser::ast::Node; +use crate::parser::constants::{Associativity, BINARY_OPERATORS, UNARY_OPERATORS}; +use crate::parser::error::ParserError::{FailedToParse, UnexpectedToken}; +use crate::parser::error::ParserResult; +use crate::parser::parser::Parser; + +#[derive(Debug)] +pub struct Standard; + +impl<'arena, 'token_ref> Parser<'arena, 'token_ref, Standard> { + pub fn parse(&self) -> ParserResult<&'arena Node<'arena>> { + let result = self.binary_expression(0)?; + if !self.is_done() { + let token = self.current(); + return Err(FailedToParse { + message: format!("Unterminated token {} on {:?}", token.value, token.span), + }); + } + + return Ok(result); + } + + fn binary_expression(&self, precedence: u8) -> ParserResult<&'arena Node<'arena>> { + let mut node_left = self.unary_expression()?; + let mut token = self.current(); + + while let TokenKind::Operator(operator) = &token.kind { + if self.is_done() { + break; + } + + let Some(op) = BINARY_OPERATORS.get(operator) else { + break; + }; + + if op.precedence < precedence { + break; + } + + self.next()?; + let node_right = match op.associativity { + Associativity::Left => self.binary_expression(op.precedence + 1)?, + _ => self.binary_expression(op.precedence)?, + }; + + node_left = self.node(Node::Binary { + operator: *operator, + left: node_left, + right: node_right, + }); + token = self.current(); + } + + if precedence == 0 { + if let Some(conditional_node) = + self.conditional(node_left, || self.binary_expression(0))? + { + node_left = conditional_node; + } + } + + Ok(node_left) + } + + fn unary_expression(&self) -> ParserResult<&'arena Node<'arena>> { + let token = self.current(); + + if self.depth() > 0 && token.kind == TokenKind::Identifier(Identifier::CallbackReference) { + self.next()?; + + let node = self.node(Node::Pointer); + return self.with_postfix(node, || self.binary_expression(0)); + } + + if let TokenKind::Operator(operator) = &token.kind { + let Some(unary_operator) = UNARY_OPERATORS.get(operator) else { + return Err(UnexpectedToken { + expected: "Unary token".to_string(), + received: format!("{token:?}"), + }); + }; + + self.next()?; + let expr = self.binary_expression(unary_operator.precedence)?; + let node = self.node(Node::Unary { + operator: *operator, + node: expr, + }); + + return Ok(node); + } + + if let Some(interval_node) = self.interval(|| self.binary_expression(0))? { + return Ok(interval_node); + } + + if token.kind == TokenKind::Bracket(Bracket::LeftParenthesis) { + self.next()?; + let expr = self.binary_expression(0)?; + self.expect(TokenKind::Bracket(Bracket::RightParenthesis))?; + return self.with_postfix(expr, || self.binary_expression(0)); + } + + self.literal(|| self.binary_expression(0)) + } +} diff --git a/core/expression/src/parser/standard/constants.rs b/core/expression/src/parser/standard/constants.rs deleted file mode 100644 index 2c52ab24..00000000 --- a/core/expression/src/parser/standard/constants.rs +++ /dev/null @@ -1,99 +0,0 @@ -use std::collections::HashMap; -use std::hash::BuildHasherDefault; - -use ahash::AHasher; -use once_cell::sync::Lazy; - -use crate::hashmap; -use crate::parser::definitions::Associativity::{Left, Right}; -use crate::parser::definitions::{Arity, BuiltIn, Operator}; - -type ADefHasher = BuildHasherDefault; - -pub(crate) static BUILT_INS: Lazy> = Lazy::new(|| { - hashmap! { - "len" => BuiltIn { arity: Arity::Single }, - "date" => BuiltIn { arity: Arity::Single }, - "time" => BuiltIn { arity: Arity::Single }, - "duration" => BuiltIn { arity: Arity::Single }, - "upper" => BuiltIn { arity: Arity::Single }, - "lower" => BuiltIn { arity: Arity::Single }, - "flatten" => BuiltIn { arity: Arity::Single }, - - "string" => BuiltIn { arity: Arity::Single }, - "number" => BuiltIn { arity: Arity::Single }, - "isNumeric" => BuiltIn { arity: Arity::Single }, - - "abs" => BuiltIn { arity: Arity::Single }, - "sum" => BuiltIn { arity: Arity::Single }, - "avg" => BuiltIn { arity: Arity::Single }, - "min" => BuiltIn { arity: Arity::Single }, - "max" => BuiltIn { arity: Arity::Single }, - "rand" => BuiltIn { arity: Arity::Single }, - "median" => BuiltIn { arity: Arity::Single }, - "mode" => BuiltIn { arity: Arity::Single }, - - "floor" => BuiltIn { arity: Arity::Single }, - "ceil" => BuiltIn { arity: Arity::Single }, - "round" => BuiltIn { arity: Arity::Single }, - - "year" => BuiltIn { arity: Arity::Single }, - "dayOfWeek" => BuiltIn { arity: Arity::Single }, - "dayOfMonth" => BuiltIn { arity: Arity::Single }, - "dayOfYear" => BuiltIn { arity: Arity::Single }, - "weekOfYear" => BuiltIn { arity: Arity::Single }, - "monthOfYear" => BuiltIn { arity: Arity::Single }, - "monthString" => BuiltIn { arity: Arity::Single }, - "dateString" => BuiltIn { arity: Arity::Single }, - "weekdayString" => BuiltIn { arity: Arity::Single }, - "startOf" => BuiltIn { arity: Arity::Dual }, - "endOf" => BuiltIn { arity: Arity::Dual }, - - "startsWith" => BuiltIn { arity: Arity::Dual }, - "endsWith" => BuiltIn { arity: Arity::Dual }, - "contains" => BuiltIn { arity: Arity::Dual }, - "matches" => BuiltIn { arity: Arity::Dual }, - "extract" => BuiltIn { arity: Arity::Dual }, - - "all" => BuiltIn { arity: Arity::Closure }, - "some" => BuiltIn { arity: Arity::Closure }, - "none" => BuiltIn { arity: Arity::Closure }, - "filter" => BuiltIn { arity: Arity::Closure }, - "map" => BuiltIn { arity: Arity::Closure }, - "count" => BuiltIn { arity: Arity::Closure }, - "one" => BuiltIn { arity: Arity::Closure }, - "flatMap" => BuiltIn { arity: Arity::Closure }, - } -}); - -pub(crate) static BINARY_OPERATORS: Lazy> = - Lazy::new(|| { - hashmap! { - "or" => Operator { precedence: 10, associativity: Left }, - "and" => Operator { precedence: 15, associativity: Left }, - "==" => Operator { precedence: 20, associativity: Left }, - "!=" => Operator { precedence: 20, associativity: Left }, - "<" => Operator { precedence: 20, associativity: Left }, - ">" => Operator { precedence: 20, associativity: Left }, - "<=" => Operator { precedence: 20, associativity: Left }, - ">=" => Operator { precedence: 20, associativity: Left }, - "not in" => Operator { precedence: 20, associativity: Left }, - "in" => Operator { precedence: 20, associativity: Left }, - "+" => Operator { precedence: 30, associativity: Left }, - "-" => Operator { precedence: 30, associativity: Left }, - "*" => Operator { precedence: 60, associativity: Left }, - "/" => Operator { precedence: 60, associativity: Left }, - "%" => Operator { precedence: 60, associativity: Left }, - "^" => Operator { precedence: 70, associativity: Right }, - } - }); - -pub(crate) static UNARY_OPERATORS: Lazy> = - Lazy::new(|| { - hashmap! { - "not" => Operator { precedence: 50, associativity: Left }, - "!" => Operator { precedence: 50, associativity: Left }, - "+" => Operator { precedence: 200, associativity: Left }, - "-" => Operator { precedence: 200, associativity: Left }, - } - }); diff --git a/core/expression/src/parser/standard/mod.rs b/core/expression/src/parser/standard/mod.rs deleted file mode 100644 index 32dcbd95..00000000 --- a/core/expression/src/parser/standard/mod.rs +++ /dev/null @@ -1,399 +0,0 @@ -use std::cell::Cell; - -use bumpalo::Bump; - -use crate::ast::Node; -use crate::lexer::token::{Token, TokenKind}; -use crate::parser::definitions::{Arity, Associativity}; -use crate::parser::error::ParserError::{ - FailedToParse, MemoryFailure, UnexpectedToken, UnknownBuiltIn, -}; -use crate::parser::error::ParserResult; -use crate::parser::iter::ParserIterator; -use crate::parser::standard::constants::{BINARY_OPERATORS, BUILT_INS, UNARY_OPERATORS}; - -mod constants; - -#[derive(Debug)] -pub struct StandardParser<'a, 'b> -where - 'b: 'a, -{ - iterator: ParserIterator<'a, 'b>, - bump: &'b Bump, - depth: Cell, -} - -impl<'a, 'b> StandardParser<'a, 'b> -where - 'b: 'a, -{ - pub fn try_new(tokens: &'a Vec, bump: &'b Bump) -> ParserResult { - Ok(Self { - iterator: ParserIterator::try_new(tokens, bump)?, - bump, - depth: Cell::new(0), - }) - } - - pub fn parse(&self) -> ParserResult<&'b Node<'b>> { - let result = self.expression(0)?; - if !self.iterator.is_done() { - let token = self.iterator.current(); - return Err(FailedToParse { - message: format!("Unterminated token {} on {:?}", token.value, token.span), - }); - } - - return Ok(result); - } - - fn expression(&self, precedence: u8) -> ParserResult<&'b Node<'b>> { - let mut node_left = self.primary_expression()?; - let mut token = self.iterator.current(); - - while !self.iterator.is_done() { - if token.kind == TokenKind::Operator { - if let Some(op) = BINARY_OPERATORS.get(token.value) { - if op.precedence >= precedence { - self.iterator.next()?; - let node_right = match op.associativity { - Associativity::Left => self.expression(op.precedence + 1)?, - _ => self.expression(op.precedence)?, - }; - - node_left = self.iterator.node(Node::Binary { - operator: self.iterator.str_value(token.value), - left: node_left, - right: node_right, - })?; - token = self.iterator.current(); - continue; - } - } - } - - break; - } - - if precedence == 0 { - node_left = self.conditional(node_left)?; - } - - Ok(node_left) - } - - fn primary_expression(&self) -> ParserResult<&'b Node<'b>> { - let token = self.iterator.current(); - if token.kind == TokenKind::Operator { - if let Some(op) = UNARY_OPERATORS.get(token.value) { - self.iterator.next()?; - let expr = self.expression(op.precedence)?; - let node = self.iterator.node(Node::Unary { - operator: self.iterator.str_value(token.value), - node: expr, - })?; - - return self.postfix(node); - } - } - - if let Some(interval_node) = self.interval()? { - return self.postfix(interval_node); - } - - if token.kind == TokenKind::Bracket && token.value == "(" { - self.iterator.next()?; - let expr = self.expression(0)?; - self.iterator.expect(TokenKind::Bracket, Some(&[")"]))?; - return self.postfix(expr); - } - - if self.depth.get() > 0 { - if token.kind == TokenKind::Operator && (token.value == "#" || token.value == ".") { - if token.value == "#" { - self.iterator.next()?; - } - let node = self.iterator.node(Node::Pointer)?; - return self.postfix(node); - } - } else if token.kind == TokenKind::Operator && (token.value == "#" || token.value == ".") { - return Err(UnexpectedToken { - expected: "anything but Operator(#, .)".to_string(), - received: format!("{token:?}"), - }); - } - - self.literal() - } - - fn conditional(&self, node: &'b Node<'b>) -> ParserResult<&'b Node<'b>> { - let mut nd = self.iterator.node(node.clone())?; - let mut expr1: &'b Node; - let mut expr2: &'b Node; - - while self.iterator.current().kind == TokenKind::Operator - && self.iterator.current().value == "?" - { - self.iterator.next()?; - - let token = self.iterator.current(); - if token.kind != TokenKind::Operator && token.value != ":" { - expr1 = self.expression(0)?; - self.iterator.expect(TokenKind::Operator, Some(&[":"]))?; - expr2 = self.expression(0)?; - } else { - self.iterator.next()?; - expr1 = node; - expr2 = self.expression(0)?; - } - - nd = self.iterator.node(Node::Conditional { - condition: nd, - on_true: expr1, - on_false: expr2, - })?; - } - - Ok(nd) - } - - fn literal(&self) -> ParserResult<&'b Node<'b>> { - let node: &'b Node; - let token = self.iterator.current(); - - match token.kind { - TokenKind::Identifier => { - self.iterator.next()?; - match token.value { - "true" | "false" => return self.iterator.bool(token), - "null" => return self.iterator.null(token), - _ => node = self.identifier_expression(token)?, - } - } - TokenKind::Number => return self.iterator.number(token), - TokenKind::String => return self.iterator.string(token), - _ => { - if token.kind == TokenKind::Bracket && token.value == "[" { - node = self.array(token)?; - } else { - return Err(UnexpectedToken { - expected: "identifier, string, number or opening bracket".to_string(), - received: format!("{token:?}"), - }); - } - } - } - - self.postfix(node) - } - - fn interval(&self) -> ParserResult>> { - // Performance optimisation: skip if expression does not contain an interval for faster evaluation - if !self.iterator.has_interval() { - return Ok(None); - } - - if self.iterator.current().kind != TokenKind::Bracket { - return Ok(None); - } - - let initial_position = self.iterator.position(); - let left_bracket = self.iterator.current().value; - if let Err(_) = self.iterator.expect(TokenKind::Bracket, None) { - self.iterator.set_position(initial_position)?; - return Ok(None); - }; - - let Ok(left) = self.primary_expression() else { - self.iterator.set_position(initial_position)?; - return Ok(None); - }; - - if let Err(_) = self.iterator.expect(TokenKind::Operator, Some(&[".."])) { - self.iterator.set_position(initial_position)?; - return Ok(None); - }; - - let Ok(right) = self.primary_expression() else { - self.iterator.set_position(initial_position)?; - return Ok(None); - }; - - let right_bracket = self.iterator.current().value; - - if let Err(_) = self.iterator.expect(TokenKind::Bracket, None) { - self.iterator.set_position(initial_position)?; - return Ok(None); - }; - - let interval_node = self.iterator.node(Node::Interval { - left_bracket: self.iterator.str_value(left_bracket), - left, - right, - right_bracket: self.iterator.str_value(right_bracket), - })?; - - Ok(Some(interval_node)) - } - - fn identifier_expression(&self, token: &Token) -> ParserResult<&'b Node<'b>> { - if self.iterator.current().kind != TokenKind::Bracket - || self.iterator.current().value != "(" - { - return self - .iterator - .node(Node::Identifier(self.iterator.str_value(token.value))); - } - - let builtin = BUILT_INS.get(token.value).ok_or_else(|| UnknownBuiltIn { - token: token.value.to_string(), - })?; - - self.iterator.expect(TokenKind::Bracket, Some(&["("]))?; - - return match builtin.arity { - Arity::Single => { - let arg = self.expression(0)?; - self.iterator.expect(TokenKind::Bracket, Some(&[")"]))?; - - self.iterator.node(Node::BuiltIn { - name: self.iterator.str_value(token.value), - arguments: self.bump.alloc_slice_copy(&[arg]), - }) - } - Arity::Dual => { - let arg1 = self.expression(0)?; - self.iterator.expect(TokenKind::Operator, Some(&[","]))?; - let arg2 = self.expression(0)?; - self.iterator.expect(TokenKind::Bracket, Some(&[")"]))?; - - self.iterator.node(Node::BuiltIn { - name: self.iterator.str_value(token.value), - arguments: self.bump.alloc_slice_copy(&[arg1, arg2]), - }) - } - Arity::Closure => { - let arg1 = self.expression(0)?; - self.iterator.expect(TokenKind::Operator, Some(&[","]))?; - let arg2 = self.closure()?; - self.iterator.expect(TokenKind::Bracket, Some(&[")"]))?; - - self.iterator.node(Node::BuiltIn { - name: self.iterator.str_value(token.value), - arguments: self.bump.alloc_slice_copy(&[arg1, arg2]), - }) - } - }; - } - - fn array(&self, _token: &Token) -> ParserResult<&'b Node<'b>> { - let mut nodes = Vec::new(); - self.iterator.expect(TokenKind::Bracket, Some(&["["]))?; - - while !(self.iterator.current().kind == TokenKind::Bracket - && self.iterator.current().value == "]") - { - if !nodes.is_empty() { - self.iterator.expect(TokenKind::Operator, Some(&[","]))?; - if self.iterator.current().value == "]" { - break; - } - } - - nodes.push(self.expression(0)?); - } - - self.iterator.expect(TokenKind::Bracket, Some(&["]"]))?; - let node = Node::Array(self.bump.alloc_slice_copy(nodes.as_slice())); - - self.iterator.node(node) - } - - fn closure(&self) -> ParserResult<&'b Node<'b>> { - self.depth.set(self.depth.get() + 1); - let node = self.expression(0)?; - self.depth.set(self.depth.get() - 1); - - return self.iterator.node(Node::Closure(node)); - } - - fn postfix(&self, node: &'b Node<'b>) -> ParserResult<&'b Node<'b>> { - let mut postfix_token = self.iterator.current(); - let mut nd = self.iterator.node(node.clone())?; - - while postfix_token.kind == TokenKind::Bracket || postfix_token.kind == TokenKind::Operator - { - if postfix_token.value == "." { - self.iterator.next()?; - let property_token = self.iterator.current(); - self.iterator.next()?; - - if property_token.kind != TokenKind::Identifier - && (property_token.kind != TokenKind::Operator - || !is_valid_identifier(property_token.value)) - { - return Err(UnexpectedToken { - expected: "member identifier token".to_string(), - received: format!("{postfix_token:?}"), - }); - } - - let property = self - .iterator - .node(Node::String(self.iterator.str_value(property_token.value)))?; - nd = self.iterator.node(Node::Member { node: nd, property })?; - } else if postfix_token.value == "[" { - self.iterator.next()?; - let mut from: Option<&'b Node<'b>> = None; - let mut to: Option<&'b Node<'b>> = None; - - let mut c = self.iterator.current(); - if c.kind == TokenKind::Operator && c.value == ":" { - self.iterator.next()?; - c = self.iterator.current(); - - if c.kind != TokenKind::Bracket && c.value != "]" { - to = Some(self.expression(0)?); - } - - nd = self.iterator.node(Node::Slice { node: nd, to, from })?; - self.iterator.expect(TokenKind::Bracket, Some(&["]"]))?; - } else { - from = Some(self.expression(0)?); - c = self.iterator.current(); - - if c.kind == TokenKind::Operator && c.value == ":" { - self.iterator.next()?; - c = self.iterator.current(); - - if c.kind != TokenKind::Bracket && c.value != "]" { - to = Some(self.expression(0)?); - } - - nd = self.iterator.node(Node::Slice { node: nd, from, to })?; - self.iterator.expect(TokenKind::Bracket, Some(&["]"]))?; - } else { - // Slice operator [:] was not found, - // it should be just an index node. - nd = self.iterator.node(Node::Member { - node: nd, - property: from.ok_or(MemoryFailure)?, - })?; - self.iterator.expect(TokenKind::Bracket, Some(&["]"]))?; - } - } - } else { - break; - } - - postfix_token = self.iterator.current(); - } - - Ok(nd) - } -} - -fn is_valid_identifier(str: &str) -> bool { - matches!(str, "and" | "or" | "in" | "not") -} diff --git a/core/expression/src/parser/unary.rs b/core/expression/src/parser/unary.rs new file mode 100644 index 00000000..f3867f7a --- /dev/null +++ b/core/expression/src/parser/unary.rs @@ -0,0 +1,313 @@ +use crate::lexer::{Bracket, ComparisonOperator, Identifier, LogicalOperator, Operator, TokenKind}; +use crate::parser::ast::Node; +use crate::parser::builtin::BuiltInFunction; +use crate::parser::constants::{Associativity, BINARY_OPERATORS, UNARY_OPERATORS}; +use crate::parser::error::ParserError::{FailedToParse, UnexpectedToken}; +use crate::parser::error::{ParserError, ParserResult}; +use crate::parser::parser::Parser; +use crate::parser::unary::UnaryNodeBehaviour::CompareWithReference; + +#[derive(Debug)] +pub struct Unary; + +const ROOT_NODE: Node<'static> = Node::Identifier("$"); + +impl<'arena, 'token_ref> Parser<'arena, 'token_ref, Unary> { + pub fn parse(&self) -> ParserResult<&'arena Node<'arena>> { + let result = self.root_expression()?; + if !self.is_done() { + let token = self.current(); + return Err(FailedToParse { + message: format!("Unterminated token {} on {:?}", token.value, token.span), + }); + } + + return Ok(result); + } + + fn root_expression(&self) -> ParserResult<&'arena Node<'arena>> { + let mut left_node = self.expression_pair()?; + while !self.is_done() { + let current_token = self.current(); + let join_operator = match ¤t_token.kind { + TokenKind::Operator(Operator::Logical(LogicalOperator::And)) => { + Operator::Logical(LogicalOperator::And) + } + TokenKind::Operator(Operator::Logical(LogicalOperator::Or)) + | TokenKind::Operator(Operator::Comma) => Operator::Logical(LogicalOperator::Or), + _ => { + return Err(ParserError::MemoryFailure); + } + }; + + self.next()?; + let right_node = self.expression_pair()?; + left_node = self.node(Node::Binary { + left: left_node, + operator: join_operator, + right: right_node, + }); + } + + Ok(left_node) + } + + fn expression_pair(&self) -> ParserResult<&'arena Node<'arena>> { + let mut left_node = &ROOT_NODE; + let initial_token = self.current(); + if let TokenKind::Operator(Operator::Comparison(_)) = &initial_token.kind { + // Skips + } else { + left_node = self.binary_expression(0)?; + } + + let current_token = self.current(); + match ¤t_token.kind { + TokenKind::Operator(Operator::Comparison(comparison)) => { + self.next()?; + let right_node = self.binary_expression(0)?; + left_node = self.node(Node::Binary { + left: left_node, + operator: Operator::Comparison(*comparison), + right: right_node, + }); + } + _ => { + let behaviour = UnaryNodeBehaviour::from(left_node); + match behaviour { + CompareWithReference(comparator) => { + left_node = self.node(Node::Binary { + left: &ROOT_NODE, + operator: Operator::Comparison(comparator), + right: left_node, + }) + } + UnaryNodeBehaviour::AsBoolean => { + left_node = self.node(Node::BuiltIn { + kind: BuiltInFunction::Bool, + arguments: self.bump.alloc_slice_clone(&[left_node]), + }) + } + } + } + } + + Ok(left_node) + } + + fn binary_expression(&self, precedence: u8) -> ParserResult<&'arena Node<'arena>> { + let mut node_left = self.unary_expression()?; + let mut token = self.current(); + + while let TokenKind::Operator(operator) = &token.kind { + if self.is_done() { + break; + } + + if matches!( + operator, + Operator::Comma + | Operator::Logical(LogicalOperator::And) + | Operator::Logical(LogicalOperator::Or) + ) { + break; + } + + let Some(op) = BINARY_OPERATORS.get(operator) else { + break; + }; + + if op.precedence < precedence { + break; + } + + self.next()?; + let node_right = match op.associativity { + Associativity::Left => self.binary_expression(op.precedence + 1)?, + _ => self.binary_expression(op.precedence)?, + }; + + node_left = self.node(Node::Binary { + operator: *operator, + left: node_left, + right: node_right, + }); + token = self.current(); + } + + if precedence == 0 { + if let Some(conditional_node) = + self.conditional(node_left, || self.binary_expression(0))? + { + node_left = conditional_node; + } + } + + Ok(node_left) + } + + fn unary_expression(&self) -> ParserResult<&'arena Node<'arena>> { + let token = self.current(); + + if self.depth() > 0 && token.kind == TokenKind::Identifier(Identifier::CallbackReference) { + self.next()?; + + let node = self.node(Node::Pointer); + return self.with_postfix(node, || self.binary_expression(0)); + } + + if let TokenKind::Operator(operator) = &token.kind { + let Some(unary_operator) = UNARY_OPERATORS.get(operator) else { + return Err(UnexpectedToken { + expected: "Unary token".to_string(), + received: format!("{token:?}"), + }); + }; + + self.next()?; + let expr = self.binary_expression(unary_operator.precedence)?; + let node = self.node(Node::Unary { + operator: *operator, + node: expr, + }); + + return Ok(node); + } + + if let Some(interval_node) = self.interval(|| self.binary_expression(0))? { + return Ok(interval_node); + } + + if token.kind == TokenKind::Bracket(Bracket::LeftParenthesis) { + self.next()?; + let expr = self.binary_expression(0)?; + self.expect(TokenKind::Bracket(Bracket::RightParenthesis))?; + return self.with_postfix(expr, || self.binary_expression(0)); + } + + self.literal(|| self.binary_expression(0)) + } +} + +/// Dictates the behaviour of nodes in unary mode. +/// If `CompareWithReference` is set, node will attempt to make the comparison with the reference, +/// essentially making it (in case of Equal operator) `$ == nodeValue`, or (in case of In operator) +/// `$ in nodeValue`. +/// +/// Using `AsBoolean` will cast the nodeValue to boolean and skip comparison with reference ($). +/// You may still use references in such case directly, e.g. `contains($, 'hello')`. +/// +/// Rationale behind this is to avoid scenarios where e.g. $ = false and expression is +/// `contains($, 'needle')`. If we didn't ignore the reference, unary expression will be +/// reduced to `$ == contains($, 'needle')` which will be truthy when $ does not +/// contain needle. +#[derive(Debug, PartialEq)] +enum UnaryNodeBehaviour { + CompareWithReference(ComparisonOperator), + AsBoolean, +} + +impl From<&Node<'_>> for UnaryNodeBehaviour { + fn from(value: &Node) -> Self { + use ComparisonOperator::*; + use UnaryNodeBehaviour::*; + + match value { + Node::Null => CompareWithReference(Equal), + Node::Bool(_) => CompareWithReference(Equal), + Node::Number(_) => CompareWithReference(Equal), + Node::String(_) => CompareWithReference(Equal), + Node::Pointer => AsBoolean, + Node::Array(_) => CompareWithReference(In), + Node::Identifier(_) => CompareWithReference(Equal), + Node::Closure(_) => AsBoolean, + Node::Member { .. } => CompareWithReference(Equal), + Node::Slice { .. } => CompareWithReference(In), + Node::Interval { .. } => CompareWithReference(In), + Node::Conditional { + on_true, on_false, .. + } => { + let a = UnaryNodeBehaviour::from(*on_true); + let b = UnaryNodeBehaviour::from(*on_false); + + return if a == b { + a + } else { + CompareWithReference(Equal) + }; + } + Node::Unary { node, .. } => UnaryNodeBehaviour::from(*node), + Node::Binary { + left, + operator, + right, + } => match operator { + Operator::Arithmetic(_) => { + let a = UnaryNodeBehaviour::from(*left); + let b = UnaryNodeBehaviour::from(*right); + + return if a == b { + a + } else { + CompareWithReference(Equal) + }; + } + Operator::Logical(_) => AsBoolean, + Operator::Comparison(_) => AsBoolean, + Operator::Range => CompareWithReference(In), + Operator::Slice => CompareWithReference(In), + Operator::Comma => AsBoolean, + Operator::Dot => AsBoolean, + Operator::QuestionMark => AsBoolean, + }, + Node::BuiltIn { kind, .. } => match kind { + BuiltInFunction::Len => CompareWithReference(Equal), + BuiltInFunction::Upper => CompareWithReference(Equal), + BuiltInFunction::Lower => CompareWithReference(Equal), + BuiltInFunction::Abs => CompareWithReference(Equal), + BuiltInFunction::Sum => CompareWithReference(Equal), + BuiltInFunction::Avg => CompareWithReference(Equal), + BuiltInFunction::Min => CompareWithReference(Equal), + BuiltInFunction::Max => CompareWithReference(Equal), + BuiltInFunction::Rand => CompareWithReference(Equal), + BuiltInFunction::Median => CompareWithReference(Equal), + BuiltInFunction::Mode => CompareWithReference(Equal), + BuiltInFunction::Floor => CompareWithReference(Equal), + BuiltInFunction::Ceil => CompareWithReference(Equal), + BuiltInFunction::Round => CompareWithReference(Equal), + BuiltInFunction::String => CompareWithReference(Equal), + BuiltInFunction::Number => CompareWithReference(Equal), + BuiltInFunction::Bool => CompareWithReference(Equal), + BuiltInFunction::Date => CompareWithReference(Equal), + BuiltInFunction::Time => CompareWithReference(Equal), + BuiltInFunction::Duration => CompareWithReference(Equal), + BuiltInFunction::Year => CompareWithReference(Equal), + BuiltInFunction::DayOfWeek => CompareWithReference(Equal), + BuiltInFunction::DayOfMonth => CompareWithReference(Equal), + BuiltInFunction::DayOfYear => CompareWithReference(Equal), + BuiltInFunction::WeekOfYear => CompareWithReference(Equal), + BuiltInFunction::MonthOfYear => CompareWithReference(Equal), + BuiltInFunction::MonthString => CompareWithReference(Equal), + BuiltInFunction::DateString => CompareWithReference(Equal), + BuiltInFunction::WeekdayString => CompareWithReference(Equal), + BuiltInFunction::StartOf => CompareWithReference(Equal), + BuiltInFunction::Count => CompareWithReference(Equal), + BuiltInFunction::EndOf => CompareWithReference(Equal), + BuiltInFunction::Flatten => CompareWithReference(In), + BuiltInFunction::Extract => CompareWithReference(In), + BuiltInFunction::Filter => CompareWithReference(In), + BuiltInFunction::Map => CompareWithReference(In), + BuiltInFunction::FlatMap => CompareWithReference(In), + BuiltInFunction::Contains => AsBoolean, + BuiltInFunction::StartsWith => AsBoolean, + BuiltInFunction::EndsWith => AsBoolean, + BuiltInFunction::Matches => AsBoolean, + BuiltInFunction::IsNumeric => AsBoolean, + BuiltInFunction::All => AsBoolean, + BuiltInFunction::Some => AsBoolean, + BuiltInFunction::None => AsBoolean, + BuiltInFunction::One => AsBoolean, + }, + } + } +} diff --git a/core/expression/src/parser/unary/constants.rs b/core/expression/src/parser/unary/constants.rs deleted file mode 100644 index d5d71fd2..00000000 --- a/core/expression/src/parser/unary/constants.rs +++ /dev/null @@ -1,61 +0,0 @@ -use phf::{phf_map, Map}; - -use crate::parser::definitions::Associativity::{Left, Right}; -use crate::parser::definitions::{Arity, BuiltIn, Operator}; - -pub(crate) const BUILT_INS: Map<&'static str, BuiltIn> = phf_map! { - "date" => BuiltIn { arity: Arity::Single }, - "time" => BuiltIn { arity: Arity::Single }, - "duration" => BuiltIn { arity: Arity::Single }, - - "string" => BuiltIn { arity: Arity::Single }, - "number" => BuiltIn { arity: Arity::Single }, - "isNumeric" => BuiltIn { arity: Arity::Single }, - - "upper" => BuiltIn { arity: Arity::Single }, - "lower" => BuiltIn { arity: Arity::Single }, - - "abs" => BuiltIn { arity: Arity::Single }, - "sum" => BuiltIn { arity: Arity::Single }, - "avg" => BuiltIn { arity: Arity::Single }, - "min" => BuiltIn { arity: Arity::Single }, - "max" => BuiltIn { arity: Arity::Single }, - "rand" => BuiltIn { arity: Arity::Single }, - "median" => BuiltIn { arity: Arity::Single }, - "mode" => BuiltIn { arity: Arity::Single }, - - "floor" => BuiltIn { arity: Arity::Single }, - "ceil" => BuiltIn { arity: Arity::Single }, - "round" => BuiltIn { arity: Arity::Single }, - - "flatten" => BuiltIn { arity: Arity::Single }, - - "year" => BuiltIn { arity: Arity::Single }, - "dayOfWeek" => BuiltIn { arity: Arity::Single }, - "dayOfMonth" => BuiltIn { arity: Arity::Single }, - "dayOfYear" => BuiltIn { arity: Arity::Single }, - "weekOfYear" => BuiltIn { arity: Arity::Single }, - "monthOfYear" => BuiltIn { arity: Arity::Single }, - "monthString" => BuiltIn { arity: Arity::Single }, - "dateString" => BuiltIn { arity: Arity::Single }, - "weekdayString" => BuiltIn { arity: Arity::Single }, -}; - -pub(crate) const STANDARD_OPERATORS: Map<&'static str, Operator> = phf_map! { - "or" => Operator { precedence: 10, associativity: Left }, - "and" => Operator { precedence: 15, associativity: Left }, - "==" => Operator { precedence: 20, associativity: Left }, - "!=" => Operator { precedence: 20, associativity: Left }, - "<" => Operator { precedence: 20, associativity: Left }, - ">" => Operator { precedence: 20, associativity: Left }, - "<=" => Operator { precedence: 20, associativity: Left }, - ">=" => Operator { precedence: 20, associativity: Left }, - "not in" => Operator { precedence: 20, associativity: Left }, - "in" => Operator { precedence: 20, associativity: Left }, - "+" => Operator { precedence: 30, associativity: Left }, - "-" => Operator { precedence: 30, associativity: Left }, - "*" => Operator { precedence: 60, associativity: Left }, - "/" => Operator { precedence: 60, associativity: Left }, - "%" => Operator { precedence: 60, associativity: Left }, - "^" => Operator { precedence: 70, associativity: Right }, -}; diff --git a/core/expression/src/parser/unary/mod.rs b/core/expression/src/parser/unary/mod.rs deleted file mode 100644 index b469de64..00000000 --- a/core/expression/src/parser/unary/mod.rs +++ /dev/null @@ -1,269 +0,0 @@ -use bumpalo::Bump; - -use crate::ast::Node; -use crate::lexer::token::{Token, TokenKind}; -use crate::parser::definitions::{Arity, Associativity}; -use crate::parser::error::{ParserError, ParserResult}; -use crate::parser::iter::ParserIterator; -use crate::parser::unary::constants::{BUILT_INS, STANDARD_OPERATORS}; - -mod constants; - -pub struct UnaryParser<'a, 'b> -where - 'b: 'a, -{ - iterator: ParserIterator<'a, 'b>, - bump: &'b Bump, -} - -const MAIN_NODE: Node<'static> = Node::Identifier("$"); - -impl<'a, 'b> UnaryParser<'a, 'b> -where - 'b: 'a, -{ - pub fn try_new(tokens: &'a Vec, bump: &'b Bump) -> ParserResult { - Ok(Self { - iterator: ParserIterator::try_new(tokens, bump)?, - bump, - }) - } - - pub fn parse(&self) -> ParserResult<&'b Node<'b>> { - self.expression(0, true) - } - - fn expression(&self, precedence: u8, root: bool) -> ParserResult<&'b Node<'b>> { - let mut node_left: &'b Node<'b> = self.iterator.node(MAIN_NODE)?; - let mut token = self.iterator.current(); - - while !self.iterator.is_done() { - match token.kind { - TokenKind::Operator => { - if token.value == "," { - self.iterator.next()?; - let node_right: &'b Node<'b> = self.expression(0, true)?; - - token = self.iterator.current(); - node_left = self.iterator.node(Node::Binary { - left: node_left, - operator: "or", - right: node_right, - })?; - } else if let Some(op) = STANDARD_OPERATORS.get(token.value) { - if op.precedence >= precedence { - self.iterator.next()?; - let node_right = match op.associativity { - Associativity::Left => self.expression(op.precedence + 1, false)?, - Associativity::Right => self.expression(op.precedence, false)?, - }; - - node_left = self.iterator.node(Node::Binary { - left: node_left, - operator: self.iterator.str_value(token.value), - right: node_right, - })?; - token = self.iterator.current(); - } - } else { - return Err(ParserError::FailedToParse { - message: format!( - "Unexpected operator {} on {:?}", - token.value, token.span - ), - }); - } - } - TokenKind::Identifier | TokenKind::String | TokenKind::Number => { - let node_right = self.literal()?; - if !root { - return Ok(node_right); - } - - token = self.iterator.current(); - node_left = self.iterator.node(Node::Binary { - left: node_left, - operator: "==", - right: node_right, - })?; - } - TokenKind::Bracket => { - let node_right: &Node; - - if let Some(interval) = self.interval(node_left)? { - node_left = interval; - } else if token.value == "[" { - let should_wrap = !self.iterator.lookup_back( - 1, - TokenKind::Operator, - Some(&["not in", "in"]), - ); - node_right = self.array(token)?; - - if should_wrap { - node_left = self.iterator.node(Node::Binary { - left: node_left, - right: node_right, - operator: "in", - })?; - } else { - node_left = node_right; - } - } else { - return Err(ParserError::FailedToParse { - message: format!( - "Unexpected bracket {} on {:?}", - token.value, token.span - ), - }); - } - } - } - } - - Ok(node_left) - } - - fn interval(&self, node: &'b Node<'b>) -> ParserResult>> { - // Performance optimisation: skip if expression does not contain an interval for faster evaluation - if !self.iterator.has_interval() { - return Ok(None); - } - - let current_token = self.iterator.current(); - if current_token.kind != TokenKind::Bracket { - return Ok(None); - } - - let initial_position = self.iterator.position(); - let should_wrap = - !self - .iterator - .lookup_back(1, TokenKind::Operator, Some(&["not in", "in"])); - - let left_bracket = self.iterator.current().value; - if let Err(_) = self.iterator.expect(TokenKind::Bracket, None) { - self.iterator.set_position(initial_position)?; - return Ok(None); - } - - let Ok(left) = self.literal() else { - self.iterator.set_position(initial_position)?; - return Ok(None); - }; - - if let Err(_) = self.iterator.expect(TokenKind::Operator, Some(&[".."])) { - self.iterator.set_position(initial_position)?; - return Ok(None); - } - - let Ok(right) = self.literal() else { - self.iterator.set_position(initial_position)?; - return Ok(None); - }; - - let right_bracket = self.iterator.current().value; - - if let Err(_) = self.iterator.expect(TokenKind::Bracket, None) { - self.iterator.set_position(initial_position)?; - return Ok(None); - } - - let interval_node = self.iterator.node(Node::Interval { - left, - left_bracket: self.iterator.str_value(left_bracket), - right, - right_bracket: self.iterator.str_value(right_bracket), - })?; - - if should_wrap { - return Ok(Some(self.iterator.node(Node::Binary { - left: node, - right: interval_node, - operator: "in", - })?)); - } - - Ok(Some(interval_node)) - } - - fn array(&self, _token: &Token) -> ParserResult<&'b Node<'b>> { - let mut nodes = Vec::new(); - - self.iterator.expect(TokenKind::Bracket, Some(&["["]))?; - while self.iterator.current().kind != TokenKind::Bracket - && self.iterator.current().value != "]" - { - if !nodes.is_empty() { - self.iterator.expect(TokenKind::Operator, Some(&[","]))?; - if self.iterator.current().value == "]" { - break; - } - } - - nodes.push(self.literal()?); - } - - self.iterator.expect(TokenKind::Bracket, Some(&["]"]))?; - let node = Node::Array(self.bump.alloc_slice_copy(nodes.as_slice())); - self.iterator.node(node) - } - - fn literal(&self) -> ParserResult<&'b Node<'b>> { - let token = self.iterator.current(); - - match token.kind { - TokenKind::Identifier => { - self.iterator.next()?; - match token.value { - "true" | "false" => self.iterator.bool(token), - "null" => self.iterator.null(token), - _ => self.builtin(token), - } - } - TokenKind::Number => self.iterator.number(token), - TokenKind::String => self.iterator.string(token), - _ => Err(ParserError::UnexpectedToken { - expected: "one of [identifier, number, string]".to_string(), - received: format!("{:?}", token.kind), - }), - } - } - - fn builtin(&self, token: &Token<'a>) -> ParserResult<&'b Node<'b>> { - let current_token = self.iterator.current(); - let valid_token = current_token.kind == TokenKind::Bracket && current_token.value == "("; - - if !valid_token { - return Err(ParserError::UnknownBuiltIn { - token: token.value.to_string(), - }); - } - - let built_in = BUILT_INS - .get(token.value) - .ok_or_else(|| ParserError::UnknownBuiltIn { - token: token.value.to_string(), - })?; - - self.iterator.expect(TokenKind::Bracket, Some(&["("]))?; - - match built_in.arity { - Arity::Single => { - let arg = self.literal()?; - let node = self.iterator.node(Node::BuiltIn { - name: self.iterator.str_value(token.value), - arguments: self.bump.alloc_slice_copy(&[arg]), - })?; - - self.iterator.expect(TokenKind::Bracket, Some(&[")"]))?; - - Ok(node) - } - _ => Err(ParserError::UnsupportedBuiltIn { - token: token.value.to_string(), - }), - } - } -} diff --git a/core/expression/src/vm/error.rs b/core/expression/src/vm/error.rs new file mode 100644 index 00000000..9ef60f52 --- /dev/null +++ b/core/expression/src/vm/error.rs @@ -0,0 +1,21 @@ +use thiserror::Error; + +#[derive(Debug, PartialEq, Eq, Clone, Error)] +pub enum VMError { + #[error("Unsupported opcode type")] + OpcodeErr { opcode: String, message: String }, + + #[error("Opcode out of bounds")] + OpcodeOutOfBounds { index: usize, bytecode: String }, + + #[error("Stack out of bounds")] + StackOutOfBounds { stack: String }, + + #[error("Failed to parse date time")] + ParseDateTimeErr { timestamp: String }, + + #[error("Number conversion error")] + NumberConversionError, +} + +pub(crate) type VMResult = Result; diff --git a/core/expression/src/helpers.rs b/core/expression/src/vm/helpers.rs similarity index 95% rename from core/expression/src/helpers.rs rename to core/expression/src/vm/helpers.rs index 6d9a7ca9..c4fdc979 100644 --- a/core/expression/src/helpers.rs +++ b/core/expression/src/vm/helpers.rs @@ -1,10 +1,9 @@ +use crate::vm::error::{VMError, VMResult}; use chrono::{ DateTime, Datelike, Days, NaiveDate, NaiveDateTime, NaiveTime, Timelike, Utc, Weekday, }; use once_cell::sync::Lazy; -use crate::vm::VMError; - #[allow(clippy::unwrap_used)] static ZERO_TIME: Lazy = Lazy::new(|| NaiveTime::from_hms_opt(0, 0, 0).unwrap()); @@ -14,22 +13,20 @@ static TIME_HMS: &str = "%H:%M:%S"; static TIME_HM: &str = "%H:%M"; static TIME_H: &str = "%H"; -pub(crate) fn date_time(str: &str) -> Result { +pub(crate) fn date_time(str: &str) -> VMResult { if str == "now" { return Ok(Utc::now().naive_utc()); } - let zero_time = ZERO_TIME.to_owned(); - NaiveDateTime::parse_from_str(str, DATE_TIME) - .or(NaiveDate::parse_from_str(str, DATE).map(|c| c.and_time(zero_time))) + .or(NaiveDate::parse_from_str(str, DATE).map(|c| c.and_time(*ZERO_TIME))) .or(DateTime::parse_from_rfc3339(str).map(|dt| dt.naive_utc())) .map_err(|_| VMError::ParseDateTimeErr { timestamp: str.to_string(), }) } -pub(crate) fn time(str: &str) -> Result { +pub(crate) fn time(str: &str) -> VMResult { let now = Utc::now(); if str == "now" { diff --git a/core/expression/src/vm/mod.rs b/core/expression/src/vm/mod.rs new file mode 100644 index 00000000..8fac0eff --- /dev/null +++ b/core/expression/src/vm/mod.rs @@ -0,0 +1,11 @@ +//! Virtual Machine - Evaluation of Opcodes +//! +//! The VM (Virtual Machine) module executes the generated machine-readable opcodes. +mod error; +mod helpers; +mod variable; +mod vm; + +pub use error::VMError; +pub use variable::Variable; +pub use vm::VM; diff --git a/core/expression/src/opcodes.rs b/core/expression/src/vm/variable.rs similarity index 69% rename from core/expression/src/opcodes.rs rename to core/expression/src/vm/variable.rs index c5239052..733528dc 100644 --- a/core/expression/src/opcodes.rs +++ b/core/expression/src/vm/variable.rs @@ -1,5 +1,8 @@ use std::str::FromStr; +use crate::vm::error::VMError; +use crate::vm::helpers::date_time; +use bumpalo::collections::Vec as BumpVec; use bumpalo::Bump; use chrono::NaiveDateTime; use hashbrown::hash_map::DefaultHashBuilder; @@ -7,19 +10,24 @@ use hashbrown::BumpWrapper; use rust_decimal::prelude::ToPrimitive; use rust_decimal::Decimal; use serde_json::{Map, Number, Value}; +use strum_macros::Display; -use crate::helpers::date_time; -use crate::vm::VMError; -use crate::vm::VMError::{OpcodeErr, ParseDateTimeErr}; - -#[derive(Debug)] -pub enum Variable<'a> { +/// In-memory representation of expression variables +#[derive(Debug, PartialEq, Eq, Clone, Display)] +pub enum Variable<'arena> { Null, Bool(bool), Number(Decimal), - String(&'a str), - Array(&'a [&'a Variable<'a>]), - Object(hashbrown::HashMap<&'a str, &'a Variable<'a>, DefaultHashBuilder, BumpWrapper<'a>>), + String(&'arena str), + Array(&'arena [&'arena Variable<'arena>]), + Object( + hashbrown::HashMap< + &'arena str, + &'arena Variable<'arena>, + DefaultHashBuilder, + BumpWrapper<'arena>, + >, + ), } impl<'a> Variable<'a> { @@ -30,17 +38,15 @@ impl<'a> Variable<'a> { pub fn from_serde(v: &Value, bump: &'a Bump) -> Self { match v { Value::String(str) => Variable::String(bump.alloc_str(str)), - Value::Number(f) => { - Variable::Number(Decimal::from_str_exact(f.to_string().as_str()).unwrap()) - } + Value::Number(f) => Variable::Number(Decimal::from_str_exact(f.as_str()).unwrap()), Value::Bool(b) => Variable::Bool(*b), Value::Array(v) => { - let mut arr: Vec<&'a Variable> = Vec::with_capacity(v.len()); + let mut arr = BumpVec::with_capacity_in(v.len(), bump); for i in v { - arr.push(bump.alloc(Variable::from_serde(i, bump))); + arr.push(&*bump.alloc(Variable::from_serde(i, bump))); } - Variable::Array(bump.alloc_slice_copy(arr.as_slice())) + Variable::Array(arr.into_bump_slice()) } Value::Object(bt) => { let mut tree: hashbrown::HashMap<&'a str, &'a Variable, _, _> = @@ -66,6 +72,13 @@ impl<'a> Variable<'a> { } } + pub(crate) fn as_bool(&self) -> Option { + match self { + Variable::Bool(b) => Some(*b), + _ => None, + } + } + pub(crate) fn type_name(&self) -> &str { match self { Variable::Null => "null", @@ -78,94 +91,16 @@ impl<'a> Variable<'a> { } } -#[derive(Debug)] -pub enum Opcode<'a> { - Push(Variable<'a>), - Pop, - Rot, - Fetch, - FetchEnv(&'a str), - Negate, - Not, - Equal, - Jump(usize), - JumpIfTrue(usize), - JumpIfFalse(usize), - JumpIfEnd(usize), - JumpBackward(usize), - In, - Less, - More, - LessOrEqual, - MoreOrEqual, - Abs, - Average, - Median, - Mode, - Min, - Max, - Round, - Floor, - Ceil, - Sum, - Random, - Add, - Subtract, - Multiply, - Divide, - Modulo, - Exponent, - Interval { - left_bracket: &'a str, - right_bracket: &'a str, - }, - Contains, - DateFunction(&'a str), - DateManipulation(&'a str), - Uppercase, - Lowercase, - StartsWith, - EndsWith, - Matches, - Extract, - Slice, - Array, - Len, - ParseDateTime, - ParseTime, - ParseDuration, - IncrementIt, - IncrementCount, - GetCount, - GetLen, - Pointer, - Begin, - End, - Flatten, - TypeConversion(TypeConversionKind), - TypeCheck(TypeCheckKind), -} - -#[derive(Debug)] -pub enum TypeConversionKind { - Number, - String, -} - -#[derive(Debug)] -pub enum TypeCheckKind { - Numeric, -} - impl TryFrom<&Variable<'_>> for Value { - type Error = (); + type Error = VMError; fn try_from(value: &Variable<'_>) -> Result { match value { Variable::Null => Ok(Value::Null), Variable::Bool(b) => Ok(Value::Bool(*b)), Variable::Number(n) => Ok(Value::Number( - Number::from_str(n.normalize().to_string().as_str()).map_err(|_| ())?, + Number::from_str(n.normalize().to_string().as_str()) + .map_err(|_| VMError::NumberConversionError)?, )), Variable::String(s) => Ok(Value::String(s.to_string())), Variable::Array(arr) => { @@ -178,10 +113,8 @@ impl TryFrom<&Variable<'_>> for Value { } Variable::Object(obj) => { let mut t = Map::new(); - - for k in obj.keys() { - let v = *obj.get(k).ok_or(())?; - t.insert(k.to_string(), Value::try_from(v)?); + for (key, value) in obj { + t.insert(key.to_string(), Value::try_from(*value)?); } Ok(Value::Object(t)) @@ -197,16 +130,16 @@ impl TryFrom<&Variable<'_>> for NaiveDateTime { match value { Variable::String(a) => date_time(a), Variable::Number(a) => NaiveDateTime::from_timestamp_opt( - a.to_i64().ok_or_else(|| OpcodeErr { + a.to_i64().ok_or_else(|| VMError::OpcodeErr { opcode: "DateManipulation".into(), message: "Failed to extract date".into(), })?, 0, ) - .ok_or_else(|| ParseDateTimeErr { + .ok_or_else(|| VMError::ParseDateTimeErr { timestamp: a.to_string(), }), - _ => Err(OpcodeErr { + _ => Err(VMError::OpcodeErr { opcode: "DateManipulation".into(), message: "Unsupported type".into(), }), diff --git a/core/expression/src/vm.rs b/core/expression/src/vm/vm.rs similarity index 87% rename from core/expression/src/vm.rs rename to core/expression/src/vm/vm.rs index 880c9a14..6b92b8dc 100644 --- a/core/expression/src/vm.rs +++ b/core/expression/src/vm/vm.rs @@ -1,5 +1,8 @@ use std::collections::HashMap; +use crate::compiler::{Opcode, TypeCheckKind, TypeConversionKind}; +use bumpalo::collections::String as BumpString; +use bumpalo::collections::Vec as BumpVec; use bumpalo::Bump; use chrono::NaiveDateTime; use chrono::{Datelike, Timelike}; @@ -10,57 +13,66 @@ use regex_lite::Regex; use rust_decimal::prelude::ToPrimitive; use rust_decimal::{Decimal, MathematicalOps}; use rust_decimal_macros::dec; -use thiserror::Error; -use crate::helpers::{date_time, date_time_end_of, date_time_start_of, time}; -use crate::opcodes::Variable::{Array, Bool, Null, Number, Object, String}; -use crate::opcodes::{IntervalObject, Opcode, TypeCheckKind, TypeConversionKind, Variable}; -use crate::vm::VMError::{OpcodeErr, OpcodeOutOfBounds, ParseDateTimeErr, StackOutOfBounds}; +use crate::vm::error::VMResult; +use crate::vm::variable::{IntervalObject, Variable}; + +use crate::vm::error::VMError::*; +use crate::vm::helpers::{date_time, date_time_end_of, date_time_start_of, time}; +use crate::vm::variable::Variable::*; const NULL_VAR: &'static Variable = &Null; -#[derive(Debug, Error)] -pub enum VMError { - #[error("Unsupported opcode type")] - OpcodeErr { - opcode: std::string::String, - message: std::string::String, - }, - - #[error("Opcode out of bounds")] - OpcodeOutOfBounds { - index: usize, - bytecode: std::string::String, - }, - - #[error("Stack out of bounds")] - StackOutOfBounds { stack: std::string::String }, - - #[error("Failed to parse date time")] - ParseDateTimeErr { timestamp: std::string::String }, -} - -pub struct Scope<'a> { - array: &'a Variable<'a>, +#[derive(Debug)] +pub struct Scope<'arena> { + array: &'arena Variable<'arena>, len: usize, iter: usize, count: usize, } -pub struct VM<'a> { - scopes: &'a mut Vec>, - stack: &'a mut Vec<&'a Variable<'a>>, - ip: usize, - bytecode: &'a Vec<&'a Opcode<'a>>, - bump: &'a Bump, +#[derive(Debug)] +pub struct VM<'arena> { + scopes: Vec>, + stack: Vec<&'arena Variable<'arena>>, } -impl<'a> VM<'a> { +impl<'arena> VM<'arena> { + pub fn new() -> Self { + Self { + scopes: Default::default(), + stack: Default::default(), + } + } + + pub fn run( + &mut self, + bytecode: &[Opcode<'arena>], + bump: &'arena Bump, + env: &Variable<'arena>, + ) -> VMResult<&Variable> { + self.stack.clear(); + self.scopes.clear(); + + let s = VMInner::new(bytecode, &mut self.stack, &mut self.scopes, bump).run(env); + Ok(s?) + } +} + +struct VMInner<'arena, 'parent_ref, 'bytecode_ref> { + scopes: &'parent_ref mut Vec>, + stack: &'parent_ref mut Vec<&'arena Variable<'arena>>, + bytecode: &'bytecode_ref [Opcode<'arena>], + bump: &'arena Bump, + ip: usize, +} + +impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'arena, 'parent_ref, 'bytecode_ref> { pub fn new( - bytecode: &'a Vec<&'a Opcode<'a>>, - stack: &'a mut Vec<&'a Variable<'a>>, - scopes: &'a mut Vec>, - bump: &'a Bump, + bytecode: &'bytecode_ref [Opcode<'arena>], + stack: &'parent_ref mut Vec<&'arena Variable<'arena>>, + scopes: &'parent_ref mut Vec>, + bump: &'arena Bump, ) -> Self { Self { scopes, @@ -71,21 +83,21 @@ impl<'a> VM<'a> { } } - fn push(&mut self, var: Variable<'a>) { + fn push(&mut self, var: Variable<'arena>) { self.stack.push(self.bump.alloc(var)); } - fn pop(&mut self) -> Result<&'a Variable<'a>, VMError> { + fn pop(&mut self) -> VMResult<&'arena Variable<'arena>> { self.stack.pop().ok_or_else(|| StackOutOfBounds { stack: format!("{:?}", self.stack), }) } - fn push_ref(&mut self, var: &'a Variable<'a>) { + fn push_ref(&mut self, var: &'arena Variable<'arena>) { self.stack.push(var); } - pub fn run(&mut self, env: &'a Variable<'a>) -> Result<&'a Variable<'a>, VMError> { + pub fn run(&mut self, env: &Variable<'arena>) -> VMResult<&'arena Variable<'arena>> { if self.ip != 0 { self.ip = 0; } @@ -103,7 +115,7 @@ impl<'a> VM<'a> { match op { Opcode::Push(v) => { - self.push_ref(v); + self.push(v.clone()); } Opcode::Pop => { self.pop()?; @@ -157,7 +169,7 @@ impl<'a> VM<'a> { let a = self.pop()?; match a { Number(n) => { - self.stack.push(self.bump.alloc(Number(-*n))); + self.push(Number(-*n)); } _ => { return Err(OpcodeErr { @@ -170,7 +182,7 @@ impl<'a> VM<'a> { Opcode::Not => { let a = self.pop()?; match a { - Bool(b) => self.stack.push(self.bump.alloc(Bool(!(*b)))), + Bool(b) => self.push(Bool(!(*b))), _ => { return Err(OpcodeErr { opcode: "Not".into(), @@ -184,19 +196,19 @@ impl<'a> VM<'a> { let a = self.pop()?; match (a, b) { (Number(a), Number(b)) => { - self.stack.push(self.bump.alloc(Bool(a == b))); + self.push(Bool(a == b)); } (Bool(a), Bool(b)) => { - self.stack.push(self.bump.alloc(Bool(a == b))); + self.push(Bool(a == b)); } (String(a), String(b)) => { - self.stack.push(self.bump.alloc(Bool(a == b))); + self.push(Bool(a == b)); } (Null, Null) => { - self.stack.push(self.bump.alloc(Bool(true))); + self.push(Bool(true)); } _ => { - self.stack.push(self.bump.alloc(Bool(false))); + self.push(Bool(false)); } } } @@ -254,7 +266,7 @@ impl<'a> VM<'a> { _ => false, }); - self.stack.push(self.bump.alloc(Bool(is_in))); + self.push(Bool(is_in)); } (Number(v), Object(_)) => { let interval = @@ -302,8 +314,7 @@ impl<'a> VM<'a> { let open_stmt = is_open && (first || second); let closed_stmt = !is_open && first && second; - self.stack - .push(self.bump.alloc(Bool(open_stmt || closed_stmt))); + self.push(Bool(open_stmt || closed_stmt)); } _ => { return Err(OpcodeErr { @@ -319,7 +330,7 @@ impl<'a> VM<'a> { _ => false, }); - self.stack.push(self.bump.alloc(Bool(is_in))); + self.push(Bool(is_in)); } (Bool(a), Array(arr)) => { let is_in = arr.iter().any(|b| match b { @@ -327,7 +338,7 @@ impl<'a> VM<'a> { _ => false, }); - self.stack.push(self.bump.alloc(Bool(is_in))); + self.push(Bool(is_in)); } (Null, Array(arr)) => { let is_in = arr.iter().any(|b| match b { @@ -335,7 +346,7 @@ impl<'a> VM<'a> { _ => false, }); - self.stack.push(self.bump.alloc(Bool(is_in))); + self.push(Bool(is_in)); } _ => { return Err(OpcodeErr { @@ -350,7 +361,7 @@ impl<'a> VM<'a> { let a = self.pop()?; match (a, b) { - (Number(a), Number(b)) => self.stack.push(self.bump.alloc(Bool(*a < *b))), + (Number(a), Number(b)) => self.push(Bool(*a < *b)), _ => { return Err(OpcodeErr { opcode: "Less".into(), @@ -364,7 +375,7 @@ impl<'a> VM<'a> { let a = self.pop()?; match (a, b) { - (Number(a), Number(b)) => self.stack.push(self.bump.alloc(Bool(*a > *b))), + (Number(a), Number(b)) => self.push(Bool(*a > *b)), _ => { return Err(OpcodeErr { opcode: "More".into(), @@ -378,7 +389,7 @@ impl<'a> VM<'a> { let a = self.pop()?; match (a, b) { - (Number(a), Number(b)) => self.stack.push(self.bump.alloc(Bool(*a <= *b))), + (Number(a), Number(b)) => self.push(Bool(*a <= *b)), _ => { return Err(OpcodeErr { opcode: "LessOrEqual".into(), @@ -392,7 +403,7 @@ impl<'a> VM<'a> { let a = self.pop()?; match (a, b) { - (Number(a), Number(b)) => self.stack.push(self.bump.alloc(Bool(*a >= *b))), + (Number(a), Number(b)) => self.push(Bool(*a >= *b)), _ => { return Err(OpcodeErr { opcode: "MoreOrEqual".into(), @@ -405,7 +416,7 @@ impl<'a> VM<'a> { let a = self.pop()?; match a { - Number(a) => self.stack.push(self.bump.alloc(Number(a.abs()))), + Number(a) => self.push(Number(a.abs())), _ => { return Err(OpcodeErr { opcode: "Abs".into(), @@ -418,7 +429,7 @@ impl<'a> VM<'a> { let var = self.pop()?; match var { - Number(a) => self.stack.push(self.bump.alloc(Number(a.round()))), + Number(a) => self.push(Number(a.round())), _ => { return Err(OpcodeErr { opcode: "Round".into(), @@ -431,7 +442,7 @@ impl<'a> VM<'a> { let var = self.pop()?; match var { - Number(a) => self.stack.push(self.bump.alloc(Number(a.ceil()))), + Number(a) => self.push(Number(a.ceil())), _ => { return Err(OpcodeErr { opcode: "Ceil".into(), @@ -444,7 +455,7 @@ impl<'a> VM<'a> { let var = self.pop()?; match var { - Number(a) => self.stack.push(self.bump.alloc(Number(a.floor()))), + Number(a) => self.push(Number(a.floor())), _ => { return Err(OpcodeErr { opcode: "Floor".into(), @@ -463,8 +474,7 @@ impl<'a> VM<'a> { })?; let random_number = fastrand::i64(0..=upper_range); - self.stack - .push(self.bump.alloc(Number(Decimal::from(random_number)))); + self.push(Number(Decimal::from(random_number))); } _ => { return Err(OpcodeErr { @@ -492,7 +502,7 @@ impl<'a> VM<'a> { })?; let avg = sum / Decimal::from(arr.len()); - self.stack.push(self.bump.alloc(Number(avg))); + self.push(Number(avg)); } _ => { return Err(OpcodeErr { @@ -537,7 +547,7 @@ impl<'a> VM<'a> { message: "Array out of bounds".into(), })?; - self.push(Number((*center_num).clone())); + self.push(Number(**center_num)); } else { let center_left = num_arr.get(center - 1).ok_or_else(|| OpcodeErr { opcode: "Median".into(), @@ -630,7 +640,7 @@ impl<'a> VM<'a> { }), })?; - self.stack.push(self.bump.alloc(Number(min))); + self.push(Number(min)); } _ => { return Err(OpcodeErr { @@ -673,7 +683,7 @@ impl<'a> VM<'a> { }), })?; - self.stack.push(self.bump.alloc(Number(max))); + self.push(Number(max)); } _ => { return Err(OpcodeErr { @@ -700,7 +710,7 @@ impl<'a> VM<'a> { }), })?; - self.stack.push(self.bump.alloc(Number(sum))); + self.push(Number(sum)); } _ => { return Err(OpcodeErr { @@ -715,14 +725,14 @@ impl<'a> VM<'a> { let a = self.pop()?; match (a, b) { - (Number(a), Number(b)) => self.stack.push(self.bump.alloc(Number(a + b))), + (Number(a), Number(b)) => self.push(Number(a + b)), (String(a), String(b)) => { - let mut str1 = std::string::String::with_capacity(a.len() + b.len()); + let mut str1 = + BumpString::with_capacity_in(a.len() + b.len(), self.bump); str1.push_str(a); str1.push_str(b); - self.stack - .push(self.bump.alloc(String(self.bump.alloc_str(&str1)))) + self.push(String(str1.into_bump_str())); } _ => { return Err(OpcodeErr { @@ -737,7 +747,7 @@ impl<'a> VM<'a> { let a = self.pop()?; match (a, b) { - (Number(a), Number(b)) => self.stack.push(self.bump.alloc(Number(a - b))), + (Number(a), Number(b)) => self.push(Number(a - b)), _ => { return Err(OpcodeErr { opcode: "Subtract".into(), @@ -751,7 +761,7 @@ impl<'a> VM<'a> { let a = self.pop()?; match (a, b) { - (Number(a), Number(b)) => self.stack.push(self.bump.alloc(Number(a * b))), + (Number(a), Number(b)) => self.push(Number(a * b)), _ => { return Err(OpcodeErr { opcode: "Multiply".into(), @@ -765,7 +775,7 @@ impl<'a> VM<'a> { let a = self.pop()?; match (a, b) { - (Number(a), Number(b)) => self.stack.push(self.bump.alloc(Number(a / b))), + (Number(a), Number(b)) => self.push(Number(a / b)), _ => { return Err(OpcodeErr { opcode: "Divide".into(), @@ -779,7 +789,7 @@ impl<'a> VM<'a> { let a = self.pop()?; match (a, b) { - (Number(a), Number(b)) => self.stack.push(self.bump.alloc(Number(a % b))), + (Number(a), Number(b)) => self.push(Number(a % b)), _ => { return Err(OpcodeErr { opcode: "Modulo".into(), @@ -794,7 +804,7 @@ impl<'a> VM<'a> { match (a, b) { (Number(a), Number(b)) => { - self.stack.push(self.bump.alloc(Number(a.powd(*b)))) + self.push(Number(a.powd(*b))); } _ => { return Err(OpcodeErr { @@ -836,8 +846,7 @@ impl<'a> VM<'a> { match a { String(a) => { let str = a.to_uppercase(); - self.stack - .push(self.bump.alloc(String(self.bump.alloc_str(&str)))) + self.push(String(self.bump.alloc_str(&str))); } _ => { return Err(OpcodeErr { @@ -853,8 +862,7 @@ impl<'a> VM<'a> { match a { String(a) => { let str = a.to_lowercase(); - self.stack - .push(self.bump.alloc(String(self.bump.alloc_str(&str)))) + self.push(String(self.bump.alloc_str(&str))); } _ => { return Err(OpcodeErr { @@ -870,7 +878,7 @@ impl<'a> VM<'a> { match (a, b) { (String(a), String(b)) => { - self.stack.push(self.bump.alloc(Bool(a.contains(b)))) + self.push(Bool(a.contains(b))); } _ => match a { Array(arr) => { @@ -882,7 +890,7 @@ impl<'a> VM<'a> { _ => false, }); - self.stack.push(self.bump.alloc(Bool(is_in))); + self.push(Bool(is_in)); } _ => { return Err(OpcodeErr { @@ -899,7 +907,7 @@ impl<'a> VM<'a> { match (a, b) { (String(a), String(b)) => { - self.stack.push(self.bump.alloc(Bool(a.starts_with(b)))) + self.push(Bool(a.starts_with(b))); } _ => { return Err(OpcodeErr { @@ -915,7 +923,7 @@ impl<'a> VM<'a> { match (a, b) { (String(a), String(b)) => { - self.stack.push(self.bump.alloc(Bool(a.ends_with(b)))) + self.push(Bool(a.ends_with(b))); } _ => { return Err(OpcodeErr { @@ -966,14 +974,12 @@ impl<'a> VM<'a> { .iter() .map(|c| c.map(|c| c.as_str())) .filter_map(|c| c) - .map(|s| { - self.bump.alloc(String(self.bump.alloc_str(s))) as &Variable - }) + .map(|s| &*self.bump.alloc(String(self.bump.alloc_str(s)))) .collect::>() }) .unwrap_or_default(); - self.push(Array(self.bump.alloc_slice_copy(captures.as_slice()))); + self.push(Array(self.bump.alloc_slice_clone(captures.as_slice()))); } Opcode::DateManipulation(operation) => { let timestamp = self.pop()?; @@ -999,7 +1005,7 @@ impl<'a> VM<'a> { } }; - self.stack.push(self.bump.alloc(var)); + self.push(var); } Opcode::DateFunction(name) => { let unit_var = self.pop()?; @@ -1053,7 +1059,7 @@ impl<'a> VM<'a> { message: "Index out of range".into(), })?; - self.stack.push(self.bump.alloc(Array(slice))) + self.push(Array(slice)); } String(s) => { let slice = s.get(from..=to).ok_or_else(|| OpcodeErr { @@ -1061,7 +1067,7 @@ impl<'a> VM<'a> { message: "Index out of range".into(), })?; - self.stack.push(self.bump.alloc(String(slice))) + self.push(String(slice)); } _ => { return Err(OpcodeErr { @@ -1081,33 +1087,26 @@ impl<'a> VM<'a> { } Opcode::Array => { let size = self.pop()?; - let mut arr = Vec::new(); - match size { - Number(s) => { - let to = s.round().to_usize().ok_or_else(|| OpcodeErr { - opcode: "Array".into(), - message: "Failed to extract argument".into(), - })?; + let Number(s) = size else { + return Err(OpcodeErr { + opcode: "Array".into(), + message: "Unsupported type".into(), + }); + }; - for _ in 0..to { - arr.push(self.pop()?); - } - } - _ => { - return Err(OpcodeErr { - opcode: "Array".into(), - message: "Unsupported type".into(), - }) - } + let to = s.round().to_usize().ok_or_else(|| OpcodeErr { + opcode: "Array".into(), + message: "Failed to extract argument".into(), + })?; + + let mut arr = BumpVec::with_capacity_in(to, &self.bump); + for _ in 0..to { + arr.push(self.pop()?); } - arr.reverse(); - self.stack.push( - self.bump - .alloc(Array(self.bump.alloc_slice_copy(arr.as_slice()))), - ); + self.push(Array(arr.into_bump_slice())); } Opcode::Len => { let current = self.stack.last().ok_or_else(|| OpcodeErr { @@ -1126,7 +1125,7 @@ impl<'a> VM<'a> { } }; - self.stack.push(self.bump.alloc(Number(len.into()))) + self.push(Number(len.into())); } Opcode::Flatten => { let current = self.pop()?; @@ -1137,16 +1136,15 @@ impl<'a> VM<'a> { }); }; - let mut flat_arr = Vec::new(); + let mut flat_arr = BumpVec::new_in(&self.bump); + flat_arr.reserve(arr.len()); + arr.iter().for_each(|&v| match v { Array(arr) => arr.iter().for_each(|&v| flat_arr.push(v)), _ => flat_arr.push(v), }); - self.stack.push( - self.bump - .alloc(Array(self.bump.alloc_slice_copy(flat_arr.as_slice()))), - ) + self.push(Array(flat_arr.into_bump_slice())); } Opcode::ParseDateTime => { let a = self.pop()?; @@ -1164,7 +1162,7 @@ impl<'a> VM<'a> { } }; - self.stack.push(self.bump.alloc(Number(ts.into()))) + self.push(Number(ts.into())); } Opcode::ParseTime => { let a = self.pop()?; @@ -1181,7 +1179,8 @@ impl<'a> VM<'a> { }) } }; - self.stack.push(self.bump.alloc(Number(ts.into()))) + + self.push(Number(ts.into())); } Opcode::ParseDuration => { let a = self.pop()?; @@ -1204,7 +1203,7 @@ impl<'a> VM<'a> { } }; - self.stack.push(self.bump.alloc(Number(dur.into()))); + self.push(Number(dur.into())); } Opcode::TypeCheck(check) => { let var = self.pop()?; @@ -1232,6 +1231,7 @@ impl<'a> VM<'a> { (TypeConversionKind::String, Bool(v)) => { make_string(v.to_string().as_str()) } + (TypeConversionKind::String, Null) => make_string("null"), (TypeConversionKind::String, _) => { return Err(OpcodeErr { opcode: "TypeConversion".into(), @@ -1264,6 +1264,23 @@ impl<'a> VM<'a> { ), }) } + (TypeConversionKind::Bool, Number(n)) => { + self.bump.alloc(Bool(!n.is_zero())) + } + (TypeConversionKind::Bool, String(s)) => { + let value = match *s { + "true" => true, + "false" => false, + _ => s.is_empty(), + }; + + self.bump.alloc(Bool(value)) + } + (TypeConversionKind::Bool, Bool(_)) => var, + (TypeConversionKind::Bool, Null) => self.bump.alloc(Bool(false)), + (TypeConversionKind::Bool, Object(_) | Array(_)) => { + self.bump.alloc(Bool(true)) + } }; self.push_ref(converted_var); diff --git a/core/expression/tests/data/unary.csv b/core/expression/tests/data/unary.csv index 0b3b6e5d..e4a0bad3 100644 --- a/core/expression/tests/data/unary.csv +++ b/core/expression/tests/data/unary.csv @@ -11,32 +11,92 @@ false;{ "$": false }; true < 10;{$: 5};true >= 10;{$: 10};true <= 5;{$: 5};true -[-10..0];{$: 5};true -(-10..0];{$: 5};false -[-10..0);{$: 5};true -(-10..0);{$: 5};false -[-15..-5];{$: 5};false +[-10..0];{$: 0};true +(-10..0];{$: -10};false +[-10..0);{$: -10};true +(-10..0);{$: 0};false +[-15..-5];{$: -4.99};false [-5..5];{$: 5};true -[-6..6];{$: 5};false +[-6..6];{$: 5};true [-5..5];{$: 5};true -[0..0];{$: 5};true -> 5 and < 10;{$: 5};true -<= 10 and > 5;{$: 5};true +[0..0];{$: 5};false +> 5 and < 10;{$: 5};false +<= 10 and > 5;{$: 10};true >= 10 or < 5;{$: 10};true <= 5 or >= 10;{$: 10};true -[-10..0] and > -5;{$: -5};true -(-10..0] or > 0;{$: 5};true +[-10..0] and > -5;{$: -5};false +[-10..0] and > -5;{$: -4.99};true +(-10..0] or > 0;{$: -10};false [-10..0) and > -5;{$: -5};false (-10..0) or > 0;{$: 5};true -[-15..-5] and <= -10;{$: -5};false +[-15..-5] and <= -10;{$: -10};true [-5..5] or > 0;{$: 5};true -[-5..5] or < 0;{$: 5};false +[-5..5] or < 0;{$: 5.01};false [-5..5] and > 0;{$: 5};true -[0..0] and < 5;{$: 5};true -> 10 or < -5;{$: 5};true -> 10, < -5;{$: 5};true -<= -10 and >= -15;{$: -5};true +[0..0] and < 5;{$: 0};true +> 10 or < -5;{$: 5};false +> 10, < -5;{$: 15};true +<= -10 and >= -15;{$: -5};false -# String + +# Numeric expressions +>= 5 + 2; { "$": 7 }; true +< 5 * 3; { "$": 15 }; false +>= 2 * 3; { "$": 6 }; true +<= 5 / 2; { "$": 2.5 }; true +!= 10 - 3; { "$": 7 }; false +5 % 2; { "$": 1 }; true +abs(-5); { "$": 5 }; true +floor(4.8); { "$": 4 }; true +ceil(4.1); { "$": 5 }; true +round(3.5); { "$": 4 }; true +max([3, 5, 8]); { "$": 8 }; true +min([3, 5, 8]); { "$": 3 }; true + +# More complex scenarios +(5 + 3) * 2; { "$": 16 }; true +(10 - 4) / 2; { "$": 3 }; true +abs(-5) * (3 + 2); { "$": 25 }; true +2 ^ 3; { "$": 8 }; true +max([3, 5, 8]) - min([2, 7, 4]); { "$": 6 }; true +floor(8.9) + ceil(4.1) * 2; { "$": 18 }; true +rand(100) >= 0 and rand(100) <= 100; { "$": true }; true +sum([1, 2, 3, 4, 5]) / len([1, 2, 3, 4, 5]); { "$": 3 }; true +median([4, 2, 7, 5, 3]); { "$": 4 }; true +mode([1, 2, 2, 3, 3, 3, 4, 4, 4, 4]); { "$": 4 }; true + +# String expressions 'GB','US';{$: 'US'};true -'GB','US';{$: 'FR'};false \ No newline at end of file +'GB','US';{$: 'FR'};false +'hello' + " " + "world"; { "$": 'hello world' }; true +len($) == 13; { "$": "Hello, World!" }; true +lower("Hello, World!"); { "$": "HELLO, WORLD!" }; false +upper("Hello, World!"); { "$": "HELLO, WORLD!" }; true +startsWith($, "Hello"); { "$": "Hello, World!" }; true +startsWith($, "World"); { "$": "Hello, World!" }; false +endsWith($, "World!"); { "$": "Hello, World!" }; true +endsWith($, "Hello!"); { "$": "Hello, World!" }; false +contains($, "lo"); { "$": "Hello, World!" }; true +contains($, "foo"); { "$": "Hello, World!" }; false +matches($, "H[a-z]+, W[a-z]+!"); { "$": "Hello, World!" }; true +matches($, "[0-9]+"); { "$": "Hello, World!" }; false + +# String Slice +$[0:5] == 'sample'; { "$": 'sample_string' }; true +$[7:12] == 'string'; { "$": 'sample_string' }; true +$[7:] == 'string'; { "$": 'sample_string' }; true +$[:5] == 'sample'; { "$": 'sample_string' }; true + +# Type conversion +string(true); { "$": 'true' }; true +string(false); { "$": 'false' }; true +string(123); { "$": '123' }; true + +number('123.45'); { "$": 123.45 }; true +number('-56.78'); { "$": -56.78 }; true +number(true); { "$": 1 }; true + +bool('true'); { "$": true }; true +bool('false'); { "$": false }; true +bool(1); { "$": true }; true +bool(0); { "$": false }; true \ No newline at end of file diff --git a/core/expression/tests/isolate.rs b/core/expression/tests/isolate.rs index 153dae7f..c00fa386 100644 --- a/core/expression/tests/isolate.rs +++ b/core/expression/tests/isolate.rs @@ -1,11 +1,10 @@ use anyhow::Context; use bumpalo::Bump; +use serde_json::{json, Value}; use std::ops::Index; -use serde_json::{json, Value}; - -use zen_expression::isolate::Isolate; -use zen_expression::opcodes::Variable; +use zen_expression::vm::Variable; +use zen_expression::Isolate; struct TestEnv { env: Value, @@ -562,13 +561,23 @@ fn isolate_standard_test() { }, ]); - let isolate = Isolate::default(); + let mut isolate = Isolate::new(); for TestEnv { env, cases } in tests { - isolate.inject_env(&env); + isolate.set_environment(&env); for TestCase { expr, result } in cases { - assert_eq!(result, isolate.run_standard(expr).unwrap(), "{}", expr); + let isolate_result = isolate.run_standard(expr); + let Ok(response) = isolate_result else { + assert!( + false, + "Expression failed: {expr}. Error: {:?}", + isolate_result.unwrap_err() + ); + continue; + }; + + assert_eq!(result, response, "{}", expr); } } } @@ -672,14 +681,14 @@ fn isolate_unary_tests() { }, ]); - let isolate = Isolate::default(); + let mut isolate = Isolate::new(); for UnaryTestEnv { env, cases, reference, } in tests { - isolate.inject_env(&env); + isolate.set_environment(&env); isolate.set_reference(reference).unwrap(); for TestCase { expr, result } in cases { @@ -703,7 +712,7 @@ fn variable_serde_test() { #[test] fn isolate_test_decimals() { - let isolate = Isolate::default(); + let mut isolate = Isolate::new(); let result = isolate.run_standard("9223372036854775807").unwrap(); assert_eq!(result, Value::from(9223372036854775807i64)); @@ -728,10 +737,10 @@ fn test_standard_csv() { let output: Value = serde_json5::from_str(output_str).unwrap(); - let isolate = Isolate::default(); + let mut isolate = Isolate::new(); if !input_str.is_empty() { let input: Value = serde_json5::from_str(input_str).unwrap(); - isolate.inject_env(&input); + isolate.set_environment(&input); } let result = isolate @@ -745,3 +754,40 @@ fn test_standard_csv() { ); } } + +#[test] +fn test_unary_csv() { + let csv_data = include_str!("data/unary.csv"); + let mut r = csv::ReaderBuilder::new() + .delimiter(b';') + .from_reader(csv_data.as_bytes()); + + while let Some(maybe_row) = r.records().next() { + let Ok(row) = maybe_row else { + continue; + }; + + let (expression, input_str, output_str) = (row.index(0), row.index(1), row.index(2)); + if expression.starts_with("#") { + continue; + } + + let output: Value = serde_json5::from_str(output_str).unwrap(); + + let mut isolate = Isolate::new(); + if !input_str.is_empty() { + let input: Value = serde_json5::from_str(input_str).unwrap(); + isolate.set_environment(&input); + } + + let result = isolate + .run_unary(expression) + .context(format!("Expression: {expression}")) + .unwrap(); + + assert_eq!( + result, output, + "Expression {expression}. Expected: {output}, got: {result}" + ); + } +} diff --git a/core/expression/tests/lexer.rs b/core/expression/tests/lexer.rs index 847100fd..6ec68210 100644 --- a/core/expression/tests/lexer.rs +++ b/core/expression/tests/lexer.rs @@ -1,5 +1,7 @@ -use zen_expression::lexer::token::{Token, TokenKind}; -use zen_expression::lexer::Lexer; +use zen_expression::lexer::{ + ArithmeticOperator, Bracket, ComparisonOperator, Identifier, Lexer, LogicalOperator, Operator, + Token, TokenKind, +}; struct LexerTest { test: &'static str, @@ -17,6 +19,14 @@ fn lexer_test() { value: "hello", }]), }, + LexerTest { + test: "null", + result: Vec::from([Token { + kind: TokenKind::Identifier(Identifier::Null), + span: (0, 4), + value: "null", + }]), + }, LexerTest { test: "'double' 'single' 'abc'", result: Vec::from([ @@ -106,12 +116,12 @@ fn lexer_test() { value: "1_000_000", }, Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (25, 28), value: "_42", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Arithmetic(ArithmeticOperator::Subtract)), span: (29, 30), value: "-", }, @@ -126,37 +136,37 @@ fn lexer_test() { test: "a and orb().val", result: Vec::from([ Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (0, 1), value: "a", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Logical(LogicalOperator::And)), span: (2, 5), value: "and", }, Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (6, 9), value: "orb", }, Token { - kind: TokenKind::Bracket, + kind: TokenKind::Bracket(Bracket::LeftParenthesis), span: (9, 10), value: "(", }, Token { - kind: TokenKind::Bracket, + kind: TokenKind::Bracket(Bracket::RightParenthesis), span: (10, 11), value: ")", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Dot), span: (11, 12), value: ".", }, Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (12, 15), value: "val", }, @@ -166,17 +176,17 @@ fn lexer_test() { test: "foo.bar", result: Vec::from([ Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (0, 3), value: "foo", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Dot), span: (3, 4), value: ".", }, Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (4, 7), value: "bar", }, @@ -186,32 +196,32 @@ fn lexer_test() { test: "foo .bar == .baz", result: Vec::from([ Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (0, 3), value: "foo", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Dot), span: (4, 5), value: ".", }, Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (5, 8), value: "bar", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Comparison(ComparisonOperator::Equal)), span: (9, 11), value: "==", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Dot), span: (12, 13), value: ".", }, Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (13, 16), value: "baz", }, @@ -221,17 +231,17 @@ fn lexer_test() { test: "func()", result: Vec::from([ Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (0, 4), value: "func", }, Token { - kind: TokenKind::Bracket, + kind: TokenKind::Bracket(Bracket::LeftParenthesis), span: (4, 5), value: "(", }, Token { - kind: TokenKind::Bracket, + kind: TokenKind::Bracket(Bracket::RightParenthesis), span: (5, 6), value: ")", }, @@ -241,47 +251,47 @@ fn lexer_test() { test: "not abc not in i not(false) not ", result: Vec::from([ Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Logical(LogicalOperator::Not)), span: (0, 3), value: "not", }, Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (4, 7), value: "abc", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Comparison(ComparisonOperator::NotIn)), span: (8, 14), value: "not in", }, Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (15, 16), value: "i", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Logical(LogicalOperator::Not)), span: (17, 20), value: "not", }, Token { - kind: TokenKind::Bracket, + kind: TokenKind::Bracket(Bracket::LeftParenthesis), span: (20, 21), value: "(", }, Token { - kind: TokenKind::Identifier, + kind: TokenKind::Boolean(false), span: (21, 26), value: "false", }, Token { - kind: TokenKind::Bracket, + kind: TokenKind::Bracket(Bracket::RightParenthesis), span: (26, 27), value: ")", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Logical(LogicalOperator::Not)), span: (28, 31), value: "not", }, @@ -291,12 +301,12 @@ fn lexer_test() { test: "not in_var", result: Vec::from([ Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Logical(LogicalOperator::Not)), span: (0, 3), value: "not", }, Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (4, 10), value: "in_var", }, @@ -306,7 +316,7 @@ fn lexer_test() { test: "[1..5)", result: Vec::from([ Token { - kind: TokenKind::Bracket, + kind: TokenKind::Bracket(Bracket::LeftSquareBracket), span: (0, 1), value: "[", }, @@ -316,7 +326,7 @@ fn lexer_test() { value: "1", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Range), span: (2, 4), value: "..", }, @@ -326,7 +336,7 @@ fn lexer_test() { value: "5", }, Token { - kind: TokenKind::Bracket, + kind: TokenKind::Bracket(Bracket::RightParenthesis), span: (5, 6), value: ")", }, @@ -336,22 +346,24 @@ fn lexer_test() { test: "product.price > 500 ? 'hello' : 'world'", result: Vec::from([ Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (0, 7), value: "product", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Dot), span: (7, 8), value: ".", }, Token { - kind: TokenKind::Identifier, + kind: TokenKind::Identifier(Identifier::Variable), span: (8, 13), value: "price", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Comparison( + ComparisonOperator::GreaterThan, + )), span: (14, 15), value: ">", }, @@ -361,7 +373,7 @@ fn lexer_test() { value: "500", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::QuestionMark), span: (20, 21), value: "?", }, @@ -371,7 +383,7 @@ fn lexer_test() { value: "hello", }, Token { - kind: TokenKind::Operator, + kind: TokenKind::Operator(Operator::Slice), span: (31, 32), value: ":", }, @@ -384,12 +396,12 @@ fn lexer_test() { }, ]); - let lexer = Lexer::new(); + let mut lexer = Lexer::new(); for LexerTest { test, result } in tests { let tokens = lexer.tokenize(test); - assert!(tokens.is_ok()); - assert_eq!(tokens.unwrap().borrow().as_slice(), result); + + assert_eq!(tokens.unwrap(), result.as_slice()); } } diff --git a/core/expression/tests/standard.rs b/core/expression/tests/standard.rs index b17dec43..350c962e 100644 --- a/core/expression/tests/standard.rs +++ b/core/expression/tests/standard.rs @@ -3,9 +3,10 @@ use rust_decimal::Decimal; use rust_decimal_macros::dec; -use zen_expression::ast::Node; -use zen_expression::lexer::Lexer; -use zen_expression::parser::StandardParser; +use zen_expression::lexer::{ + ArithmeticOperator, ComparisonOperator, Lexer, LogicalOperator, Operator, +}; +use zen_expression::parser::{Node, Parser}; struct StandardTest { src: &'static str, @@ -70,7 +71,7 @@ fn standard_test() { StandardTest { src: "-3", result: &Node::Unary { - operator: "-", + operator: Operator::Arithmetic(ArithmeticOperator::Subtract), node: &Node::Number(D3), }, }, @@ -78,7 +79,7 @@ fn standard_test() { src: "1 - 2", result: &Node::Binary { left: &Node::Number(D1), - operator: "-", + operator: Operator::Arithmetic(ArithmeticOperator::Subtract), right: &Node::Number(D2), }, }, @@ -87,21 +88,21 @@ fn standard_test() { result: &Node::Binary { left: &Node::Binary { left: &Node::Number(D1), - operator: "-", + operator: Operator::Arithmetic(ArithmeticOperator::Subtract), right: &Node::Number(D2), }, - operator: "*", + operator: Operator::Arithmetic(ArithmeticOperator::Multiply), right: &Node::Number(D3), }, }, StandardTest { src: "a or b or c", result: &Node::Binary { - operator: "or", + operator: Operator::Logical(LogicalOperator::Or), left: &Node::Binary { left: &Node::Identifier("a"), right: &Node::Identifier("b"), - operator: "or", + operator: Operator::Logical(LogicalOperator::Or), }, right: &Node::Identifier("c"), }, @@ -109,23 +110,23 @@ fn standard_test() { StandardTest { src: "a or b and c", result: &Node::Binary { - operator: "or", + operator: Operator::Logical(LogicalOperator::Or), left: &Node::Identifier("a"), right: &Node::Binary { left: &Node::Identifier("b"), right: &Node::Identifier("c"), - operator: "and", + operator: Operator::Logical(LogicalOperator::And), }, }, }, StandardTest { src: "(a or b) and c", result: &Node::Binary { - operator: "and", + operator: Operator::Logical(LogicalOperator::And), left: &Node::Binary { left: &Node::Identifier("a"), right: &Node::Identifier("b"), - operator: "or", + operator: Operator::Logical(LogicalOperator::Or), }, right: &Node::Identifier("c"), }, @@ -134,11 +135,11 @@ fn standard_test() { src: "2^4 - 1", result: &Node::Binary { left: &Node::Binary { - operator: "^", + operator: Operator::Arithmetic(ArithmeticOperator::Power), left: &Node::Number(D2), right: &Node::Number(D4), }, - operator: "-", + operator: Operator::Arithmetic(ArithmeticOperator::Subtract), right: &Node::Number(D1), }, }, @@ -184,21 +185,21 @@ fn standard_test() { result: &Node::Binary { left: &Node::String("a"), right: &Node::String("b"), - operator: "==", + operator: Operator::Comparison(ComparisonOperator::Equal), }, }, StandardTest { src: "+0 != -1", result: &Node::Binary { left: &Node::Unary { - operator: "+", + operator: Operator::Arithmetic(ArithmeticOperator::Add), node: &Node::Number(D0), }, right: &Node::Unary { - operator: "-", + operator: Operator::Arithmetic(ArithmeticOperator::Subtract), node: &Node::Number(D1), }, - operator: "!=", + operator: Operator::Comparison(ComparisonOperator::NotEqual), }, }, StandardTest { @@ -220,7 +221,7 @@ fn standard_test() { src: "x not in (1..9]", result: &Node::Binary { left: &Node::Identifier("x"), - operator: "not in", + operator: Operator::Comparison(ComparisonOperator::NotIn), right: &Node::Interval { left_bracket: "(", left: &Node::Number(D1), @@ -232,7 +233,7 @@ fn standard_test() { StandardTest { src: "not in_var", result: &Node::Unary { - operator: "not", + operator: Operator::Logical(LogicalOperator::Not), node: &Node::Identifier("in_var"), }, }, @@ -276,22 +277,30 @@ fn standard_test() { src: "0 in []", result: &Node::Binary { left: &Node::Number(D0), - operator: "in", + operator: Operator::Comparison(ComparisonOperator::In), right: &Node::Array(&[]), }, }, ]); - let lexer = Lexer::new(); + let mut lexer = Lexer::new(); let mut bump = Bump::new(); for StandardTest { src, result } in tests { - let t_res = lexer.tokenize(src).unwrap(); - let tokens = t_res.borrow(); - let unary_parser = StandardParser::try_new(tokens.as_ref(), &bump).unwrap(); - let ast = unary_parser.parse(); - assert_eq!(ast.unwrap(), result); + let tokens = lexer.tokenize(src).unwrap(); + let unary_parser = Parser::try_new(tokens, &bump).unwrap().standard(); + let parser_result = unary_parser.parse(); + let Ok(ast) = parser_result else { + assert!( + false, + "Failed on expression: {}. Error: {:?}.", + src, + parser_result.unwrap_err() + ); + return; + }; + assert_eq!(ast, result, "Failed on expression: {}", src); bump.reset(); } } @@ -300,14 +309,13 @@ fn standard_test() { fn failure_tests() { let tests: Vec<&str> = Vec::from(["a + b ++", "null.nested.property", "false.nested.property"]); - let lexer = Lexer::new(); + let mut lexer = Lexer::new(); let mut bump = Bump::new(); for test in tests { - let t_res = lexer.tokenize(test).unwrap(); - let tokens = t_res.borrow(); - let unary_parser = StandardParser::try_new(tokens.as_ref(), &bump).unwrap(); - let ast = unary_parser.parse(); + let tokens = lexer.tokenize(test).unwrap(); + let parser = Parser::try_new(tokens, &bump).unwrap().standard(); + let ast = parser.parse(); assert!(ast.is_err()); bump.reset(); diff --git a/core/expression/tests/unary.rs b/core/expression/tests/unary.rs index ba130b99..eee00197 100644 --- a/core/expression/tests/unary.rs +++ b/core/expression/tests/unary.rs @@ -2,9 +2,8 @@ use bumpalo::Bump; use rust_decimal::Decimal; use rust_decimal_macros::dec; -use zen_expression::ast::Node; -use zen_expression::lexer::Lexer; -use zen_expression::parser::UnaryParser; +use zen_expression::lexer::{ComparisonOperator, Lexer, LogicalOperator, Operator}; +use zen_expression::parser::{BuiltInFunction, Node, Parser}; struct UnaryTest { src: &'static str, @@ -25,7 +24,7 @@ fn unary_test() { src: "'str'", result: &Node::Binary { left: &Node::Identifier("$"), - operator: "==", + operator: Operator::Comparison(ComparisonOperator::Equal), right: &Node::String("str"), }, }, @@ -33,38 +32,38 @@ fn unary_test() { src: "20.5", result: &Node::Binary { left: &Node::Identifier("$"), - operator: "==", + operator: Operator::Comparison(ComparisonOperator::Equal), right: &Node::Number(D20P5), }, }, UnaryTest { src: "'a', 'b', 'c'", result: &Node::Binary { - operator: "or", + operator: Operator::Logical(LogicalOperator::Or), left: &Node::Binary { - left: &Node::Identifier("$"), - operator: "==", - right: &Node::String("a"), - }, - right: &Node::Binary { - operator: "or", + operator: Operator::Logical(LogicalOperator::Or), left: &Node::Binary { left: &Node::Identifier("$"), - operator: "==", - right: &Node::String("b"), + operator: Operator::Comparison(ComparisonOperator::Equal), + right: &Node::String("a"), }, right: &Node::Binary { left: &Node::Identifier("$"), - operator: "==", - right: &Node::String("c"), + operator: Operator::Comparison(ComparisonOperator::Equal), + right: &Node::String("b"), }, }, + right: &Node::Binary { + left: &Node::Identifier("$"), + operator: Operator::Comparison(ComparisonOperator::Equal), + right: &Node::String("c"), + }, }, }, UnaryTest { src: "[1..10]", result: &Node::Binary { - operator: "in", + operator: Operator::Comparison(ComparisonOperator::In), left: &Node::Identifier("$"), right: &Node::Interval { left_bracket: "[", @@ -77,7 +76,7 @@ fn unary_test() { UnaryTest { src: "in [1..10]", result: &Node::Binary { - operator: "in", + operator: Operator::Comparison(ComparisonOperator::In), left: &Node::Identifier("$"), right: &Node::Interval { left_bracket: "[", @@ -90,7 +89,7 @@ fn unary_test() { UnaryTest { src: "not in [1..10]", result: &Node::Binary { - operator: "not in", + operator: Operator::Comparison(ComparisonOperator::NotIn), left: &Node::Identifier("$"), right: &Node::Interval { left_bracket: "[", @@ -103,7 +102,7 @@ fn unary_test() { UnaryTest { src: "[1, 2, 3]", result: &Node::Binary { - operator: "in", + operator: Operator::Comparison(ComparisonOperator::In), left: &Node::Identifier("$"), right: &Node::Array(&[&Node::Number(D1), &Node::Number(D2), &Node::Number(D3)]), }, @@ -111,10 +110,10 @@ fn unary_test() { UnaryTest { src: "date('2022-01-01')", result: &Node::Binary { - operator: "==", + operator: Operator::Comparison(ComparisonOperator::Equal), left: &Node::Identifier("$"), right: &Node::BuiltIn { - name: "date", + kind: BuiltInFunction::Date, arguments: &[&Node::String("2022-01-01")], }, }, @@ -122,10 +121,10 @@ fn unary_test() { UnaryTest { src: "time('14:00:00')", result: &Node::Binary { - operator: "==", + operator: Operator::Comparison(ComparisonOperator::Equal), left: &Node::Identifier("$"), right: &Node::BuiltIn { - name: "time", + kind: BuiltInFunction::Time, arguments: &[&Node::String("14:00:00")], }, }, @@ -133,21 +132,20 @@ fn unary_test() { UnaryTest { src: "< 50", result: &Node::Binary { - operator: "<", + operator: Operator::Comparison(ComparisonOperator::LessThan), left: &Node::Identifier("$"), right: &Node::Number(D50), }, }, ]); - let lexer = Lexer::new(); + let mut lexer = Lexer::new(); let mut bump = Bump::new(); for UnaryTest { src, result } in tests { - let t_res = lexer.tokenize(src).unwrap(); - let tokens = t_res.borrow(); - let unary_parser = UnaryParser::try_new(tokens.as_ref(), &bump).unwrap(); - let ast = unary_parser.parse().unwrap(); + let tokens = lexer.tokenize(src).unwrap(); + let parser = Parser::try_new(tokens, &bump).unwrap().unary(); + let ast = parser.parse().unwrap(); assert_eq!(ast, result); bump.reset(); @@ -156,15 +154,14 @@ fn unary_test() { #[test] fn failure_tests() { - let tests: Vec<&str> = Vec::from(["a + b ++", "a +++ b +--= fa", "null.a", "false.b"]); + let tests: Vec<&str> = Vec::from(["a + b ++", "null.a", "false.b"]); - let lexer = Lexer::new(); + let mut lexer = Lexer::new(); let mut bump = Bump::new(); for test in tests { - let t_res = lexer.tokenize(test).unwrap(); - let tokens = t_res.borrow(); - let unary_parser = UnaryParser::try_new(tokens.as_ref(), &bump).unwrap(); + let tokens = lexer.tokenize(test).unwrap(); + let unary_parser = Parser::try_new(tokens, &bump).unwrap().standard(); let ast = unary_parser.parse(); assert!(ast.is_err()); diff --git a/core/expression_repl/Cargo.toml b/core/expression_repl/Cargo.toml index 97879c04..20b90d72 100644 --- a/core/expression_repl/Cargo.toml +++ b/core/expression_repl/Cargo.toml @@ -10,4 +10,5 @@ publish = false colored = "2.0.4" rustyline = "12.0.0" serde_json = "1.0.108" -zen-expression = { path = "../expression", version = "0.16.1" } +zen-expression = { path = "../expression" } + diff --git a/core/expression_repl/src/main.rs b/core/expression_repl/src/main.rs index 71ee44d5..1a6bc92c 100644 --- a/core/expression_repl/src/main.rs +++ b/core/expression_repl/src/main.rs @@ -3,7 +3,7 @@ use rustyline::config::Configurer; use rustyline::{DefaultEditor, Result}; use serde_json::Value; -use zen_expression::isolate::Isolate; +use zen_expression::Isolate; trait PrettyPrint { fn pretty_print(&self) -> String; @@ -47,7 +47,7 @@ fn main() -> Result<()> { break; }; - let isolate = Isolate::default(); + let mut isolate = Isolate::new(); let result = isolate.run_standard(line.as_str()); match result {