From 3e7cdfcde1d44df7b0b34dce40d858ef1be585a4 Mon Sep 17 00:00:00 2001 From: stefan-gorules <127550877+stefan-gorules@users.noreply.github.com> Date: Mon, 23 Sep 2024 09:23:48 +0200 Subject: [PATCH] feat: intellisense (#244) * local: fault tolerant parser * fix tests * add intellisense * fix minor issues * improve support * fix fmt * fix * fix: improve error messages * improve error handling and ast * fix lint * fix tests * exclude zen-nodejs from rust tests --- .github/workflows/rust.yaml | 4 +- core/engine/src/handler/table/zen.rs | 12 +- core/expression/Cargo.toml | 2 +- core/expression/benches/standard.rs | 2 +- core/expression/benches/unary.rs | 2 +- core/expression/src/compiler/compiler.rs | 2 + core/expression/src/compiler/error.rs | 3 + core/expression/src/intellisense/mod.rs | 153 +++ core/expression/src/intellisense/scope.rs | 8 + core/expression/src/intellisense/types/mod.rs | 2 + .../src/intellisense/types/provider.rs | 715 +++++++++++++ .../src/intellisense/types/type_info.rs | 51 + core/expression/src/isolate.rs | 14 +- core/expression/src/lexer/token.rs | 17 +- core/expression/src/lib.rs | 1 + core/expression/src/parser/ast.rs | 152 ++- core/expression/src/parser/error.rs | 26 +- core/expression/src/parser/mod.rs | 4 + core/expression/src/parser/parser.rs | 972 ++++++++++++------ core/expression/src/parser/result.rs | 28 + core/expression/src/parser/standard.rs | 122 ++- core/expression/src/parser/unary.rs | 238 +++-- core/expression/src/variable/mod.rs | 5 +- core/expression/src/variable/types/conv.rs | 94 ++ core/expression/src/variable/types/mod.rs | 40 + core/expression/src/variable/types/util.rs | 199 ++++ core/expression/src/vm/vm.rs | 2 + core/expression/tests/isolate.rs | 7 +- core/expression/tests/standard.rs | 35 +- core/expression/tests/unary.rs | 14 +- core/expression_repl/src/main.rs | 2 +- 31 files changed, 2399 insertions(+), 529 deletions(-) create mode 100644 core/expression/src/intellisense/mod.rs create mode 100644 core/expression/src/intellisense/scope.rs create mode 100644 core/expression/src/intellisense/types/mod.rs create mode 100644 core/expression/src/intellisense/types/provider.rs create mode 100644 core/expression/src/intellisense/types/type_info.rs create mode 100644 core/expression/src/parser/result.rs create mode 100644 core/expression/src/variable/types/conv.rs create mode 100644 core/expression/src/variable/types/mod.rs create mode 100644 core/expression/src/variable/types/util.rs diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index dedc0b8e..c7088035 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -52,8 +52,8 @@ jobs: - uses: actions/checkout@v3 - name: Install Rust run: rustup install 1.80 - - run: cargo test --workspace --all-features --exclude zen-ffi - - run: cargo test --workspace --all-features --exclude zen-ffi --release + - run: cargo test --workspace --all-features --exclude zen-ffi --exclude zen-nodejs + - run: cargo test --workspace --all-features --exclude zen-ffi --exclude zen-nodejs --release build: name: cargo +${{ matrix.rust }} build diff --git a/core/engine/src/handler/table/zen.rs b/core/engine/src/handler/table/zen.rs index a9f20046..fc640425 100644 --- a/core/engine/src/handler/table/zen.rs +++ b/core/engine/src/handler/table/zen.rs @@ -153,15 +153,15 @@ impl<'a> DecisionTableHandler<'a> { for input in &content.inputs { let rule_value = rule.get(input.id.as_str())?; - let mut input_identifier = input.id.clone(); - if let Some(input_field) = &input.field { - input_identifier = format!("{input_field}[{input_identifier}]"); - } + let Some(input_field) = &input.field else { + continue; + }; - if let Some(reference) = self.isolate.get_reference(input_identifier.as_str()) { - reference_map.insert(input_identifier.clone(), reference); + if let Some(reference) = self.isolate.get_reference(input_field.as_str()) { + reference_map.insert(input_field.clone(), reference); } + let input_identifier = format!("{input_field}[{}]", &input.id); expressions.insert(input_identifier, rule_value.clone()); } diff --git a/core/expression/Cargo.toml b/core/expression/Cargo.toml index 5b605758..fa82eb1c 100644 --- a/core/expression/Cargo.toml +++ b/core/expression/Cargo.toml @@ -16,7 +16,7 @@ fastrand = { workspace = true } once_cell = { workspace = true } regex = { workspace = true, optional = true } regex-lite = { workspace = true, optional = true } -serde = { workspace = true } +serde = { workspace = true, features = ["rc", "derive"] } serde_json = { workspace = true, features = ["arbitrary_precision"] } strum = { workspace = true } strum_macros = { workspace = true } diff --git a/core/expression/benches/standard.rs b/core/expression/benches/standard.rs index 90d67bf1..9cdf15d9 100644 --- a/core/expression/benches/standard.rs +++ b/core/expression/benches/standard.rs @@ -12,7 +12,7 @@ fn bench_source(b: &mut Bencher, src: &'static str) { b.iter(|| { let std_parser = Parser::try_new(tokens, &bump).unwrap().standard(); - criterion::black_box(std_parser.parse().unwrap()); + criterion::black_box(std_parser.parse()); bump.reset(); }); diff --git a/core/expression/benches/unary.rs b/core/expression/benches/unary.rs index 12443d3c..a6d4e5c4 100644 --- a/core/expression/benches/unary.rs +++ b/core/expression/benches/unary.rs @@ -11,7 +11,7 @@ fn bench_source(b: &mut Bencher, src: &'static str) { b.iter(|| { let unary_parser = Parser::try_new(tokens, &bump).unwrap().unary(); - criterion::black_box(unary_parser.parse().unwrap()); + criterion::black_box(unary_parser.parse()); bump.reset(); }) diff --git a/core/expression/src/compiler/compiler.rs b/core/expression/src/compiler/compiler.rs index b7503509..13209453 100644 --- a/core/expression/src/compiler/compiler.rs +++ b/core/expression/src/compiler/compiler.rs @@ -132,6 +132,7 @@ impl<'arena, 'bytecode_ref> CompilerInner<'arena, 'bytecode_ref> { } Node::Identifier(v) => Ok(self.emit(Opcode::FetchEnv(v))), Node::Closure(v) => self.compile_node(v), + Node::Parenthesized(v) => self.compile_node(v), Node::Member { node, property } => { self.compile_node(node)?; self.compile_node(property)?; @@ -588,6 +589,7 @@ impl<'arena, 'bytecode_ref> CompilerInner<'arena, 'bytecode_ref> { Ok(self.emit(Opcode::End)) } }, + Node::Error { .. } => Err(CompilerError::UnexpectedErrorNode), } } } diff --git a/core/expression/src/compiler/error.rs b/core/expression/src/compiler/error.rs index fffe938f..d3085a74 100644 --- a/core/expression/src/compiler/error.rs +++ b/core/expression/src/compiler/error.rs @@ -10,6 +10,9 @@ pub enum CompilerError { #[error("Argument not found for builtin {builtin} at index {index}")] ArgumentNotFound { builtin: String, index: usize }, + + #[error("Unexpected error node")] + UnexpectedErrorNode, } pub(crate) type CompilerResult = Result; diff --git a/core/expression/src/intellisense/mod.rs b/core/expression/src/intellisense/mod.rs new file mode 100644 index 00000000..e84f7503 --- /dev/null +++ b/core/expression/src/intellisense/mod.rs @@ -0,0 +1,153 @@ +use crate::arena::UnsafeArena; +use crate::intellisense::scope::IntelliSenseScope; +use crate::intellisense::types::provider::TypesProvider; +use crate::lexer::Lexer; +use crate::parser::{Node, Parser}; +use crate::variable::VariableType; +use serde::Serialize; +use std::cell::RefCell; +use std::rc::Rc; + +mod scope; +mod types; + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct IntelliSenseToken { + pub span: (u32, u32), + pub kind: Rc, + pub node_kind: &'static str, + pub error: Option, +} + +pub struct IntelliSense<'arena> { + arena: UnsafeArena<'arena>, + lexer: Lexer<'arena>, +} + +impl<'arena> IntelliSense<'arena> { + pub fn new() -> Self { + Self { + arena: UnsafeArena::new(), + lexer: Lexer::new(), + } + } + + pub fn type_check( + &mut self, + source: &'arena str, + data: &VariableType, + ) -> Option> { + let arena = self.arena.get(); + + let tokens = self.lexer.tokenize(source).ok()?; + let parser = Parser::try_new(tokens, &arena).map(|p| p.standard()).ok()?; + + let parser_result = parser.with_metadata().parse(); + let ast = parser_result.root; + let metadata = parser_result.metadata?; + + let type_data = TypesProvider::generate( + ast, + IntelliSenseScope { + pointer_data: data, + root_data: data, + current_data: data, + }, + ); + + let results = RefCell::new(Vec::new()); + ast.walk(|node| { + let addr = node as *const Node as usize; + let mut r = results.borrow_mut(); + let typ = type_data.get_type(node); + + r.push(IntelliSenseToken { + span: node + .span() + .or_else(|| metadata.get(&addr).map(|s| s.span)) + .unwrap_or_default(), + node_kind: node.into(), + error: typ.map(|t| t.error.clone()).flatten(), + kind: typ + .map(|t| t.kind.clone()) + .unwrap_or_else(|| Rc::new(VariableType::Any)), + }); + }); + + self.arena.with_mut(|a| a.reset()); + Some(results.into_inner()) + } + + pub fn type_check_unary( + &mut self, + source: &'arena str, + data: &VariableType, + ) -> Option> { + let arena = self.arena.get(); + + let tokens = self.lexer.tokenize(source).ok()?; + let parser = Parser::try_new(tokens, &arena).map(|p| p.unary()).ok()?; + + let parser_result = parser.with_metadata().parse(); + let ast = parser_result.root; + let metadata = parser_result.metadata?; + + let type_data = TypesProvider::generate( + ast, + IntelliSenseScope { + pointer_data: data, + root_data: data, + current_data: data, + }, + ); + + let results = RefCell::new(Vec::new()); + ast.walk(|node| { + let addr = node as *const Node as usize; + let mut r = results.borrow_mut(); + let typ = type_data.get_type(node); + + r.push(IntelliSenseToken { + span: metadata.get(&addr).map(|s| s.span).unwrap_or_default(), + node_kind: node.into(), + error: typ.map(|t| t.error.clone()).flatten(), + kind: typ + .map(|t| t.kind.clone()) + .unwrap_or_else(|| Rc::new(VariableType::Any)), + }); + }); + + self.arena.with_mut(|a| a.reset()); + Some(results.into_inner()) + } +} + +#[cfg(test)] +mod tests { + use crate::intellisense::IntelliSense; + use crate::variable::VariableType; + use serde_json::json; + + #[test] + fn sample_test() { + let mut is = IntelliSense::new(); + + let data = json!({ "customer": { "firstName": "John", "lastName": "Doe", "array": [{"a": 5}, {"a": 6}] } }); + let data_type: VariableType = data.into(); + + let typ = is.type_check("customer.array[0]", &data_type); + println!("{:?}", typ); + } + + #[test] + fn sample_test_unary() { + let mut is = IntelliSense::new(); + + let data = json!({ "customer": { "firstName": "John", "lastName": "Doe" }, "$": 10}); + let data_type: VariableType = data.into(); + + let typ = is.type_check_unary("> 10", &data_type); + println!("{typ:?}"); + } +} diff --git a/core/expression/src/intellisense/scope.rs b/core/expression/src/intellisense/scope.rs new file mode 100644 index 00000000..8cf40777 --- /dev/null +++ b/core/expression/src/intellisense/scope.rs @@ -0,0 +1,8 @@ +use crate::variable::VariableType; + +#[derive(Clone, Debug)] +pub struct IntelliSenseScope<'a> { + pub root_data: &'a VariableType, + pub current_data: &'a VariableType, + pub pointer_data: &'a VariableType, +} diff --git a/core/expression/src/intellisense/types/mod.rs b/core/expression/src/intellisense/types/mod.rs new file mode 100644 index 00000000..bf6a4364 --- /dev/null +++ b/core/expression/src/intellisense/types/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod provider; +mod type_info; diff --git a/core/expression/src/intellisense/types/provider.rs b/core/expression/src/intellisense/types/provider.rs new file mode 100644 index 00000000..d3af6275 --- /dev/null +++ b/core/expression/src/intellisense/types/provider.rs @@ -0,0 +1,715 @@ +use crate::intellisense::scope::IntelliSenseScope; +use crate::intellisense::types::type_info::TypeInfo; +use crate::lexer::{ArithmeticOperator, ComparisonOperator, LogicalOperator, Operator}; +use crate::parser::{Arity, BuiltInFunction, Node}; +use crate::variable::VariableType; +use serde_json::{Number, Value}; +use std::collections::HashMap; +use std::rc::Rc; + +#[derive(Debug)] +pub struct TypesProvider { + types: HashMap, +} + +impl TypesProvider { + pub fn generate(root: &Node, scope: IntelliSenseScope) -> Self { + let mut s = Self { + types: HashMap::new(), + }; + + s.determine(root, scope, false); + s + } + + pub fn get_type(&self, node: &Node) -> Option<&TypeInfo> { + let addr = node_address(node); + self.types.get(&addr) + } + + fn set_type(&mut self, node: &Node, type_info: TypeInfo) { + let addr = node_address(node); + self.types.insert(addr, type_info); + } + + fn update_type(&mut self, node: &Node, updater: F) + where + F: FnOnce(&mut TypeInfo), + { + let addr = node_address(node); + if let Some(reference) = self.types.get_mut(&addr) { + updater(reference) + } + } + + fn set_error(&mut self, node: &Node, message: String) { + self.update_type(node, |typ| { + typ.error = Some(message); + }); + } + + fn determine(&mut self, node: &Node, scope: IntelliSenseScope, detailed: bool) -> TypeInfo { + #[allow(non_snake_case)] + let V = |vt: VariableType| TypeInfo::from(vt); + #[allow(non_snake_case)] + let Const = |v: Value| TypeInfo::from(VariableType::Constant(Rc::new(v))); + #[allow(non_snake_case)] + let Error = |error: String| TypeInfo { + kind: Rc::from(VariableType::Any), + error: Some(error), + }; + + let node_type = match node { + Node::Null => V(VariableType::Null), + Node::Bool(b) => match detailed { + true => Const(Value::Bool(*b)), + false => V(VariableType::Bool), + }, + Node::Number(n) => match detailed { + true => Const(Value::Number(Number::from_string_unchecked( + n.normalize().to_string(), + ))), + false => V(VariableType::Number), + }, + Node::String(s) => match detailed { + true => Const(Value::String(s.to_string())), + false => V(VariableType::String), + }, + Node::TemplateString(_) => V(VariableType::String), + + Node::Pointer => V(scope.pointer_data.clone()), + Node::Root => V(scope.root_data.clone()), + + Node::Slice { node, from, to } => { + if let Some(f) = from { + let from_type = self.determine(f, scope.clone(), false); + if !from_type.satisfies(&VariableType::Number) { + self.set_error(node, format!("Invalid slice index: expected a `number`, but found `{from_type}`.")); + } + } + + if let Some(t) = to { + let to_type = self.determine(t, scope.clone(), false); + if !to_type.satisfies(&VariableType::Number) { + self.set_error( + node, + format!( + "Invalid slice index: expected a `number`, but found `{to_type}`." + ), + ); + } + } + + let node_type = self.determine(node, scope.clone(), false); + match node_type.kind.as_ref() { + VariableType::Any => V(VariableType::Any), + VariableType::String => V(VariableType::String), + VariableType::Array(inner) => TypeInfo::from(inner.clone()), + VariableType::Constant(c) => match c.as_ref() { + Value::String(_) => V(VariableType::String), + Value::Array(inner) => match VariableType::from(inner).array_item() { + Some(item) => TypeInfo::from(item), + None => Error("Array expected".to_string()), + }, + _ => { + Error("Slice operation is only allowed on `string | any[]`".to_string()) + } + }, + _ => Error("Slice operation is only allowed on `string | any[]`".to_string()), + } + } + + Node::Array(items) => { + let mut type_list: Vec> = items + .iter() + .map(|n| self.determine(n, scope.clone(), false).kind) + .collect(); + let first = type_list.pop(); + let all_same = type_list.iter().all(|t| Some(t) == first.as_ref()); + + match (first, all_same) { + (Some(typ), true) => V(VariableType::Array(typ)), + _ => V(VariableType::Array(Rc::new(VariableType::Any))), + } + } + + Node::Object(obj) => { + let obj_type = obj + .iter() + .filter_map(|(k, v)| { + let key_type = self.determine(k, scope.clone(), true); + Some(( + key_type.kind.as_const_str()?.to_string(), + self.determine(v, scope.clone(), false).kind, + )) + }) + .collect(); + + V(VariableType::Object(obj_type)) + } + Node::Identifier(i) => TypeInfo::from(scope.root_data.get(&VariableType::Constant( + Rc::from(Value::String(i.to_string())), + ))), + Node::Member { node, property } => { + let node_type = self.determine(node, scope.clone(), true); + let property_type = self.determine(property, scope.clone(), true); + + match node_type.kind.as_ref() { + VariableType::Any => V(VariableType::Any), + VariableType::Null => V(VariableType::Null), + VariableType::Array(inner) => { + if !property_type.satisfies(&VariableType::Number) { + self.set_error( + property, + format!("Expression of type `{property_type}` cannot be used to index `{node_type}`."), + ); + } + + TypeInfo::from(inner.clone()) + } + VariableType::Object(obj) => { + if !property_type.satisfies(&VariableType::String) { + self.set_error( + property, + format!("Expression of type `{property_type}` cannot be used to index `{node_type}`."), + ); + } + + match property_type.as_const_str() { + None => V(VariableType::Any), + Some(key) => TypeInfo::from( + obj.get(key).cloned().unwrap_or(Rc::new(VariableType::Any)), + ), + } + } + VariableType::Constant(c) => match c.as_ref() { + Value::Null => V(VariableType::Null), + Value::Array(arr) => { + if !property_type.satisfies(&VariableType::Number) { + self.set_error( + property, + format!("Expression of type `{property_type}` cannot be used to index `{node_type}`."), + ); + } + + match VariableType::from(arr).array_item() { + Some(item) => TypeInfo::from(item), + None => Error("Expected an array".to_string()), + } + } + Value::Object(obj) => { + if !property_type.satisfies(&VariableType::String) { + self.set_error( + property, + format!("Expression of type `{property_type}` cannot be used to index `{node_type}`."), + ); + } + + match property_type.as_const_str() { + None => V(VariableType::Any), + Some(key) => V(obj + .get(key) + .cloned() + .map(VariableType::from) + .unwrap_or(VariableType::Any)), + } + } + _ => Error(format!("Expression of type `{property_type}` cannot be used to index `{node_type}`.")), + }, + _ => Error(format!("Expression of type `{property_type}` cannot be used to index `{node_type}`.")), + } + } + Node::Binary { + left, + right, + operator, + } => { + let left_type = self.determine(left, scope.clone(), false); + let right_type = self.determine(right, scope.clone(), false); + + match operator { + Operator::Arithmetic(arith) => match arith { + ArithmeticOperator::Add => match (left_type.omit_const(), right_type.omit_const()) { + (VariableType::Number, VariableType::Number) => V(VariableType::Number), + (VariableType::String, VariableType::String) => V(VariableType::String), + (VariableType::Any, VariableType::Number | VariableType::String) => V(VariableType::Any), + (VariableType::Number | VariableType::String, VariableType::Any) => V(VariableType::Any), + _ => Error(format!( + "Operator `{operator}` cannot be applied to types `{left_type}` and `{right_type}`." + )), + }, + ArithmeticOperator::Subtract + | ArithmeticOperator::Multiply + | ArithmeticOperator::Divide + | ArithmeticOperator::Modulus + | ArithmeticOperator::Power => match (left_type.omit_const(), right_type.omit_const()) { + (VariableType::Number | VariableType::Any, VariableType::Number | VariableType::Any) => V(VariableType::Number), + _ => Error(format!( + "Operator `{operator}` cannot be applied to types `{left_type}` and `{right_type}`." + )), + }, + }, + Operator::Logical(l) => match l { + LogicalOperator::And | LogicalOperator::Or | LogicalOperator::Not => { + match (left_type.omit_const(), right_type.omit_const()) { + (VariableType::Bool | VariableType::Any, VariableType::Bool | VariableType::Any) => V(VariableType::Bool), + _ => Error(format!( + "Operator `{operator}` cannot be applied to types `{left_type}` and `{right_type}`." + )), + } + } + LogicalOperator::NullishCoalescing => TypeInfo::from(right_type.kind), + }, + Operator::Comparison(comp) => match comp { + ComparisonOperator::Equal => V(VariableType::Bool), + ComparisonOperator::NotEqual => V(VariableType::Bool), + ComparisonOperator::LessThan + | ComparisonOperator::GreaterThan + | ComparisonOperator::LessThanOrEqual + | ComparisonOperator::GreaterThanOrEqual => match (left_type.omit_const(), right_type.omit_const()) { + (VariableType::Number | VariableType::Any, VariableType::Number | VariableType::Any) => V(VariableType::Bool), + _ => Error(format!( + "Operator `{operator}` cannot be applied to types `{left_type}` and `{right_type}`." + )), + }, + ComparisonOperator::In | ComparisonOperator::NotIn => match (left_type.kind.as_ref(), right_type.kind.as_ref()) { + (_, VariableType::Array(_)) => V(VariableType::Bool), + (_, VariableType::Object(_)) => V(VariableType::Bool), + (VariableType::Any, _) => V(VariableType::Bool), + (_, VariableType::Any) => V(VariableType::Bool), + _ => Error(format!( + "Operator `{operator}` cannot be applied to types `{left_type}` and `{right_type}`." + )) + } + }, + _ => V(VariableType::Any), + } + } + Node::Conditional { + condition, + on_true, + on_false, + } => { + let condition_type = self.determine(condition, scope.clone(), false); + if !condition_type.satisfies(&VariableType::Bool) { + self.set_error( + condition, + format!("Ternary operator cannot be applied to type `{condition_type}`."), + ); + } + + let true_type = self.determine(on_true, scope.clone(), false); + let false_type = self.determine(on_false, scope.clone(), false); + + V(true_type.kind.merge(false_type.kind.as_ref())) + } + Node::Unary { node, operator } => { + let node_type = self.determine(node, scope.clone(), false); + + match operator { + Operator::Arithmetic(arith) => match arith { + ArithmeticOperator::Add | ArithmeticOperator::Subtract => { + if !node_type.satisfies(&VariableType::Number) { + self.set_error(node, format!("Operator `{operator}` cannot be applied to type `{node_type}`.")) + } + + V(VariableType::Number) + } + ArithmeticOperator::Multiply + | ArithmeticOperator::Divide + | ArithmeticOperator::Modulus + | ArithmeticOperator::Power => Error("Unsupported operator".to_string()), + }, + Operator::Logical(logical) => match logical { + LogicalOperator::Not => { + if !node_type.satisfies(&VariableType::Bool) { + self.set_error(node, format!("Operator `{operator}` cannot be applied to type `{node_type}`.")) + } + + V(VariableType::Bool) + } + LogicalOperator::And + | LogicalOperator::Or + | LogicalOperator::NullishCoalescing => { + Error("Unsupported operator".to_string()) + } + }, + Operator::Comparison(_) + | Operator::Range + | Operator::Comma + | Operator::Slice + | Operator::Dot + | Operator::QuestionMark => Error("Unsupported operator".to_string()), + } + } + Node::Interval { left, right, .. } => { + let left_type = self.determine(left, scope.clone(), false); + if !left_type.satisfies(&VariableType::Number) { + self.set_error( + left, + format!("Interval cannot be created from type `{left_type}`."), + ) + } + + let right_type = self.determine(right, scope.clone(), false); + if !right_type.satisfies(&VariableType::Number) { + self.set_error( + right, + format!("Interval cannot be created from type `{right_type}`."), + ) + } + + V(VariableType::Any) + } + Node::BuiltIn { arguments, kind } => { + let mut type_list: Vec> = arguments + .iter() + .map(|n| self.determine(n, scope.clone(), false).kind) + .collect(); + + let arg_len = match kind.arity() { + Arity::Single => 1, + Arity::Closure | Arity::Dual => 2, + }; + + if type_list.len() != arg_len { + self.set_type( + node, + Error(format!( + "Expected {arg_len} arguments, but got {}.", + type_list.len() + )), + ); + } + + if kind.arity() == Arity::Closure { + let ptr_type = type_list[0].array_item().unwrap_or_default(); + let new_type = self.determine( + arguments[1], + IntelliSenseScope { + pointer_data: &ptr_type, + current_data: scope.current_data, + root_data: scope.root_data, + }, + false, + ); + + type_list[1] = new_type.kind; + } + + match kind { + BuiltInFunction::Len => { + if !type_list[0].satisfies(&VariableType::String) + && !type_list[0] + .satisfies(&VariableType::Array(VariableType::Any.into())) + { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `string | any[]`.", type_list[0])); + } + + V(VariableType::Number) + } + BuiltInFunction::Contains => { + match (type_list[0].omit_const(), type_list[1].omit_const()) { + (VariableType::String, VariableType::String) + | (VariableType::Any, _) + | (_, VariableType::Any) => { + // ok + } + (VariableType::Array(vt), b) => { + if !b.satisfies(&vt) { + self.set_error(arguments[1], format!("Argument of type `{b}` is not assignable to parameter of type `{vt}`.")); + } + } + _ => self.set_error(node, "Unsupported call signature.".to_string()), + } + + V(VariableType::Bool) + } + BuiltInFunction::Upper | BuiltInFunction::Lower => { + if !type_list[0].satisfies(&VariableType::String) { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[0])); + } + + V(VariableType::String) + } + BuiltInFunction::StartsWith + | BuiltInFunction::EndsWith + | BuiltInFunction::Matches => { + if !type_list[0].satisfies(&VariableType::String) { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[0])); + } + + if !type_list[1].satisfies(&VariableType::String) { + self.set_error(arguments[1], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[1])); + } + + V(VariableType::Bool) + } + BuiltInFunction::Extract | BuiltInFunction::Split => { + if !type_list[0].satisfies(&VariableType::String) { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[0])); + } + + if !type_list[1].satisfies(&VariableType::String) { + self.set_error(arguments[1], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[1])); + } + + V(VariableType::Array(Rc::new(VariableType::String))) + } + BuiltInFunction::FuzzyMatch => { + if !type_list[0].satisfies(&VariableType::String) + && !type_list[0] + .satisfies(&VariableType::Array(Rc::new(VariableType::String))) + { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `string | string[]`.", type_list[0])); + } + + if !type_list[1].satisfies(&VariableType::String) { + self.set_error(arguments[1], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[1])); + } + + V(VariableType::Bool) + } + BuiltInFunction::Abs + | BuiltInFunction::Rand + | BuiltInFunction::Floor + | BuiltInFunction::Ceil + | BuiltInFunction::Round => { + if !type_list[0].satisfies(&VariableType::Number) { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `number`.", type_list[0])); + } + + V(VariableType::Number) + } + BuiltInFunction::Sum + | BuiltInFunction::Avg + | BuiltInFunction::Min + | BuiltInFunction::Max + | BuiltInFunction::Median + | BuiltInFunction::Mode => { + if !type_list[0] + .satisfies(&VariableType::Array(Rc::new(VariableType::Number))) + { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `number[]`.", type_list[0])); + } + + V(VariableType::Number) + } + BuiltInFunction::IsNumeric => V(VariableType::Bool), + BuiltInFunction::String => V(VariableType::String), + BuiltInFunction::Number => V(VariableType::Number), + BuiltInFunction::Bool => V(VariableType::Bool), + BuiltInFunction::Type => V(VariableType::String), + BuiltInFunction::Date => { + if !type_list[0].satisfies(&VariableType::Number) + || !type_list[0].satisfies(&VariableType::String) + { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `number | string`.", type_list[0])); + } + + V(VariableType::Number) + } + BuiltInFunction::Time | BuiltInFunction::Duration => { + if !type_list[0].satisfies(&VariableType::String) { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[0])); + } + + V(VariableType::Number) + } + BuiltInFunction::Year + | BuiltInFunction::DayOfWeek + | BuiltInFunction::DayOfMonth + | BuiltInFunction::DayOfYear + | BuiltInFunction::WeekOfYear + | BuiltInFunction::MonthOfYear => { + if !type_list[0].satisfies(&VariableType::Number) { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `number`.", type_list[0])); + } + + V(VariableType::Number) + } + BuiltInFunction::MonthString + | BuiltInFunction::DateString + | BuiltInFunction::WeekdayString => { + if !type_list[0].satisfies(&VariableType::Number) { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `number`.", type_list[0])); + } + + V(VariableType::String) + } + BuiltInFunction::StartOf | BuiltInFunction::EndOf => { + if !type_list[0].satisfies(&VariableType::Number) { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `number`.", type_list[0])); + } + + if !type_list[1].satisfies(&VariableType::String) { + self.set_error(arguments[1], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[1])); + } + + V(VariableType::Number) + } + BuiltInFunction::Keys => { + if !type_list[0].satisfies_object() { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `object`.", type_list[0])); + } + + V(VariableType::Array(Rc::new(VariableType::String))) + } + BuiltInFunction::Values => match type_list[0].as_ref() { + VariableType::Any | VariableType::Object(_) => { + V(VariableType::Array(VariableType::Any.into())) + } + VariableType::Constant(c) => match c.as_ref() { + Value::Object(obj) => { + let s: Vec = obj.values().cloned().collect(); + V(s.into()) + } + _ => { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `object`.", type_list[0])); + V(VariableType::Array(VariableType::Any.into())) + } + }, + _ => { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `object`", type_list[0])); + V(VariableType::Array(VariableType::Any.into())) + } + }, + BuiltInFunction::All => { + if !type_list[0].satisfies_array() { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `any[]`.", type_list[0])); + } + + if !type_list[1].satisfies(&VariableType::Bool) { + self.set_error( + arguments[1], + format!( + "Callback must return a `bool`, but its return type is `{}`.", + type_list[1] + ), + ); + } + + V(VariableType::Bool) + } + BuiltInFunction::Some => { + if !type_list[0].satisfies_array() { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `any[]`.", type_list[0])); + } + + if !type_list[1].satisfies(&VariableType::Bool) { + self.set_error( + arguments[1], + format!( + "Callback must return a `bool`, but its return type is `{}`.", + type_list[1] + ), + ); + } + + V(VariableType::Bool) + } + BuiltInFunction::None => { + if !type_list[0].satisfies_array() { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `any[]`.", type_list[0])); + } + + if !type_list[1].satisfies(&VariableType::Bool) { + self.set_error( + arguments[1], + format!( + "Callback must return a `bool`, but its return type is `{}`.", + type_list[1] + ), + ); + } + + V(VariableType::Bool) + } + BuiltInFunction::Filter => { + if !type_list[0].satisfies_array() { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `any[]`.", type_list[0])); + } + + if !type_list[1].satisfies(&VariableType::Bool) { + self.set_error( + arguments[1], + format!( + "Callback must return a `bool`, but its return type is `{}`.", + type_list[1] + ), + ); + } + + TypeInfo::from(type_list[0].clone()) + } + BuiltInFunction::Map => { + if !type_list[0].satisfies_array() { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `any[]`.", type_list[0])); + } + + V(VariableType::Array(type_list[1].clone())) + } + BuiltInFunction::Count => { + if !type_list[0].satisfies_array() { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `any[]`.", type_list[0])); + } + + if !type_list[1].satisfies(&VariableType::Bool) { + self.set_error( + arguments[1], + format!( + "Callback must return a `bool`, but its return type is `{}`.", + type_list[1] + ), + ); + } + + V(VariableType::Number) + } + BuiltInFunction::One => { + if !type_list[0].satisfies_array() { + self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `any[]`.", type_list[0])); + } + + if !type_list[1].satisfies(&VariableType::Bool) { + self.set_error( + arguments[1], + format!( + "Callback must return a `bool`, but its return type is `{}`.", + type_list[1] + ), + ); + } + + V(VariableType::Bool) + } + BuiltInFunction::FlatMap => V(VariableType::Any), + BuiltInFunction::Flatten => V(VariableType::Any), + } + } + Node::Closure(c) => self.determine(c, scope.clone(), false), + Node::Parenthesized(c) => self.determine(c, scope.clone(), false), + Node::Error { node, error } => match node { + None => TypeInfo { + kind: Rc::new(VariableType::Any), + error: Some(error.to_string()), + }, + Some(n) => { + let typ = self.determine(n, scope.clone(), false); + TypeInfo { + kind: typ.kind, + error: Some(error.to_string()), + } + } + }, + }; + + self.set_type(node, node_type.clone()); + node_type + } +} + +#[allow(unused)] +fn node_address(node: &Node) -> usize { + node as *const Node as usize +} diff --git a/core/expression/src/intellisense/types/type_info.rs b/core/expression/src/intellisense/types/type_info.rs new file mode 100644 index 00000000..b1e028e0 --- /dev/null +++ b/core/expression/src/intellisense/types/type_info.rs @@ -0,0 +1,51 @@ +use crate::variable::VariableType; +use std::fmt::{Display, Formatter}; +use std::ops::Deref; +use std::rc::Rc; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TypeInfo { + pub(crate) kind: Rc, + pub(crate) error: Option, +} + +impl Deref for TypeInfo { + type Target = VariableType; + + fn deref(&self) -> &Self::Target { + &self.kind + } +} + +impl Display for TypeInfo { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.kind) + } +} + +impl Default for TypeInfo { + fn default() -> Self { + Self { + kind: Rc::new(VariableType::Any), + error: None, + } + } +} + +impl From for TypeInfo { + fn from(value: VariableType) -> Self { + Self { + kind: Rc::new(value), + error: None, + } + } +} + +impl From> for TypeInfo { + fn from(value: Rc) -> Self { + Self { + kind: value, + error: None, + } + } +} diff --git a/core/expression/src/isolate.rs b/core/expression/src/isolate.rs index 65aeb05c..1334cb96 100644 --- a/core/expression/src/isolate.rs +++ b/core/expression/src/isolate.rs @@ -116,13 +116,14 @@ impl<'a> Isolate<'a> { .map_err(|source| IsolateError::ParserError { source })? .standard(); - let ast = parser - .parse() + let parser_result = parser.parse(); + parser_result + .error() .map_err(|source| IsolateError::ParserError { source })?; let bytecode = self .compiler - .compile(ast) + .compile(parser_result.root) .map_err(|source| IsolateError::CompilerError { source })?; let result = self @@ -150,13 +151,14 @@ impl<'a> Isolate<'a> { .map_err(|source| IsolateError::ParserError { source })? .unary(); - let ast = parser - .parse() + let parser_result = parser.parse(); + parser_result + .error() .map_err(|source| IsolateError::ParserError { source })?; let bytecode = self .compiler - .compile(ast) + .compile(parser_result.root) .map_err(|source| IsolateError::CompilerError { source })?; let result = self diff --git a/core/expression/src/lexer/token.rs b/core/expression/src/lexer/token.rs index a920cc18..1c02b636 100644 --- a/core/expression/src/lexer/token.rs +++ b/core/expression/src/lexer/token.rs @@ -65,7 +65,7 @@ impl Display for TemplateString { } } -#[derive(Debug, PartialEq, Eq, Clone, Copy, Display)] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum Operator { Arithmetic(ArithmeticOperator), Logical(LogicalOperator), @@ -77,6 +77,21 @@ pub enum Operator { QuestionMark, // ? } +impl Display for Operator { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Operator::Arithmetic(a) => write!(f, "{a}"), + Operator::Logical(l) => write!(f, "{l}"), + Operator::Comparison(c) => write!(f, "{c}"), + Operator::Range => write!(f, ".."), + Operator::Comma => write!(f, ","), + Operator::Slice => write!(f, ":"), + Operator::Dot => write!(f, "."), + Operator::QuestionMark => write!(f, "?"), + } + } +} + impl FromStr for Operator { type Err = strum::ParseError; diff --git a/core/expression/src/lib.rs b/core/expression/src/lib.rs index 17f0b01d..4bb4581f 100644 --- a/core/expression/src/lib.rs +++ b/core/expression/src/lib.rs @@ -54,6 +54,7 @@ mod isolate; mod arena; pub mod compiler; mod function; +pub mod intellisense; pub mod lexer; pub mod parser; pub mod variable; diff --git a/core/expression/src/parser/ast.rs b/core/expression/src/parser/ast.rs index 97b0497e..1715ad20 100644 --- a/core/expression/src/parser/ast.rs +++ b/core/expression/src/parser/ast.rs @@ -1,9 +1,11 @@ -use rust_decimal::Decimal; - use crate::lexer::Operator; use crate::parser::builtin::BuiltInFunction; +use rust_decimal::Decimal; +use std::cell::Cell; +use strum_macros::IntoStaticStr; +use thiserror::Error; -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, IntoStaticStr)] pub enum Node<'a> { Null, Bool(bool), @@ -15,6 +17,7 @@ pub enum Node<'a> { Object(&'a [(&'a Node<'a>, &'a Node<'a>)]), Identifier(&'a str), Closure(&'a Node<'a>), + Parenthesized(&'a Node<'a>), Root, Member { node: &'a Node<'a>, @@ -49,4 +52,147 @@ pub enum Node<'a> { kind: BuiltInFunction, arguments: &'a [&'a Node<'a>], }, + Error { + node: Option<&'a Node<'a>>, + error: AstNodeError<'a>, + }, +} + +impl<'a> Node<'a> { + pub fn walk(&self, mut func: F) + where + F: FnMut(&Self) + Clone, + { + { + func(self); + }; + + match self { + Node::Null => {} + Node::Bool(_) => {} + Node::Number(_) => {} + Node::String(_) => {} + Node::Pointer => {} + Node::Identifier(_) => {} + Node::Root => {} + Node::Error { node, .. } => { + if let Some(n) = node { + n.walk(func.clone()) + } + } + Node::TemplateString(parts) => parts.iter().for_each(|n| n.walk(func.clone())), + Node::Array(parts) => parts.iter().for_each(|n| n.walk(func.clone())), + Node::Object(obj) => obj.iter().for_each(|(k, v)| { + k.walk(func.clone()); + v.walk(func.clone()); + }), + Node::Closure(closure) => closure.walk(func.clone()), + Node::Parenthesized(c) => c.walk(func.clone()), + Node::Member { node, property } => { + node.walk(func.clone()); + property.walk(func.clone()); + } + Node::Slice { node, to, from } => { + node.walk(func.clone()); + if let Some(to) = to { + to.walk(func.clone()); + } + + if let Some(from) = from { + from.walk(func.clone()); + } + } + Node::Interval { left, right, .. } => { + left.walk(func.clone()); + right.walk(func.clone()); + } + Node::Unary { node, .. } => { + node.walk(func); + } + Node::Binary { left, right, .. } => { + left.walk(func.clone()); + right.walk(func.clone()); + } + Node::BuiltIn { arguments, .. } => { + arguments.iter().for_each(|n| n.walk(func.clone())); + } + Node::Conditional { + on_true, + condition, + on_false, + } => { + condition.walk(func.clone()); + on_true.walk(func.clone()); + on_false.walk(func.clone()); + } + }; + } + + pub fn first_error(&self) -> Option { + let error_cell = Cell::new(None); + self.walk(|n| { + if let Node::Error { error, .. } = n { + error_cell.set(Some(error.clone())) + } + }); + + error_cell.into_inner() + } + + pub fn has_error(&self) -> bool { + self.first_error().is_some() + } + + pub(crate) fn span(&self) -> Option<(u32, u32)> { + match self { + Node::Error { error, .. } => match error { + AstNodeError::UnknownBuiltIn { span, .. } => Some(span.clone()), + AstNodeError::UnexpectedIdentifier { span, .. } => Some(span.clone()), + AstNodeError::UnexpectedToken { span, .. } => Some(span.clone()), + AstNodeError::InvalidNumber { span, .. } => Some(span.clone()), + AstNodeError::InvalidBoolean { span, .. } => Some(span.clone()), + AstNodeError::InvalidProperty { span, .. } => Some(span.clone()), + AstNodeError::MissingToken { position, .. } => { + Some((*position as u32, *position as u32)) + } + AstNodeError::Custom { span, .. } => Some(span.clone()), + }, + _ => None, + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Error)] +pub enum AstNodeError<'a> { + #[error("Unknown built in: {name} at ({}, {})", span.0, span.1)] + UnknownBuiltIn { name: &'a str, span: (u32, u32) }, + + #[error("Unexpected identifier: {received} at ({}, {}); Expected {expected}.", span.0, span.1)] + UnexpectedIdentifier { + received: &'a str, + expected: &'a str, + span: (u32, u32), + }, + + #[error("Unexpected token: {received} at ({}, {}); Expected {expected}.", span.0, span.1)] + UnexpectedToken { + received: &'a str, + expected: &'a str, + span: (u32, u32), + }, + + #[error("Invalid number: {number} at ({}, {})", span.0, span.1)] + InvalidNumber { number: &'a str, span: (u32, u32) }, + + #[error("Invalid boolean: {boolean} at ({}, {})", span.0, span.1)] + InvalidBoolean { boolean: &'a str, span: (u32, u32) }, + + #[error("Invalid property: {property} at ({}, {})", span.0, span.1)] + InvalidProperty { property: &'a str, span: (u32, u32) }, + + #[error("Missing expected token: {expected} at {position}")] + MissingToken { expected: &'a str, position: usize }, + + #[error("{message} at ({}, {})", span.0, span.1)] + Custom { message: &'a str, span: (u32, u32) }, } diff --git a/core/expression/src/parser/error.rs b/core/expression/src/parser/error.rs index 7643002c..88b831e2 100644 --- a/core/expression/src/parser/error.rs +++ b/core/expression/src/parser/error.rs @@ -2,27 +2,9 @@ use thiserror::Error; #[derive(Debug, PartialEq, Eq, Clone, Error)] pub enum ParserError { - #[error("Unexpected token: received {received} instead of {expected} at ({}, {})", span.0, span.1)] - UnexpectedToken { - expected: String, - received: String, - span: (u32, u32), - }, + #[error("{0}")] + NodeError(String), - #[error("Failed to parse: {message} at ({}, {})", span.0, span.1)] - FailedToParse { message: String, span: (u32, u32) }, - - #[error("Unknown built in: {name} at ({}, {})", span.0, span.1)] - UnknownBuiltIn { name: String, span: (u32, u32) }, - - #[error("Unsupported built in: {name} at ({}, {})", span.0, span.1)] - UnsupportedBuiltIn { name: String, span: (u32, u32) }, - - #[error("Token out of bounds")] - TokenOutOfBounds, - - #[error("Memory failure")] - MemoryFailure, + #[error("Incomplete parser output")] + Incomplete, } - -pub(crate) type ParserResult = Result; diff --git a/core/expression/src/parser/mod.rs b/core/expression/src/parser/mod.rs index 1d5b3afe..6f1dadaf 100644 --- a/core/expression/src/parser/mod.rs +++ b/core/expression/src/parser/mod.rs @@ -10,12 +10,16 @@ mod builtin; mod constants; mod error; mod parser; +mod result; mod standard; mod unary; +pub(crate) use builtin::Arity; + pub use ast::Node; pub use builtin::BuiltInFunction; pub use error::ParserError; pub use parser::Parser; +pub use result::{NodeMetadata, ParserResult}; pub use standard::Standard; pub use unary::Unary; diff --git a/core/expression/src/parser/parser.rs b/core/expression/src/parser/parser.rs index d3ca0500..3784f0d3 100644 --- a/core/expression/src/parser/parser.rs +++ b/core/expression/src/parser/parser.rs @@ -1,20 +1,37 @@ -use std::cell::Cell; -use std::fmt::Debug; -use std::marker::PhantomData; - use bumpalo::collections::Vec as BumpVec; use bumpalo::Bump; +use nohash_hasher::BuildNoHashHasher; use rust_decimal::Decimal; +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; +use std::fmt::Debug; +use std::marker::PhantomData; +use std::ops::Deref; use crate::lexer::{ Bracket, ComparisonOperator, Identifier, Operator, QuotationMark, TemplateString, Token, TokenKind, }; -use crate::parser::ast::Node; +use crate::parser::ast::{AstNodeError, Node}; use crate::parser::builtin::{Arity, BuiltInFunction}; -use crate::parser::error::{ParserError, ParserResult}; +use crate::parser::error::ParserError; use crate::parser::standard::Standard; use crate::parser::unary::Unary; +use crate::parser::NodeMetadata; + +macro_rules! expect { + ($self:ident, $token:expr) => { + if let Some(error_node) = $self.expect($token) { + return error_node; + } + }; +} +macro_rules! afmt { + ($self:expr, $($arg:tt)*) => {{ + let formatted = format!($($arg)*); + $self.bump.alloc_str(formatted.as_str()) + }} +} #[derive(Debug)] pub struct BaseParser; @@ -22,13 +39,14 @@ pub struct BaseParser; #[derive(Debug)] pub struct Parser<'arena, 'token_ref, Flavor> { tokens: &'token_ref [Token<'arena>], - current: Cell<&'token_ref Token<'arena>>, + current: Cell>>, pub(crate) bump: &'arena Bump, - is_done: Cell, position: Cell, depth: Cell, marker_flavor: PhantomData, has_range_operator: bool, + pub(crate) node_metadata: + Option>>>, } impl<'arena, 'token_ref> Parser<'arena, 'token_ref, BaseParser> { @@ -36,7 +54,7 @@ impl<'arena, 'token_ref> Parser<'arena, 'token_ref, BaseParser> { tokens: &'token_ref [Token<'arena>], bump: &'arena Bump, ) -> Result { - let current = tokens.get(0).ok_or(ParserError::TokenOutOfBounds)?; + let current = tokens.get(0); let has_range_operator = tokens .iter() .any(|t| t.kind == TokenKind::Operator(Operator::Range)); @@ -47,8 +65,8 @@ impl<'arena, 'token_ref> Parser<'arena, 'token_ref, BaseParser> { current: Cell::new(current), depth: Cell::new(0), position: Cell::new(0), - is_done: Cell::new(false), has_range_operator, + node_metadata: None, marker_flavor: PhantomData, }) } @@ -60,8 +78,8 @@ impl<'arena, 'token_ref> Parser<'arena, 'token_ref, BaseParser> { current: self.current, depth: self.depth, position: self.position, - is_done: self.is_done, has_range_operator: self.has_range_operator, + node_metadata: self.node_metadata, marker_flavor: PhantomData, } } @@ -73,30 +91,60 @@ impl<'arena, 'token_ref> Parser<'arena, 'token_ref, BaseParser> { current: self.current, depth: self.depth, position: self.position, - is_done: self.is_done, has_range_operator: self.has_range_operator, + node_metadata: self.node_metadata, marker_flavor: PhantomData, } } } impl<'arena, 'token_ref, Flavor> Parser<'arena, 'token_ref, Flavor> { - pub(crate) fn current(&self) -> &Token<'arena> { + pub fn with_metadata(mut self) -> Parser<'arena, 'token_ref, Flavor> { + self.node_metadata = Some(Default::default()); + self + } + + pub(crate) fn current(&self) -> Option<&Token<'arena>> { self.current.get() } + pub(crate) fn current_kind(&self) -> Option<&TokenKind> { + self.current.get().map(|token| &token.kind) + } + + fn token_start(&self) -> u32 { + match self.current() { + None => self.tokens.last().map(|t| t.span.1).unwrap_or_default(), + Some(t) => t.span.0, + } + } + + #[allow(dead_code)] + fn token_end(&self) -> u32 { + match self.current() { + None => self.tokens.last().map(|t| t.span.1).unwrap_or_default(), + Some(t) => t.span.1, + } + } + + pub(crate) fn prev_token_end(&self) -> u32 { + match self.tokens.get(self.position() - 1) { + None => self.token_start(), + Some(t) => t.span.1, + } + } + fn position(&self) -> usize { self.position.get() } - fn set_position(&self, position: usize) -> ParserResult<()> { - let Some(token) = self.tokens.get(position) else { - return Err(ParserError::TokenOutOfBounds); - }; + fn set_position(&self, position: usize) -> bool { + let target_token = self.tokens.get(position); self.position.set(position); - self.current.set(token); - Ok(()) + self.current.set(target_token); + + target_token.is_some() } pub(crate) fn depth(&self) -> u8 { @@ -104,259 +152,459 @@ impl<'arena, 'token_ref, Flavor> Parser<'arena, 'token_ref, Flavor> { } pub(crate) fn is_done(&self) -> bool { - self.is_done.get() + self.current.get().is_none() } - pub(crate) fn next(&self) -> ParserResult<()> { - self.position.set(self.position.get() + 1); + pub(crate) fn node(&self, node: Node<'arena>, gen_metadata: F) -> &'arena Node<'arena> + where + F: FnOnce(MetadataHelper<'_, 'arena>) -> NodeMetadata, + { + let node = self.bump.alloc(node); + if let Some(node_metadata) = &self.node_metadata { + let metadata = { + let nm = node_metadata.borrow(); + gen_metadata(MetadataHelper { + node_metadata: nm.deref(), + arena: PhantomData::<&'arena ()>, + }) + }; - if let Some(token) = self.tokens.get(self.position.get()) { - self.current.set(token); - Ok(()) - } else { - if self.is_done.get() { - return Err(ParserError::TokenOutOfBounds); - } + let mut nm = node_metadata.borrow_mut(); + nm.insert(node as *const Node as usize, metadata); + }; - self.is_done.set(true); - Ok(()) - } + node } - pub(crate) fn expect(&self, kind: TokenKind) -> Result<(), ParserError> { + pub(crate) fn error(&self, error: AstNodeError<'arena>) -> &'arena Node<'arena> { + self.node(Node::Error { error, node: None }, |_| NodeMetadata { + span: (self.prev_token_end(), self.prev_token_end()), + }) + } + + pub(crate) fn error_with_node( + &self, + error: AstNodeError<'arena>, + node: &'arena Node<'arena>, + ) -> &'arena Node<'arena> { + self.node( + Node::Error { + error, + node: Some(node), + }, + |_| NodeMetadata { span: (0, 0) }, + ) + } + + pub(crate) fn next(&self) { + let new_position = self.position.get() + 1; + + self.position.set(new_position); + self.current.set(self.tokens.get(new_position)); + } + + pub(crate) fn expect(&self, kind: TokenKind) -> Option<&'arena Node<'arena>> { let token = self.current(); - if token.kind != kind { - return Err(ParserError::UnexpectedToken { - expected: kind.to_string(), - received: token.kind.to_string(), + if token.is_some_and(|t| t.kind == kind) { + self.next(); + return None; + } + + Some( + self.error(AstNodeError::UnexpectedToken { + expected: afmt!(self, "{kind}"), + received: token + .map(|t| afmt!(self, "{}", t.kind)) + .unwrap_or_else(|| afmt!(self, "None")), + span: token.map(|t| t.span).unwrap_or((0, 0)), + }), + ) + } + + pub(crate) fn number(&self) -> &'arena Node<'arena> { + let Some(token) = self.current() else { + return self.error(AstNodeError::MissingToken { + expected: afmt!(self, "Number"), + position: self.position(), + }); + }; + + let Ok(decimal) = Decimal::from_str_exact(token.value) else { + return self.error(AstNodeError::InvalidNumber { + number: afmt!(self, "{}", token.value), + span: token.span, + }); + }; + + self.next(); + self.node(Node::Number(decimal), |_| NodeMetadata { span: token.span }) + } + + pub(crate) fn bool(&self) -> &'arena Node<'arena> { + let Some(token) = self.current() else { + return self.error(AstNodeError::MissingToken { + expected: afmt!(self, "Boolean"), + position: self.position(), + }); + }; + + let TokenKind::Boolean(boolean) = token.kind else { + return self.error(AstNodeError::InvalidBoolean { + boolean: afmt!(self, "{}", token.value), + span: token.span, + }); + }; + + self.next(); + self.node(Node::Bool(boolean), |_| NodeMetadata { span: token.span }) + } + + pub(crate) fn null(&self) -> &'arena Node<'arena> { + let Some(token) = self.current() else { + return self.error(AstNodeError::MissingToken { + expected: afmt!(self, "Null"), + position: self.position(), + }); + }; + + if token.kind != TokenKind::Identifier(Identifier::Null) { + return self.error(AstNodeError::UnexpectedIdentifier { + expected: afmt!(self, "Null"), + received: afmt!(self, "{}", token.value), span: token.span, }); } - self.next() + self.next(); + self.node(Node::Null, |_| NodeMetadata { span: token.span }) } - pub(crate) fn number(&self) -> ParserResult>> { - let Ok(decimal) = Decimal::from_str_exact(self.current().value) else { - return Ok(None); + pub(crate) fn simple_string(&self, quote_mark: &QuotationMark) -> &'arena Node<'arena> { + expect!(self, TokenKind::QuotationMark(quote_mark.clone())); + let string_value = self.current(); + + let error_literal = self.expect(TokenKind::Literal); + let error_mark_end = self.expect(TokenKind::QuotationMark(quote_mark.clone())); + + error_literal + .or(error_mark_end) + .or(string_value + .map(|t| self.node(Node::String(t.value), |_| NodeMetadata { span: t.span }))) + .unwrap_or_else(|| { + self.error(AstNodeError::Custom { + message: afmt!( + self, + "Failed to parse string `{}`", + string_value.map(|s| s.value).unwrap_or_default() + ), + span: string_value + .map(|s| s.span) + .unwrap_or((self.prev_token_end(), self.prev_token_end())), + }) + }) + } + + pub(crate) fn template_string(&self, expression_parser: F) -> &'arena Node<'arena> + where + F: Fn() -> &'arena Node<'arena>, + { + expect!(self, TokenKind::QuotationMark(QuotationMark::Backtick)); + + let Some(mut current_token) = self.current() else { + return self.error(AstNodeError::MissingToken { + expected: afmt!(self, "Backtick (`)"), + position: self.position(), + }); }; - self.next()?; - Ok(Some(self.node(Node::Number(decimal)))) - } + let mut span = (current_token.span.0, 0u32); - pub(crate) fn simple_string( - &self, - quote_mark: &QuotationMark, - ) -> ParserResult<&'arena Node<'arena>> { - self.expect(TokenKind::QuotationMark(quote_mark.clone()))?; - let string_value = self.current().value; - - self.expect(TokenKind::Literal)?; - self.expect(TokenKind::QuotationMark(quote_mark.clone()))?; - - Ok(self.node(Node::String(string_value))) - } - - pub(crate) fn template_string( - &self, - expression_parser: F, - ) -> ParserResult<&'arena Node<'arena>> - where - F: Fn() -> ParserResult<&'arena Node<'arena>>, - { - self.expect(TokenKind::QuotationMark(QuotationMark::Backtick))?; - - let mut current_token = self.current(); let mut nodes = BumpVec::new_in(self.bump); while TokenKind::QuotationMark(QuotationMark::Backtick) != current_token.kind { match current_token.kind { TokenKind::TemplateString(template) => match template { TemplateString::ExpressionStart => { - self.next()?; - nodes.push(expression_parser()?); + self.next(); + nodes.push(expression_parser()); } TemplateString::ExpressionEnd => { - self.next()?; + self.next(); } }, TokenKind::Literal => { - nodes.push(self.node(Node::String(current_token.value))); - self.next()?; + nodes.push( + self.node(Node::String(current_token.value), |_| NodeMetadata { + span: current_token.span, + }), + ); + self.next(); } _ => { - return Err(ParserError::UnexpectedToken { - expected: "Valid TemplateString token".to_string(), - received: current_token.kind.to_string(), + return self.error(AstNodeError::UnexpectedToken { + expected: afmt!(self, "Valid TemplateString token"), + received: afmt!(self, "{}", current_token.kind), span: current_token.span, }) } } - current_token = self.current(); + if let Some(ct) = self.current() { + current_token = ct; + span.1 = ct.span.1; + } else { + break; + } } - self.expect(TokenKind::QuotationMark(QuotationMark::Backtick))?; - - Ok(self.node(Node::TemplateString(nodes.into_bump_slice()))) + expect!(self, TokenKind::QuotationMark(QuotationMark::Backtick)); + self.node(Node::TemplateString(nodes.into_bump_slice()), |_| { + NodeMetadata { span } + }) } - pub(crate) fn bool(&self) -> ParserResult>> { - let current_token = self.current(); - let TokenKind::Boolean(boolean) = current_token.kind else { - return Ok(None); - }; - - self.next()?; - Ok(Some(self.node(Node::Bool(boolean)))) - } - - pub(crate) fn null(&self) -> ParserResult>> { - let current_token = self.current(); - if current_token.kind != TokenKind::Identifier(Identifier::Null) { - return Ok(None); - } - - self.next()?; - Ok(Some(self.node(Node::Null))) - } - - pub(crate) fn node(&self, node: Node<'arena>) -> &'arena Node<'arena> { - self.bump.alloc(node) - } - - // Higher level constructs - pub(crate) fn with_postfix( &self, node: &'arena Node<'arena>, expression_parser: F, - ) -> ParserResult<&'arena Node<'arena>> + ) -> &'arena Node<'arena> where - F: Fn() -> ParserResult<&'arena Node<'arena>>, + F: Fn() -> &'arena Node<'arena>, { - let postfix_token = self.current(); + let Some(postfix_token) = self.current() else { + return node; + }; + let postfix_kind = PostfixKind::from(postfix_token); let processed_token = match postfix_kind { - PostfixKind::Other => return Ok(node), + PostfixKind::Other => return node, PostfixKind::MemberAccess => { - self.next()?; + self.next(); let property_token = self.current(); - self.next()?; + self.next(); - if !is_valid_property(property_token) { - return Err(ParserError::UnexpectedToken { - expected: "valid property".to_string(), - received: postfix_token.kind.to_string(), - span: postfix_token.span, - }); - } + let property = match property_token { + None => self.error_with_node( + AstNodeError::Custom { + message: afmt!(self, "Expected a property"), + span: (self.prev_token_end(), self.prev_token_end()), + }, + node, + ), + Some(t) => match is_valid_property(t) { + true => self.node(Node::String(t.value), |_| NodeMetadata { span: t.span }), + false => self.error_with_node( + AstNodeError::InvalidProperty { + property: afmt!(self, "{}", t.value), + span: t.span, + }, + node, + ), + }, + }; - let property = self.node(Node::String(property_token.value)); - Ok(self.node(Node::Member { node, property })) + self.node(Node::Member { node, property }, |h| NodeMetadata { + span: h.span(node, property).unwrap_or_default(), + }) } PostfixKind::PropertyAccess => { - self.next()?; + self.next(); let mut from: Option<&'arena Node<'arena>> = None; let mut to: Option<&'arena Node<'arena>> = None; - let mut c = self.current(); + let Some(mut c) = self.current() else { + return self.error_with_node( + AstNodeError::Custom { + message: afmt!(self, "Expected a property"), + span: (self.prev_token_end(), self.prev_token_end()), + }, + node, + ); + }; + if c.kind == TokenKind::Operator(Operator::Slice) { - self.next()?; - c = self.current(); + self.next(); + + let Some(cc) = self.current() else { + return self.error_with_node( + AstNodeError::Custom { + message: afmt!(self, "Unexpected token"), + span: (self.prev_token_end(), self.prev_token_end()), + }, + node, + ); + }; + c = cc; if c.kind != TokenKind::Bracket(Bracket::RightSquareBracket) { - to = Some(expression_parser()?); + to = Some(expression_parser()); } - self.expect(TokenKind::Bracket(Bracket::RightSquareBracket))?; - Ok(self.node(Node::Slice { node, to, from })) + expect!(self, TokenKind::Bracket(Bracket::RightSquareBracket)); + self.node(Node::Slice { node, to, from }, |h| NodeMetadata { + span: ( + h.metadata(node).map(|m| m.span.0).unwrap_or_default(), + self.prev_token_end(), + ), + }) } else { - from = Some(expression_parser()?); - c = self.current(); + let from_node = expression_parser(); + from = Some(from_node); + let Some(cc) = self.current() else { + return self.error_with_node( + AstNodeError::Custom { + message: afmt!(self, "Unexpected token"), + span: (self.prev_token_end(), self.prev_token_end()), + }, + self.node( + Node::Member { + node, + property: from_node, + }, + |h| NodeMetadata { + span: h.span(node, from_node).unwrap_or_default(), + }, + ), + ); + }; + c = cc; if c.kind == TokenKind::Operator(Operator::Slice) { - self.next()?; - c = self.current(); + self.next(); + let Some(cc) = self.current() else { + return self.error_with_node( + AstNodeError::Custom { + message: afmt!(self, "Invalid slice syntax"), + span: (self.prev_token_end(), self.prev_token_end()), + }, + self.node(Node::Slice { node, from, to }, |h| NodeMetadata { + span: h.span(node, from_node).unwrap_or_default(), + }), + ); + }; + c = cc; if c.kind != TokenKind::Bracket(Bracket::RightSquareBracket) { - to = Some(expression_parser()?); + to = Some(expression_parser()); } - self.expect(TokenKind::Bracket(Bracket::RightSquareBracket))?; - Ok(self.node(Node::Slice { node, from, to })) + expect!(self, TokenKind::Bracket(Bracket::RightSquareBracket)); + self.node(Node::Slice { node, from, to }, |h| NodeMetadata { + span: ( + h.metadata(node).map(|m| m.span.0).unwrap_or_default(), + self.prev_token_end(), + ), + }) } else { // Slice operator [:] was not found, // it should be just an index node. - self.expect(TokenKind::Bracket(Bracket::RightSquareBracket))?; - Ok(self.node(Node::Member { - node, - property: from.ok_or(ParserError::MemoryFailure)?, - })) + expect!(self, TokenKind::Bracket(Bracket::RightSquareBracket)); + self.node( + Node::Member { + node, + property: from.unwrap_or_else(|| { + return self.error_with_node( + AstNodeError::Custom { + message: afmt!(self, "Invalid index property"), + span: (self.prev_token_end(), self.prev_token_end()), + }, + node, + ); + }), + }, + |h| NodeMetadata { + span: ( + h.metadata(node).map(|m| m.span.0).unwrap_or_default(), + self.prev_token_end(), + ), + }, + ) } } } - }?; + }; self.with_postfix(processed_token, expression_parser) } /// Closure - pub(crate) fn closure(&self, expression_parser: F) -> ParserResult<&'arena Node<'arena>> + pub(crate) fn closure(&self, expression_parser: F) -> &'arena Node<'arena> where - F: Fn() -> ParserResult<&'arena Node<'arena>>, + F: Fn() -> &'arena Node<'arena>, { + let start = self.token_start(); + self.depth.set(self.depth.get() + 1); - let node = expression_parser()?; + let node = expression_parser(); self.depth.set(self.depth.get() - 1); - Ok(self.node(Node::Closure(node))) + self.node(Node::Closure(node), |_| NodeMetadata { + span: (start, self.prev_token_end()), + }) } /// Identifier expression /// Either or - pub(crate) fn identifier( - &self, - expression_parser: F, - ) -> ParserResult>> + pub(crate) fn identifier(&self, expression_parser: F) -> &'arena Node<'arena> where - F: Fn() -> ParserResult<&'arena Node<'arena>>, + F: Fn() -> &'arena Node<'arena>, { - match &self.current().kind { + let Some(token) = self.current() else { + return self.error(AstNodeError::MissingToken { + expected: afmt!(self, "Identifier"), + position: self.position(), + }); + }; + + match token.kind { TokenKind::Identifier(_) | TokenKind::Literal => { // ok } - _ => return Ok(None), + _ => { + return self.error(AstNodeError::Custom { + message: afmt!(self, "Expected an `identifier`, received `{}`.", token.kind), + span: token.span, + }); + } } - let identifier_token = self.current(); - self.next()?; + let Some(identifier_token) = self.current() else { + return self.error(AstNodeError::Custom { + message: afmt!(self, "Expected an `identifier`."), + span: (self.prev_token_end(), self.prev_token_end()), + }); + }; + self.next(); + let current_token = self.current(); - if current_token.kind != TokenKind::Bracket(Bracket::LeftParenthesis) { + if current_token.map(|t| t.kind) != Some(TokenKind::Bracket(Bracket::LeftParenthesis)) { let identifier_node = match identifier_token.kind { - TokenKind::Identifier(Identifier::RootReference) => self.node(Node::Root), - _ => self.node(Node::Identifier(identifier_token.value)), + TokenKind::Identifier(Identifier::RootReference) => { + self.node(Node::Root, |_| NodeMetadata { + span: identifier_token.span, + }) + } + _ => self.node(Node::Identifier(identifier_token.value), |_| NodeMetadata { + span: identifier_token.span, + }), }; - return self - .with_postfix(identifier_node, expression_parser) - .map(Some); + return self.with_postfix(identifier_node, expression_parser); } // Potentially it might be a built-in expression - let builtin = BuiltInFunction::try_from(identifier_token.value).map_err(|_| { - ParserError::UnknownBuiltIn { - name: identifier_token.value.to_string(), + let Ok(builtin) = BuiltInFunction::try_from(identifier_token.value) else { + return self.error(AstNodeError::UnknownBuiltIn { + name: afmt!(self, "{}", identifier_token.value), span: identifier_token.span, - } - })?; + }); + }; - self.next()?; + self.next(); let builtin_node = match builtin.arity() { Arity::Single => { - let arg = expression_parser()?; - self.expect(TokenKind::Bracket(Bracket::RightParenthesis))?; + let arg = expression_parser(); + self.expect(TokenKind::Bracket(Bracket::RightParenthesis)); Node::BuiltIn { kind: builtin, @@ -364,10 +612,10 @@ impl<'arena, 'token_ref, Flavor> Parser<'arena, 'token_ref, Flavor> { } } Arity::Dual => { - let arg1 = expression_parser()?; - self.expect(TokenKind::Operator(Operator::Comma))?; - let arg2 = expression_parser()?; - self.expect(TokenKind::Bracket(Bracket::RightParenthesis))?; + let arg1 = expression_parser(); + self.expect(TokenKind::Operator(Operator::Comma)); + let arg2 = expression_parser(); + self.expect(TokenKind::Bracket(Bracket::RightParenthesis)); Node::BuiltIn { kind: builtin, @@ -375,10 +623,11 @@ impl<'arena, 'token_ref, Flavor> Parser<'arena, 'token_ref, Flavor> { } } Arity::Closure => { - let arg1 = expression_parser()?; - self.expect(TokenKind::Operator(Operator::Comma))?; - let arg2 = self.closure(&expression_parser)?; - self.expect(TokenKind::Bracket(Bracket::RightParenthesis))?; + let arg1 = expression_parser(); + + self.expect(TokenKind::Operator(Operator::Comma)); + let arg2 = self.closure(&expression_parser); + self.expect(TokenKind::Bracket(Bracket::RightParenthesis)); Node::BuiltIn { kind: builtin, @@ -387,212 +636,267 @@ impl<'arena, 'token_ref, Flavor> Parser<'arena, 'token_ref, Flavor> { } }; - self.with_postfix(self.node(builtin_node), expression_parser) - .map(Some) + self.with_postfix( + self.node(builtin_node, |_| NodeMetadata { + span: (identifier_token.span.0, self.prev_token_end()), + }), + expression_parser, + ) } /// Interval node - pub(crate) fn interval( - &self, - expression_parser: F, - ) -> ParserResult>> + pub(crate) fn interval(&self, expression_parser: F) -> Option<&'arena Node<'arena>> where - F: Fn() -> ParserResult<&'arena Node<'arena>>, + F: Fn() -> &'arena Node<'arena>, { // Performance optimisation: skip if expression does not contain an interval for faster evaluation if !self.has_range_operator { - return Ok(None); + return None; } - let TokenKind::Bracket(_) = &self.current().kind else { - return Ok(None); + let TokenKind::Bracket(_) = &self.current()?.kind else { + return None; }; let initial_position = self.position(); - let left_bracket = self.current().value; + let left_bracket = self.current()?.value; - let TokenKind::Bracket(_) = &self.current().kind else { - self.set_position(initial_position)?; - return Ok(None); - }; - self.next()?; - - let Ok(left) = expression_parser() else { - self.set_position(initial_position)?; - return Ok(None); + let TokenKind::Bracket(_) = &self.current()?.kind else { + self.set_position(initial_position); + return None; }; - if let Err(_) = self.expect(TokenKind::Operator(Operator::Range)) { - self.set_position(initial_position)?; - return Ok(None); + self.next(); + let left = expression_parser(); + if left.has_error() { + self.set_position(initial_position); + return None; }; - let Ok(right) = expression_parser() else { - self.set_position(initial_position)?; - return Ok(None); + if let Some(_) = self.expect(TokenKind::Operator(Operator::Range)) { + self.set_position(initial_position); + return None; }; - let right_bracket = self.current().value; - - let TokenKind::Bracket(_) = &self.current().kind else { - self.set_position(initial_position)?; - return Ok(None); + let right = expression_parser(); + if right.has_error() { + self.set_position(initial_position); + return None; }; - self.next()?; - let interval_node = self.node(Node::Interval { - left_bracket, - left, - right, - right_bracket, - }); + let right_bracket = self.current()?.value; + let TokenKind::Bracket(_) = &self.current()?.kind else { + self.set_position(initial_position); + return None; + }; - self.with_postfix(interval_node, expression_parser) - .map(Some) + self.next(); + + let interval_node = self.node( + Node::Interval { + left_bracket, + left, + right, + right_bracket, + }, + |_| NodeMetadata { + span: (initial_position as u32, self.position() as u32), + }, + ); + + Some(self.with_postfix(interval_node, expression_parser)) } /// Array nodes - pub(crate) fn array( - &self, - expression_parser: F, - ) -> ParserResult>> + pub(crate) fn array(&self, expression_parser: F) -> &'arena Node<'arena> where - F: Fn() -> ParserResult<&'arena Node<'arena>>, + F: Fn() -> &'arena Node<'arena>, { - let current_token = self.current(); + let Some(current_token) = self.current() else { + return self.error(AstNodeError::MissingToken { + expected: afmt!(self, "Array"), + position: self.position(), + }); + }; + if current_token.kind != TokenKind::Bracket(Bracket::LeftSquareBracket) { - return Ok(None); + return self.error(AstNodeError::UnexpectedToken { + expected: afmt!(self, "{}", TokenKind::Bracket(Bracket::LeftSquareBracket)), + received: afmt!(self, "{}", current_token.kind), + span: current_token.span, + }); } - self.next()?; + self.next(); let mut nodes = BumpVec::new_in(self.bump); - while !(self.current().kind == TokenKind::Bracket(Bracket::RightSquareBracket)) { + while !(self.current().map(|t| t.kind) + == Some(TokenKind::Bracket(Bracket::RightSquareBracket))) + { if !nodes.is_empty() { - self.expect(TokenKind::Operator(Operator::Comma))?; - if self.current().kind == TokenKind::Bracket(Bracket::RightSquareBracket) { + expect!(self, TokenKind::Operator(Operator::Comma)); + if self.current().map(|t| t.kind) + == Some(TokenKind::Bracket(Bracket::RightSquareBracket)) + { break; } } - nodes.push(expression_parser()?); + nodes.push(expression_parser()); } - self.expect(TokenKind::Bracket(Bracket::RightSquareBracket))?; + expect!(self, TokenKind::Bracket(Bracket::RightSquareBracket)); + let node = Node::Array(nodes.into_bump_slice()); - self.with_postfix(self.node(node), expression_parser) - .map(Some) + self.with_postfix( + self.node(node, |_| NodeMetadata { + span: (current_token.span.0, self.prev_token_end()), + }), + expression_parser, + ) } - pub(crate) fn object(&self, expression_parser: F) -> ParserResult<&'arena Node<'arena>> + pub(crate) fn object(&self, expression_parser: F) -> &'arena Node<'arena> where - F: Fn() -> ParserResult<&'arena Node<'arena>>, + F: Fn() -> &'arena Node<'arena>, { - self.expect(TokenKind::Bracket(Bracket::LeftCurlyBracket))?; + let span_start = self.token_start(); + expect!(self, TokenKind::Bracket(Bracket::LeftCurlyBracket)); let mut key_value_pairs = BumpVec::new_in(self.bump); - if let TokenKind::Bracket(Bracket::RightCurlyBracket) = self.current().kind { - self.next()?; - return Ok(self.node(Node::Object(key_value_pairs.into_bump_slice()))); + if let Some(TokenKind::Bracket(Bracket::RightCurlyBracket)) = self.current().map(|t| t.kind) + { + self.next(); + return self.node(Node::Object(key_value_pairs.into_bump_slice()), |_| { + NodeMetadata { + span: (span_start, self.prev_token_end()), + } + }); } loop { - let key = self.object_key(&expression_parser)?; - self.expect(TokenKind::Operator(Operator::Slice))?; - let value = expression_parser()?; + let key = self.object_key(&expression_parser); + expect!(self, TokenKind::Operator(Operator::Slice)); + let value = expression_parser(); key_value_pairs.push((key, value)); - let current_token = self.current(); + let Some(current_token) = self.current() else { + break; + }; + match current_token.kind { TokenKind::Operator(Operator::Comma) => { - self.expect(TokenKind::Operator(Operator::Comma))?; + expect!(self, TokenKind::Operator(Operator::Comma)); } TokenKind::Bracket(Bracket::RightCurlyBracket) => break, _ => { - return Err(ParserError::UnexpectedToken { - expected: "RightCurlyBracket or Comma".to_string(), - received: current_token.kind.to_string(), + return self.error(AstNodeError::Custom { + message: afmt!(self, "Invalid object syntax"), span: current_token.span, }) } } } - self.expect(TokenKind::Bracket(Bracket::RightCurlyBracket))?; - Ok(self.node(Node::Object(key_value_pairs.into_bump_slice()))) + expect!(self, TokenKind::Bracket(Bracket::RightCurlyBracket)); + self.node(Node::Object(key_value_pairs.into_bump_slice()), |_| { + NodeMetadata { + span: (span_start, self.prev_token_end()), + } + }) } - pub(crate) fn object_key(&self, expression_parser: F) -> ParserResult<&'arena Node<'arena>> + pub(crate) fn object_key(&self, expression_parser: F) -> &'arena Node<'arena> where - F: Fn() -> ParserResult<&'arena Node<'arena>>, + F: Fn() -> &'arena Node<'arena>, { - let key_token = self.current(); + let Some(key_token) = self.current() else { + return self.error(AstNodeError::Custom { + message: afmt!(self, "Expected an object key"), + span: (self.prev_token_end(), self.prev_token_end()), + }); + }; let key = match key_token.kind { TokenKind::Identifier(identifier) => { - self.next()?; - self.node(Node::String(identifier.into())) + self.next(); + self.node(Node::String(identifier.into()), |_| NodeMetadata { + span: key_token.span, + }) } TokenKind::Boolean(boolean) => match boolean { true => { - self.next()?; - self.node(Node::String("true")) + self.next(); + self.node(Node::String("true"), |_| NodeMetadata { + span: key_token.span, + }) } false => { - self.next()?; - self.node(Node::String("false")) + self.next(); + self.node(Node::String("false"), |_| NodeMetadata { + span: key_token.span, + }) } }, TokenKind::Number => { - self.next()?; - self.node(Node::String(key_token.value)) + self.next(); + self.node(Node::String(key_token.value), |_| NodeMetadata { + span: key_token.span, + }) } TokenKind::Literal => { - self.next()?; - self.node(Node::String(key_token.value)) + self.next(); + self.node(Node::String(key_token.value), |_| NodeMetadata { + span: key_token.span, + }) } TokenKind::Bracket(bracket) => match bracket { Bracket::LeftSquareBracket => { - self.expect(TokenKind::Bracket(Bracket::LeftSquareBracket))?; - let token = expression_parser()?; - self.expect(TokenKind::Bracket(Bracket::RightSquareBracket))?; + expect!(self, TokenKind::Bracket(Bracket::LeftSquareBracket)); + let token = expression_parser(); + expect!(self, TokenKind::Bracket(Bracket::RightSquareBracket)); token } _ => { - return Err(ParserError::FailedToParse { - message: "Operator is not supported as object key".to_string(), + return self.error(AstNodeError::Custom { + message: afmt!(self, "Operator is not supported as object key"), span: key_token.span, }) } }, TokenKind::QuotationMark(qm) => match qm { - QuotationMark::SingleQuote => self.simple_string(&QuotationMark::SingleQuote)?, - QuotationMark::DoubleQuote => self.simple_string(&QuotationMark::DoubleQuote)?, + QuotationMark::SingleQuote => self.simple_string(&QuotationMark::SingleQuote), + QuotationMark::DoubleQuote => self.simple_string(&QuotationMark::DoubleQuote), QuotationMark::Backtick => { - return Err(ParserError::FailedToParse { - message: "TemplateString expression not supported as object key" - .to_string(), + return self.error(AstNodeError::Custom { + message: afmt!( + self, + "TemplateString expression not supported as object key" + ), span: key_token.span, }) } }, TokenKind::TemplateString(_) => { - return Err(ParserError::FailedToParse { - message: "TemplateString expression not supported as object key".to_string(), + return self.error(AstNodeError::Custom { + message: afmt!( + self, + "TemplateString expression not supported as object key" + ), span: key_token.span, }) } TokenKind::Operator(_) => { - return Err(ParserError::FailedToParse { - message: "Operator is not supported as object key".to_string(), + return self.error(AstNodeError::Custom { + message: afmt!(self, "Operator is not supported as object key"), span: key_token.span, }) } }; - Ok(key) + key } /// Conditional @@ -601,20 +905,25 @@ impl<'arena, 'token_ref, Flavor> Parser<'arena, 'token_ref, Flavor> { &self, condition: &'arena Node<'arena>, expression_parser: F, - ) -> ParserResult>> + ) -> Option<&'arena Node<'arena>> where - F: Fn() -> ParserResult<&'arena Node<'arena>>, + F: Fn() -> &'arena Node<'arena>, { - let current_token = self.current(); + let Some(current_token) = self.current() else { + return None; + }; if current_token.kind != TokenKind::Operator(Operator::QuestionMark) { - return Ok(None); + return None; } - self.next()?; + self.next(); - let on_true = expression_parser()?; - self.expect(TokenKind::Operator(Operator::Slice))?; - let on_false = expression_parser()?; + let on_true = expression_parser(); + if let Some(error_node) = self.expect(TokenKind::Operator(Operator::Slice)) { + return Some(error_node); + } + + let on_false = expression_parser(); let conditional_node = Node::Conditional { condition, @@ -622,44 +931,31 @@ impl<'arena, 'token_ref, Flavor> Parser<'arena, 'token_ref, Flavor> { on_false, }; - Ok(Some(self.node(conditional_node))) + Some(self.node(conditional_node, |_| NodeMetadata { + span: (current_token.span.0, self.prev_token_end()), + })) } /// Literal - number, string, array etc. - pub(crate) fn literal(&self, expression_parser: F) -> ParserResult<&'arena Node<'arena>> + pub(crate) fn literal(&self, expression_parser: F) -> &'arena Node<'arena> where - F: Fn() -> ParserResult<&'arena Node<'arena>>, + F: Fn() -> &'arena Node<'arena>, { - let current_token = self.current(); + let Some(current_token) = self.current() else { + return self.error(AstNodeError::Custom { + message: afmt!(self, "Expected a literal"), + span: (self.prev_token_end(), self.prev_token_end()), + }); + }; + match ¤t_token.kind { TokenKind::Identifier(identifier) => match identifier { - Identifier::Null => self.null()?.ok_or_else(|| ParserError::FailedToParse { - message: "Failed to parse null identifier".to_string(), - span: current_token.span, - }), - _ => { - self.identifier(&expression_parser)? - .ok_or_else(|| ParserError::FailedToParse { - message: "Failed to parse identifier".to_string(), - span: current_token.span, - }) - } + Identifier::Null => self.null(), + _ => self.identifier(&expression_parser), }, - TokenKind::Literal => { - self.identifier(&expression_parser)? - .ok_or_else(|| ParserError::FailedToParse { - message: "Failed to parse literal".to_string(), - span: current_token.span, - }) - } - TokenKind::Boolean(_) => self.bool()?.ok_or_else(|| ParserError::FailedToParse { - message: "Failed to parse boolean".to_string(), - span: current_token.span, - }), - TokenKind::Number => self.number()?.ok_or_else(|| ParserError::FailedToParse { - message: "Failed to parse number".to_string(), - span: current_token.span, - }), + TokenKind::Literal => self.identifier(&expression_parser), + TokenKind::Boolean(_) => self.bool(), + TokenKind::Number => self.number(), TokenKind::QuotationMark(quote_mark) => match quote_mark { QuotationMark::SingleQuote | QuotationMark::DoubleQuote => { self.simple_string(quote_mark) @@ -670,33 +966,28 @@ impl<'arena, 'token_ref, Flavor> Parser<'arena, 'token_ref, Flavor> { Bracket::LeftParenthesis | Bracket::RightParenthesis | Bracket::RightSquareBracket => { - self.interval(&expression_parser)? - .ok_or_else(|| ParserError::FailedToParse { - message: "Failed to parse interval".to_string(), - span: current_token.span, + self.interval(&expression_parser).unwrap_or_else(|| { + self.error(AstNodeError::Custom { + message: afmt!(self, "Invalid syntax"), + span: (self.prev_token_end(), self.prev_token_end()), }) + }) } Bracket::LeftSquareBracket => self .interval(&expression_parser) - .transpose() - .or_else(|| self.array(&expression_parser).transpose()) - .transpose()? - .ok_or_else(|| ParserError::FailedToParse { - message: "Invalid bracket".to_string(), - span: current_token.span, - }), + .unwrap_or_else(|| self.array(&expression_parser)), Bracket::LeftCurlyBracket => self.object(&expression_parser), - Bracket::RightCurlyBracket => Err(ParserError::FailedToParse { - message: "Unexpected RightCurlyBracket token".to_string(), + Bracket::RightCurlyBracket => self.error(AstNodeError::Custom { + message: afmt!(self, "Unexpected RightCurlyBracket token"), span: current_token.span, }), }, - TokenKind::Operator(_) => Err(ParserError::FailedToParse { - message: "Unexpected Operator token".to_string(), + TokenKind::Operator(_) => self.error(AstNodeError::Custom { + message: afmt!(self, "Unexpected Operator token"), span: current_token.span, }), - TokenKind::TemplateString(_) => Err(ParserError::FailedToParse { - message: "Unexpected TemplateString token".to_string(), + TokenKind::TemplateString(_) => self.error(AstNodeError::Custom { + message: afmt!(self, "Unexpected TemplateString token"), span: current_token.span, }), } @@ -732,3 +1023,26 @@ impl From<&Token<'_>> for PostfixKind { } } } + +pub(crate) struct MetadataHelper<'a, 'arena> { + node_metadata: &'a HashMap>, + arena: PhantomData<&'arena ()>, +} + +impl<'a, 'arena> MetadataHelper<'a, 'arena> { + pub(crate) fn span( + &self, + left: &'arena Node<'arena>, + right: &'arena Node<'arena>, + ) -> Option<(u32, u32)> { + Some((self.metadata(left)?.span.0, self.metadata(right)?.span.1)) + } + + pub(crate) fn metadata(&self, n: &'arena Node<'arena>) -> Option<&NodeMetadata> { + self.node_metadata.get(&self.address(n)) + } + + fn address(&self, n: &'arena Node<'arena>) -> usize { + n as *const Node as usize + } +} diff --git a/core/expression/src/parser/result.rs b/core/expression/src/parser/result.rs new file mode 100644 index 00000000..c3ec4cc2 --- /dev/null +++ b/core/expression/src/parser/result.rs @@ -0,0 +1,28 @@ +use crate::parser::{Node, ParserError}; +use nohash_hasher::BuildNoHashHasher; +use std::collections::HashMap; + +#[derive(Debug)] +pub struct ParserResult<'a> { + pub root: &'a Node<'a>, + pub is_complete: bool, + pub metadata: Option>>, +} + +#[derive(Debug, Clone)] +pub struct NodeMetadata { + pub span: (u32, u32), +} + +impl<'a> ParserResult<'a> { + pub fn error(&self) -> Result<(), ParserError> { + if !self.is_complete { + return Err(ParserError::Incomplete); + } + + match self.root.first_error() { + None => Ok(()), + Some(err) => Err(ParserError::NodeError(err.to_string())), + } + } +} diff --git a/core/expression/src/parser/standard.rs b/core/expression/src/parser/standard.rs index 183a27bd..0c3ac790 100644 --- a/core/expression/src/parser/standard.rs +++ b/core/expression/src/parser/standard.rs @@ -1,30 +1,29 @@ use crate::lexer::{Bracket, Identifier, TokenKind}; -use crate::parser::ast::Node; +use crate::parser::ast::{AstNodeError, Node}; use crate::parser::constants::{Associativity, BINARY_OPERATORS, UNARY_OPERATORS}; -use crate::parser::error::ParserError::{FailedToParse, UnexpectedToken}; -use crate::parser::error::ParserResult; use crate::parser::parser::Parser; +use crate::parser::result::ParserResult; +use crate::parser::NodeMetadata; #[derive(Debug)] pub struct Standard; impl<'arena, 'token_ref> Parser<'arena, 'token_ref, Standard> { - pub fn parse(&self) -> ParserResult<&'arena Node<'arena>> { - let result = self.binary_expression(0)?; - if !self.is_done() { - let token = self.current(); - return Err(FailedToParse { - message: format!("Unterminated token {}", token.value), - span: token.span, - }); - } + pub fn parse(&self) -> ParserResult<'arena> { + let root = self.binary_expression(0); - return Ok(result); + ParserResult { + root, + is_complete: self.is_done(), + metadata: self.node_metadata.clone().map(|t| t.into_inner()), + } } - fn binary_expression(&self, precedence: u8) -> ParserResult<&'arena Node<'arena>> { - let mut node_left = self.unary_expression()?; - let mut token = self.current(); + fn binary_expression(&self, precedence: u8) -> &'arena Node<'arena> { + let mut node_left = self.unary_expression(); + let Some(mut token) = self.current() else { + return node_left; + }; while let TokenKind::Operator(operator) = &token.kind { if self.is_done() { @@ -39,68 +38,99 @@ impl<'arena, 'token_ref> Parser<'arena, 'token_ref, Standard> { break; } - self.next()?; + self.next(); let node_right = match op.associativity { - Associativity::Left => self.binary_expression(op.precedence + 1)?, - _ => self.binary_expression(op.precedence)?, + Associativity::Left => self.binary_expression(op.precedence + 1), + _ => self.binary_expression(op.precedence), }; - node_left = self.node(Node::Binary { - operator: *operator, - left: node_left, - right: node_right, - }); - token = self.current(); + node_left = self.node( + Node::Binary { + operator: *operator, + left: node_left, + right: node_right, + }, + |h| NodeMetadata { + span: h.span(node_left, node_right).unwrap_or_default(), + }, + ); + + let Some(t) = self.current() else { + break; + }; + token = t; } if precedence == 0 { if let Some(conditional_node) = - self.conditional(node_left, || self.binary_expression(0))? + self.conditional(node_left, || self.binary_expression(0)) { node_left = conditional_node; } } - Ok(node_left) + node_left } - fn unary_expression(&self) -> ParserResult<&'arena Node<'arena>> { - let token = self.current(); + fn unary_expression(&self) -> &'arena Node<'arena> { + let Some(token) = self.current() else { + return self.error(AstNodeError::Custom { + message: self.bump.alloc_str("Unexpected end of unary expression"), + span: (self.prev_token_end(), self.prev_token_end()), + }); + }; if self.depth() > 0 && token.kind == TokenKind::Identifier(Identifier::CallbackReference) { - self.next()?; + self.next(); - let node = self.node(Node::Pointer); + let node = self.node(Node::Pointer, |_| NodeMetadata { span: token.span }); return self.with_postfix(node, || self.binary_expression(0)); } if let TokenKind::Operator(operator) = &token.kind { let Some(unary_operator) = UNARY_OPERATORS.get(operator) else { - return Err(UnexpectedToken { - expected: "UnaryOperator".to_string(), - received: token.kind.to_string(), + return self.error(AstNodeError::UnexpectedToken { + expected: "UnaryOperator", + received: self.bump.alloc_str(token.kind.to_string().as_str()), span: token.span, }); }; - self.next()?; - let expr = self.binary_expression(unary_operator.precedence)?; - let node = self.node(Node::Unary { - operator: *operator, - node: expr, - }); + self.next(); + let expr = self.binary_expression(unary_operator.precedence); + let node = self.node( + Node::Unary { + operator: *operator, + node: expr, + }, + |h| NodeMetadata { + span: ( + token.span.0, + h.metadata(expr).map(|n| n.span.1).unwrap_or_default(), + ), + }, + ); - return Ok(node); + return node; } - if let Some(interval_node) = self.interval(|| self.binary_expression(0))? { - return Ok(interval_node); + if let Some(interval_node) = self.interval(|| self.binary_expression(0)) { + return interval_node; } if token.kind == TokenKind::Bracket(Bracket::LeftParenthesis) { - self.next()?; - let expr = self.binary_expression(0)?; - self.expect(TokenKind::Bracket(Bracket::RightParenthesis))?; + let p_start = self.current().map(|s| s.span.0); + + self.next(); + let binary_node = self.binary_expression(0); + if let Some(error_node) = self.expect(TokenKind::Bracket(Bracket::RightParenthesis)) { + return error_node; + }; + + let expr = self.node(Node::Parenthesized(binary_node), |_| NodeMetadata { + span: (p_start.unwrap_or_default(), self.prev_token_end()), + }); + return self.with_postfix(expr, || self.binary_expression(0)); } diff --git a/core/expression/src/parser/unary.rs b/core/expression/src/parser/unary.rs index 88059418..940f29f2 100644 --- a/core/expression/src/parser/unary.rs +++ b/core/expression/src/parser/unary.rs @@ -1,11 +1,10 @@ use crate::lexer::{Bracket, ComparisonOperator, Identifier, LogicalOperator, Operator, TokenKind}; -use crate::parser::ast::Node; +use crate::parser::ast::{AstNodeError, Node}; use crate::parser::builtin::BuiltInFunction; use crate::parser::constants::{Associativity, BINARY_OPERATORS, UNARY_OPERATORS}; -use crate::parser::error::ParserError::{FailedToParse, UnexpectedToken}; -use crate::parser::error::{ParserError, ParserResult}; use crate::parser::parser::Parser; use crate::parser::unary::UnaryNodeBehaviour::CompareWithReference; +use crate::parser::{NodeMetadata, ParserResult}; #[derive(Debug)] pub struct Unary; @@ -13,23 +12,24 @@ pub struct Unary; const ROOT_NODE: Node<'static> = Node::Identifier("$"); impl<'arena, 'token_ref> Parser<'arena, 'token_ref, Unary> { - pub fn parse(&self) -> ParserResult<&'arena Node<'arena>> { - let result = self.root_expression()?; - if !self.is_done() { - let token = self.current(); - return Err(FailedToParse { - message: format!("Unterminated token {}", token.value), - span: token.span, - }); - } + pub fn parse(&self) -> ParserResult<'arena> { + let root = self.root_expression(); - return Ok(result); + ParserResult { + root, + is_complete: self.is_done(), + metadata: self.node_metadata.clone().map(|t| t.into_inner()), + } } - fn root_expression(&self) -> ParserResult<&'arena Node<'arena>> { - let mut left_node = self.expression_pair()?; + fn root_expression(&self) -> &'arena Node<'arena> { + let mut left_node = self.expression_pair(); + while !self.is_done() { - let current_token = self.current(); + let Some(current_token) = self.current() else { + break; + }; + let join_operator = match ¤t_token.kind { TokenKind::Operator(Operator::Logical(LogicalOperator::And)) => { Operator::Logical(LogicalOperator::And) @@ -37,68 +37,104 @@ impl<'arena, 'token_ref> Parser<'arena, 'token_ref, Unary> { TokenKind::Operator(Operator::Logical(LogicalOperator::Or)) | TokenKind::Operator(Operator::Comma) => Operator::Logical(LogicalOperator::Or), _ => { - return Err(ParserError::MemoryFailure); + return self.error(AstNodeError::Custom { + message: self.bump.alloc_str( + format!("Invalid join operator `{}`", current_token.kind).as_str(), + ), + span: current_token.span, + }) } }; - self.next()?; - let right_node = self.expression_pair()?; - left_node = self.node(Node::Binary { - left: left_node, - operator: join_operator, - right: right_node, - }); + self.next(); + let right_node = self.expression_pair(); + left_node = self.node( + Node::Binary { + left: left_node, + operator: join_operator, + right: right_node, + }, + |h| NodeMetadata { + span: h.span(left_node, right_node).unwrap_or_default(), + }, + ); } - Ok(left_node) + left_node } - fn expression_pair(&self) -> ParserResult<&'arena Node<'arena>> { + fn expression_pair(&self) -> &'arena Node<'arena> { let mut left_node = &ROOT_NODE; - let initial_token = self.current(); - if let TokenKind::Operator(Operator::Comparison(_)) = &initial_token.kind { + let current_token = self.current(); + + if let Some(TokenKind::Operator(Operator::Comparison(_))) = self.current_kind() { // Skips } else { - left_node = self.binary_expression(0)?; + left_node = self.binary_expression(0); } - let current_token = self.current(); - match ¤t_token.kind { - TokenKind::Operator(Operator::Comparison(comparison)) => { - self.next()?; - let right_node = self.binary_expression(0)?; - left_node = self.node(Node::Binary { - left: left_node, - operator: Operator::Comparison(*comparison), - right: right_node, - }); + match self.current_kind() { + Some(TokenKind::Operator(Operator::Comparison(comparison))) => { + self.next(); + let right_node = self.binary_expression(0); + left_node = self.node( + Node::Binary { + left: left_node, + operator: Operator::Comparison(*comparison), + right: right_node, + }, + |h| NodeMetadata { + span: ( + current_token.map(|t| t.span.0).unwrap_or_default(), + h.metadata(right_node).map(|n| n.span.1).unwrap_or_default(), + ), + }, + ); } _ => { let behaviour = UnaryNodeBehaviour::from(left_node); match behaviour { CompareWithReference(comparator) => { - left_node = self.node(Node::Binary { - left: &ROOT_NODE, - operator: Operator::Comparison(comparator), - right: left_node, - }) + left_node = self.node( + Node::Binary { + left: &ROOT_NODE, + operator: Operator::Comparison(comparator), + right: left_node, + }, + |h| NodeMetadata { + span: ( + current_token.map(|t| t.span.0).unwrap_or_default(), + h.metadata(left_node).map(|n| n.span.1).unwrap_or_default(), + ), + }, + ) } UnaryNodeBehaviour::AsBoolean => { - left_node = self.node(Node::BuiltIn { - kind: BuiltInFunction::Bool, - arguments: self.bump.alloc_slice_clone(&[left_node]), - }) + left_node = self.node( + Node::BuiltIn { + kind: BuiltInFunction::Bool, + arguments: self.bump.alloc_slice_clone(&[left_node]), + }, + |h| NodeMetadata { + span: ( + current_token.map(|t| t.span.0).unwrap_or_default(), + h.metadata(left_node).map(|n| n.span.1).unwrap_or_default(), + ), + }, + ) } } } } - Ok(left_node) + left_node } - fn binary_expression(&self, precedence: u8) -> ParserResult<&'arena Node<'arena>> { - let mut node_left = self.unary_expression()?; - let mut token = self.current(); + fn binary_expression(&self, precedence: u8) -> &'arena Node<'arena> { + let mut node_left = self.unary_expression(); + let Some(mut token) = self.current() else { + return node_left; + }; while let TokenKind::Operator(operator) = &token.kind { if self.is_done() { @@ -122,68 +158,96 @@ impl<'arena, 'token_ref> Parser<'arena, 'token_ref, Unary> { break; } - self.next()?; + self.next(); let node_right = match op.associativity { - Associativity::Left => self.binary_expression(op.precedence + 1)?, - _ => self.binary_expression(op.precedence)?, + Associativity::Left => self.binary_expression(op.precedence + 1), + _ => self.binary_expression(op.precedence), }; - node_left = self.node(Node::Binary { - operator: *operator, - left: node_left, - right: node_right, - }); - token = self.current(); + node_left = self.node( + Node::Binary { + operator: *operator, + left: node_left, + right: node_right, + }, + |h| NodeMetadata { + span: h.span(node_left, node_right).unwrap_or_default(), + }, + ); + + let Some(t) = self.current() else { + break; + }; + token = t; } if precedence == 0 { if let Some(conditional_node) = - self.conditional(node_left, || self.binary_expression(0))? + self.conditional(node_left, || self.binary_expression(0)) { node_left = conditional_node; } } - Ok(node_left) + node_left } - fn unary_expression(&self) -> ParserResult<&'arena Node<'arena>> { - let token = self.current(); + fn unary_expression(&self) -> &'arena Node<'arena> { + let Some(token) = self.current() else { + return self.literal(|| self.binary_expression(0)); + }; if self.depth() > 0 && token.kind == TokenKind::Identifier(Identifier::CallbackReference) { - self.next()?; + self.next(); - let node = self.node(Node::Pointer); + let node = self.node(Node::Pointer, |_| NodeMetadata { span: token.span }); return self.with_postfix(node, || self.binary_expression(0)); } if let TokenKind::Operator(operator) = &token.kind { let Some(unary_operator) = UNARY_OPERATORS.get(operator) else { - return Err(UnexpectedToken { - expected: "UnaryOperator".to_string(), - received: token.kind.to_string(), + return self.error(AstNodeError::UnexpectedToken { + expected: self.bump.alloc_str("UnaryOperator"), + received: self.bump.alloc_str(token.kind.to_string().as_str()), span: token.span, }); }; - self.next()?; - let expr = self.binary_expression(unary_operator.precedence)?; - let node = self.node(Node::Unary { - operator: *operator, - node: expr, - }); + self.next(); + let expr = self.binary_expression(unary_operator.precedence); + let node = self.node( + Node::Unary { + operator: *operator, + node: expr, + }, + |h| NodeMetadata { + span: ( + token.span.0, + h.metadata(expr).map(|n| n.span.1).unwrap_or_default(), + ), + }, + ); - return Ok(node); + return node; } - if let Some(interval_node) = self.interval(|| self.binary_expression(0))? { - return Ok(interval_node); + if let Some(interval_node) = self.interval(|| self.binary_expression(0)) { + return interval_node; } if token.kind == TokenKind::Bracket(Bracket::LeftParenthesis) { - self.next()?; - let expr = self.binary_expression(0)?; - self.expect(TokenKind::Bracket(Bracket::RightParenthesis))?; + let p_start = self.current().map(|s| s.span.0); + + self.next(); + let binary_node = self.binary_expression(0); + if let Some(error_node) = self.expect(TokenKind::Bracket(Bracket::RightParenthesis)) { + return error_node; + }; + + let expr = self.node(Node::Parenthesized(binary_node), |_| NodeMetadata { + span: (p_start.unwrap_or_default(), self.prev_token_end()), + }); + return self.with_postfix(expr, || self.binary_expression(0)); } @@ -235,13 +299,14 @@ impl From<&Node<'_>> for UnaryNodeBehaviour { let a = UnaryNodeBehaviour::from(*on_true); let b = UnaryNodeBehaviour::from(*on_false); - return if a == b { + if a == b { a } else { CompareWithReference(Equal) - }; + } } Node::Unary { node, .. } => UnaryNodeBehaviour::from(*node), + Node::Parenthesized(n) => UnaryNodeBehaviour::from(*n), Node::Binary { left, operator, @@ -251,11 +316,11 @@ impl From<&Node<'_>> for UnaryNodeBehaviour { let a = UnaryNodeBehaviour::from(*left); let b = UnaryNodeBehaviour::from(*right); - return if a == b { + if a == b { a } else { CompareWithReference(Equal) - }; + } } Operator::Logical(_) => AsBoolean, Operator::Comparison(_) => AsBoolean, @@ -318,6 +383,7 @@ impl From<&Node<'_>> for UnaryNodeBehaviour { BuiltInFunction::One => AsBoolean, BuiltInFunction::Type => CompareWithReference(Equal), }, + Node::Error { .. } => AsBoolean, } } } diff --git a/core/expression/src/variable/mod.rs b/core/expression/src/variable/mod.rs index 7345dd6a..31aaec6a 100644 --- a/core/expression/src/variable/mod.rs +++ b/core/expression/src/variable/mod.rs @@ -11,11 +11,13 @@ mod conv; mod de; mod map; mod ser; +mod types; use crate::vm::helpers::date_time; use crate::vm::VMError; -#[allow(unused_imports)] + pub use conv::ToVariable; +pub use types::VariableType; #[derive(Debug, PartialEq, Eq, Display)] pub enum Variable<'arena> { @@ -163,6 +165,7 @@ impl TryFrom<&Variable<'_>> for NaiveDateTime { fn try_from(value: &Variable<'_>) -> Result { match value { Variable::String(a) => date_time(a), + #[allow(deprecated)] Variable::Number(a) => NaiveDateTime::from_timestamp_opt( a.to_i64().ok_or_else(|| VMError::OpcodeErr { opcode: "DateManipulation".into(), diff --git a/core/expression/src/variable/types/conv.rs b/core/expression/src/variable/types/conv.rs new file mode 100644 index 00000000..250e049b --- /dev/null +++ b/core/expression/src/variable/types/conv.rs @@ -0,0 +1,94 @@ +use crate::variable::types::VariableType; +use serde_json::Value; +use std::borrow::Cow; +use std::ops::Deref; +use std::rc::Rc; + +impl<'a> From> for VariableType { + fn from(value: Cow<'a, Value>) -> Self { + match value.deref() { + Value::Null => VariableType::Null, + Value::Bool(_) => VariableType::Bool, + Value::Number(_) => VariableType::Number, + Value::String(_) => VariableType::String, + Value::Array(_) => { + let Value::Array(arr) = value.into_owned() else { + panic!("unexpected type of value, expected array"); + }; + + VariableType::from(arr) + } + Value::Object(_) => { + let Value::Object(obj) = value.into_owned() else { + panic!("unexpected type of value, expected object"); + }; + + VariableType::Object( + obj.into_iter() + .map(|(k, v)| (k, Rc::new(v.into()))) + .collect(), + ) + } + } + } +} + +impl From for VariableType { + fn from(value: Value) -> Self { + VariableType::from(Cow::Owned(value)).into() + } +} + +impl From<&Value> for VariableType { + fn from(value: &Value) -> Self { + VariableType::from(Cow::Borrowed(value)).into() + } +} + +impl From> for VariableType { + fn from(arr: Vec) -> Self { + if arr.len() == 0 { + return VariableType::Array(Rc::new(VariableType::Any)); + } + + let result_type = arr + .into_iter() + .fold(None, |acc: Option, b| match acc { + Some(a) => Some(a.merge(&VariableType::from(b))), + None => Some(VariableType::from(b)), + }); + + VariableType::Array(Rc::new(result_type.unwrap_or(VariableType::Any))) + } +} + +impl From<&Vec> for VariableType { + fn from(arr: &Vec) -> Self { + if arr.len() == 0 { + return VariableType::Array(Rc::new(VariableType::Any)); + } + + let result_type = arr + .iter() + .fold(None, |acc: Option, b| match acc { + Some(a) => Some(a.merge(&VariableType::from(b))), + None => Some(VariableType::from(b)), + }); + + VariableType::Array(Rc::new(result_type.unwrap_or(VariableType::Any))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_value_to_value_kind() { + assert_eq!(VariableType::from(json!(null)), VariableType::Null); + assert_eq!(VariableType::from(json!(true)), VariableType::Bool); + assert_eq!(VariableType::from(json!(42)), VariableType::Number); + assert_eq!(VariableType::from(json!("hello")), VariableType::String); + } +} diff --git a/core/expression/src/variable/types/mod.rs b/core/expression/src/variable/types/mod.rs new file mode 100644 index 00000000..7538a92d --- /dev/null +++ b/core/expression/src/variable/types/mod.rs @@ -0,0 +1,40 @@ +mod conv; +mod util; + +use serde::Serialize; +use std::collections::HashMap; +use std::fmt::Display; +use std::rc::Rc; + +#[derive(Debug, Clone, Eq, PartialEq, Serialize)] +pub enum VariableType { + Any, + Null, + Bool, + String, + Number, + Constant(Rc), + Array(Rc), + Object(HashMap>), +} + +impl Default for VariableType { + fn default() -> Self { + VariableType::Null + } +} + +impl Display for VariableType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + VariableType::Any => write!(f, "any"), + VariableType::Null => write!(f, "null"), + VariableType::Bool => write!(f, "bool"), + VariableType::String => write!(f, "string"), + VariableType::Number => write!(f, "number"), + VariableType::Constant(c) => write!(f, "{c}"), + VariableType::Array(v) => write!(f, "{v}[]"), + VariableType::Object(_) => write!(f, "object"), + } + } +} diff --git a/core/expression/src/variable/types/util.rs b/core/expression/src/variable/types/util.rs new file mode 100644 index 00000000..ce1d341e --- /dev/null +++ b/core/expression/src/variable/types/util.rs @@ -0,0 +1,199 @@ +use crate::variable::types::VariableType; +use serde_json::Value; +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::rc::Rc; + +impl VariableType { + pub fn array_item(&self) -> Option> { + match self { + VariableType::Array(item) => Some(item.clone()), + _ => None, + } + } + + pub fn as_const_str(&self) -> Option<&str> { + match self { + VariableType::Constant(c) => match c.as_ref() { + Value::String(s) => Some(s.as_str()), + _ => None, + }, + _ => None, + } + } + + pub fn omit_const(&self) -> VariableType { + match self { + VariableType::Constant(v) => VariableType::from(v.as_ref()), + _ => self.clone(), + } + } + + pub fn get(&self, vt: &VariableType) -> Rc { + match self { + VariableType::Array(inner) => inner.clone(), + VariableType::Object(obj) => match vt.as_const_str() { + None => Rc::new(VariableType::Any), + Some(key) => obj.get(key).cloned().unwrap_or(Rc::new(VariableType::Any)), + }, + VariableType::Any => Rc::new(VariableType::Any), + VariableType::Constant(c) => match c.as_ref() { + Value::Array(arr) => { + let arr_type = VariableType::from(arr.clone()); + arr_type.array_item().unwrap_or(Rc::new(VariableType::Any)) + } + Value::Object(obj) => match vt.as_const_str() { + None => Rc::new(VariableType::Any), + Some(key) => obj + .get(key) + .map(|v| Rc::new(v.into())) + .unwrap_or(Rc::new(VariableType::Any)), + }, + _ => Rc::from(VariableType::Null), + }, + _ => Rc::from(VariableType::Null), + } + } + + pub fn satisfies(&self, constraint: &Self) -> bool { + match (self, constraint) { + (VariableType::Any, _) | (_, VariableType::Any) => true, + (VariableType::Null, VariableType::Null) => true, + (VariableType::Bool, VariableType::Bool) => true, + (VariableType::String, VariableType::String) => true, + (VariableType::Number, VariableType::Number) => true, + (VariableType::Array(a1), VariableType::Array(a2)) => a1 == a2, + (VariableType::Object(o1), VariableType::Object(o2)) => o1 + .iter() + .all(|(k, v)| o2.get(k).is_some_and(|tv| v.satisfies(tv))), + (VariableType::Constant(c1), VariableType::Constant(c2)) => c1 == c2, + (VariableType::Constant(c), _) => { + let self_kind: VariableType = c.as_ref().into(); + self_kind.satisfies(constraint) + } + (_, _) => false, + } + } + + pub fn satisfies_array(&self) -> bool { + match self { + VariableType::Any | VariableType::Array(_) => true, + VariableType::Constant(c) => match c.as_ref() { + Value::Array(_) => true, + _ => false, + }, + _ => false, + } + } + + pub fn satisfies_object(&self) -> bool { + match self { + VariableType::Any | VariableType::Object(_) => true, + VariableType::Constant(c) => match c.as_ref() { + Value::Object(_) => true, + _ => false, + }, + _ => false, + } + } + + pub fn merge(&self, other: &Self) -> Self { + match (&self, other) { + (VariableType::Any, _) | (_, VariableType::Any) => VariableType::Any, + (VariableType::Null, VariableType::Null) => VariableType::Null, + (VariableType::Bool, VariableType::Bool) => VariableType::Bool, + (VariableType::String, VariableType::String) => VariableType::String, + (VariableType::Number, VariableType::Number) => VariableType::Number, + (VariableType::Array(a1), VariableType::Array(a2)) => { + if Rc::ptr_eq(&a1, &a2) { + VariableType::Array(a1.clone()) + } else { + VariableType::Array(Rc::new(a1.merge(a2))) + } + } + (VariableType::Constant(c1), VariableType::Constant(c2)) => { + if Rc::ptr_eq(&c1, &c2) { + VariableType::Constant(c1.clone()) + } else if c1 == c2 { + VariableType::Constant(c1.clone()) + } else { + let vt1 = VariableType::from(c1.as_ref()); + let vt2 = VariableType::from(c2.as_ref()); + + vt1.merge(&vt2) + } + } + (VariableType::Object(o1), VariableType::Object(o2)) => { + let cap = o1.capacity().max(o2.capacity()); + + let map = o1.iter().chain(o2.iter()).fold( + HashMap::>::with_capacity(cap), + |mut acc, (k, v)| { + match acc.entry(k.clone()) { + Entry::Occupied(mut occ) => { + let current = occ.get(); + let merged = v.merge(current.as_ref()); + occ.insert(Rc::new(merged)); + } + Entry::Vacant(vac) => { + vac.insert(v.clone()); + } + } + + acc + }, + ); + + VariableType::Object(map) + } + (_, _) => VariableType::Any, + } + } +} + +#[cfg(test)] +mod tests { + use crate::variable::VariableType; + use std::rc::Rc; + + #[test] + fn merge_simple() { + assert_eq!( + VariableType::Number.merge(&VariableType::Number), + VariableType::Number + ); + assert_eq!( + VariableType::String.merge(&VariableType::String), + VariableType::String + ); + assert_eq!( + VariableType::Bool.merge(&VariableType::Bool), + VariableType::Bool + ); + assert_eq!( + VariableType::Null.merge(&VariableType::Null), + VariableType::Null + ); + assert_eq!( + VariableType::Any.merge(&VariableType::Any), + VariableType::Any + ); + } + + #[test] + fn merge_array() { + assert_eq!( + VariableType::Array(Rc::new(VariableType::Number)) + .merge(&VariableType::Array(Rc::new(VariableType::Number))), + VariableType::Array(Rc::new(VariableType::Number)) + ); + } + + #[test] + fn merge_mixed() { + assert_eq!( + VariableType::Number.merge(&VariableType::String), + VariableType::Any + ); + } +} diff --git a/core/expression/src/vm/vm.rs b/core/expression/src/vm/vm.rs index f8d24476..e391edd2 100644 --- a/core/expression/src/vm/vm.rs +++ b/core/expression/src/vm/vm.rs @@ -1210,6 +1210,7 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'arena, 'parent_ref, 'bytecode_ message: "Failed to run DateFunction".into(), })?; + #[allow(deprecated)] self.push(Number(s.timestamp().into())); } Opcode::Slice => { @@ -1358,6 +1359,7 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'arena, 'parent_ref, 'bytecode_ Opcode::ParseDateTime => { let a = self.pop()?; let ts = match a { + #[allow(deprecated)] String(a) => date_time(a)?.timestamp(), Number(a) => a.to_i64().ok_or_else(|| OpcodeErr { opcode: "ParseDateTime".into(), diff --git a/core/expression/tests/isolate.rs b/core/expression/tests/isolate.rs index 452440b8..57ca3e2d 100644 --- a/core/expression/tests/isolate.rs +++ b/core/expression/tests/isolate.rs @@ -780,11 +780,12 @@ fn test_standard_csv() { isolate.set_environment(&input); } - let result = isolate + let maybe_result = isolate .run_standard(expression) - .context(format!("Expression: {expression}")) - .unwrap(); + .context(format!("Expression: {expression}")); + assert!(maybe_result.is_ok(), "{}", maybe_result.unwrap_err()); + let result = maybe_result.unwrap(); assert_eq!( result, output, "Expression {expression}. Expected: {output}, got: {result}" diff --git a/core/expression/tests/standard.rs b/core/expression/tests/standard.rs index d8908658..1e500f7d 100644 --- a/core/expression/tests/standard.rs +++ b/core/expression/tests/standard.rs @@ -86,11 +86,11 @@ fn standard_test() { StandardTest { src: "(1 - 2) * 3", result: &Node::Binary { - left: &Node::Binary { + left: &Node::Parenthesized(&Node::Binary { left: &Node::Number(D1), operator: Operator::Arithmetic(ArithmeticOperator::Subtract), right: &Node::Number(D2), - }, + }), operator: Operator::Arithmetic(ArithmeticOperator::Multiply), right: &Node::Number(D3), }, @@ -123,11 +123,11 @@ fn standard_test() { src: "(a or b) and c", result: &Node::Binary { operator: Operator::Logical(LogicalOperator::And), - left: &Node::Binary { + left: &Node::Parenthesized(&Node::Binary { left: &Node::Identifier("a"), right: &Node::Identifier("b"), operator: Operator::Logical(LogicalOperator::Or), - }, + }), right: &Node::Identifier("c"), }, }, @@ -298,17 +298,19 @@ fn standard_test() { let tokens = lexer.tokenize(src).unwrap(); let unary_parser = Parser::try_new(tokens, &bump).unwrap().standard(); let parser_result = unary_parser.parse(); - let Ok(ast) = parser_result else { - assert!( - false, - "Failed on expression: {}. Error: {:?}.", - src, - parser_result.unwrap_err() - ); - return; - }; + // let Ok(ast) = parser_result else { + // assert!( + // false, + // "Failed on expression: {}. Error: {:?}.", + // src, + // parser_result.unwrap_err() + // ); + // return; + // }; + + assert!(parser_result.error().is_ok(), "Expression failed: {src}"); + assert_eq!(parser_result.root, result, "Failed on expression: {}", src); - assert_eq!(ast, result, "Failed on expression: {}", src); bump.reset(); } } @@ -323,8 +325,9 @@ fn failure_tests() { for test in tests { let tokens = lexer.tokenize(test).unwrap(); let parser = Parser::try_new(tokens, &bump).unwrap().standard(); - let ast = parser.parse(); - assert!(ast.is_err()); + let parser_result = parser.parse(); + + assert!(parser_result.error().is_err(), "{parser_result:?}"); bump.reset(); } diff --git a/core/expression/tests/unary.rs b/core/expression/tests/unary.rs index eee00197..e6721c69 100644 --- a/core/expression/tests/unary.rs +++ b/core/expression/tests/unary.rs @@ -145,8 +145,10 @@ fn unary_test() { for UnaryTest { src, result } in tests { let tokens = lexer.tokenize(src).unwrap(); let parser = Parser::try_new(tokens, &bump).unwrap().unary(); - let ast = parser.parse().unwrap(); - assert_eq!(ast, result); + let parser_result = parser.parse(); + + assert!(parser_result.error().is_ok(), "Parser failed"); + assert_eq!(parser_result.root, result); bump.reset(); } @@ -162,8 +164,12 @@ fn failure_tests() { for test in tests { let tokens = lexer.tokenize(test).unwrap(); let unary_parser = Parser::try_new(tokens, &bump).unwrap().standard(); - let ast = unary_parser.parse(); - assert!(ast.is_err()); + let parser_result = unary_parser.parse(); + + assert!( + parser_result.error().is_err(), + "Parsing expected to fail for: {test}" + ); bump.reset(); } diff --git a/core/expression_repl/src/main.rs b/core/expression_repl/src/main.rs index a291ef6b..8f3e6935 100644 --- a/core/expression_repl/src/main.rs +++ b/core/expression_repl/src/main.rs @@ -49,7 +49,7 @@ fn main() -> Result<()> { let mut isolate = Isolate::new(); isolate.set_environment( - &json!({ "customer": { "firstName": "John", "lastName": "Doe", "age": 20 } }), + &json!({ "customer": { "firstName": "John", "lastName": "Doe", "age": 20 }, "hello": true, "$": 10 }), ); let result = isolate.run_standard(line.as_str());