feat: intellisense (#244)

* local: fault tolerant parser

* fix tests

* add intellisense

* fix minor issues

* improve support

* fix fmt

* fix

* fix: improve error messages

* improve error handling and ast

* fix lint

* fix tests

* exclude zen-nodejs from rust tests
This commit is contained in:
stefan-gorules
2024-09-23 09:23:48 +02:00
committed by GitHub
parent 35265f76a2
commit 3e7cdfcde1
31 changed files with 2399 additions and 529 deletions
+2 -2
View File
@@ -52,8 +52,8 @@ jobs:
- uses: actions/checkout@v3
- name: Install Rust
run: rustup install 1.80
- run: cargo test --workspace --all-features --exclude zen-ffi
- run: cargo test --workspace --all-features --exclude zen-ffi --release
- run: cargo test --workspace --all-features --exclude zen-ffi --exclude zen-nodejs
- run: cargo test --workspace --all-features --exclude zen-ffi --exclude zen-nodejs --release
build:
name: cargo +${{ matrix.rust }} build
+6 -6
View File
@@ -153,15 +153,15 @@ impl<'a> DecisionTableHandler<'a> {
for input in &content.inputs {
let rule_value = rule.get(input.id.as_str())?;
let mut input_identifier = input.id.clone();
if let Some(input_field) = &input.field {
input_identifier = format!("{input_field}[{input_identifier}]");
}
let Some(input_field) = &input.field else {
continue;
};
if let Some(reference) = self.isolate.get_reference(input_identifier.as_str()) {
reference_map.insert(input_identifier.clone(), reference);
if let Some(reference) = self.isolate.get_reference(input_field.as_str()) {
reference_map.insert(input_field.clone(), reference);
}
let input_identifier = format!("{input_field}[{}]", &input.id);
expressions.insert(input_identifier, rule_value.clone());
}
+1 -1
View File
@@ -16,7 +16,7 @@ fastrand = { workspace = true }
once_cell = { workspace = true }
regex = { workspace = true, optional = true }
regex-lite = { workspace = true, optional = true }
serde = { workspace = true }
serde = { workspace = true, features = ["rc", "derive"] }
serde_json = { workspace = true, features = ["arbitrary_precision"] }
strum = { workspace = true }
strum_macros = { workspace = true }
+1 -1
View File
@@ -12,7 +12,7 @@ fn bench_source(b: &mut Bencher, src: &'static str) {
b.iter(|| {
let std_parser = Parser::try_new(tokens, &bump).unwrap().standard();
criterion::black_box(std_parser.parse().unwrap());
criterion::black_box(std_parser.parse());
bump.reset();
});
+1 -1
View File
@@ -11,7 +11,7 @@ fn bench_source(b: &mut Bencher, src: &'static str) {
b.iter(|| {
let unary_parser = Parser::try_new(tokens, &bump).unwrap().unary();
criterion::black_box(unary_parser.parse().unwrap());
criterion::black_box(unary_parser.parse());
bump.reset();
})
+2
View File
@@ -132,6 +132,7 @@ impl<'arena, 'bytecode_ref> CompilerInner<'arena, 'bytecode_ref> {
}
Node::Identifier(v) => Ok(self.emit(Opcode::FetchEnv(v))),
Node::Closure(v) => self.compile_node(v),
Node::Parenthesized(v) => self.compile_node(v),
Node::Member { node, property } => {
self.compile_node(node)?;
self.compile_node(property)?;
@@ -588,6 +589,7 @@ impl<'arena, 'bytecode_ref> CompilerInner<'arena, 'bytecode_ref> {
Ok(self.emit(Opcode::End))
}
},
Node::Error { .. } => Err(CompilerError::UnexpectedErrorNode),
}
}
}
+3
View File
@@ -10,6 +10,9 @@ pub enum CompilerError {
#[error("Argument not found for builtin {builtin} at index {index}")]
ArgumentNotFound { builtin: String, index: usize },
#[error("Unexpected error node")]
UnexpectedErrorNode,
}
pub(crate) type CompilerResult<T> = Result<T, CompilerError>;
+153
View File
@@ -0,0 +1,153 @@
use crate::arena::UnsafeArena;
use crate::intellisense::scope::IntelliSenseScope;
use crate::intellisense::types::provider::TypesProvider;
use crate::lexer::Lexer;
use crate::parser::{Node, Parser};
use crate::variable::VariableType;
use serde::Serialize;
use std::cell::RefCell;
use std::rc::Rc;
mod scope;
mod types;
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct IntelliSenseToken {
pub span: (u32, u32),
pub kind: Rc<VariableType>,
pub node_kind: &'static str,
pub error: Option<String>,
}
pub struct IntelliSense<'arena> {
arena: UnsafeArena<'arena>,
lexer: Lexer<'arena>,
}
impl<'arena> IntelliSense<'arena> {
pub fn new() -> Self {
Self {
arena: UnsafeArena::new(),
lexer: Lexer::new(),
}
}
pub fn type_check(
&mut self,
source: &'arena str,
data: &VariableType,
) -> Option<Vec<IntelliSenseToken>> {
let arena = self.arena.get();
let tokens = self.lexer.tokenize(source).ok()?;
let parser = Parser::try_new(tokens, &arena).map(|p| p.standard()).ok()?;
let parser_result = parser.with_metadata().parse();
let ast = parser_result.root;
let metadata = parser_result.metadata?;
let type_data = TypesProvider::generate(
ast,
IntelliSenseScope {
pointer_data: data,
root_data: data,
current_data: data,
},
);
let results = RefCell::new(Vec::new());
ast.walk(|node| {
let addr = node as *const Node as usize;
let mut r = results.borrow_mut();
let typ = type_data.get_type(node);
r.push(IntelliSenseToken {
span: node
.span()
.or_else(|| metadata.get(&addr).map(|s| s.span))
.unwrap_or_default(),
node_kind: node.into(),
error: typ.map(|t| t.error.clone()).flatten(),
kind: typ
.map(|t| t.kind.clone())
.unwrap_or_else(|| Rc::new(VariableType::Any)),
});
});
self.arena.with_mut(|a| a.reset());
Some(results.into_inner())
}
pub fn type_check_unary(
&mut self,
source: &'arena str,
data: &VariableType,
) -> Option<Vec<IntelliSenseToken>> {
let arena = self.arena.get();
let tokens = self.lexer.tokenize(source).ok()?;
let parser = Parser::try_new(tokens, &arena).map(|p| p.unary()).ok()?;
let parser_result = parser.with_metadata().parse();
let ast = parser_result.root;
let metadata = parser_result.metadata?;
let type_data = TypesProvider::generate(
ast,
IntelliSenseScope {
pointer_data: data,
root_data: data,
current_data: data,
},
);
let results = RefCell::new(Vec::new());
ast.walk(|node| {
let addr = node as *const Node as usize;
let mut r = results.borrow_mut();
let typ = type_data.get_type(node);
r.push(IntelliSenseToken {
span: metadata.get(&addr).map(|s| s.span).unwrap_or_default(),
node_kind: node.into(),
error: typ.map(|t| t.error.clone()).flatten(),
kind: typ
.map(|t| t.kind.clone())
.unwrap_or_else(|| Rc::new(VariableType::Any)),
});
});
self.arena.with_mut(|a| a.reset());
Some(results.into_inner())
}
}
#[cfg(test)]
mod tests {
use crate::intellisense::IntelliSense;
use crate::variable::VariableType;
use serde_json::json;
#[test]
fn sample_test() {
let mut is = IntelliSense::new();
let data = json!({ "customer": { "firstName": "John", "lastName": "Doe", "array": [{"a": 5}, {"a": 6}] } });
let data_type: VariableType = data.into();
let typ = is.type_check("customer.array[0]", &data_type);
println!("{:?}", typ);
}
#[test]
fn sample_test_unary() {
let mut is = IntelliSense::new();
let data = json!({ "customer": { "firstName": "John", "lastName": "Doe" }, "$": 10});
let data_type: VariableType = data.into();
let typ = is.type_check_unary("> 10", &data_type);
println!("{typ:?}");
}
}
@@ -0,0 +1,8 @@
use crate::variable::VariableType;
#[derive(Clone, Debug)]
pub struct IntelliSenseScope<'a> {
pub root_data: &'a VariableType,
pub current_data: &'a VariableType,
pub pointer_data: &'a VariableType,
}
@@ -0,0 +1,2 @@
pub(crate) mod provider;
mod type_info;
@@ -0,0 +1,715 @@
use crate::intellisense::scope::IntelliSenseScope;
use crate::intellisense::types::type_info::TypeInfo;
use crate::lexer::{ArithmeticOperator, ComparisonOperator, LogicalOperator, Operator};
use crate::parser::{Arity, BuiltInFunction, Node};
use crate::variable::VariableType;
use serde_json::{Number, Value};
use std::collections::HashMap;
use std::rc::Rc;
#[derive(Debug)]
pub struct TypesProvider {
types: HashMap<usize, TypeInfo>,
}
impl TypesProvider {
pub fn generate(root: &Node, scope: IntelliSenseScope) -> Self {
let mut s = Self {
types: HashMap::new(),
};
s.determine(root, scope, false);
s
}
pub fn get_type(&self, node: &Node) -> Option<&TypeInfo> {
let addr = node_address(node);
self.types.get(&addr)
}
fn set_type(&mut self, node: &Node, type_info: TypeInfo) {
let addr = node_address(node);
self.types.insert(addr, type_info);
}
fn update_type<F>(&mut self, node: &Node, updater: F)
where
F: FnOnce(&mut TypeInfo),
{
let addr = node_address(node);
if let Some(reference) = self.types.get_mut(&addr) {
updater(reference)
}
}
fn set_error(&mut self, node: &Node, message: String) {
self.update_type(node, |typ| {
typ.error = Some(message);
});
}
fn determine(&mut self, node: &Node, scope: IntelliSenseScope, detailed: bool) -> TypeInfo {
#[allow(non_snake_case)]
let V = |vt: VariableType| TypeInfo::from(vt);
#[allow(non_snake_case)]
let Const = |v: Value| TypeInfo::from(VariableType::Constant(Rc::new(v)));
#[allow(non_snake_case)]
let Error = |error: String| TypeInfo {
kind: Rc::from(VariableType::Any),
error: Some(error),
};
let node_type = match node {
Node::Null => V(VariableType::Null),
Node::Bool(b) => match detailed {
true => Const(Value::Bool(*b)),
false => V(VariableType::Bool),
},
Node::Number(n) => match detailed {
true => Const(Value::Number(Number::from_string_unchecked(
n.normalize().to_string(),
))),
false => V(VariableType::Number),
},
Node::String(s) => match detailed {
true => Const(Value::String(s.to_string())),
false => V(VariableType::String),
},
Node::TemplateString(_) => V(VariableType::String),
Node::Pointer => V(scope.pointer_data.clone()),
Node::Root => V(scope.root_data.clone()),
Node::Slice { node, from, to } => {
if let Some(f) = from {
let from_type = self.determine(f, scope.clone(), false);
if !from_type.satisfies(&VariableType::Number) {
self.set_error(node, format!("Invalid slice index: expected a `number`, but found `{from_type}`."));
}
}
if let Some(t) = to {
let to_type = self.determine(t, scope.clone(), false);
if !to_type.satisfies(&VariableType::Number) {
self.set_error(
node,
format!(
"Invalid slice index: expected a `number`, but found `{to_type}`."
),
);
}
}
let node_type = self.determine(node, scope.clone(), false);
match node_type.kind.as_ref() {
VariableType::Any => V(VariableType::Any),
VariableType::String => V(VariableType::String),
VariableType::Array(inner) => TypeInfo::from(inner.clone()),
VariableType::Constant(c) => match c.as_ref() {
Value::String(_) => V(VariableType::String),
Value::Array(inner) => match VariableType::from(inner).array_item() {
Some(item) => TypeInfo::from(item),
None => Error("Array expected".to_string()),
},
_ => {
Error("Slice operation is only allowed on `string | any[]`".to_string())
}
},
_ => Error("Slice operation is only allowed on `string | any[]`".to_string()),
}
}
Node::Array(items) => {
let mut type_list: Vec<Rc<VariableType>> = items
.iter()
.map(|n| self.determine(n, scope.clone(), false).kind)
.collect();
let first = type_list.pop();
let all_same = type_list.iter().all(|t| Some(t) == first.as_ref());
match (first, all_same) {
(Some(typ), true) => V(VariableType::Array(typ)),
_ => V(VariableType::Array(Rc::new(VariableType::Any))),
}
}
Node::Object(obj) => {
let obj_type = obj
.iter()
.filter_map(|(k, v)| {
let key_type = self.determine(k, scope.clone(), true);
Some((
key_type.kind.as_const_str()?.to_string(),
self.determine(v, scope.clone(), false).kind,
))
})
.collect();
V(VariableType::Object(obj_type))
}
Node::Identifier(i) => TypeInfo::from(scope.root_data.get(&VariableType::Constant(
Rc::from(Value::String(i.to_string())),
))),
Node::Member { node, property } => {
let node_type = self.determine(node, scope.clone(), true);
let property_type = self.determine(property, scope.clone(), true);
match node_type.kind.as_ref() {
VariableType::Any => V(VariableType::Any),
VariableType::Null => V(VariableType::Null),
VariableType::Array(inner) => {
if !property_type.satisfies(&VariableType::Number) {
self.set_error(
property,
format!("Expression of type `{property_type}` cannot be used to index `{node_type}`."),
);
}
TypeInfo::from(inner.clone())
}
VariableType::Object(obj) => {
if !property_type.satisfies(&VariableType::String) {
self.set_error(
property,
format!("Expression of type `{property_type}` cannot be used to index `{node_type}`."),
);
}
match property_type.as_const_str() {
None => V(VariableType::Any),
Some(key) => TypeInfo::from(
obj.get(key).cloned().unwrap_or(Rc::new(VariableType::Any)),
),
}
}
VariableType::Constant(c) => match c.as_ref() {
Value::Null => V(VariableType::Null),
Value::Array(arr) => {
if !property_type.satisfies(&VariableType::Number) {
self.set_error(
property,
format!("Expression of type `{property_type}` cannot be used to index `{node_type}`."),
);
}
match VariableType::from(arr).array_item() {
Some(item) => TypeInfo::from(item),
None => Error("Expected an array".to_string()),
}
}
Value::Object(obj) => {
if !property_type.satisfies(&VariableType::String) {
self.set_error(
property,
format!("Expression of type `{property_type}` cannot be used to index `{node_type}`."),
);
}
match property_type.as_const_str() {
None => V(VariableType::Any),
Some(key) => V(obj
.get(key)
.cloned()
.map(VariableType::from)
.unwrap_or(VariableType::Any)),
}
}
_ => Error(format!("Expression of type `{property_type}` cannot be used to index `{node_type}`.")),
},
_ => Error(format!("Expression of type `{property_type}` cannot be used to index `{node_type}`.")),
}
}
Node::Binary {
left,
right,
operator,
} => {
let left_type = self.determine(left, scope.clone(), false);
let right_type = self.determine(right, scope.clone(), false);
match operator {
Operator::Arithmetic(arith) => match arith {
ArithmeticOperator::Add => match (left_type.omit_const(), right_type.omit_const()) {
(VariableType::Number, VariableType::Number) => V(VariableType::Number),
(VariableType::String, VariableType::String) => V(VariableType::String),
(VariableType::Any, VariableType::Number | VariableType::String) => V(VariableType::Any),
(VariableType::Number | VariableType::String, VariableType::Any) => V(VariableType::Any),
_ => Error(format!(
"Operator `{operator}` cannot be applied to types `{left_type}` and `{right_type}`."
)),
},
ArithmeticOperator::Subtract
| ArithmeticOperator::Multiply
| ArithmeticOperator::Divide
| ArithmeticOperator::Modulus
| ArithmeticOperator::Power => match (left_type.omit_const(), right_type.omit_const()) {
(VariableType::Number | VariableType::Any, VariableType::Number | VariableType::Any) => V(VariableType::Number),
_ => Error(format!(
"Operator `{operator}` cannot be applied to types `{left_type}` and `{right_type}`."
)),
},
},
Operator::Logical(l) => match l {
LogicalOperator::And | LogicalOperator::Or | LogicalOperator::Not => {
match (left_type.omit_const(), right_type.omit_const()) {
(VariableType::Bool | VariableType::Any, VariableType::Bool | VariableType::Any) => V(VariableType::Bool),
_ => Error(format!(
"Operator `{operator}` cannot be applied to types `{left_type}` and `{right_type}`."
)),
}
}
LogicalOperator::NullishCoalescing => TypeInfo::from(right_type.kind),
},
Operator::Comparison(comp) => match comp {
ComparisonOperator::Equal => V(VariableType::Bool),
ComparisonOperator::NotEqual => V(VariableType::Bool),
ComparisonOperator::LessThan
| ComparisonOperator::GreaterThan
| ComparisonOperator::LessThanOrEqual
| ComparisonOperator::GreaterThanOrEqual => match (left_type.omit_const(), right_type.omit_const()) {
(VariableType::Number | VariableType::Any, VariableType::Number | VariableType::Any) => V(VariableType::Bool),
_ => Error(format!(
"Operator `{operator}` cannot be applied to types `{left_type}` and `{right_type}`."
)),
},
ComparisonOperator::In | ComparisonOperator::NotIn => match (left_type.kind.as_ref(), right_type.kind.as_ref()) {
(_, VariableType::Array(_)) => V(VariableType::Bool),
(_, VariableType::Object(_)) => V(VariableType::Bool),
(VariableType::Any, _) => V(VariableType::Bool),
(_, VariableType::Any) => V(VariableType::Bool),
_ => Error(format!(
"Operator `{operator}` cannot be applied to types `{left_type}` and `{right_type}`."
))
}
},
_ => V(VariableType::Any),
}
}
Node::Conditional {
condition,
on_true,
on_false,
} => {
let condition_type = self.determine(condition, scope.clone(), false);
if !condition_type.satisfies(&VariableType::Bool) {
self.set_error(
condition,
format!("Ternary operator cannot be applied to type `{condition_type}`."),
);
}
let true_type = self.determine(on_true, scope.clone(), false);
let false_type = self.determine(on_false, scope.clone(), false);
V(true_type.kind.merge(false_type.kind.as_ref()))
}
Node::Unary { node, operator } => {
let node_type = self.determine(node, scope.clone(), false);
match operator {
Operator::Arithmetic(arith) => match arith {
ArithmeticOperator::Add | ArithmeticOperator::Subtract => {
if !node_type.satisfies(&VariableType::Number) {
self.set_error(node, format!("Operator `{operator}` cannot be applied to type `{node_type}`."))
}
V(VariableType::Number)
}
ArithmeticOperator::Multiply
| ArithmeticOperator::Divide
| ArithmeticOperator::Modulus
| ArithmeticOperator::Power => Error("Unsupported operator".to_string()),
},
Operator::Logical(logical) => match logical {
LogicalOperator::Not => {
if !node_type.satisfies(&VariableType::Bool) {
self.set_error(node, format!("Operator `{operator}` cannot be applied to type `{node_type}`."))
}
V(VariableType::Bool)
}
LogicalOperator::And
| LogicalOperator::Or
| LogicalOperator::NullishCoalescing => {
Error("Unsupported operator".to_string())
}
},
Operator::Comparison(_)
| Operator::Range
| Operator::Comma
| Operator::Slice
| Operator::Dot
| Operator::QuestionMark => Error("Unsupported operator".to_string()),
}
}
Node::Interval { left, right, .. } => {
let left_type = self.determine(left, scope.clone(), false);
if !left_type.satisfies(&VariableType::Number) {
self.set_error(
left,
format!("Interval cannot be created from type `{left_type}`."),
)
}
let right_type = self.determine(right, scope.clone(), false);
if !right_type.satisfies(&VariableType::Number) {
self.set_error(
right,
format!("Interval cannot be created from type `{right_type}`."),
)
}
V(VariableType::Any)
}
Node::BuiltIn { arguments, kind } => {
let mut type_list: Vec<Rc<VariableType>> = arguments
.iter()
.map(|n| self.determine(n, scope.clone(), false).kind)
.collect();
let arg_len = match kind.arity() {
Arity::Single => 1,
Arity::Closure | Arity::Dual => 2,
};
if type_list.len() != arg_len {
self.set_type(
node,
Error(format!(
"Expected {arg_len} arguments, but got {}.",
type_list.len()
)),
);
}
if kind.arity() == Arity::Closure {
let ptr_type = type_list[0].array_item().unwrap_or_default();
let new_type = self.determine(
arguments[1],
IntelliSenseScope {
pointer_data: &ptr_type,
current_data: scope.current_data,
root_data: scope.root_data,
},
false,
);
type_list[1] = new_type.kind;
}
match kind {
BuiltInFunction::Len => {
if !type_list[0].satisfies(&VariableType::String)
&& !type_list[0]
.satisfies(&VariableType::Array(VariableType::Any.into()))
{
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `string | any[]`.", type_list[0]));
}
V(VariableType::Number)
}
BuiltInFunction::Contains => {
match (type_list[0].omit_const(), type_list[1].omit_const()) {
(VariableType::String, VariableType::String)
| (VariableType::Any, _)
| (_, VariableType::Any) => {
// ok
}
(VariableType::Array(vt), b) => {
if !b.satisfies(&vt) {
self.set_error(arguments[1], format!("Argument of type `{b}` is not assignable to parameter of type `{vt}`."));
}
}
_ => self.set_error(node, "Unsupported call signature.".to_string()),
}
V(VariableType::Bool)
}
BuiltInFunction::Upper | BuiltInFunction::Lower => {
if !type_list[0].satisfies(&VariableType::String) {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[0]));
}
V(VariableType::String)
}
BuiltInFunction::StartsWith
| BuiltInFunction::EndsWith
| BuiltInFunction::Matches => {
if !type_list[0].satisfies(&VariableType::String) {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[0]));
}
if !type_list[1].satisfies(&VariableType::String) {
self.set_error(arguments[1], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[1]));
}
V(VariableType::Bool)
}
BuiltInFunction::Extract | BuiltInFunction::Split => {
if !type_list[0].satisfies(&VariableType::String) {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[0]));
}
if !type_list[1].satisfies(&VariableType::String) {
self.set_error(arguments[1], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[1]));
}
V(VariableType::Array(Rc::new(VariableType::String)))
}
BuiltInFunction::FuzzyMatch => {
if !type_list[0].satisfies(&VariableType::String)
&& !type_list[0]
.satisfies(&VariableType::Array(Rc::new(VariableType::String)))
{
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `string | string[]`.", type_list[0]));
}
if !type_list[1].satisfies(&VariableType::String) {
self.set_error(arguments[1], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[1]));
}
V(VariableType::Bool)
}
BuiltInFunction::Abs
| BuiltInFunction::Rand
| BuiltInFunction::Floor
| BuiltInFunction::Ceil
| BuiltInFunction::Round => {
if !type_list[0].satisfies(&VariableType::Number) {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `number`.", type_list[0]));
}
V(VariableType::Number)
}
BuiltInFunction::Sum
| BuiltInFunction::Avg
| BuiltInFunction::Min
| BuiltInFunction::Max
| BuiltInFunction::Median
| BuiltInFunction::Mode => {
if !type_list[0]
.satisfies(&VariableType::Array(Rc::new(VariableType::Number)))
{
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `number[]`.", type_list[0]));
}
V(VariableType::Number)
}
BuiltInFunction::IsNumeric => V(VariableType::Bool),
BuiltInFunction::String => V(VariableType::String),
BuiltInFunction::Number => V(VariableType::Number),
BuiltInFunction::Bool => V(VariableType::Bool),
BuiltInFunction::Type => V(VariableType::String),
BuiltInFunction::Date => {
if !type_list[0].satisfies(&VariableType::Number)
|| !type_list[0].satisfies(&VariableType::String)
{
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `number | string`.", type_list[0]));
}
V(VariableType::Number)
}
BuiltInFunction::Time | BuiltInFunction::Duration => {
if !type_list[0].satisfies(&VariableType::String) {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[0]));
}
V(VariableType::Number)
}
BuiltInFunction::Year
| BuiltInFunction::DayOfWeek
| BuiltInFunction::DayOfMonth
| BuiltInFunction::DayOfYear
| BuiltInFunction::WeekOfYear
| BuiltInFunction::MonthOfYear => {
if !type_list[0].satisfies(&VariableType::Number) {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `number`.", type_list[0]));
}
V(VariableType::Number)
}
BuiltInFunction::MonthString
| BuiltInFunction::DateString
| BuiltInFunction::WeekdayString => {
if !type_list[0].satisfies(&VariableType::Number) {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `number`.", type_list[0]));
}
V(VariableType::String)
}
BuiltInFunction::StartOf | BuiltInFunction::EndOf => {
if !type_list[0].satisfies(&VariableType::Number) {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `number`.", type_list[0]));
}
if !type_list[1].satisfies(&VariableType::String) {
self.set_error(arguments[1], format!("Argument of type `{}` is not assignable to parameter of type `string`.", type_list[1]));
}
V(VariableType::Number)
}
BuiltInFunction::Keys => {
if !type_list[0].satisfies_object() {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `object`.", type_list[0]));
}
V(VariableType::Array(Rc::new(VariableType::String)))
}
BuiltInFunction::Values => match type_list[0].as_ref() {
VariableType::Any | VariableType::Object(_) => {
V(VariableType::Array(VariableType::Any.into()))
}
VariableType::Constant(c) => match c.as_ref() {
Value::Object(obj) => {
let s: Vec<Value> = obj.values().cloned().collect();
V(s.into())
}
_ => {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `object`.", type_list[0]));
V(VariableType::Array(VariableType::Any.into()))
}
},
_ => {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `object`", type_list[0]));
V(VariableType::Array(VariableType::Any.into()))
}
},
BuiltInFunction::All => {
if !type_list[0].satisfies_array() {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `any[]`.", type_list[0]));
}
if !type_list[1].satisfies(&VariableType::Bool) {
self.set_error(
arguments[1],
format!(
"Callback must return a `bool`, but its return type is `{}`.",
type_list[1]
),
);
}
V(VariableType::Bool)
}
BuiltInFunction::Some => {
if !type_list[0].satisfies_array() {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `any[]`.", type_list[0]));
}
if !type_list[1].satisfies(&VariableType::Bool) {
self.set_error(
arguments[1],
format!(
"Callback must return a `bool`, but its return type is `{}`.",
type_list[1]
),
);
}
V(VariableType::Bool)
}
BuiltInFunction::None => {
if !type_list[0].satisfies_array() {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `any[]`.", type_list[0]));
}
if !type_list[1].satisfies(&VariableType::Bool) {
self.set_error(
arguments[1],
format!(
"Callback must return a `bool`, but its return type is `{}`.",
type_list[1]
),
);
}
V(VariableType::Bool)
}
BuiltInFunction::Filter => {
if !type_list[0].satisfies_array() {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `any[]`.", type_list[0]));
}
if !type_list[1].satisfies(&VariableType::Bool) {
self.set_error(
arguments[1],
format!(
"Callback must return a `bool`, but its return type is `{}`.",
type_list[1]
),
);
}
TypeInfo::from(type_list[0].clone())
}
BuiltInFunction::Map => {
if !type_list[0].satisfies_array() {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `any[]`.", type_list[0]));
}
V(VariableType::Array(type_list[1].clone()))
}
BuiltInFunction::Count => {
if !type_list[0].satisfies_array() {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `any[]`.", type_list[0]));
}
if !type_list[1].satisfies(&VariableType::Bool) {
self.set_error(
arguments[1],
format!(
"Callback must return a `bool`, but its return type is `{}`.",
type_list[1]
),
);
}
V(VariableType::Number)
}
BuiltInFunction::One => {
if !type_list[0].satisfies_array() {
self.set_error(arguments[0], format!("Argument of type `{}` is not assignable to parameter of type `any[]`.", type_list[0]));
}
if !type_list[1].satisfies(&VariableType::Bool) {
self.set_error(
arguments[1],
format!(
"Callback must return a `bool`, but its return type is `{}`.",
type_list[1]
),
);
}
V(VariableType::Bool)
}
BuiltInFunction::FlatMap => V(VariableType::Any),
BuiltInFunction::Flatten => V(VariableType::Any),
}
}
Node::Closure(c) => self.determine(c, scope.clone(), false),
Node::Parenthesized(c) => self.determine(c, scope.clone(), false),
Node::Error { node, error } => match node {
None => TypeInfo {
kind: Rc::new(VariableType::Any),
error: Some(error.to_string()),
},
Some(n) => {
let typ = self.determine(n, scope.clone(), false);
TypeInfo {
kind: typ.kind,
error: Some(error.to_string()),
}
}
},
};
self.set_type(node, node_type.clone());
node_type
}
}
#[allow(unused)]
fn node_address(node: &Node) -> usize {
node as *const Node as usize
}
@@ -0,0 +1,51 @@
use crate::variable::VariableType;
use std::fmt::{Display, Formatter};
use std::ops::Deref;
use std::rc::Rc;
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct TypeInfo {
pub(crate) kind: Rc<VariableType>,
pub(crate) error: Option<String>,
}
impl Deref for TypeInfo {
type Target = VariableType;
fn deref(&self) -> &Self::Target {
&self.kind
}
}
impl Display for TypeInfo {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.kind)
}
}
impl Default for TypeInfo {
fn default() -> Self {
Self {
kind: Rc::new(VariableType::Any),
error: None,
}
}
}
impl From<VariableType> for TypeInfo {
fn from(value: VariableType) -> Self {
Self {
kind: Rc::new(value),
error: None,
}
}
}
impl From<Rc<VariableType>> for TypeInfo {
fn from(value: Rc<VariableType>) -> Self {
Self {
kind: value,
error: None,
}
}
}
+8 -6
View File
@@ -116,13 +116,14 @@ impl<'a> Isolate<'a> {
.map_err(|source| IsolateError::ParserError { source })?
.standard();
let ast = parser
.parse()
let parser_result = parser.parse();
parser_result
.error()
.map_err(|source| IsolateError::ParserError { source })?;
let bytecode = self
.compiler
.compile(ast)
.compile(parser_result.root)
.map_err(|source| IsolateError::CompilerError { source })?;
let result = self
@@ -150,13 +151,14 @@ impl<'a> Isolate<'a> {
.map_err(|source| IsolateError::ParserError { source })?
.unary();
let ast = parser
.parse()
let parser_result = parser.parse();
parser_result
.error()
.map_err(|source| IsolateError::ParserError { source })?;
let bytecode = self
.compiler
.compile(ast)
.compile(parser_result.root)
.map_err(|source| IsolateError::CompilerError { source })?;
let result = self
+16 -1
View File
@@ -65,7 +65,7 @@ impl Display for TemplateString {
}
}
#[derive(Debug, PartialEq, Eq, Clone, Copy, Display)]
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum Operator {
Arithmetic(ArithmeticOperator),
Logical(LogicalOperator),
@@ -77,6 +77,21 @@ pub enum Operator {
QuestionMark, // ?
}
impl Display for Operator {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Operator::Arithmetic(a) => write!(f, "{a}"),
Operator::Logical(l) => write!(f, "{l}"),
Operator::Comparison(c) => write!(f, "{c}"),
Operator::Range => write!(f, ".."),
Operator::Comma => write!(f, ","),
Operator::Slice => write!(f, ":"),
Operator::Dot => write!(f, "."),
Operator::QuestionMark => write!(f, "?"),
}
}
}
impl FromStr for Operator {
type Err = strum::ParseError;
+1
View File
@@ -54,6 +54,7 @@ mod isolate;
mod arena;
pub mod compiler;
mod function;
pub mod intellisense;
pub mod lexer;
pub mod parser;
pub mod variable;
+149 -3
View File
@@ -1,9 +1,11 @@
use rust_decimal::Decimal;
use crate::lexer::Operator;
use crate::parser::builtin::BuiltInFunction;
use rust_decimal::Decimal;
use std::cell::Cell;
use strum_macros::IntoStaticStr;
use thiserror::Error;
#[derive(Debug, PartialEq, Clone)]
#[derive(Debug, PartialEq, Clone, IntoStaticStr)]
pub enum Node<'a> {
Null,
Bool(bool),
@@ -15,6 +17,7 @@ pub enum Node<'a> {
Object(&'a [(&'a Node<'a>, &'a Node<'a>)]),
Identifier(&'a str),
Closure(&'a Node<'a>),
Parenthesized(&'a Node<'a>),
Root,
Member {
node: &'a Node<'a>,
@@ -49,4 +52,147 @@ pub enum Node<'a> {
kind: BuiltInFunction,
arguments: &'a [&'a Node<'a>],
},
Error {
node: Option<&'a Node<'a>>,
error: AstNodeError<'a>,
},
}
impl<'a> Node<'a> {
pub fn walk<F>(&self, mut func: F)
where
F: FnMut(&Self) + Clone,
{
{
func(self);
};
match self {
Node::Null => {}
Node::Bool(_) => {}
Node::Number(_) => {}
Node::String(_) => {}
Node::Pointer => {}
Node::Identifier(_) => {}
Node::Root => {}
Node::Error { node, .. } => {
if let Some(n) = node {
n.walk(func.clone())
}
}
Node::TemplateString(parts) => parts.iter().for_each(|n| n.walk(func.clone())),
Node::Array(parts) => parts.iter().for_each(|n| n.walk(func.clone())),
Node::Object(obj) => obj.iter().for_each(|(k, v)| {
k.walk(func.clone());
v.walk(func.clone());
}),
Node::Closure(closure) => closure.walk(func.clone()),
Node::Parenthesized(c) => c.walk(func.clone()),
Node::Member { node, property } => {
node.walk(func.clone());
property.walk(func.clone());
}
Node::Slice { node, to, from } => {
node.walk(func.clone());
if let Some(to) = to {
to.walk(func.clone());
}
if let Some(from) = from {
from.walk(func.clone());
}
}
Node::Interval { left, right, .. } => {
left.walk(func.clone());
right.walk(func.clone());
}
Node::Unary { node, .. } => {
node.walk(func);
}
Node::Binary { left, right, .. } => {
left.walk(func.clone());
right.walk(func.clone());
}
Node::BuiltIn { arguments, .. } => {
arguments.iter().for_each(|n| n.walk(func.clone()));
}
Node::Conditional {
on_true,
condition,
on_false,
} => {
condition.walk(func.clone());
on_true.walk(func.clone());
on_false.walk(func.clone());
}
};
}
pub fn first_error(&self) -> Option<AstNodeError> {
let error_cell = Cell::new(None);
self.walk(|n| {
if let Node::Error { error, .. } = n {
error_cell.set(Some(error.clone()))
}
});
error_cell.into_inner()
}
pub fn has_error(&self) -> bool {
self.first_error().is_some()
}
pub(crate) fn span(&self) -> Option<(u32, u32)> {
match self {
Node::Error { error, .. } => match error {
AstNodeError::UnknownBuiltIn { span, .. } => Some(span.clone()),
AstNodeError::UnexpectedIdentifier { span, .. } => Some(span.clone()),
AstNodeError::UnexpectedToken { span, .. } => Some(span.clone()),
AstNodeError::InvalidNumber { span, .. } => Some(span.clone()),
AstNodeError::InvalidBoolean { span, .. } => Some(span.clone()),
AstNodeError::InvalidProperty { span, .. } => Some(span.clone()),
AstNodeError::MissingToken { position, .. } => {
Some((*position as u32, *position as u32))
}
AstNodeError::Custom { span, .. } => Some(span.clone()),
},
_ => None,
}
}
}
#[derive(Debug, PartialEq, Eq, Clone, Error)]
pub enum AstNodeError<'a> {
#[error("Unknown built in: {name} at ({}, {})", span.0, span.1)]
UnknownBuiltIn { name: &'a str, span: (u32, u32) },
#[error("Unexpected identifier: {received} at ({}, {}); Expected {expected}.", span.0, span.1)]
UnexpectedIdentifier {
received: &'a str,
expected: &'a str,
span: (u32, u32),
},
#[error("Unexpected token: {received} at ({}, {}); Expected {expected}.", span.0, span.1)]
UnexpectedToken {
received: &'a str,
expected: &'a str,
span: (u32, u32),
},
#[error("Invalid number: {number} at ({}, {})", span.0, span.1)]
InvalidNumber { number: &'a str, span: (u32, u32) },
#[error("Invalid boolean: {boolean} at ({}, {})", span.0, span.1)]
InvalidBoolean { boolean: &'a str, span: (u32, u32) },
#[error("Invalid property: {property} at ({}, {})", span.0, span.1)]
InvalidProperty { property: &'a str, span: (u32, u32) },
#[error("Missing expected token: {expected} at {position}")]
MissingToken { expected: &'a str, position: usize },
#[error("{message} at ({}, {})", span.0, span.1)]
Custom { message: &'a str, span: (u32, u32) },
}
+4 -22
View File
@@ -2,27 +2,9 @@ use thiserror::Error;
#[derive(Debug, PartialEq, Eq, Clone, Error)]
pub enum ParserError {
#[error("Unexpected token: received {received} instead of {expected} at ({}, {})", span.0, span.1)]
UnexpectedToken {
expected: String,
received: String,
span: (u32, u32),
},
#[error("{0}")]
NodeError(String),
#[error("Failed to parse: {message} at ({}, {})", span.0, span.1)]
FailedToParse { message: String, span: (u32, u32) },
#[error("Unknown built in: {name} at ({}, {})", span.0, span.1)]
UnknownBuiltIn { name: String, span: (u32, u32) },
#[error("Unsupported built in: {name} at ({}, {})", span.0, span.1)]
UnsupportedBuiltIn { name: String, span: (u32, u32) },
#[error("Token out of bounds")]
TokenOutOfBounds,
#[error("Memory failure")]
MemoryFailure,
#[error("Incomplete parser output")]
Incomplete,
}
pub(crate) type ParserResult<T> = Result<T, ParserError>;
+4
View File
@@ -10,12 +10,16 @@ mod builtin;
mod constants;
mod error;
mod parser;
mod result;
mod standard;
mod unary;
pub(crate) use builtin::Arity;
pub use ast::Node;
pub use builtin::BuiltInFunction;
pub use error::ParserError;
pub use parser::Parser;
pub use result::{NodeMetadata, ParserResult};
pub use standard::Standard;
pub use unary::Unary;
File diff suppressed because it is too large Load Diff
+28
View File
@@ -0,0 +1,28 @@
use crate::parser::{Node, ParserError};
use nohash_hasher::BuildNoHashHasher;
use std::collections::HashMap;
#[derive(Debug)]
pub struct ParserResult<'a> {
pub root: &'a Node<'a>,
pub is_complete: bool,
pub metadata: Option<HashMap<usize, NodeMetadata, BuildNoHashHasher<usize>>>,
}
#[derive(Debug, Clone)]
pub struct NodeMetadata {
pub span: (u32, u32),
}
impl<'a> ParserResult<'a> {
pub fn error(&self) -> Result<(), ParserError> {
if !self.is_complete {
return Err(ParserError::Incomplete);
}
match self.root.first_error() {
None => Ok(()),
Some(err) => Err(ParserError::NodeError(err.to_string())),
}
}
}
+76 -46
View File
@@ -1,30 +1,29 @@
use crate::lexer::{Bracket, Identifier, TokenKind};
use crate::parser::ast::Node;
use crate::parser::ast::{AstNodeError, Node};
use crate::parser::constants::{Associativity, BINARY_OPERATORS, UNARY_OPERATORS};
use crate::parser::error::ParserError::{FailedToParse, UnexpectedToken};
use crate::parser::error::ParserResult;
use crate::parser::parser::Parser;
use crate::parser::result::ParserResult;
use crate::parser::NodeMetadata;
#[derive(Debug)]
pub struct Standard;
impl<'arena, 'token_ref> Parser<'arena, 'token_ref, Standard> {
pub fn parse(&self) -> ParserResult<&'arena Node<'arena>> {
let result = self.binary_expression(0)?;
if !self.is_done() {
let token = self.current();
return Err(FailedToParse {
message: format!("Unterminated token {}", token.value),
span: token.span,
});
}
pub fn parse(&self) -> ParserResult<'arena> {
let root = self.binary_expression(0);
return Ok(result);
ParserResult {
root,
is_complete: self.is_done(),
metadata: self.node_metadata.clone().map(|t| t.into_inner()),
}
}
fn binary_expression(&self, precedence: u8) -> ParserResult<&'arena Node<'arena>> {
let mut node_left = self.unary_expression()?;
let mut token = self.current();
fn binary_expression(&self, precedence: u8) -> &'arena Node<'arena> {
let mut node_left = self.unary_expression();
let Some(mut token) = self.current() else {
return node_left;
};
while let TokenKind::Operator(operator) = &token.kind {
if self.is_done() {
@@ -39,68 +38,99 @@ impl<'arena, 'token_ref> Parser<'arena, 'token_ref, Standard> {
break;
}
self.next()?;
self.next();
let node_right = match op.associativity {
Associativity::Left => self.binary_expression(op.precedence + 1)?,
_ => self.binary_expression(op.precedence)?,
Associativity::Left => self.binary_expression(op.precedence + 1),
_ => self.binary_expression(op.precedence),
};
node_left = self.node(Node::Binary {
operator: *operator,
left: node_left,
right: node_right,
});
token = self.current();
node_left = self.node(
Node::Binary {
operator: *operator,
left: node_left,
right: node_right,
},
|h| NodeMetadata {
span: h.span(node_left, node_right).unwrap_or_default(),
},
);
let Some(t) = self.current() else {
break;
};
token = t;
}
if precedence == 0 {
if let Some(conditional_node) =
self.conditional(node_left, || self.binary_expression(0))?
self.conditional(node_left, || self.binary_expression(0))
{
node_left = conditional_node;
}
}
Ok(node_left)
node_left
}
fn unary_expression(&self) -> ParserResult<&'arena Node<'arena>> {
let token = self.current();
fn unary_expression(&self) -> &'arena Node<'arena> {
let Some(token) = self.current() else {
return self.error(AstNodeError::Custom {
message: self.bump.alloc_str("Unexpected end of unary expression"),
span: (self.prev_token_end(), self.prev_token_end()),
});
};
if self.depth() > 0 && token.kind == TokenKind::Identifier(Identifier::CallbackReference) {
self.next()?;
self.next();
let node = self.node(Node::Pointer);
let node = self.node(Node::Pointer, |_| NodeMetadata { span: token.span });
return self.with_postfix(node, || self.binary_expression(0));
}
if let TokenKind::Operator(operator) = &token.kind {
let Some(unary_operator) = UNARY_OPERATORS.get(operator) else {
return Err(UnexpectedToken {
expected: "UnaryOperator".to_string(),
received: token.kind.to_string(),
return self.error(AstNodeError::UnexpectedToken {
expected: "UnaryOperator",
received: self.bump.alloc_str(token.kind.to_string().as_str()),
span: token.span,
});
};
self.next()?;
let expr = self.binary_expression(unary_operator.precedence)?;
let node = self.node(Node::Unary {
operator: *operator,
node: expr,
});
self.next();
let expr = self.binary_expression(unary_operator.precedence);
let node = self.node(
Node::Unary {
operator: *operator,
node: expr,
},
|h| NodeMetadata {
span: (
token.span.0,
h.metadata(expr).map(|n| n.span.1).unwrap_or_default(),
),
},
);
return Ok(node);
return node;
}
if let Some(interval_node) = self.interval(|| self.binary_expression(0))? {
return Ok(interval_node);
if let Some(interval_node) = self.interval(|| self.binary_expression(0)) {
return interval_node;
}
if token.kind == TokenKind::Bracket(Bracket::LeftParenthesis) {
self.next()?;
let expr = self.binary_expression(0)?;
self.expect(TokenKind::Bracket(Bracket::RightParenthesis))?;
let p_start = self.current().map(|s| s.span.0);
self.next();
let binary_node = self.binary_expression(0);
if let Some(error_node) = self.expect(TokenKind::Bracket(Bracket::RightParenthesis)) {
return error_node;
};
let expr = self.node(Node::Parenthesized(binary_node), |_| NodeMetadata {
span: (p_start.unwrap_or_default(), self.prev_token_end()),
});
return self.with_postfix(expr, || self.binary_expression(0));
}
+152 -86
View File
@@ -1,11 +1,10 @@
use crate::lexer::{Bracket, ComparisonOperator, Identifier, LogicalOperator, Operator, TokenKind};
use crate::parser::ast::Node;
use crate::parser::ast::{AstNodeError, Node};
use crate::parser::builtin::BuiltInFunction;
use crate::parser::constants::{Associativity, BINARY_OPERATORS, UNARY_OPERATORS};
use crate::parser::error::ParserError::{FailedToParse, UnexpectedToken};
use crate::parser::error::{ParserError, ParserResult};
use crate::parser::parser::Parser;
use crate::parser::unary::UnaryNodeBehaviour::CompareWithReference;
use crate::parser::{NodeMetadata, ParserResult};
#[derive(Debug)]
pub struct Unary;
@@ -13,23 +12,24 @@ pub struct Unary;
const ROOT_NODE: Node<'static> = Node::Identifier("$");
impl<'arena, 'token_ref> Parser<'arena, 'token_ref, Unary> {
pub fn parse(&self) -> ParserResult<&'arena Node<'arena>> {
let result = self.root_expression()?;
if !self.is_done() {
let token = self.current();
return Err(FailedToParse {
message: format!("Unterminated token {}", token.value),
span: token.span,
});
}
pub fn parse(&self) -> ParserResult<'arena> {
let root = self.root_expression();
return Ok(result);
ParserResult {
root,
is_complete: self.is_done(),
metadata: self.node_metadata.clone().map(|t| t.into_inner()),
}
}
fn root_expression(&self) -> ParserResult<&'arena Node<'arena>> {
let mut left_node = self.expression_pair()?;
fn root_expression(&self) -> &'arena Node<'arena> {
let mut left_node = self.expression_pair();
while !self.is_done() {
let current_token = self.current();
let Some(current_token) = self.current() else {
break;
};
let join_operator = match &current_token.kind {
TokenKind::Operator(Operator::Logical(LogicalOperator::And)) => {
Operator::Logical(LogicalOperator::And)
@@ -37,68 +37,104 @@ impl<'arena, 'token_ref> Parser<'arena, 'token_ref, Unary> {
TokenKind::Operator(Operator::Logical(LogicalOperator::Or))
| TokenKind::Operator(Operator::Comma) => Operator::Logical(LogicalOperator::Or),
_ => {
return Err(ParserError::MemoryFailure);
return self.error(AstNodeError::Custom {
message: self.bump.alloc_str(
format!("Invalid join operator `{}`", current_token.kind).as_str(),
),
span: current_token.span,
})
}
};
self.next()?;
let right_node = self.expression_pair()?;
left_node = self.node(Node::Binary {
left: left_node,
operator: join_operator,
right: right_node,
});
self.next();
let right_node = self.expression_pair();
left_node = self.node(
Node::Binary {
left: left_node,
operator: join_operator,
right: right_node,
},
|h| NodeMetadata {
span: h.span(left_node, right_node).unwrap_or_default(),
},
);
}
Ok(left_node)
left_node
}
fn expression_pair(&self) -> ParserResult<&'arena Node<'arena>> {
fn expression_pair(&self) -> &'arena Node<'arena> {
let mut left_node = &ROOT_NODE;
let initial_token = self.current();
if let TokenKind::Operator(Operator::Comparison(_)) = &initial_token.kind {
let current_token = self.current();
if let Some(TokenKind::Operator(Operator::Comparison(_))) = self.current_kind() {
// Skips
} else {
left_node = self.binary_expression(0)?;
left_node = self.binary_expression(0);
}
let current_token = self.current();
match &current_token.kind {
TokenKind::Operator(Operator::Comparison(comparison)) => {
self.next()?;
let right_node = self.binary_expression(0)?;
left_node = self.node(Node::Binary {
left: left_node,
operator: Operator::Comparison(*comparison),
right: right_node,
});
match self.current_kind() {
Some(TokenKind::Operator(Operator::Comparison(comparison))) => {
self.next();
let right_node = self.binary_expression(0);
left_node = self.node(
Node::Binary {
left: left_node,
operator: Operator::Comparison(*comparison),
right: right_node,
},
|h| NodeMetadata {
span: (
current_token.map(|t| t.span.0).unwrap_or_default(),
h.metadata(right_node).map(|n| n.span.1).unwrap_or_default(),
),
},
);
}
_ => {
let behaviour = UnaryNodeBehaviour::from(left_node);
match behaviour {
CompareWithReference(comparator) => {
left_node = self.node(Node::Binary {
left: &ROOT_NODE,
operator: Operator::Comparison(comparator),
right: left_node,
})
left_node = self.node(
Node::Binary {
left: &ROOT_NODE,
operator: Operator::Comparison(comparator),
right: left_node,
},
|h| NodeMetadata {
span: (
current_token.map(|t| t.span.0).unwrap_or_default(),
h.metadata(left_node).map(|n| n.span.1).unwrap_or_default(),
),
},
)
}
UnaryNodeBehaviour::AsBoolean => {
left_node = self.node(Node::BuiltIn {
kind: BuiltInFunction::Bool,
arguments: self.bump.alloc_slice_clone(&[left_node]),
})
left_node = self.node(
Node::BuiltIn {
kind: BuiltInFunction::Bool,
arguments: self.bump.alloc_slice_clone(&[left_node]),
},
|h| NodeMetadata {
span: (
current_token.map(|t| t.span.0).unwrap_or_default(),
h.metadata(left_node).map(|n| n.span.1).unwrap_or_default(),
),
},
)
}
}
}
}
Ok(left_node)
left_node
}
fn binary_expression(&self, precedence: u8) -> ParserResult<&'arena Node<'arena>> {
let mut node_left = self.unary_expression()?;
let mut token = self.current();
fn binary_expression(&self, precedence: u8) -> &'arena Node<'arena> {
let mut node_left = self.unary_expression();
let Some(mut token) = self.current() else {
return node_left;
};
while let TokenKind::Operator(operator) = &token.kind {
if self.is_done() {
@@ -122,68 +158,96 @@ impl<'arena, 'token_ref> Parser<'arena, 'token_ref, Unary> {
break;
}
self.next()?;
self.next();
let node_right = match op.associativity {
Associativity::Left => self.binary_expression(op.precedence + 1)?,
_ => self.binary_expression(op.precedence)?,
Associativity::Left => self.binary_expression(op.precedence + 1),
_ => self.binary_expression(op.precedence),
};
node_left = self.node(Node::Binary {
operator: *operator,
left: node_left,
right: node_right,
});
token = self.current();
node_left = self.node(
Node::Binary {
operator: *operator,
left: node_left,
right: node_right,
},
|h| NodeMetadata {
span: h.span(node_left, node_right).unwrap_or_default(),
},
);
let Some(t) = self.current() else {
break;
};
token = t;
}
if precedence == 0 {
if let Some(conditional_node) =
self.conditional(node_left, || self.binary_expression(0))?
self.conditional(node_left, || self.binary_expression(0))
{
node_left = conditional_node;
}
}
Ok(node_left)
node_left
}
fn unary_expression(&self) -> ParserResult<&'arena Node<'arena>> {
let token = self.current();
fn unary_expression(&self) -> &'arena Node<'arena> {
let Some(token) = self.current() else {
return self.literal(|| self.binary_expression(0));
};
if self.depth() > 0 && token.kind == TokenKind::Identifier(Identifier::CallbackReference) {
self.next()?;
self.next();
let node = self.node(Node::Pointer);
let node = self.node(Node::Pointer, |_| NodeMetadata { span: token.span });
return self.with_postfix(node, || self.binary_expression(0));
}
if let TokenKind::Operator(operator) = &token.kind {
let Some(unary_operator) = UNARY_OPERATORS.get(operator) else {
return Err(UnexpectedToken {
expected: "UnaryOperator".to_string(),
received: token.kind.to_string(),
return self.error(AstNodeError::UnexpectedToken {
expected: self.bump.alloc_str("UnaryOperator"),
received: self.bump.alloc_str(token.kind.to_string().as_str()),
span: token.span,
});
};
self.next()?;
let expr = self.binary_expression(unary_operator.precedence)?;
let node = self.node(Node::Unary {
operator: *operator,
node: expr,
});
self.next();
let expr = self.binary_expression(unary_operator.precedence);
let node = self.node(
Node::Unary {
operator: *operator,
node: expr,
},
|h| NodeMetadata {
span: (
token.span.0,
h.metadata(expr).map(|n| n.span.1).unwrap_or_default(),
),
},
);
return Ok(node);
return node;
}
if let Some(interval_node) = self.interval(|| self.binary_expression(0))? {
return Ok(interval_node);
if let Some(interval_node) = self.interval(|| self.binary_expression(0)) {
return interval_node;
}
if token.kind == TokenKind::Bracket(Bracket::LeftParenthesis) {
self.next()?;
let expr = self.binary_expression(0)?;
self.expect(TokenKind::Bracket(Bracket::RightParenthesis))?;
let p_start = self.current().map(|s| s.span.0);
self.next();
let binary_node = self.binary_expression(0);
if let Some(error_node) = self.expect(TokenKind::Bracket(Bracket::RightParenthesis)) {
return error_node;
};
let expr = self.node(Node::Parenthesized(binary_node), |_| NodeMetadata {
span: (p_start.unwrap_or_default(), self.prev_token_end()),
});
return self.with_postfix(expr, || self.binary_expression(0));
}
@@ -235,13 +299,14 @@ impl From<&Node<'_>> for UnaryNodeBehaviour {
let a = UnaryNodeBehaviour::from(*on_true);
let b = UnaryNodeBehaviour::from(*on_false);
return if a == b {
if a == b {
a
} else {
CompareWithReference(Equal)
};
}
}
Node::Unary { node, .. } => UnaryNodeBehaviour::from(*node),
Node::Parenthesized(n) => UnaryNodeBehaviour::from(*n),
Node::Binary {
left,
operator,
@@ -251,11 +316,11 @@ impl From<&Node<'_>> for UnaryNodeBehaviour {
let a = UnaryNodeBehaviour::from(*left);
let b = UnaryNodeBehaviour::from(*right);
return if a == b {
if a == b {
a
} else {
CompareWithReference(Equal)
};
}
}
Operator::Logical(_) => AsBoolean,
Operator::Comparison(_) => AsBoolean,
@@ -318,6 +383,7 @@ impl From<&Node<'_>> for UnaryNodeBehaviour {
BuiltInFunction::One => AsBoolean,
BuiltInFunction::Type => CompareWithReference(Equal),
},
Node::Error { .. } => AsBoolean,
}
}
}
+4 -1
View File
@@ -11,11 +11,13 @@ mod conv;
mod de;
mod map;
mod ser;
mod types;
use crate::vm::helpers::date_time;
use crate::vm::VMError;
#[allow(unused_imports)]
pub use conv::ToVariable;
pub use types::VariableType;
#[derive(Debug, PartialEq, Eq, Display)]
pub enum Variable<'arena> {
@@ -163,6 +165,7 @@ impl TryFrom<&Variable<'_>> for NaiveDateTime {
fn try_from(value: &Variable<'_>) -> Result<Self, Self::Error> {
match value {
Variable::String(a) => date_time(a),
#[allow(deprecated)]
Variable::Number(a) => NaiveDateTime::from_timestamp_opt(
a.to_i64().ok_or_else(|| VMError::OpcodeErr {
opcode: "DateManipulation".into(),
@@ -0,0 +1,94 @@
use crate::variable::types::VariableType;
use serde_json::Value;
use std::borrow::Cow;
use std::ops::Deref;
use std::rc::Rc;
impl<'a> From<Cow<'a, Value>> for VariableType {
fn from(value: Cow<'a, Value>) -> Self {
match value.deref() {
Value::Null => VariableType::Null,
Value::Bool(_) => VariableType::Bool,
Value::Number(_) => VariableType::Number,
Value::String(_) => VariableType::String,
Value::Array(_) => {
let Value::Array(arr) = value.into_owned() else {
panic!("unexpected type of value, expected array");
};
VariableType::from(arr)
}
Value::Object(_) => {
let Value::Object(obj) = value.into_owned() else {
panic!("unexpected type of value, expected object");
};
VariableType::Object(
obj.into_iter()
.map(|(k, v)| (k, Rc::new(v.into())))
.collect(),
)
}
}
}
}
impl From<Value> for VariableType {
fn from(value: Value) -> Self {
VariableType::from(Cow::Owned(value)).into()
}
}
impl From<&Value> for VariableType {
fn from(value: &Value) -> Self {
VariableType::from(Cow::Borrowed(value)).into()
}
}
impl From<Vec<Value>> for VariableType {
fn from(arr: Vec<Value>) -> Self {
if arr.len() == 0 {
return VariableType::Array(Rc::new(VariableType::Any));
}
let result_type = arr
.into_iter()
.fold(None, |acc: Option<VariableType>, b| match acc {
Some(a) => Some(a.merge(&VariableType::from(b))),
None => Some(VariableType::from(b)),
});
VariableType::Array(Rc::new(result_type.unwrap_or(VariableType::Any)))
}
}
impl From<&Vec<Value>> for VariableType {
fn from(arr: &Vec<Value>) -> Self {
if arr.len() == 0 {
return VariableType::Array(Rc::new(VariableType::Any));
}
let result_type = arr
.iter()
.fold(None, |acc: Option<VariableType>, b| match acc {
Some(a) => Some(a.merge(&VariableType::from(b))),
None => Some(VariableType::from(b)),
});
VariableType::Array(Rc::new(result_type.unwrap_or(VariableType::Any)))
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn test_value_to_value_kind() {
assert_eq!(VariableType::from(json!(null)), VariableType::Null);
assert_eq!(VariableType::from(json!(true)), VariableType::Bool);
assert_eq!(VariableType::from(json!(42)), VariableType::Number);
assert_eq!(VariableType::from(json!("hello")), VariableType::String);
}
}
+40
View File
@@ -0,0 +1,40 @@
mod conv;
mod util;
use serde::Serialize;
use std::collections::HashMap;
use std::fmt::Display;
use std::rc::Rc;
#[derive(Debug, Clone, Eq, PartialEq, Serialize)]
pub enum VariableType {
Any,
Null,
Bool,
String,
Number,
Constant(Rc<serde_json::Value>),
Array(Rc<VariableType>),
Object(HashMap<String, Rc<VariableType>>),
}
impl Default for VariableType {
fn default() -> Self {
VariableType::Null
}
}
impl Display for VariableType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
VariableType::Any => write!(f, "any"),
VariableType::Null => write!(f, "null"),
VariableType::Bool => write!(f, "bool"),
VariableType::String => write!(f, "string"),
VariableType::Number => write!(f, "number"),
VariableType::Constant(c) => write!(f, "{c}"),
VariableType::Array(v) => write!(f, "{v}[]"),
VariableType::Object(_) => write!(f, "object"),
}
}
}
+199
View File
@@ -0,0 +1,199 @@
use crate::variable::types::VariableType;
use serde_json::Value;
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::rc::Rc;
impl VariableType {
pub fn array_item(&self) -> Option<Rc<VariableType>> {
match self {
VariableType::Array(item) => Some(item.clone()),
_ => None,
}
}
pub fn as_const_str(&self) -> Option<&str> {
match self {
VariableType::Constant(c) => match c.as_ref() {
Value::String(s) => Some(s.as_str()),
_ => None,
},
_ => None,
}
}
pub fn omit_const(&self) -> VariableType {
match self {
VariableType::Constant(v) => VariableType::from(v.as_ref()),
_ => self.clone(),
}
}
pub fn get(&self, vt: &VariableType) -> Rc<VariableType> {
match self {
VariableType::Array(inner) => inner.clone(),
VariableType::Object(obj) => match vt.as_const_str() {
None => Rc::new(VariableType::Any),
Some(key) => obj.get(key).cloned().unwrap_or(Rc::new(VariableType::Any)),
},
VariableType::Any => Rc::new(VariableType::Any),
VariableType::Constant(c) => match c.as_ref() {
Value::Array(arr) => {
let arr_type = VariableType::from(arr.clone());
arr_type.array_item().unwrap_or(Rc::new(VariableType::Any))
}
Value::Object(obj) => match vt.as_const_str() {
None => Rc::new(VariableType::Any),
Some(key) => obj
.get(key)
.map(|v| Rc::new(v.into()))
.unwrap_or(Rc::new(VariableType::Any)),
},
_ => Rc::from(VariableType::Null),
},
_ => Rc::from(VariableType::Null),
}
}
pub fn satisfies(&self, constraint: &Self) -> bool {
match (self, constraint) {
(VariableType::Any, _) | (_, VariableType::Any) => true,
(VariableType::Null, VariableType::Null) => true,
(VariableType::Bool, VariableType::Bool) => true,
(VariableType::String, VariableType::String) => true,
(VariableType::Number, VariableType::Number) => true,
(VariableType::Array(a1), VariableType::Array(a2)) => a1 == a2,
(VariableType::Object(o1), VariableType::Object(o2)) => o1
.iter()
.all(|(k, v)| o2.get(k).is_some_and(|tv| v.satisfies(tv))),
(VariableType::Constant(c1), VariableType::Constant(c2)) => c1 == c2,
(VariableType::Constant(c), _) => {
let self_kind: VariableType = c.as_ref().into();
self_kind.satisfies(constraint)
}
(_, _) => false,
}
}
pub fn satisfies_array(&self) -> bool {
match self {
VariableType::Any | VariableType::Array(_) => true,
VariableType::Constant(c) => match c.as_ref() {
Value::Array(_) => true,
_ => false,
},
_ => false,
}
}
pub fn satisfies_object(&self) -> bool {
match self {
VariableType::Any | VariableType::Object(_) => true,
VariableType::Constant(c) => match c.as_ref() {
Value::Object(_) => true,
_ => false,
},
_ => false,
}
}
pub fn merge(&self, other: &Self) -> Self {
match (&self, other) {
(VariableType::Any, _) | (_, VariableType::Any) => VariableType::Any,
(VariableType::Null, VariableType::Null) => VariableType::Null,
(VariableType::Bool, VariableType::Bool) => VariableType::Bool,
(VariableType::String, VariableType::String) => VariableType::String,
(VariableType::Number, VariableType::Number) => VariableType::Number,
(VariableType::Array(a1), VariableType::Array(a2)) => {
if Rc::ptr_eq(&a1, &a2) {
VariableType::Array(a1.clone())
} else {
VariableType::Array(Rc::new(a1.merge(a2)))
}
}
(VariableType::Constant(c1), VariableType::Constant(c2)) => {
if Rc::ptr_eq(&c1, &c2) {
VariableType::Constant(c1.clone())
} else if c1 == c2 {
VariableType::Constant(c1.clone())
} else {
let vt1 = VariableType::from(c1.as_ref());
let vt2 = VariableType::from(c2.as_ref());
vt1.merge(&vt2)
}
}
(VariableType::Object(o1), VariableType::Object(o2)) => {
let cap = o1.capacity().max(o2.capacity());
let map = o1.iter().chain(o2.iter()).fold(
HashMap::<String, Rc<VariableType>>::with_capacity(cap),
|mut acc, (k, v)| {
match acc.entry(k.clone()) {
Entry::Occupied(mut occ) => {
let current = occ.get();
let merged = v.merge(current.as_ref());
occ.insert(Rc::new(merged));
}
Entry::Vacant(vac) => {
vac.insert(v.clone());
}
}
acc
},
);
VariableType::Object(map)
}
(_, _) => VariableType::Any,
}
}
}
#[cfg(test)]
mod tests {
use crate::variable::VariableType;
use std::rc::Rc;
#[test]
fn merge_simple() {
assert_eq!(
VariableType::Number.merge(&VariableType::Number),
VariableType::Number
);
assert_eq!(
VariableType::String.merge(&VariableType::String),
VariableType::String
);
assert_eq!(
VariableType::Bool.merge(&VariableType::Bool),
VariableType::Bool
);
assert_eq!(
VariableType::Null.merge(&VariableType::Null),
VariableType::Null
);
assert_eq!(
VariableType::Any.merge(&VariableType::Any),
VariableType::Any
);
}
#[test]
fn merge_array() {
assert_eq!(
VariableType::Array(Rc::new(VariableType::Number))
.merge(&VariableType::Array(Rc::new(VariableType::Number))),
VariableType::Array(Rc::new(VariableType::Number))
);
}
#[test]
fn merge_mixed() {
assert_eq!(
VariableType::Number.merge(&VariableType::String),
VariableType::Any
);
}
}
+2
View File
@@ -1210,6 +1210,7 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'arena, 'parent_ref, 'bytecode_
message: "Failed to run DateFunction".into(),
})?;
#[allow(deprecated)]
self.push(Number(s.timestamp().into()));
}
Opcode::Slice => {
@@ -1358,6 +1359,7 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'arena, 'parent_ref, 'bytecode_
Opcode::ParseDateTime => {
let a = self.pop()?;
let ts = match a {
#[allow(deprecated)]
String(a) => date_time(a)?.timestamp(),
Number(a) => a.to_i64().ok_or_else(|| OpcodeErr {
opcode: "ParseDateTime".into(),
+4 -3
View File
@@ -780,11 +780,12 @@ fn test_standard_csv() {
isolate.set_environment(&input);
}
let result = isolate
let maybe_result = isolate
.run_standard(expression)
.context(format!("Expression: {expression}"))
.unwrap();
.context(format!("Expression: {expression}"));
assert!(maybe_result.is_ok(), "{}", maybe_result.unwrap_err());
let result = maybe_result.unwrap();
assert_eq!(
result, output,
"Expression {expression}. Expected: {output}, got: {result}"
+19 -16
View File
@@ -86,11 +86,11 @@ fn standard_test() {
StandardTest {
src: "(1 - 2) * 3",
result: &Node::Binary {
left: &Node::Binary {
left: &Node::Parenthesized(&Node::Binary {
left: &Node::Number(D1),
operator: Operator::Arithmetic(ArithmeticOperator::Subtract),
right: &Node::Number(D2),
},
}),
operator: Operator::Arithmetic(ArithmeticOperator::Multiply),
right: &Node::Number(D3),
},
@@ -123,11 +123,11 @@ fn standard_test() {
src: "(a or b) and c",
result: &Node::Binary {
operator: Operator::Logical(LogicalOperator::And),
left: &Node::Binary {
left: &Node::Parenthesized(&Node::Binary {
left: &Node::Identifier("a"),
right: &Node::Identifier("b"),
operator: Operator::Logical(LogicalOperator::Or),
},
}),
right: &Node::Identifier("c"),
},
},
@@ -298,17 +298,19 @@ fn standard_test() {
let tokens = lexer.tokenize(src).unwrap();
let unary_parser = Parser::try_new(tokens, &bump).unwrap().standard();
let parser_result = unary_parser.parse();
let Ok(ast) = parser_result else {
assert!(
false,
"Failed on expression: {}. Error: {:?}.",
src,
parser_result.unwrap_err()
);
return;
};
// let Ok(ast) = parser_result else {
// assert!(
// false,
// "Failed on expression: {}. Error: {:?}.",
// src,
// parser_result.unwrap_err()
// );
// return;
// };
assert!(parser_result.error().is_ok(), "Expression failed: {src}");
assert_eq!(parser_result.root, result, "Failed on expression: {}", src);
assert_eq!(ast, result, "Failed on expression: {}", src);
bump.reset();
}
}
@@ -323,8 +325,9 @@ fn failure_tests() {
for test in tests {
let tokens = lexer.tokenize(test).unwrap();
let parser = Parser::try_new(tokens, &bump).unwrap().standard();
let ast = parser.parse();
assert!(ast.is_err());
let parser_result = parser.parse();
assert!(parser_result.error().is_err(), "{parser_result:?}");
bump.reset();
}
+10 -4
View File
@@ -145,8 +145,10 @@ fn unary_test() {
for UnaryTest { src, result } in tests {
let tokens = lexer.tokenize(src).unwrap();
let parser = Parser::try_new(tokens, &bump).unwrap().unary();
let ast = parser.parse().unwrap();
assert_eq!(ast, result);
let parser_result = parser.parse();
assert!(parser_result.error().is_ok(), "Parser failed");
assert_eq!(parser_result.root, result);
bump.reset();
}
@@ -162,8 +164,12 @@ fn failure_tests() {
for test in tests {
let tokens = lexer.tokenize(test).unwrap();
let unary_parser = Parser::try_new(tokens, &bump).unwrap().standard();
let ast = unary_parser.parse();
assert!(ast.is_err());
let parser_result = unary_parser.parse();
assert!(
parser_result.error().is_err(),
"Parsing expected to fail for: {test}"
);
bump.reset();
}
+1 -1
View File
@@ -49,7 +49,7 @@ fn main() -> Result<()> {
let mut isolate = Isolate::new();
isolate.set_environment(
&json!({ "customer": { "firstName": "John", "lastName": "Doe", "age": 20 } }),
&json!({ "customer": { "firstName": "John", "lastName": "Doe", "age": 20 }, "hello": true, "$": 10 }),
);
let result = isolate.run_standard(line.as_str());