diff --git a/core/engine/src/nodes/input/dates.rs b/core/engine/src/nodes/input/dates.rs new file mode 100644 index 00000000..1f35323d --- /dev/null +++ b/core/engine/src/nodes/input/dates.rs @@ -0,0 +1,138 @@ +use serde_json::{Map, Value}; +use zen_expression::DateValue; +use zen_types::variable::Variable; + +pub(crate) struct DeclaredDates; + +impl DeclaredDates { + const DATE_KEYS: [&str; 10] = [ + "type", + "format", + "description", + "title", + "examples", + "default", + "$comment", + "readOnly", + "writeOnly", + "deprecated", + ]; + + pub(crate) fn declared(schema: &Value) -> bool { + schema.as_object().is_some_and(Self::declared_map) + } + + pub(crate) fn declared_map(schema: &Map) -> bool { + let format = schema.get("format").and_then(Value::as_str); + if matches!(format, Some("date" | "date-time")) && Self::only_keys(schema, &Self::DATE_KEYS) + { + return Self::string_type(schema.get("type")); + } + + let Some(variants) = schema.get("anyOf").and_then(Value::as_array) else { + return false; + }; + let is_null = |v: &Value| { + v.as_object().is_some_and(|o| { + o.len() == 1 && o.get("type").and_then(Value::as_str) == Some("null") + }) + }; + Self::only_keys(schema, &["anyOf", "description"]) + && variants.len() == 2 + && variants.iter().any(is_null) + && variants.iter().any(|v| !is_null(v) && Self::declared(v)) + } + + pub(crate) fn prepare(value: &Variable, schema: Option<&Value>) -> Option { + if schema.is_some_and(Self::declared) { + return match value { + Variable::String(text) => DateValue::from_text(text), + _ => None, + }; + } + let object = schema.and_then(Value::as_object); + match value { + Variable::Dynamic(_) => DateValue::source_text(value), + Variable::Object(_) => { + let properties = object + .and_then(|o| Self::structure(o, "properties")) + .and_then(Value::as_object); + Self::rewrite_fields(value, |key, child| { + Self::prepare(child, properties.and_then(|p| p.get(key))) + }) + } + Variable::Array(_) => { + let items = object.and_then(|o| Self::structure(o, "items")); + Self::rewrite_items(value, |item| Self::prepare(item, items)) + } + _ => None, + } + } + + fn structure<'s>(schema: &'s Map, key: &str) -> Option<&'s Value> { + schema.get(key).or_else(|| { + ["anyOf", "oneOf", "allOf"] + .iter() + .filter_map(|keyword| schema.get(*keyword)?.as_array()) + .flatten() + .find_map(|variant| variant.get(key)) + }) + } + + pub(crate) fn rewrite_fields( + value: &Variable, + rewrite: impl Fn(&str, &Variable) -> Option, + ) -> Option { + let object = value.as_object()?; + let changed: Vec<(String, Variable)> = object + .borrow() + .iter() + .filter_map(|(key, child)| { + let key: &str = key.as_ref(); + rewrite(key, child).map(|next| (key.to_string(), next)) + }) + .collect(); + if changed.is_empty() { + return None; + } + let mut next = object.borrow().clone(); + for (key, rewritten) in changed { + next.insert_str(&key, rewritten); + } + Some(Variable::from_object(next)) + } + + pub(crate) fn rewrite_items( + value: &Variable, + rewrite: impl Fn(&Variable) -> Option, + ) -> Option { + let array = value.as_array()?; + let array = array.borrow(); + let rewritten: Vec> = array.iter().map(&rewrite).collect(); + if rewritten.iter().all(Option::is_none) { + return None; + } + Some(Variable::from_array( + array + .iter() + .zip(rewritten) + .map(|(item, next)| next.unwrap_or_else(|| item.clone())) + .collect(), + )) + } + + fn string_type(value: Option<&Value>) -> bool { + match value { + Some(Value::String(t)) => t == "string", + Some(Value::Array(types)) if types.len() == 2 => { + let has = |name: &str| types.iter().any(|t| t.as_str() == Some(name)); + has("string") && has("null") + } + _ => false, + } + } + + fn only_keys(schema: &Map, allowed: &[&str]) -> bool { + schema.keys().all(|key| allowed.contains(&key.as_str())) + } +} diff --git a/core/engine/src/nodes/input/mod.rs b/core/engine/src/nodes/input/mod.rs index 4c32d2fc..9caf86de 100644 --- a/core/engine/src/nodes/input/mod.rs +++ b/core/engine/src/nodes/input/mod.rs @@ -1,3 +1,5 @@ +pub(crate) mod dates; + use crate::nodes::definition::NodeHandler; use crate::nodes::result::NodeResult; use crate::nodes::NodeContext; @@ -17,8 +19,9 @@ impl NodeHandler for InputNodeHandler { async fn handle(&self, ctx: NodeContext) -> NodeResult { if let Some(json_schema) = &ctx.node.schema { ctx.validate(json_schema, &ctx.input)?; - }; + } - ctx.success(ctx.input.clone()) + let output = dates::DeclaredDates::prepare(&ctx.input, ctx.node.schema.as_deref()); + ctx.success(output.unwrap_or_else(|| ctx.input.clone())) } } diff --git a/core/engine/src/nodes/validator_cache.rs b/core/engine/src/nodes/validator_cache.rs index 2b061882..4ed14fbc 100644 --- a/core/engine/src/nodes/validator_cache.rs +++ b/core/engine/src/nodes/validator_cache.rs @@ -39,6 +39,8 @@ impl ValidatorCache { let validator = Arc::new( jsonschema::options_for::() .with_draft(jsonschema::Draft::Draft7) + .with_format("date", zen_expression::DateValue::is_text) + .with_format("date-time", zen_expression::DateValue::is_text) .build(schema)?, ); w_shared.insert(key, validator.clone()); diff --git a/core/engine/src/policy/evaluator.rs b/core/engine/src/policy/evaluator.rs index 8e9843a7..5acb78c9 100644 --- a/core/engine/src/policy/evaluator.rs +++ b/core/engine/src/policy/evaluator.rs @@ -116,9 +116,13 @@ impl EvalArtifact { ) -> Result { let start = Instant::now(); - self.validate_request(req)?; + let input = self + .input_schema + .convert_dates(&req.input) + .unwrap_or_else(|| req.input.clone()); + self.validate_request(req, &input)?; - let store = req.input.depth_clone(1); + let store = input.depth_clone(1); let ref_targets: HashSet> = self .reference_fields .iter() @@ -154,13 +158,17 @@ impl EvalArtifact { }) } - fn validate_request(&self, req: &EvaluateRequest) -> Result<(), EvaluationError> { + fn validate_request( + &self, + req: &EvaluateRequest, + input: &Variable, + ) -> Result<(), EvaluationError> { for goal in &req.goals { if !self.eval_graph.contains(goal) { return Err(EvaluationError::GoalNotFound(goal.clone())); } } - let validation_errors = self.input_schema.validate(&req.input); + let validation_errors = self.input_schema.validate(input); if !validation_errors.is_empty() { return Err(EvaluationError::InputValidationFailed { errors: validation_errors, diff --git a/core/engine/src/policy/ir.rs b/core/engine/src/policy/ir.rs index 223dc88d..bec84b87 100644 --- a/core/engine/src/policy/ir.rs +++ b/core/engine/src/policy/ir.rs @@ -221,7 +221,8 @@ impl DataModelIr { visited: &mut HashSet>, ) -> VariableType { let inner = match &prop.kind { - PropertyTypeIr::String | PropertyTypeIr::Date => VariableType::String, + PropertyTypeIr::String => VariableType::String, + PropertyTypeIr::Date => VariableType::Date, PropertyTypeIr::Enum(values) => VariableType::Enum(None, enum_values_to_rc(values)), PropertyTypeIr::Number => VariableType::Number, PropertyTypeIr::Boolean => VariableType::Bool, @@ -550,7 +551,7 @@ impl std::fmt::Display for PropertyTypeIr { } PropertyTypeIr::Number => f.write_str("number"), PropertyTypeIr::Boolean => f.write_str("bool"), - PropertyTypeIr::Date => f.write_str("date (string)"), + PropertyTypeIr::Date => f.write_str("date"), PropertyTypeIr::Reference { target } => { write!(f, "reference id (string → {target})") } diff --git a/core/engine/src/policy/queries/scope.rs b/core/engine/src/policy/queries/scope.rs index 00113134..504625fd 100644 --- a/core/engine/src/policy/queries/scope.rs +++ b/core/engine/src/policy/queries/scope.rs @@ -686,7 +686,8 @@ impl Property { dictionaries: &HashMap, Arc>, ) -> VariableType { let inner = match &self.kind { - PropertyTypeIr::String | PropertyTypeIr::Date => VariableType::String, + PropertyTypeIr::String => VariableType::String, + PropertyTypeIr::Date => VariableType::Date, PropertyTypeIr::Enum(values) => { VariableType::Enum(None, crate::policy::ir::enum_values_to_rc(values)) } diff --git a/core/engine/src/policy/queries/skeleton.rs b/core/engine/src/policy/queries/skeleton.rs index 51eab938..ba60a5cf 100644 --- a/core/engine/src/policy/queries/skeleton.rs +++ b/core/engine/src/policy/queries/skeleton.rs @@ -50,9 +50,8 @@ impl SkeletonBuilder { fn default_for(ty: &VariableType) -> Value { match ty { - VariableType::String | VariableType::Date | VariableType::Interval => { - Value::String(String::new()) - } + VariableType::String | VariableType::Interval => Value::String(String::new()), + VariableType::Date => Value::String("2000-01-01".to_string()), VariableType::Number => Value::Number(0u64.into()), VariableType::Bool => Value::Bool(false), VariableType::Null | VariableType::Any => Value::Null, diff --git a/core/engine/src/policy/validator.rs b/core/engine/src/policy/validator.rs index 4f38c30e..985693e7 100644 --- a/core/engine/src/policy/validator.rs +++ b/core/engine/src/policy/validator.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use ahash::{HashMap, HashMapExt, HashSet}; use zen_expression::variable::Variable; +use crate::nodes::input::dates::DeclaredDates; use crate::policy::ir::{DataModelIr, DictionaryIr, Property, PropertyTypeIr}; use crate::policy::refs::RefPoolIndex; use crate::policy::MAX_RECURSION_DEPTH; @@ -114,6 +115,53 @@ impl InputSchema { } } +impl InputSchema { + pub(crate) fn convert_dates(&self, input: &Variable) -> Option { + DeclaredDates::rewrite_fields(input, |key, value| { + if self.ref_targets.contains(key) { + DeclaredDates::rewrite_items(value, |item| self.convert_entity(item, key, 0)) + } else if self.roots.contains(key) { + self.convert_entity(value, key, 0) + } else { + self.convert_property(value, self.globals.get(key)?, 0) + } + }) + } + + fn convert_entity(&self, value: &Variable, entity: &str, depth: usize) -> Option { + if depth >= MAX_RECURSION_DEPTH { + return None; + } + let model = self.entities.get(entity)?; + DeclaredDates::rewrite_fields(value, |key, child| { + let property = model.properties.iter().find(|p| *p.name == *key)?; + self.convert_property(child, property, depth + 1) + }) + } + + fn convert_property( + &self, + value: &Variable, + property: &Property, + depth: usize, + ) -> Option { + let convert_one = |item: &Variable| match &property.kind { + PropertyTypeIr::Date => match item { + Variable::String(text) => zen_expression::DateValue::from_text(text), + _ => None, + }, + PropertyTypeIr::Relationship { target } if self.entities.contains_key(target) => { + self.convert_entity(item, target, depth) + } + _ => None, + }; + match property.array { + true => DeclaredDates::rewrite_items(value, convert_one), + false => convert_one(value), + } + } +} + struct InputValidator<'a> { entities: &'a HashMap, Arc>, dictionaries: &'a HashMap, Arc>, @@ -218,7 +266,12 @@ impl InputValidator<'_> { } PropertyTypeIr::Number => matches!(value, Variable::Number(_)), PropertyTypeIr::Boolean => matches!(value, Variable::Bool(_)), - PropertyTypeIr::Date => matches!(value, Variable::String(_)), + PropertyTypeIr::Date => match value { + Variable::String(text) => { + text.is_empty() || zen_expression::DateValue::is_text(text) + } + other => zen_expression::DateValue::is(other), + }, PropertyTypeIr::Reference { target } => { self.validate_reference(value, target, path); return; diff --git a/core/engine/src/workspace/graph/schema.rs b/core/engine/src/workspace/graph/schema.rs index 80219fa9..154a47da 100644 --- a/core/engine/src/workspace/graph/schema.rs +++ b/core/engine/src/workspace/graph/schema.rs @@ -6,6 +6,8 @@ use ahash::{HashMap, HashMapExt}; use serde_json::{Map, Value}; use zen_expression::variable::VariableType; +use crate::nodes::input::dates::DeclaredDates; + pub(crate) type SchemaDictionaries = HashMap, VariableType>; pub(crate) struct SchemaType; @@ -131,9 +133,9 @@ impl SchemaType { .map(|items| Self::variable_type_with(items, dictionaries)) .unwrap_or(VariableType::Any), )), - "string" => match object.get("format").and_then(Value::as_str) { - Some("date" | "date-time") => VariableType::Date, - _ => VariableType::String, + "string" => match DeclaredDates::declared_map(object) { + true => VariableType::Date, + false => VariableType::String, }, "number" | "integer" => VariableType::Number, "boolean" => VariableType::Bool, diff --git a/core/expression/src/dates.rs b/core/expression/src/dates.rs new file mode 100644 index 00000000..6821485c --- /dev/null +++ b/core/expression/src/dates.rs @@ -0,0 +1,24 @@ +use crate::variable::Variable; +use crate::vm::date::VmDate; +use std::rc::Rc; + +pub struct DateValue; + +impl DateValue { + pub fn from_text(text: &str) -> Option { + VmDate::from_text(text).map(|date| Variable::Dynamic(Rc::new(date))) + } + + pub fn is_text(text: &str) -> bool { + VmDate::parses(text) + } + + pub fn source_text(value: &Variable) -> Option { + let text = value.dynamic::()?.source()?; + Some(Variable::String(text.into())) + } + + pub fn is(value: &Variable) -> bool { + matches!(value, Variable::Dynamic(dynamic) if dynamic.type_name() == "date") + } +} diff --git a/core/expression/src/functions/deprecated.rs b/core/expression/src/functions/deprecated.rs index 454a6fd1..c19d28a2 100644 --- a/core/expression/src/functions/deprecated.rs +++ b/core/expression/src/functions/deprecated.rs @@ -116,15 +116,27 @@ impl From<&DeprecatedFunction> for Rc { mod imp { use super::*; + use crate::vm::date::DynamicVariableExt; use crate::vm::helpers::DateUnit; use crate::vm::VMError; use zen_types::variable::Variable; fn __internal_convert_datetime(timestamp: &V) -> anyhow::Result { - match timestamp { - Variable::String(a) => date_time(a), + let instant = match timestamp { + Variable::Dynamic(d) => d + .as_date() + .and_then(|date| date.0) + .map(|date| date.naive_local()), + _ => None, + }; + match (timestamp.as_str(), timestamp) { + (Some(text), _) => date_time(text).or_else(|error| instant.ok_or(error)), + (None, Variable::Dynamic(_)) => instant.ok_or_else(|| VMError::OpcodeErr { + opcode: "DateManipulation".into(), + message: "Invalid date".into(), + }), #[allow(deprecated)] - Variable::Number(a) => NaiveDateTime::from_timestamp_opt( + (None, Variable::Number(a)) => NaiveDateTime::from_timestamp_opt( a.to_i64().ok_or_else(|| VMError::OpcodeErr { opcode: "DateManipulation".into(), message: "Failed to extract date".into(), @@ -152,7 +164,13 @@ mod imp { dt.timestamp() } V::Number(a) => a.to_i64().context("Number overflow")?, - _ => return Err(anyhow!("Unsupported type for date function")), + _ => + { + #[allow(deprecated)] + __internal_convert_datetime(a) + .map_err(|_| anyhow!("Unsupported type for date function"))? + .timestamp() + } }; Ok(V::Number(ts.into())) @@ -161,9 +179,15 @@ mod imp { pub fn parse_time(args: Arguments) -> anyhow::Result { let a = args.var(0)?; - let ts = match a { - V::String(a) => time(a.as_ref())?.num_seconds_from_midnight(), - V::Number(a) => a.to_u32().context("Number overflow")?, + let ts = match (a, a.as_str()) { + (V::Number(a), _) => a.to_u32().context("Number overflow")?, + (V::String(_), Some(text)) => time(text)?.num_seconds_from_midnight(), + (V::Dynamic(_), text) => match text.map(time) { + Some(Ok(time)) => time.num_seconds_from_midnight(), + _ => __internal_convert_datetime(a)? + .time() + .num_seconds_from_midnight(), + }, _ => return Err(anyhow!("Unsupported type for time function")), }; diff --git a/core/expression/src/functions/internal.rs b/core/expression/src/functions/internal.rs index 47354cd9..c9a02723 100644 --- a/core/expression/src/functions/internal.rs +++ b/core/expression/src/functions/internal.rs @@ -725,7 +725,11 @@ pub(crate) mod imp { pub fn to_bool(args: Arguments) -> anyhow::Result { let a = args.var(0)?; - let val = match a { + let a = match a.dynamic::().and_then(VmDate::source) { + Some(text) => V::String(text.into()), + None => a.clone(), + }; + let val = match &a { V::Null => false, V::Bool(v) => *v, V::Number(n) => !n.is_zero(), @@ -747,6 +751,7 @@ pub(crate) mod imp { V::Bool(v) => v.to_string().into(), V::Number(n) => n.to_string().into(), V::String(s) => s.clone(), + V::Dynamic(d) if d.as_date().is_some() => d.to_string().into(), _ => return Err(anyhow!("Cannot convert type {} to string", a.type_name())), }; @@ -791,12 +796,9 @@ pub(crate) mod imp { pub fn len(args: Arguments) -> anyhow::Result { let a = args.var(0)?; - let len = match a { - V::String(s) => s.len(), - V::Array(s) => { - let arr = s.borrow(); - arr.len() - } + let len = match (a, a.as_str()) { + (V::Array(s), _) => s.borrow().len(), + (_, Some(text)) => text.len(), _ => { return Err(anyhow!("Cannot determine len of type {}", a.type_name())); } @@ -809,9 +811,8 @@ pub(crate) mod imp { let a = args.var(0)?; let b = args.var(1)?; - let val = match (a, b) { - (V::String(a), V::String(b)) => a.contains(b.as_str()), - (V::Array(a), _) => { + let val = match (a, b, a.as_str().zip(b.as_str())) { + (V::Array(a), _, _) => { let arr = a.borrow(); arr.iter().any(|a| match (a, b) { @@ -819,9 +820,13 @@ pub(crate) mod imp { (V::String(a), V::String(b)) => a == b, (V::Bool(a), V::Bool(b)) => a == b, (V::Null, V::Null) => true, + (V::Dynamic(d), other) | (other, V::Dynamic(d)) => { + d.as_date().is_some_and(|d| d.matches(other)) + } _ => false, }) } + (_, _, Some((a, b))) => a.contains(b), _ => { return Err(anyhow!( "Cannot determine contains for type {} and {}", @@ -838,13 +843,13 @@ pub(crate) mod imp { let a = args.var(0)?; let b = args.str(1)?; - let val = match a { - V::String(a) => { - let sim = strsim::normalized_damerau_levenshtein(a.as_ref(), b.as_ref()); + let val = match (a, a.as_str()) { + (_, Some(a)) => { + let sim = strsim::normalized_damerau_levenshtein(a, b.as_ref()); // This is okay, as NDL will return [0, 1] V::Number(Decimal::from_f64(sim).unwrap_or(dec!(0))) } - V::Array(_a) => { + (V::Array(_a), _) => { let a = _a.borrow(); let mut sims = Vec::with_capacity(a.len()); for v in a.iter() { diff --git a/core/expression/src/intellisense/type_provider.rs b/core/expression/src/intellisense/type_provider.rs index dfdb50d5..72211601 100644 --- a/core/expression/src/intellisense/type_provider.rs +++ b/core/expression/src/intellisense/type_provider.rs @@ -203,7 +203,7 @@ impl TypesProvider { match node_type.kind.widen() { VariableType::Any => V(VariableType::Any), VariableType::Array(inner) => V(VariableType::Array(inner.clone())), - VariableType::String => V(VariableType::String), + VariableType::String | VariableType::Date => V(VariableType::String), _ => Error("Slice operation is only allowed on `string | any[]`".to_string()), } } @@ -347,6 +347,7 @@ impl TypesProvider { ArithmeticOperator::Add => match (left_type.widen(), right_type.widen()) { (VariableType::Number, VariableType::Number) => V(VariableType::Number), (VariableType::String, VariableType::String) => V(VariableType::String), + (VariableType::String, VariableType::Date) | (VariableType::Date, VariableType::String) => V(VariableType::String), (VariableType::Any, VariableType::Number | VariableType::String | VariableType::Any) => V(VariableType::Any), (VariableType::Number | VariableType::String, VariableType::Any) => V(VariableType::Any), _ => Error(format!( @@ -405,7 +406,7 @@ impl TypesProvider { Some(None) => {} None => { let always_false = Self::structured_comparison(&left_type, &right_type) - || (types_disjoint(&left_type, &right_type) && !left_type.is_nullable() && !right_type.is_nullable() && !left_type.is_null() && !right_type.is_null()); + || (types_disjoint(&left_type, &right_type) && !Self::date_and_string(&left_type, &right_type) && !left_type.is_nullable() && !right_type.is_nullable() && !left_type.is_null() && !right_type.is_null()); if always_false { on_fly_error.replace(format!( "Hint: Expression will always evaluate to `false` because `{left_type}` != `{right_type}`." @@ -422,7 +423,7 @@ impl TypesProvider { Some(None) => {} None => { let always_true = Self::structured_comparison(&left_type, &right_type) - || (types_disjoint(&left_type, &right_type) && !left_type.is_nullable() && !right_type.is_nullable() && !left_type.is_null() && !right_type.is_null()); + || (types_disjoint(&left_type, &right_type) && !Self::date_and_string(&left_type, &right_type) && !left_type.is_nullable() && !right_type.is_nullable() && !left_type.is_null() && !right_type.is_null()); if always_true { on_fly_error.replace(format!( "Hint: Expression will always evaluate to `true` because `{left_type}` != `{right_type}`." @@ -439,6 +440,7 @@ impl TypesProvider { | ComparisonOperator::GreaterThanOrEqual => match (left_type.deref(), right_type.deref()) { (VariableType::Date | VariableType::Any, VariableType::Date | VariableType::Any) => V(VariableType::Bool), (VariableType::Number | VariableType::Any, VariableType::Number | VariableType::Any) => V(VariableType::Bool), + (left, right) if Self::date_and_string(left, right) => V(VariableType::Bool), _ => Error(format!( "Operator `{operator}` cannot be applied to types `{left_type}` and `{right_type}`." )), @@ -449,7 +451,7 @@ impl TypesProvider { Some(Some(error)) => { on_fly_error.replace(self.coded(node, error)); } Some(None) => {} None => { - if types_disjoint(&left_type, &inner_type) { + if types_disjoint(&left_type, &inner_type) && !Self::date_and_string(&left_type, &inner_type) { let expected = match comp { ComparisonOperator::In => "false", _ => "true" @@ -703,10 +705,18 @@ impl TypesProvider { }; let typecheck = def.check_types(type_list.as_slice()); + let receiver = type_list[0].unwrap_nullable().0; + let needs_conversion = def.param_type(0) == Some(VariableType::Date) + && (receiver.widen().is_string() || matches!(receiver, VariableType::Number)); + if needs_conversion { + self.set_error(this, "Date methods require a date value. Use d(...) to convert a date string or timestamp first.".to_string()); + } for (i, arg_error) in typecheck.arguments { let code = Self::mismatch(def.param_type_str(i), &type_list[i]); if i == 0 { - self.set_coded_error(this, arg_error, code); + if !needs_conversion { + self.set_coded_error(this, arg_error, code); + } } else { self.set_coded_error(arguments[i - 1], arg_error, code); } @@ -927,6 +937,17 @@ impl TypesProvider { ))) } + fn date_and_string(left: &VariableType, right: &VariableType) -> bool { + let (left, right) = ( + left.unwrap_nullable().0.widen(), + right.unwrap_nullable().0.widen(), + ); + matches!( + (left, right), + (VariableType::Date, VariableType::String) | (VariableType::String, VariableType::Date) + ) + } + fn structured_comparison(left: &VariableType, right: &VariableType) -> bool { matches!( (left, right), diff --git a/core/expression/src/lib.rs b/core/expression/src/lib.rs index e5e9b061..a5eff1eb 100644 --- a/core/expression/src/lib.rs +++ b/core/expression/src/lib.rs @@ -60,6 +60,7 @@ mod isolate; pub mod compiler; +mod dates; mod exports; pub mod expression; pub mod functions; @@ -72,6 +73,7 @@ pub mod validate; pub mod variable; pub mod vm; +pub use dates::DateValue; pub use exports::{ compile_expression, compile_unary_expression, evaluate_expression, evaluate_unary_expression, }; diff --git a/core/expression/src/vm/date/mod.rs b/core/expression/src/vm/date/mod.rs index 30246b96..672bbb05 100644 --- a/core/expression/src/vm/date/mod.rs +++ b/core/expression/src/vm/date/mod.rs @@ -6,7 +6,10 @@ use chrono::{DateTime, SecondsFormat, Utc}; use chrono_tz::Tz; use serde_json::Value; use std::any::Any; +use std::cell::OnceCell; +use std::cmp::Ordering; use std::fmt::{Display, Formatter}; +use std::rc::Rc; use std::sync::OnceLock; // Duration is a modified copy of `humantime` @@ -14,8 +17,28 @@ mod duration; mod duration_parser; mod duration_unit; -#[derive(Debug, Clone, PartialOrd, PartialEq, Ord, Eq)] -pub(crate) struct VmDate(pub Option>); +#[derive(Debug, Clone)] +pub(crate) struct VmDate(pub Option>, Option>, OnceCell>); + +impl PartialEq for VmDate { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} + +impl Eq for VmDate {} + +impl PartialOrd for VmDate { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for VmDate { + fn cmp(&self, other: &Self) -> Ordering { + self.0.cmp(&other.0) + } +} impl DynamicVariable for VmDate { fn type_name(&self) -> &'static str { @@ -27,31 +50,37 @@ impl DynamicVariable for VmDate { } fn to_value(&self) -> Value { - match self.0 { - None => Value::String(String::from("Invalid date")), - Some(d) => Value::String(d.to_rfc3339_opts(SecondsFormat::Secs, true)), - } + Value::String(self.to_string()) + } + + fn as_text(&self) -> Option<&str> { + let date_time = self.0?; + Some(self.1.as_deref().unwrap_or_else(|| { + self.2 + .get_or_init(|| date_time.to_rfc3339_opts(SecondsFormat::Secs, true).into()) + })) } } impl Display for VmDate { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match &self.0 { - None => write!(f, "Invalid date"), - Some(d) => write!(f, "{}", d.to_rfc3339_opts(SecondsFormat::Secs, true)), + match (&self.0, &self.1) { + (None, _) => write!(f, "Invalid date"), + (Some(_), Some(text)) => write!(f, "{text}"), + (Some(d), None) => write!(f, "{}", d.to_rfc3339_opts(SecondsFormat::Secs, true)), } } } impl From>> for VmDate { fn from(value: Option>) -> Self { - Self(value) + Self(value, None, OnceCell::new()) } } impl VmDate { pub fn now() -> Self { - Self(Some(helper::now())) + Self::from(Some(helper::now())) } pub fn yesterday() -> Self { @@ -62,9 +91,43 @@ impl VmDate { Self::now().add(Duration::day()) } + pub fn from_text(text: &str) -> Option { + helper::parse_text(text) + .map(|date_time| Self(Some(date_time), Some(Rc::from(text)), OnceCell::new())) + } + + pub fn coerce(value: &Variable) -> Option { + match value { + Variable::Dynamic(d) => d.as_date().cloned(), + Variable::String(text) => Self::from_text(text), + _ => None, + } + } + + pub fn parses(text: &str) -> bool { + helper::parse_text(text).is_some() + } + + pub fn source(&self) -> Option<&str> { + self.1.as_deref() + } + + pub fn textual(value: Variable) -> Variable { + if let Variable::Dynamic(d) = &value { + if let Some(text) = d.as_text() { + return Variable::String(text.into()); + } + } + value + } + + pub fn matches(&self, other: &Variable) -> bool { + self.0.is_some() && Self::coerce(other).is_some_and(|other| other == *self) + } + /// Create a new VmDate from the current time pub fn new(var: Variable, tz_opt: Option) -> Self { - Self(helper::parse_date(var, tz_opt)) + Self::from(helper::parse_date(var, tz_opt)) } pub fn is_valid(&self) -> bool { @@ -76,7 +139,7 @@ impl VmDate { return self.clone(); }; - Self(Some(date_time.with_timezone(&timezone))) + Self::from(Some(date_time.with_timezone(&timezone))) } pub fn format(&self, format: Option<&str>) -> String { @@ -92,34 +155,34 @@ impl VmDate { pub fn add(&self, duration: Duration) -> Self { let Some(date_time) = &self.0 else { - return Self(None); + return Self::from(None); }; - Self(helper::add_duration(date_time.clone(), duration)) + Self::from(helper::add_duration(date_time.clone(), duration)) } pub fn sub(&self, duration: Duration) -> Self { let Some(date_time) = &self.0 else { - return Self(None); + return Self::from(None); }; - Self(helper::add_duration(date_time.clone(), duration.negate())) + Self::from(helper::add_duration(date_time.clone(), duration.negate())) } pub fn start_of(&self, unit: DurationUnit) -> Self { let Some(date_time) = &self.0 else { - return Self(None); + return Self::from(None); }; - Self(helper::start_of(date_time.clone(), unit)) + Self::from(helper::start_of(date_time.clone(), unit)) } pub fn end_of(&self, unit: DurationUnit) -> Self { let Some(date_time) = &self.0 else { - return Self(None); + return Self::from(None); }; - Self(helper::end_of(date_time.clone(), unit)) + Self::from(helper::end_of(date_time.clone(), unit)) } pub fn diff(&self, date_time: &Self, unit: Option) -> Option { @@ -133,10 +196,10 @@ impl VmDate { pub fn set(&self, value: u32, unit: DurationUnit) -> Self { let Some(date_time) = self.0.clone() else { - return Self(None); + return Self::from(None); }; - Self(helper::set(date_time, value, unit)) + Self::from(helper::set(date_time, value, unit)) } pub fn is_same(&self, other: &Self, unit: Option) -> bool { @@ -179,8 +242,8 @@ mod helper { use crate::vm::date::{utc_now, Duration, DurationUnit, DynamicVariableExt}; use crate::Variable; use chrono::{ - DateTime, Datelike, Days, LocalResult, Month, Months, NaiveDate, NaiveDateTime, Offset, - TimeDelta, TimeZone, Timelike, + DateTime, Datelike, Days, FixedOffset, LocalResult, Month, Months, NaiveDate, + NaiveDateTime, Offset, TimeDelta, TimeZone, Timelike, }; use chrono_tz::Tz; use rust_decimal::prelude::ToPrimitive; @@ -207,6 +270,118 @@ mod helper { utc_now().with_timezone(&tz) } + const LENIENT: [&str; 2] = ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M"]; + + const SHAPED: [(&str, &str); 11] = [ + ("9999-99-99T99:99:99", "%Y-%m-%dT%H:%M:%S%.f"), + ("9999-99-99T99:99", "%Y-%m-%dT%H:%M"), + ("9999-99-99 99:99:99", "%Y-%m-%d %H:%M:%S%.f"), + ("99999999T999999", "%Y%m%dT%H%M%S%.f"), + ("99999999T9999", "%Y%m%dT%H%M"), + ("9999/99/99 99:99:99", "%Y/%m/%d %H:%M:%S%.f"), + ("9999/99/99 99:99", "%Y/%m/%d %H:%M"), + ("99999999", "%Y%m%d"), + ("9999/99/99", "%Y/%m/%d"), + ("9999-99", "%Y-%m"), + ("9999", "%Y"), + ]; + + fn shape(value: &str) -> String { + let seconds = value + .rfind('.') + .filter(|&dot| { + dot + 1 < value.len() && value[dot + 1..].bytes().all(|b| b.is_ascii_digit()) + }) + .map_or(value, |dot| &value[..dot]); + seconds + .chars() + .map(|c| if c.is_ascii_digit() { '9' } else { c }) + .collect() + } + + fn parse_shaped(value: &str) -> Option { + let shape = shape(value); + let (_, format) = SHAPED.iter().find(|(pattern, _)| *pattern == shape)?; + match *format { + "%Y" => NaiveDate::from_ymd_opt(value.parse().ok()?, 1, 1)?.and_hms_opt(0, 0, 0), + "%Y-%m" => NaiveDate::parse_from_str(&format!("{value}-01"), "%Y-%m-%d") + .ok()? + .and_hms_opt(0, 0, 0), + format if !format.contains("%H") => NaiveDate::parse_from_str(value, format) + .ok()? + .and_hms_opt(0, 0, 0), + format => NaiveDateTime::parse_from_str(value, format).ok(), + } + } + + fn split_offset(value: &str) -> Option<(&str, FixedOffset)> { + if let Some(local) = value.strip_suffix('Z') { + return Some((local, FixedOffset::east_opt(0)?)); + } + let time = value.find('T')?; + let sign_at = value[time..].rfind(['+', '-'])? + time; + let digits: String = value[sign_at + 1..].chars().filter(|c| *c != ':').collect(); + let valid = matches!(value.len() - sign_at - 1, 2 | 4 | 5) + && matches!(digits.len(), 2 | 4) + && digits.bytes().all(|b| b.is_ascii_digit()); + if !valid { + return None; + } + let hours: i32 = digits[..2].parse().ok()?; + let minutes: i32 = match &digits[2..] { + "" => 0, + minutes => minutes.parse().ok()?, + }; + let sign = if value.as_bytes()[sign_at] == b'-' { + -1 + } else { + 1 + }; + let seconds = sign * (hours * 3600 + minutes * 60); + Some((&value[..sign_at], FixedOffset::east_opt(seconds)?)) + } + + fn resolve_local(naive: NaiveDateTime, tz: Tz) -> Option> { + tz.from_local_datetime(&naive).earliest().or_else(|| { + let before = tz + .from_local_datetime(&naive.checked_sub_signed(TimeDelta::hours(3))?) + .earliest()?; + Some(tz.from_utc_datetime(&naive.checked_sub_offset(before.offset().fix())?)) + }) + } + + fn parse_text_in(value: &str, tz: Tz) -> Option> { + if let Ok(date_time) = DateTime::parse_from_rfc3339(value) { + return Some(date_time.with_timezone(&tz)); + } + if let Some(naive) = LENIENT + .iter() + .find_map(|format| NaiveDateTime::parse_from_str(value, format).ok()) + .or_else(|| { + NaiveDate::parse_from_str(value, "%Y-%m-%d") + .ok()? + .and_hms_opt(0, 0, 0) + }) + { + return resolve_local(naive, tz); + } + match split_offset(value) { + Some((local, offset)) if local.contains('T') => { + let naive = parse_shaped(local)?; + let utc = naive.checked_sub_offset(offset)?; + (utc.year().abs() <= 9999).then(|| tz.from_utc_datetime(&utc)) + } + _ => { + let naive = parse_shaped(value)?; + (naive.year().abs() <= 9999).then(|| resolve_local(naive, tz))? + } + } + } + + pub fn parse_text(value: &str) -> Option> { + parse_text_in(value, tz()) + } + pub fn parse_date(var: Variable, tz_opt: Option) -> Option> { let tz = tz_opt.unwrap_or_else(|| tz()); @@ -221,28 +396,16 @@ mod helper { Some(date_time) } - Variable::String(str) => DateTime::parse_from_rfc3339(str.deref()) - .ok() - .map(|date_time| tz.from_local_datetime(&date_time.naive_local()).earliest()) - .or_else(|| { - NaiveDateTime::parse_from_str(str.deref(), "%Y-%m-%d %H:%M:%S") - .ok() - .or_else(|| { - NaiveDateTime::parse_from_str(str.deref(), "%Y-%m-%d %H:%M").ok() - }) - .or_else(|| { - NaiveDate::parse_from_str(str.deref(), "%Y-%m-%d") - .ok()? - .and_hms_opt(0, 0, 0) - }) - .map(|dt| tz.from_local_datetime(&dt).earliest()) - }) - .or_else(|| Some(Tz::from_str(&str.deref()).ok().map(now_tz))) - .flatten(), - Variable::Dynamic(d) => match d.as_date() { - Some(d) => d.0.clone(), - None => None, - }, + Variable::String(str) => parse_text_in(str.deref(), tz) + .or_else(|| Tz::from_str(str.deref()).ok().map(now_tz)), + Variable::Dynamic(d) => { + let date = d.as_date()?; + match (tz_opt, &date.1) { + (Some(tz), Some(text)) => parse_text_in(text, tz), + (Some(tz), None) => date.0.map(|date_time| date_time.with_timezone(&tz)), + (None, _) => date.0, + } + } _ => None, } } diff --git a/core/expression/src/vm/vm.rs b/core/expression/src/vm/vm.rs index 6a0f2e15..22ed6e5a 100644 --- a/core/expression/src/vm/vm.rs +++ b/core/expression/src/vm/vm.rs @@ -9,6 +9,7 @@ use crate::vm::date::DynamicVariableExt; use crate::vm::error::VMError::*; use crate::vm::error::VMResult; use crate::vm::interval::{VmInterval, VmIntervalData}; +use crate::vm::VmDate; use rust_decimal::prelude::{FromPrimitive, ToPrimitive}; use rust_decimal::{Decimal, MathematicalOps}; use std::rc::Rc; @@ -104,8 +105,8 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'parent_ref, 'bytecode_ref> { self.pop()?; } Opcode::Fetch => { - let b = self.pop()?; - let a = self.pop()?; + let b = VmDate::textual(self.pop()?); + let a = VmDate::textual(self.pop()?); match (a, b) { (Object(o), String(s)) => { @@ -163,7 +164,7 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'parent_ref, 'bytecode_ref> { } _ => Null, }, - FetchFastTarget::Number(num) => match v { + FetchFastTarget::Number(num) => match VmDate::textual(v) { Array(arr) => { let arr_ref = arr.borrow(); arr_ref.get(*num as usize).cloned().unwrap_or(Null) @@ -250,6 +251,10 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'parent_ref, 'bytecode_ref> { self.push(Bool(a.is_some() && b.is_some() && a == b)); } + (Dynamic(a), String(b)) | (String(b), Dynamic(a)) => { + let equal = a.as_date().is_some_and(|a| a.matches(&String(b))); + self.push(Bool(equal)); + } _ => { self.push(Bool(false)); } @@ -323,7 +328,10 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'parent_ref, 'bytecode_ref> { }, Opcode::In => { let b = self.pop()?; - let a = self.pop()?; + let a = match &b { + Object(_) => VmDate::textual(self.pop()?), + _ => self.pop()?, + }; match (a, &b) { (Number(a), Array(b)) => { @@ -381,10 +389,7 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'parent_ref, 'bytecode_ref> { }; let arr = arr.borrow(); - let is_in = arr.iter().any(|b| match b { - Dynamic(b) => Some(a) == b.as_date(), - _ => false, - }); + let is_in = arr.iter().any(|b| a.matches(b)); self.push(Bool(is_in)); } @@ -392,6 +397,9 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'parent_ref, 'bytecode_ref> { let arr = b.borrow(); let is_in = arr.iter().any(|b| match b { String(b) => &a == b, + Dynamic(d) => { + d.as_date().is_some_and(|d| d.matches(&String(a.clone()))) + } _ => false, }); @@ -455,6 +463,20 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'parent_ref, 'bytecode_ref> { self.push(Bool(compare(a, b, comparison))); } + (a @ Dynamic(_), b @ String(_)) | (a @ String(_), b @ Dynamic(_)) => { + let valid = |date: &VmDate| date.is_valid(); + let (Some(a), Some(b)) = ( + VmDate::coerce(&a).filter(valid), + VmDate::coerce(&b).filter(valid), + ) else { + return Err(OpcodeErr { + opcode: "Compare".into(), + message: "Unsupported type".into(), + }); + }; + + self.push(Bool(compare(&a, &b, comparison))); + } _ => { return Err(OpcodeErr { opcode: "Compare".into(), @@ -464,8 +486,8 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'parent_ref, 'bytecode_ref> { } } Opcode::Add => { - let b = self.pop()?; - let a = self.pop()?; + let b = VmDate::textual(self.pop()?); + let a = VmDate::textual(self.pop()?); match (a, b) { (Number(a), Number(b)) => { @@ -681,7 +703,7 @@ impl<'arena, 'parent_ref, 'bytecode_ref> VMInner<'parent_ref, 'bytecode_ref> { Opcode::Slice => { let from_var = self.pop()?; let to_var = self.pop()?; - let current = self.pop()?; + let current = VmDate::textual(self.pop()?); match (from_var, to_var) { (Number(f), Number(t)) => { diff --git a/core/expression/tests/data/date.csv b/core/expression/tests/data/date.csv index d61b052b..8ec27122 100644 --- a/core/expression/tests/data/date.csv +++ b/core/expression/tests/data/date.csv @@ -9,6 +9,63 @@ d('2023-10-15 14:30', 'Europe/Berlin');;'2023-10-15T14:30:00+02:00' d('2023-10-15 14:30:45', 'Europe/Berlin');;'2023-10-15T14:30:45+02:00' d('Europe/Berlin').isValid() and d('Europe/Berlin').isToday();;true +# ISO 8601 forms +d('2023-10-15T14:30');;'2023-10-15T14:30:00Z' +d('2023-10-15T14:30:45');;'2023-10-15T14:30:45Z' +d('2023-10-15T14:30:45.123').format('%Y-%m-%d %H:%M:%S%.3f');;'2023-10-15 14:30:45.123' +d('2023-10-15 14:30:45.5').format('%Y-%m-%d %H:%M:%S%.3f');;'2023-10-15 14:30:45.500' +d('2023-10-15T14:30Z');;'2023-10-15T14:30:00Z' +d('20231015T143045Z');;'2023-10-15T14:30:45Z' +d('20231015T1430');;'2023-10-15T14:30:00Z' +d('20231015');;'2023-10-15T00:00:00Z' +d('2023/10/15');;'2023-10-15T00:00:00Z' +d('2023/10/15 14:30');;'2023-10-15T14:30:00Z' +d('2023-10');;'2023-10-01T00:00:00Z' +d('2023');;'2023-01-01T00:00:00Z' +d('2023-10-15T14:30', 'Europe/Berlin');;'2023-10-15T14:30:00+02:00' +d('2023-13').isValid();;false +d('2023-02-30').isValid();;false +d('12345').isValid();;false +d('202310').isValid();;false +d('hello').isValid();;false +d('2023105').isValid();;false +d('2023011').isValid();;false +d('2023101T1430').isValid();;false +d('2023 1015').isValid();;false +d('+2023-1').isValid();;false +d('2023/1/5 1:2').isValid();;false +d('2023-10-15T14:30:45 Z').isValid();;false +d('-262143-01-01T00:00:00Z').isValid();;false +d('-262143-01-01', 'Asia/Tokyo').isValid();;false + +# Offsets keep the instant +d('2023-10-15T14:30+0200');;'2023-10-15T12:30:00Z' +d('2023-10-15T14:30:45-0530');;'2023-10-15T20:00:45Z' +d('2023-10-15T14:30:45+02');;'2023-10-15T12:30:45Z' +d('2023-10-15T14:30:00+05:00');;'2023-10-15T09:30:00Z' +d('2023-10-15T14:30:00+05:00').hour();;9 +d('2023-10-15T14:30:00+05:00', 'Europe/Berlin');;'2023-10-15T11:30:00+02:00' +d(d('2023-10-15T12:00:00Z'), 'Europe/Berlin');;'2023-10-15T14:00:00+02:00' +d('2023-10-15T14:30:00+05:00') == d('2023-10-15T09:30:00Z');;true + +# Local times inside a DST gap move forward by the gap +d('2024-03-31 02:30', 'Europe/Berlin');;'2024-03-31T03:30:00+02:00' +d('2024-03-31T02:30:00Z', 'Europe/Berlin');;'2024-03-31T04:30:00+02:00' + +# Dates compared with strings +d('2023-10-15') == '2023-10-15';;true +'2023-10-15' == d('2023-10-15');;true +d('2023-10-15') == '2023-10-16';;false +d('2023-10-15') != '2023-10-15';;false +d('2023-10-15') > '2023-10-14';;true +'2023-10-14' < d('2023-10-15');;true +d('2023-10-15') in ['2023-10-14', '2023-10-15'];;true +'2023-10-15' in [d('2023-10-15')];;true +contains([d('2023-10-15')], '2023-10-15');;true +string(d('2023-10-15'));;'2023-10-15T00:00:00Z' +'on ' + d('2023-10-15');;'on 2023-10-15T00:00:00Z' +year(d('2023-10-15'));;2023 + # Date manipulation d('2023-10-15').add('1d');;'2023-10-16T00:00:00Z' d('2023-10-15').add('1d 5h');;'2023-10-16T05:00:00Z' diff --git a/core/types/src/variable/mod.rs b/core/types/src/variable/mod.rs index e03f6112..1c2cff6a 100644 --- a/core/types/src/variable/mod.rs +++ b/core/types/src/variable/mod.rs @@ -50,6 +50,10 @@ pub trait DynamicVariable: Display { fn as_any(&self) -> &dyn Any; fn to_value(&self) -> Value; + + fn as_text(&self) -> Option<&str> { + None + } } impl Variable { @@ -104,6 +108,7 @@ impl Variable { pub fn as_str(&self) -> Option<&str> { match self { Variable::String(s) => Some(s.as_ref()), + Variable::Dynamic(d) => d.as_text(), _ => None, } } @@ -111,6 +116,7 @@ impl Variable { pub fn as_rc_str(&self) -> Option> { match self { Variable::String(s) => Some(Rc::from(s.as_str())), + Variable::Dynamic(d) => d.as_text().map(Rc::from), _ => None, } } diff --git a/core/types/src/variable_type/util.rs b/core/types/src/variable_type/util.rs index 90af41bc..a9c9eb01 100644 --- a/core/types/src/variable_type/util.rs +++ b/core/types/src/variable_type/util.rs @@ -68,6 +68,7 @@ impl VariableType { (VariableType::Number, VariableType::Number) => true, (VariableType::Date, VariableType::Date) => true, (VariableType::Number, VariableType::Date) => true, + (VariableType::Date, VariableType::String) => true, (_, VariableType::Date) if self.widen().is_string() => true, (VariableType::Interval, VariableType::Interval) => true, (VariableType::Array(a1), VariableType::Array(a2)) => { @@ -276,6 +277,14 @@ impl VariableType { } (VariableType::Enum(_, _), VariableType::String) | (VariableType::String, VariableType::Enum(_, _)) => VariableType::String, + ( + VariableType::Date, + VariableType::String | VariableType::Const(_) | VariableType::Enum(_, _), + ) + | ( + VariableType::String | VariableType::Const(_) | VariableType::Enum(_, _), + VariableType::Date, + ) => VariableType::String, (_, _) => VariableType::Any, };