1use std::sync::{Arc, LazyLock};
16
17use arrow_schema::Field;
18use common_function::scalars::json::json_get::JsonGetWithType;
19use common_function::scalars::udf::create_udf;
20use datafusion_common::arrow::datatypes::DataType;
21use datafusion_common::{Column, DFSchema, DataFusionError, Result, ScalarValue, TableReference};
22use datafusion_expr::expr::{BinaryExpr, ScalarFunction};
23use datafusion_expr::planner::{
24 ExprPlanner, PlannerResult, RawAggregateExpr, RawBinaryExpr, RawFieldAccessExpr, RawScalarExpr,
25 RawWindowExpr,
26};
27use datafusion_expr::type_coercion::functions::{UDFCoercionExt, fields_with_udf};
28use datafusion_expr::{
29 Expr, ExprSchemable, GetFieldAccess, Operator, ScalarUDF, WindowFunctionDefinition,
30};
31use datatypes::extension::json::is_json2_extension_type;
32use sqlparser::ast::BinaryOperator;
33
34#[derive(Debug)]
48pub(crate) struct JsonExprPlanner;
49
50impl ExprPlanner for JsonExprPlanner {
51 fn plan_binary_op(
52 &self,
53 expr: RawBinaryExpr,
54 schema: &DFSchema,
55 ) -> Result<PlannerResult<RawBinaryExpr>> {
56 let RawBinaryExpr {
57 op,
58 mut left,
59 mut right,
60 } = expr;
61
62 if !is_untyped_json_get(&left) && !is_untyped_json_get(&right) {
63 return Ok(PlannerResult::Original(RawBinaryExpr { op, left, right }));
64 }
65
66 let Some(expr_op) = parse_sql_op(&op) else {
67 return Ok(PlannerResult::Original(RawBinaryExpr { op, left, right }));
68 };
69
70 let left_type = left.get_type(schema)?;
71 let right_type = right.get_type(schema)?;
72 let left_changed = push_json_get_type_arg(&mut left, &right_type)?;
73 let right_changed = push_json_get_type_arg(&mut right, &left_type)?;
74 if left_changed || right_changed {
75 Ok(PlannerResult::Planned(Expr::BinaryExpr(BinaryExpr::new(
76 Box::new(left),
77 expr_op,
78 Box::new(right),
79 ))))
80 } else {
81 Ok(PlannerResult::Original(RawBinaryExpr { op, left, right }))
82 }
83 }
84
85 fn plan_field_access(
92 &self,
93 mut expr: RawFieldAccessExpr,
94 _schema: &DFSchema,
95 ) -> Result<PlannerResult<RawFieldAccessExpr>> {
96 let suffix = match &expr.field_access {
99 GetFieldAccess::ListIndex { key } => {
100 let Expr::Literal(ScalarValue::Int64(Some(index)), _) = key.as_ref() else {
102 return Ok(PlannerResult::Original(expr));
103 };
104 format!("[{index}]")
105 }
106 GetFieldAccess::NamedStructField { name } => {
107 let Some(name) = name.try_as_str().flatten() else {
108 return Ok(PlannerResult::Original(expr));
109 };
110 format!(".{}", json_path_field(name)?)
111 }
112 GetFieldAccess::ListRange { .. } => return Ok(PlannerResult::Original(expr)),
113 };
114 let Some(json_get) = extract_untyped_json_get(&mut expr.expr) else {
115 return Ok(PlannerResult::Original(expr));
116 };
117 let Some(Expr::Literal(ScalarValue::Utf8(Some(path)), _)) = json_get.args.get_mut(1) else {
118 return Ok(PlannerResult::Original(expr));
119 };
120
121 path.push_str(&suffix);
122 Ok(PlannerResult::Planned(expr.expr))
123 }
124
125 fn plan_compound_identifier(
126 &self,
127 field: &Field,
128 qualifier: Option<&TableReference>,
129 nested_names: &[String],
130 ) -> Result<PlannerResult<Vec<Expr>>> {
131 if !is_json2_extension_type(field) {
132 return Ok(PlannerResult::Original(Vec::new()));
133 }
134
135 static JSON_GET_UDF: LazyLock<Arc<ScalarUDF>> =
136 LazyLock::new(|| Arc::new(create_udf(Arc::new(JsonGetWithType::default()))));
137
138 let json_get = JSON_GET_UDF.clone();
139 let mut path = "$".to_string();
140 for name in nested_names {
141 path.push('.');
142 path.push_str(&json_path_field(name)?);
143 }
144
145 Ok(PlannerResult::Planned(Expr::ScalarFunction(
146 ScalarFunction::new_udf(
147 json_get,
148 vec![
149 Expr::Column(Column::from((qualifier, field))),
150 Expr::Literal(ScalarValue::Utf8(Some(path)), None),
151 ],
152 ),
153 )))
154 }
155
156 fn plan_scalar(&self, mut expr: RawScalarExpr) -> Result<PlannerResult<RawScalarExpr>> {
161 push_function_arg_types(expr.func.as_ref(), &mut expr.args)?;
162 Ok(PlannerResult::Original(expr))
163 }
164
165 fn plan_aggregate(
167 &self,
168 mut expr: RawAggregateExpr,
169 ) -> Result<PlannerResult<RawAggregateExpr>> {
170 push_function_arg_types(expr.func.as_ref(), &mut expr.args)?;
171 Ok(PlannerResult::Original(expr))
172 }
173
174 fn plan_window(&self, mut expr: RawWindowExpr) -> Result<PlannerResult<RawWindowExpr>> {
176 match &expr.func_def {
177 WindowFunctionDefinition::AggregateUDF(func) => {
178 push_function_arg_types(func.as_ref(), &mut expr.args)?;
179 }
180 WindowFunctionDefinition::WindowUDF(func) => {
181 push_function_arg_types(func.as_ref(), &mut expr.args)?;
182 }
183 }
184 Ok(PlannerResult::Original(expr))
185 }
186}
187
188fn json_path_field(name: &str) -> Result<String> {
190 if !name.is_empty() && name.chars().all(|c| c.is_alphanumeric() || c == '_') {
191 Ok(name.to_string())
192 } else {
193 serde_json::to_string(name).map_err(|e| DataFusionError::External(Box::new(e)))
195 }
196}
197
198enum JsonGetTypeResolution {
199 Fallback,
200 Typed(Vec<(usize, DataType)>),
201}
202
203fn push_function_arg_types<F>(func: &F, args: &mut [Expr]) -> Result<()>
217where
218 F: UDFCoercionExt,
219{
220 if !args.iter().any(is_untyped_json_get) {
221 return Ok(());
222 }
223
224 let fields = args.iter().map(function_arg_field).collect::<Vec<_>>();
225 match infer_json_get_types(func, args, &fields) {
226 JsonGetTypeResolution::Fallback => {
227 let Some(data_type) = fallback_json_get_type(func, args, &fields) else {
228 return Ok(());
229 };
230 for arg in args.iter_mut() {
231 if is_untyped_json_get(arg) {
232 let _ = push_json_get_type_arg(arg, &data_type)?;
233 }
234 }
235 }
236 JsonGetTypeResolution::Typed(types) => {
237 for (index, data_type) in types {
238 let _ = push_json_get_type_arg(&mut args[index], &data_type)?;
239 }
240 }
241 }
242 Ok(())
243}
244
245fn infer_json_get_types<F>(func: &F, args: &[Expr], fields: &[Arc<Field>]) -> JsonGetTypeResolution
246where
247 F: UDFCoercionExt,
248{
249 let Ok(coerced) = fields_with_udf(fields, func) else {
258 return JsonGetTypeResolution::Fallback;
259 };
260
261 let mut inferred_types = Vec::with_capacity(coerced.len());
262 for (index, (arg, field)) in args.iter().zip(coerced).enumerate() {
263 if !is_untyped_json_get(arg) || field.data_type().is_null() {
264 continue;
265 }
266 let Some(data_type) = json_get_output_type(field.data_type()) else {
267 return JsonGetTypeResolution::Fallback;
268 };
269 inferred_types.push((index, data_type));
270 }
271 if inferred_types.is_empty() {
272 JsonGetTypeResolution::Fallback
273 } else {
274 JsonGetTypeResolution::Typed(inferred_types)
275 }
276}
277
278fn fallback_json_get_type<F>(func: &F, args: &[Expr], fields: &[Arc<Field>]) -> Option<DataType>
279where
280 F: UDFCoercionExt,
281{
282 let mut candidate_fields = fields.to_vec();
285 for data_type in [
286 DataType::Utf8View,
287 DataType::Float64,
288 DataType::Int64,
289 DataType::Boolean,
290 ] {
291 for (index, arg) in args.iter().enumerate() {
292 if is_untyped_json_get(arg) {
293 candidate_fields[index] = Arc::new(
294 fields[index]
295 .as_ref()
296 .clone()
297 .with_data_type(data_type.clone()),
298 );
299 }
300 }
301 if fields_with_udf(&candidate_fields, func).is_ok() {
302 return Some(data_type);
303 }
304 }
305 None
306}
307
308fn function_arg_field(expr: &Expr) -> Arc<Field> {
309 let data_type = if is_untyped_json_get(expr) {
310 DataType::Null
311 } else if let Some(data_type) = extract_json_get_type(expr) {
312 data_type
313 } else {
314 expr.get_type(&DFSchema::empty()).unwrap_or(DataType::Null)
320 };
321 Arc::new(Field::new("", data_type, true))
322}
323
324fn json_get_output_type(data_type: &DataType) -> Option<DataType> {
325 let output_type = match data_type {
326 DataType::Boolean => DataType::Boolean,
327 data_type if data_type.is_integer() => DataType::Int64,
328 data_type if data_type.is_floating() => DataType::Float64,
329 DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => DataType::Float64,
330 data_type if data_type.is_string() => DataType::Utf8View,
331 _ => return None,
332 };
333 Some(output_type)
334}
335
336macro_rules! is_untyped_json_get_func {
337 ($func:expr) => {
338 $func
339 .func
340 .name()
341 .eq_ignore_ascii_case(JsonGetWithType::NAME)
342 && $func.args.len() == 2
343 };
344}
345
346macro_rules! is_typed_json_get_func {
347 ($func:expr) => {
348 $func
349 .func
350 .name()
351 .eq_ignore_ascii_case(JsonGetWithType::NAME)
352 && $func.args.len() == 3
353 };
354}
355
356fn extract_untyped_json_get(expr: &mut Expr) -> Option<&mut ScalarFunction> {
357 match expr {
358 Expr::ScalarFunction(f) if is_untyped_json_get_func!(f) => Some(f),
359 _ => None,
360 }
361}
362
363fn extract_json_get_type(expr: &Expr) -> Option<DataType> {
364 match expr {
365 Expr::ScalarFunction(f) if is_typed_json_get_func!(f) => f
366 .args
367 .get(2)
368 .and_then(|x| x.as_literal())
369 .map(|x| x.data_type()),
370 _ => None,
371 }
372}
373
374fn is_untyped_json_get(expr: &Expr) -> bool {
375 matches!(
376 expr,
377 Expr::ScalarFunction(f) if is_untyped_json_get_func!(f)
378 )
379}
380
381fn push_json_get_type_arg(expr: &mut Expr, data_type: &DataType) -> Result<bool> {
382 let Some(json_get) = extract_untyped_json_get(expr) else {
383 return Ok(false);
384 };
385
386 if data_type.is_string() {
389 return Ok(false);
390 }
391 let with_type = ScalarValue::try_new_null(data_type).map(|x| Expr::Literal(x, None))?;
392 json_get.args.push(with_type);
393 Ok(true)
394}
395
396fn parse_sql_op(op: &BinaryOperator) -> Option<Operator> {
397 match *op {
398 BinaryOperator::Plus => Some(Operator::Plus),
399 BinaryOperator::Minus => Some(Operator::Minus),
400 BinaryOperator::Multiply => Some(Operator::Multiply),
401 BinaryOperator::Divide => Some(Operator::Divide),
402 BinaryOperator::Modulo => Some(Operator::Modulo),
403 BinaryOperator::Gt => Some(Operator::Gt),
404 BinaryOperator::GtEq => Some(Operator::GtEq),
405 BinaryOperator::Lt => Some(Operator::Lt),
406 BinaryOperator::LtEq => Some(Operator::LtEq),
407 BinaryOperator::Eq => Some(Operator::Eq),
408 BinaryOperator::NotEq => Some(Operator::NotEq),
409 BinaryOperator::And => Some(Operator::And),
410 BinaryOperator::Or => Some(Operator::Or),
411 BinaryOperator::BitwiseAnd => Some(Operator::BitwiseAnd),
412 BinaryOperator::BitwiseOr => Some(Operator::BitwiseOr),
413 BinaryOperator::BitwiseXor => Some(Operator::BitwiseXor),
414 _ => None,
415 }
416}
417
418#[cfg(test)]
419mod tests {
420 use arrow_schema::Fields;
421 use datafusion::functions_aggregate::count::count_udaf;
422 use datafusion::functions_aggregate::sum::sum_udaf;
423 use datafusion_expr::WindowFrame;
424 use datafusion_functions::core::coalesce;
425 use datafusion_functions::math::{abs, power};
426 use datatypes::extension::json::Json2ExtensionType;
427
428 use super::*;
429
430 fn json_get_expr(base: Expr, path: &str) -> Expr {
431 let json_get = Arc::new(create_udf(Arc::new(JsonGetWithType::default())));
432 Expr::ScalarFunction(ScalarFunction::new_udf(
433 json_get,
434 vec![
435 base,
436 Expr::Literal(ScalarValue::Utf8(Some(path.to_string())), None),
437 ],
438 ))
439 }
440
441 #[test]
442 fn test_plan_binary_op() -> Result<()> {
443 let planner = JsonExprPlanner;
444 let schema = DFSchema::from_unqualified_fields(
445 Fields::from(vec![Field::new("value", DataType::Int64, true)]),
446 Default::default(),
447 )?;
448
449 let planned = planner.plan_binary_op(
450 RawBinaryExpr {
451 op: BinaryOperator::Eq,
452 left: json_get_expr(
453 Expr::Literal(ScalarValue::Binary(Some(b"{\"a\": 1}".to_vec())), None),
454 "a",
455 ),
456 right: Expr::Column(Column::new_unqualified("value")),
457 },
458 &schema,
459 )?;
460
461 match planned {
462 PlannerResult::Planned(Expr::BinaryExpr(expr)) => {
463 assert_eq!(expr.op, Operator::Eq);
464
465 match expr.left.as_ref() {
466 Expr::ScalarFunction(func) => {
467 assert_eq!(func.func.name(), JsonGetWithType::NAME);
468 assert_eq!(func.args.len(), 3);
469 assert_eq!(func.args[2], Expr::Literal(ScalarValue::Int64(None), None));
470 }
471 other => panic!("expected json_get on left side, got {other:?}"),
472 }
473
474 assert_eq!(
475 expr.right.as_ref(),
476 &Expr::Column(Column::new_unqualified("value"))
477 );
478 }
479 other => panic!("expected planned binary expression, got {other:?}"),
480 }
481
482 let original = planner.plan_binary_op(
483 RawBinaryExpr {
484 op: BinaryOperator::StringConcat,
485 left: Expr::Column(Column::new_unqualified("value")),
486 right: Expr::Literal(ScalarValue::Utf8(Some("x".to_string())), None),
487 },
488 &schema,
489 )?;
490
491 match original {
492 PlannerResult::Original(expr) => {
493 assert!(matches!(expr.op, BinaryOperator::StringConcat));
494 assert_eq!(expr.left, Expr::Column(Column::new_unqualified("value")));
495 assert_eq!(
496 expr.right,
497 Expr::Literal(ScalarValue::Utf8(Some("x".to_string())), None)
498 );
499 }
500 other => panic!(
501 "expected original expression for unsupported operator, got {:?}",
502 other,
503 ),
504 }
505
506 Ok(())
507 }
508
509 #[test]
510 fn test_plan_list_index() -> Result<()> {
511 let planner = JsonExprPlanner;
512 let planned = planner.plan_field_access(
513 RawFieldAccessExpr {
514 field_access: GetFieldAccess::ListIndex {
515 key: Box::new(Expr::Literal(ScalarValue::Int64(Some(0)), None)),
516 },
517 expr: json_get_expr(Expr::Column(Column::new_unqualified("j")), "list"),
518 },
519 &DFSchema::empty(),
520 )?;
521 let PlannerResult::Planned(Expr::ScalarFunction(func)) = planned else {
522 unreachable!()
523 };
524 assert_eq!(func.func.name(), JsonGetWithType::NAME);
525 assert_eq!(func.args.len(), 2);
526 assert_eq!(
527 func.args[1],
528 Expr::Literal(ScalarValue::Utf8(Some("list[0]".to_string())), None)
529 );
530 Ok(())
531 }
532
533 #[test]
534 fn test_plan_field_after_list_index() -> Result<()> {
535 let planner = JsonExprPlanner;
536 let planned = planner.plan_field_access(
537 RawFieldAccessExpr {
538 field_access: GetFieldAccess::NamedStructField {
539 name: ScalarValue::Utf8(Some("a.b".to_string())),
540 },
541 expr: json_get_expr(Expr::Column(Column::new_unqualified("j")), "list[0]"),
542 },
543 &DFSchema::empty(),
544 )?;
545 let PlannerResult::Planned(Expr::ScalarFunction(func)) = planned else {
546 unreachable!()
547 };
548 assert_eq!(
549 func.args[1],
550 Expr::Literal(
551 ScalarValue::Utf8(Some(r#"list[0]."a.b""#.to_string())),
552 None
553 )
554 );
555 Ok(())
556 }
557
558 #[test]
559 fn test_plan_compound_identifier() -> Result<()> {
560 let planner = JsonExprPlanner;
561 let qualifier = TableReference::bare("events");
562 let nested_names = vec!["payload".to_string(), "cpu".to_string()];
563
564 let planned = planner.plan_compound_identifier(
565 &Field::new("labels", DataType::Struct(Fields::empty()), true)
566 .with_extension_type(Json2ExtensionType::default()),
567 Some(&qualifier),
568 &nested_names,
569 )?;
570
571 match planned {
572 PlannerResult::Planned(Expr::ScalarFunction(func)) => {
573 assert_eq!(func.func.name(), JsonGetWithType::NAME);
574 assert_eq!(func.args.len(), 2);
575 assert_eq!(
576 func.args[0],
577 Expr::Column(Column::new(Some(qualifier.clone()), "labels"))
578 );
579 assert_eq!(
580 func.args[1],
581 Expr::Literal(ScalarValue::Utf8(Some("$.payload.cpu".to_string())), None)
582 );
583 }
584 other => panic!("expected json_get scalar function, got {other:?}"),
585 }
586
587 for (key, path) in [
588 ("http.status_code", r#"$."http.status_code""#),
589 ("a\"b", r#"$."a\"b""#),
590 ("a\\b", r#"$."a\\b""#),
591 ] {
592 let planned = planner.plan_compound_identifier(
593 &Field::new("labels", DataType::Struct(Fields::empty()), true)
594 .with_extension_type(Json2ExtensionType::default()),
595 Some(&qualifier),
596 &[key.to_string()],
597 )?;
598 let PlannerResult::Planned(Expr::ScalarFunction(func)) = planned else {
599 unreachable!()
600 };
601 assert_eq!(
602 func.args[1],
603 Expr::Literal(ScalarValue::Utf8(Some(path.to_string())), None)
604 );
605 }
606
607 let PlannerResult::Planned(Expr::ScalarFunction(func)) = planner.plan_compound_identifier(
608 &Field::new("labels", DataType::Struct(Fields::empty()), true)
609 .with_extension_type(Json2ExtensionType::default()),
610 Some(&qualifier),
611 &["resource".to_string(), "http.status_code".to_string()],
612 )?
613 else {
614 unreachable!()
615 };
616 assert_eq!(
617 func.args[1],
618 Expr::Literal(
619 ScalarValue::Utf8(Some(r#"$.resource."http.status_code""#.to_string())),
620 None
621 )
622 );
623
624 let original = planner.plan_compound_identifier(
625 &Field::new("plain", DataType::Utf8, true),
626 Some(&qualifier),
627 &nested_names,
628 )?;
629
630 match original {
631 PlannerResult::Original(exprs) => assert!(exprs.is_empty()),
632 other => panic!(
633 "expected original empty result for non-json field, got {:?}",
634 other,
635 ),
636 }
637
638 Ok(())
639 }
640
641 #[test]
642 fn test_plan_functions() -> Result<()> {
643 let planner = JsonExprPlanner;
644 let json_get = || json_get_expr(Expr::Column(Column::new_unqualified("j")), "a.b");
645
646 let PlannerResult::Original(scalar) = planner.plan_scalar(RawScalarExpr {
647 func: abs(),
648 args: vec![json_get()],
649 })?
650 else {
651 unreachable!();
652 };
653 assert_eq!(
654 Some(DataType::Float64),
655 extract_json_get_type(&scalar.args[0])
656 );
657
658 let PlannerResult::Original(scalar) = planner.plan_scalar(RawScalarExpr {
659 func: power(),
660 args: vec![
661 json_get(),
662 Expr::Column(Column::new_unqualified("exponent")),
663 ],
664 })?
665 else {
666 unreachable!();
667 };
668 assert_eq!(
669 Some(DataType::Float64),
670 extract_json_get_type(&scalar.args[0])
671 );
672
673 let PlannerResult::Original(aggregate) = planner.plan_aggregate(RawAggregateExpr {
674 func: sum_udaf(),
675 args: vec![json_get()],
676 distinct: false,
677 filter: None,
678 order_by: vec![],
679 null_treatment: None,
680 })?
681 else {
682 unreachable!();
683 };
684 assert_eq!(
685 Some(DataType::Float64),
686 extract_json_get_type(&aggregate.args[0])
687 );
688
689 let PlannerResult::Original(count) = planner.plan_aggregate(RawAggregateExpr {
690 func: count_udaf(),
691 args: vec![json_get()],
692 distinct: false,
693 filter: None,
694 order_by: vec![],
695 null_treatment: None,
696 })?
697 else {
698 unreachable!();
699 };
700 assert_eq!(None, extract_json_get_type(&count.args[0]));
701
702 let PlannerResult::Original(window) = planner.plan_window(RawWindowExpr {
703 func_def: WindowFunctionDefinition::AggregateUDF(sum_udaf()),
704 args: vec![json_get()],
705 partition_by: vec![],
706 order_by: vec![],
707 window_frame: WindowFrame::new(None),
708 filter: None,
709 null_treatment: None,
710 distinct: false,
711 })?
712 else {
713 unreachable!();
714 };
715 assert_eq!(
716 Some(DataType::Float64),
717 extract_json_get_type(&window.args[0])
718 );
719 Ok(())
720 }
721
722 #[test]
723 fn test_plan_function_with_mixed_json_get_types() -> Result<()> {
724 let planner = JsonExprPlanner;
725 let json_get = || json_get_expr(Expr::Column(Column::new_unqualified("j")), "a.b");
726 let mut typed = json_get();
727 push_json_get_type_arg(&mut typed, &DataType::Float64)?;
728
729 let PlannerResult::Original(scalar) = planner.plan_scalar(RawScalarExpr {
730 func: coalesce(),
731 args: vec![json_get(), typed],
732 })?
733 else {
734 unreachable!();
735 };
736 assert_eq!(
737 Some(DataType::Float64),
738 extract_json_get_type(&scalar.args[0])
739 );
740 assert_eq!(
741 Some(DataType::Float64),
742 extract_json_get_type(&scalar.args[1])
743 );
744 Ok(())
745 }
746}