datatypes/
json.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Data conversion between greptime's StructType and Json
16//!
17//! The idea of this module is to provide utilities to convert serde_json::Value to greptime's StructType and vice versa.
18//!
19//! The struct will carry all the fields of the Json object. We will not flatten any json object in this implementation.
20//!
21
22pub mod value;
23
24use std::collections::BTreeMap;
25use std::collections::btree_map::Entry;
26
27use serde::{Deserialize, Serialize};
28use serde_json::{Map, Value as Json};
29use snafu::ResultExt;
30
31use crate::data_type::ConcreteDataType;
32use crate::error::{self, Result, UnsupportedJsonTypeSnafu};
33use crate::json::value::{JsonValue, JsonVariant};
34use crate::schema::ColumnDefaultConstraint;
35use crate::types::json_type::JsonNativeType;
36use crate::value::{ListValue, StructValue, Value};
37
38/// JSON2 settings stored in column schema metadata and represented through
39/// Arrow extension metadata.
40#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
41pub struct JsonSettings {
42    #[serde(default)]
43    pub type_hints: Vec<JsonTypeHint>,
44}
45
46/// Declares selected JSON2 subpaths as typed fields.
47///
48/// These hints let JSON2 encode frequently used subpaths in a typed layout, so
49/// queries over those subpaths can get behavior and performance closer to
50/// ordinary columns.
51#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
52pub struct JsonTypeHint {
53    /// JSON2 subpath for a typed field.
54    ///
55    /// Each item is one JSON object key. For example, `["user", "age"]`
56    /// represents `user.age`.
57    ///
58    /// Array traversal is not currently supported. For example, a hint cannot
59    /// describe `events[0].name` or fields shared by all items in `events[*]`.
60    pub path: Vec<String>,
61    #[serde(rename = "type")]
62    pub data_type: ConcreteDataType,
63    pub nullable: bool,
64    #[serde(skip_serializing_if = "Option::is_none")]
65    pub default_constraint: Option<ColumnDefaultConstraint>,
66    pub inverted_index: bool,
67}
68
69/// Context for JSON encoding/decoding that tracks the current key path.
70#[derive(Clone, Debug)]
71pub struct JsonContext<'a> {
72    /// Current key path from the JSON2 root.
73    pub path: Vec<String>,
74    /// Settings for JSON encoding/decoding.
75    pub settings: &'a JsonSettings,
76}
77
78impl JsonSettings {
79    pub fn new(type_hints: Vec<JsonTypeHint>) -> Self {
80        Self { type_hints }
81    }
82
83    /// Decode an encoded StructValue back into a serde_json::Value.
84    pub fn decode(&self, value: Value) -> Result<Json> {
85        let mut context = JsonContext {
86            path: Vec::new(),
87            settings: self,
88        };
89        decode_value_with_context(value, &mut context)
90    }
91
92    /// Encode a serde_json::Value into a Value::Json using current settings.
93    pub fn encode(&self, json: Json) -> Result<Value> {
94        let mut context = JsonContext {
95            path: Vec::new(),
96            settings: self,
97        };
98        encode_json_with_context(json, &mut context).map(|v| Value::Json(Box::new(v)))
99    }
100}
101
102impl<'a> JsonContext<'a> {
103    fn type_hint(&self) -> Option<&'a JsonTypeHint> {
104        self.settings
105            .type_hints
106            .iter()
107            .find(|hint| hint.path == self.path)
108    }
109}
110
111fn with_key_context<T>(
112    context: &mut JsonContext,
113    key: &str,
114    f: impl FnOnce(&mut JsonContext) -> Result<T>,
115) -> Result<T> {
116    context.path.push(key.to_string());
117    let result = f(context);
118    context.path.pop();
119    result
120}
121
122/// Main encoding function with key path tracking
123fn encode_json_with_context(json: Json, context: &mut JsonContext) -> Result<JsonValue> {
124    if context.path.is_empty() && !matches!(json, Json::Object(_)) {
125        return UnsupportedJsonTypeSnafu.fail();
126    }
127
128    match json {
129        Json::Object(json_object) => encode_json_object_with_context(json_object, context),
130        Json::Array(json_array) => encode_json_array_with_context(json_array, context),
131        _ => encode_json_value_with_context(json, context),
132    }
133}
134
135fn encode_json_object_with_context<'a>(
136    json_object: Map<String, Json>,
137    context: &mut JsonContext<'a>,
138) -> Result<JsonValue> {
139    let mut object = BTreeMap::new();
140    for (key, value) in json_object {
141        let value = with_key_context(context, &key, |context| {
142            if let Some(hint) = context.type_hint() {
143                encode_json_value_with_hint(value, hint, context)
144            } else {
145                encode_json_value_with_context(value, context)
146            }
147        })?;
148
149        object.insert(key, value.into_variant());
150    }
151
152    apply_missing_type_hints(&mut object, context)?;
153
154    Ok(JsonValue::new(JsonVariant::Object(object)))
155}
156
157fn apply_missing_type_hints(
158    object: &mut BTreeMap<String, JsonVariant>,
159    context: &mut JsonContext,
160) -> Result<()> {
161    for hint in &context.settings.type_hints {
162        if hint.path.len() > context.path.len() && hint.path.starts_with(&context.path) {
163            let depth = context.path.len();
164            let key = &hint.path[depth];
165            with_key_context(context, key, |context| {
166                insert_missing_type_hint(object, context, hint, depth)
167            })?;
168        }
169    }
170    Ok(())
171}
172
173fn insert_missing_type_hint(
174    object: &mut BTreeMap<String, JsonVariant>,
175    field_context: &mut JsonContext,
176    hint: &JsonTypeHint,
177    depth: usize,
178) -> Result<()> {
179    let key = &hint.path[depth];
180    let is_leaf = depth + 1 == hint.path.len();
181
182    if is_leaf {
183        if !object.contains_key(key) {
184            let value = encode_missing_type_hint_value(hint, field_context)?;
185            object.insert(key.clone(), value.into_variant());
186        }
187        return Ok(());
188    }
189
190    match object.entry(key.clone()) {
191        Entry::Occupied(mut entry) => match entry.get_mut() {
192            JsonVariant::Object(child) => {
193                insert_missing_type_hint(child, field_context, hint, depth + 1)
194            }
195            _ => error::InvalidJsonSnafu {
196                value: format!(
197                    "JSON2 type hint path {} expects object at {}",
198                    hint.path.join("."),
199                    field_context.path.join(".")
200                ),
201            }
202            .fail(),
203        },
204        Entry::Vacant(entry) => {
205            let mut child = BTreeMap::new();
206            insert_missing_type_hint(&mut child, field_context, hint, depth + 1)?;
207            entry.insert(JsonVariant::Object(child));
208            Ok(())
209        }
210    }
211}
212
213fn encode_missing_type_hint_value(
214    hint: &JsonTypeHint,
215    context: &mut JsonContext,
216) -> Result<JsonValue> {
217    if let Some(default_constraint) = &hint.default_constraint {
218        let value = default_constraint.create_default(&hint.data_type, hint.nullable)?;
219        let json = decode_primitive_value(value)?;
220        return encode_json_value_with_hint(json, hint, context);
221    }
222
223    if hint.nullable {
224        Ok(JsonValue::null())
225    } else {
226        error::InvalidJsonSnafu {
227            value: format!(
228                "missing non-null JSON2 type hint path {}",
229                hint.path.join(".")
230            ),
231        }
232        .fail()
233    }
234}
235
236fn encode_json_value_with_hint(
237    json: Json,
238    hint: &JsonTypeHint,
239    context: &mut JsonContext,
240) -> Result<JsonValue> {
241    if json.is_null() {
242        return if hint.nullable {
243            Ok(JsonValue::null())
244        } else {
245            error::InvalidJsonSnafu {
246                value: format!(
247                    "JSON2 type hint path {} is not nullable",
248                    context.path.join(".")
249                ),
250            }
251            .fail()
252        };
253    }
254
255    let invalid_type = || {
256        error::InvalidJsonSnafu {
257            value: format!(
258                "JSON value at {} does not match JSON2 type hint {}",
259                context.path.join("."),
260                hint.data_type
261            ),
262        }
263        .fail()
264    };
265
266    match (&hint.data_type, json) {
267        (ConcreteDataType::String(_), Json::String(v)) => Ok(v.into()),
268        (
269            ConcreteDataType::Int8(_)
270            | ConcreteDataType::Int16(_)
271            | ConcreteDataType::Int32(_)
272            | ConcreteDataType::Int64(_),
273            Json::Number(v),
274        ) => match v.as_i64() {
275            Some(v) => Ok(v.into()),
276            None => invalid_type(),
277        },
278        (
279            ConcreteDataType::UInt8(_)
280            | ConcreteDataType::UInt16(_)
281            | ConcreteDataType::UInt32(_)
282            | ConcreteDataType::UInt64(_),
283            Json::Number(v),
284        ) => match v.as_u64() {
285            Some(v) => Ok(v.into()),
286            None => invalid_type(),
287        },
288        (ConcreteDataType::Float32(_) | ConcreteDataType::Float64(_), Json::Number(v)) => {
289            match v.as_f64() {
290                Some(v) => Ok(v.into()),
291                None => invalid_type(),
292            }
293        }
294        (ConcreteDataType::Boolean(_), Json::Bool(v)) => Ok(v.into()),
295        _ => invalid_type(),
296    }
297}
298
299fn encode_json_array_with_context<'a>(
300    json_array: Vec<Json>,
301    context: &mut JsonContext<'a>,
302) -> Result<JsonValue> {
303    let json_array_len = json_array.len();
304    let mut items = Vec::with_capacity(json_array_len);
305
306    for (index, value) in json_array.into_iter().enumerate() {
307        let item_value = with_key_context(context, &index.to_string(), |context| {
308            encode_json_value_with_context(value, context)
309        })?;
310        items.push(item_value);
311    }
312
313    // In specification, it's valid for a JSON array to have different types of items, for example,
314    // ["a string", 1]. However, in implementation, the `JsonValue` will be converted to Arrow list
315    // array, which requires all items have exactly the same type. So we merge out the maybe
316    // different item types to a unified type, and align all the item values to it.
317
318    let merged_item_type = if let Some((first, rests)) = items.split_first() {
319        let mut merged = first.json_type().clone();
320        for rest in rests.iter().map(|x| x.json_type()) {
321            if matches!(merged, JsonNativeType::Variant) {
322                break;
323            }
324            merged.merge(rest);
325        }
326        Some(merged)
327    } else {
328        None
329    };
330    if let Some(unified_item_type) = merged_item_type {
331        for item in &mut items {
332            item.try_align(&unified_item_type)?;
333        }
334    }
335    let items = items
336        .into_iter()
337        .map(|x| x.into_variant())
338        .collect::<Vec<_>>();
339    Ok(JsonValue::new(JsonVariant::Array(items)))
340}
341
342/// Helper function to encode a JSON value to a Value and determine its ConcreteDataType with context
343fn encode_json_value_with_context(json: Json, context: &mut JsonContext) -> Result<JsonValue> {
344    match json {
345        Json::Null => Ok(JsonValue::null()),
346        Json::Bool(b) => Ok(b.into()),
347        Json::Number(n) => {
348            if let Some(i) = n.as_i64() {
349                Ok(i.into())
350            } else if let Some(u) = n.as_u64() {
351                if u <= i64::MAX as u64 {
352                    Ok((u as i64).into())
353                } else {
354                    Ok(u.into())
355                }
356            } else if let Some(f) = n.as_f64() {
357                Ok(f.into())
358            } else {
359                // Fallback to string representation
360                Ok(n.to_string().into())
361            }
362        }
363        Json::String(s) => Ok(s.into()),
364        Json::Array(arr) => encode_json_array_with_context(arr, context),
365        Json::Object(obj) => encode_json_object_with_context(obj, context),
366    }
367}
368
369/// Main decoding function with key path tracking
370fn decode_value_with_context(value: Value, context: &mut JsonContext) -> Result<Json> {
371    match value {
372        Value::Struct(struct_value) => decode_struct_with_context(struct_value, context),
373        Value::List(list_value) => decode_list_with_context(list_value, context),
374        _ => decode_primitive_value(value),
375    }
376}
377
378/// Decode a structured value to JSON object
379fn decode_struct_with_context<'a>(
380    struct_value: StructValue,
381    context: &mut JsonContext<'a>,
382) -> Result<Json> {
383    let mut json_object = Map::with_capacity(struct_value.len());
384
385    let (items, fields) = struct_value.into_parts();
386
387    for (field, field_value) in fields.fields().iter().zip(items) {
388        let json_value = with_key_context(context, field.name(), |context| {
389            decode_value_with_context(field_value, context)
390        })?;
391        json_object.insert(field.name().to_string(), json_value);
392    }
393
394    Ok(Json::Object(json_object))
395}
396
397/// Decode a list value to JSON array
398fn decode_list_with_context(list_value: ListValue, context: &mut JsonContext) -> Result<Json> {
399    let mut json_array = Vec::with_capacity(list_value.len());
400
401    let data_items = list_value.take_items();
402
403    for (index, item) in data_items.into_iter().enumerate() {
404        let json_value = with_key_context(context, &index.to_string(), |context| {
405            decode_value_with_context(item, context)
406        })?;
407        json_array.push(json_value);
408    }
409
410    Ok(Json::Array(json_array))
411}
412
413/// Decode primitive value to JSON
414fn decode_primitive_value(value: Value) -> Result<Json> {
415    match value {
416        Value::Null => Ok(Json::Null),
417        Value::Boolean(b) => Ok(Json::Bool(b)),
418        Value::UInt8(v) => Ok(Json::from(v)),
419        Value::UInt16(v) => Ok(Json::from(v)),
420        Value::UInt32(v) => Ok(Json::from(v)),
421        Value::UInt64(v) => Ok(Json::from(v)),
422        Value::Int8(v) => Ok(Json::from(v)),
423        Value::Int16(v) => Ok(Json::from(v)),
424        Value::Int32(v) => Ok(Json::from(v)),
425        Value::Int64(v) => Ok(Json::from(v)),
426        Value::Float32(v) => Ok(Json::from(v.0)),
427        Value::Float64(v) => Ok(Json::from(v.0)),
428        Value::String(s) => Ok(Json::String(s.as_utf8().to_string())),
429        Value::Binary(b) => serde_json::to_value(b.as_ref()).context(error::SerializeSnafu),
430        Value::Date(v) => Ok(Json::from(v.val())),
431        Value::Timestamp(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
432        Value::Time(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
433        Value::IntervalYearMonth(v) => {
434            serde_json::to_value(v.to_i32()).context(error::SerializeSnafu)
435        }
436        Value::IntervalDayTime(v) => {
437            serde_json::to_value(v.to_i64()).context(error::SerializeSnafu)
438        }
439        Value::IntervalMonthDayNano(v) => {
440            serde_json::to_value(v.to_i128()).context(error::SerializeSnafu)
441        }
442        Value::Duration(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
443        Value::Decimal128(v) => serde_json::to_value(v.to_string()).context(error::SerializeSnafu),
444        Value::Struct(_) | Value::List(_) | Value::Json(_) => {
445            // These should be handled by the context-aware functions
446            Err(error::InvalidJsonSnafu {
447                value: "Structured values should be handled by context-aware decoding".to_string(),
448            }
449            .build())
450        }
451    }
452}
453
454#[cfg(test)]
455mod tests {
456    use std::sync::Arc;
457
458    use serde_json::json;
459
460    use super::*;
461    use crate::data_type::ConcreteDataType;
462    use crate::types::{ListType, StructField, StructType};
463
464    fn struct_field_value<'a>(struct_value: &'a StructValue, field_name: &str) -> &'a Value {
465        let index = struct_value
466            .struct_type()
467            .fields()
468            .iter()
469            .position(|field| field.name() == field_name)
470            .expect("field exists");
471        &struct_value.items()[index]
472    }
473
474    #[test]
475    fn test_json_settings_forward_compatibility() {
476        let json_str = r#"{
477            "type_hints": [
478                {
479                    "path": ["user", "age"],
480                    "type": {
481                        "Int64": {}
482                    },
483                    "nullable": false,
484                    "default_constraint": {
485                        "Value": {
486                            "Int64": 18
487                        }
488                    },
489                    "inverted_index": true
490                },
491                {
492                    "path": ["user", "name"],
493                    "type": {
494                        "String": {
495                            "size_type": "Utf8"
496                        }
497                    },
498                    "nullable": true,
499                    "inverted_index": false
500                }
501            ]
502        }"#;
503
504        let deserialized = serde_json::from_str::<JsonSettings>(json_str).unwrap();
505
506        assert_eq!(
507            deserialized,
508            JsonSettings::new(vec![
509                JsonTypeHint {
510                    path: vec!["user".to_string(), "age".to_string()],
511                    data_type: ConcreteDataType::int64_datatype(),
512                    nullable: false,
513                    default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
514                    inverted_index: true,
515                },
516                JsonTypeHint {
517                    path: vec!["user".to_string(), "name".to_string()],
518                    data_type: ConcreteDataType::string_datatype(),
519                    nullable: true,
520                    default_constraint: None,
521                    inverted_index: false,
522                },
523            ])
524        );
525    }
526
527    #[test]
528    fn test_json_settings_ser_de() {
529        let settings = JsonSettings::new(vec![
530            JsonTypeHint {
531                path: vec!["user".to_string(), "age".to_string()],
532                data_type: ConcreteDataType::int64_datatype(),
533                nullable: false,
534                default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
535                inverted_index: true,
536            },
537            JsonTypeHint {
538                path: vec!["user".to_string(), "name".to_string()],
539                data_type: ConcreteDataType::string_datatype(),
540                nullable: true,
541                default_constraint: None,
542                inverted_index: false,
543            },
544        ]);
545
546        let serialized = serde_json::to_string(&settings).unwrap();
547        let deserialized = serde_json::from_str::<JsonSettings>(&serialized).unwrap();
548
549        assert_eq!(settings, deserialized);
550    }
551
552    #[test]
553    fn test_encode_root_non_object_json() {
554        let settings = JsonSettings::default();
555        let cases = [
556            ("null", Json::Null),
557            ("boolean", Json::Bool(true)),
558            ("integer", Json::from(42)),
559            ("float", Json::from(3.15)),
560            ("string", Json::String("hello".to_string())),
561            ("array", json!([1, 2, 3])),
562            ("mixed array", json!([1, "hello", true, 3.15])),
563            ("empty array", json!([])),
564        ];
565
566        for (name, json) in cases {
567            let err = settings.encode(json).unwrap_err();
568            assert!(
569                matches!(err, crate::error::Error::UnsupportedJsonType { .. }),
570                "{name}: {err:?}"
571            );
572        }
573    }
574
575    #[test]
576    fn test_encode_json_object() {
577        let json = json!({
578            "name": "John",
579            "age": 30,
580            "active": true
581        });
582
583        let settings = JsonSettings::default();
584        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
585        let Value::Struct(result) = result else {
586            panic!("Expected Struct value");
587        };
588        assert_eq!(result.items().len(), 3);
589
590        let items = result.items();
591        let struct_type = result.struct_type();
592
593        // Check that we have the expected fields
594        let fields = struct_type.fields();
595        let field_names: Vec<&str> = fields.iter().map(|f| f.name()).collect();
596        assert!(field_names.contains(&"name"));
597        assert!(field_names.contains(&"age"));
598        assert!(field_names.contains(&"active"));
599
600        // Find and check each field
601        for (i, field) in struct_type.fields().iter().enumerate() {
602            match field.name() {
603                "name" => {
604                    assert_eq!(items[i], Value::String("John".into()));
605                    assert_eq!(field.data_type(), &ConcreteDataType::string_datatype());
606                }
607                "age" => {
608                    assert_eq!(items[i], Value::Int64(30));
609                    assert_eq!(field.data_type(), &ConcreteDataType::int64_datatype());
610                }
611                "active" => {
612                    assert_eq!(items[i], Value::Boolean(true));
613                    assert_eq!(field.data_type(), &ConcreteDataType::boolean_datatype());
614                }
615                _ => panic!("Unexpected field: {}", field.name()),
616            }
617        }
618    }
619
620    #[test]
621    fn test_encode_json_nested_object() {
622        let json = json!({
623            "person": {
624                "name": "Alice",
625                "age": 25
626            },
627            "scores": [95, 87, 92]
628        });
629
630        let settings = JsonSettings::default();
631        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
632        let Value::Struct(result) = result else {
633            panic!("Expected Struct value");
634        };
635        assert_eq!(result.items().len(), 2);
636
637        let items = result.items();
638        let struct_type = result.struct_type();
639
640        // Check person field (nested struct)
641        let person_index = struct_type
642            .fields()
643            .iter()
644            .position(|f| f.name() == "person")
645            .unwrap();
646        if let Value::Struct(person_struct) = &items[person_index] {
647            assert_eq!(person_struct.items().len(), 2);
648            let fields = person_struct.struct_type().fields();
649            let person_fields: Vec<&str> = fields.iter().map(|f| f.name()).collect();
650            assert!(person_fields.contains(&"name"));
651            assert!(person_fields.contains(&"age"));
652        } else {
653            panic!("Expected Struct value for person field");
654        }
655
656        // Check scores field (list)
657        let scores_index = struct_type
658            .fields()
659            .iter()
660            .position(|f| f.name() == "scores")
661            .unwrap();
662        if let Value::List(scores_list) = &items[scores_index] {
663            assert_eq!(scores_list.items().len(), 3);
664            assert_eq!(scores_list.items()[0], Value::Int64(95));
665            assert_eq!(scores_list.items()[1], Value::Int64(87));
666            assert_eq!(scores_list.items()[2], Value::Int64(92));
667        } else {
668            panic!("Expected List value for scores field");
669        }
670    }
671
672    #[test]
673    fn test_encode_json_structured() {
674        let json = json!({
675            "name": "Bob",
676            "age": 35
677        });
678
679        let settings = JsonSettings::default();
680        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
681
682        if let Value::Struct(struct_value) = result {
683            assert_eq!(struct_value.items().len(), 2);
684            let fields = struct_value.struct_type().fields();
685            let field_names: Vec<&str> = fields.iter().map(|f| f.name()).collect();
686            assert!(field_names.contains(&"name"));
687            assert!(field_names.contains(&"age"));
688        } else {
689            panic!("Expected Struct value");
690        }
691    }
692
693    #[test]
694    fn test_encode_json_respects_type_hint() {
695        let settings = JsonSettings::new(vec![JsonTypeHint {
696            path: vec!["age".to_string()],
697            data_type: ConcreteDataType::int64_datatype(),
698            nullable: false,
699            default_constraint: None,
700            inverted_index: false,
701        }]);
702
703        let result = settings
704            .encode(json!({
705                "name": "Alice",
706                "age": 42
707            }))
708            .unwrap()
709            .into_json_inner()
710            .unwrap();
711
712        let Value::Struct(struct_value) = result else {
713            panic!("Expected Struct value");
714        };
715        assert_eq!(struct_field_value(&struct_value, "age"), &Value::Int64(42));
716
717        let err = settings
718            .encode(json!({
719                "age": "42"
720            }))
721            .unwrap_err();
722        assert!(err.to_string().contains("does not match JSON2 type hint"));
723    }
724
725    #[test]
726    fn test_encode_json_respects_unsigned_type_hint() {
727        let settings = JsonSettings::new(vec![JsonTypeHint {
728            path: vec!["count".to_string()],
729            data_type: ConcreteDataType::uint64_datatype(),
730            nullable: false,
731            default_constraint: None,
732            inverted_index: false,
733        }]);
734
735        let result = settings
736            .encode(json!({
737                "count": u64::MAX
738            }))
739            .unwrap()
740            .into_json_inner()
741            .unwrap();
742
743        let Value::Struct(struct_value) = result else {
744            panic!("Expected Struct value");
745        };
746        assert_eq!(
747            struct_field_value(&struct_value, "count"),
748            &Value::UInt64(u64::MAX)
749        );
750
751        let err = settings
752            .encode(json!({
753                "count": -1
754            }))
755            .unwrap_err();
756        assert!(err.to_string().contains("does not match JSON2 type hint"));
757    }
758
759    #[test]
760    fn test_encode_json_fills_missing_type_hint_with_default() {
761        let settings = JsonSettings::new(vec![JsonTypeHint {
762            path: vec!["user".to_string(), "age".to_string()],
763            data_type: ConcreteDataType::int64_datatype(),
764            nullable: false,
765            default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
766            inverted_index: false,
767        }]);
768
769        let result = settings
770            .encode(json!({}))
771            .unwrap()
772            .into_json_inner()
773            .unwrap();
774
775        let Value::Struct(root) = result else {
776            panic!("Expected Struct value");
777        };
778        let Value::Struct(user) = struct_field_value(&root, "user") else {
779            panic!("Expected user Struct value");
780        };
781        assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
782    }
783
784    #[test]
785    fn test_encode_json_fills_missing_nullable_type_hint_with_null() {
786        let settings = JsonSettings::new(vec![JsonTypeHint {
787            path: vec!["user".to_string(), "name".to_string()],
788            data_type: ConcreteDataType::string_datatype(),
789            nullable: true,
790            default_constraint: None,
791            inverted_index: false,
792        }]);
793
794        let result = settings
795            .encode(json!({ "user": {} }))
796            .unwrap()
797            .into_json_inner()
798            .unwrap();
799
800        let Value::Struct(root) = result else {
801            panic!("Expected Struct value");
802        };
803        let Value::Struct(user) = struct_field_value(&root, "user") else {
804            panic!("Expected user Struct value");
805        };
806        assert_eq!(struct_field_value(user, "name"), &Value::Null);
807    }
808
809    #[test]
810    fn test_encode_json_rejects_missing_non_null_type_hint() {
811        let settings = JsonSettings::new(vec![JsonTypeHint {
812            path: vec!["user".to_string(), "age".to_string()],
813            data_type: ConcreteDataType::int64_datatype(),
814            nullable: false,
815            default_constraint: None,
816            inverted_index: false,
817        }]);
818
819        let err = settings.encode(json!({})).unwrap_err();
820        assert!(
821            err.to_string()
822                .contains("missing non-null JSON2 type hint path user.age")
823        );
824    }
825
826    #[test]
827    fn test_encode_json_merges_missing_type_hint_prefix() {
828        let settings = JsonSettings::new(vec![
829            JsonTypeHint {
830                path: vec!["user".to_string(), "age".to_string()],
831                data_type: ConcreteDataType::int64_datatype(),
832                nullable: false,
833                default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
834                inverted_index: false,
835            },
836            JsonTypeHint {
837                path: vec!["user".to_string(), "name".to_string()],
838                data_type: ConcreteDataType::string_datatype(),
839                nullable: false,
840                default_constraint: Some(ColumnDefaultConstraint::Value(Value::String(
841                    "unknown".into(),
842                ))),
843                inverted_index: false,
844            },
845        ]);
846
847        let result = settings
848            .encode(json!({}))
849            .unwrap()
850            .into_json_inner()
851            .unwrap();
852
853        let Value::Struct(root) = result else {
854            panic!("Expected Struct value");
855        };
856        let Value::Struct(user) = struct_field_value(&root, "user") else {
857            panic!("Expected user Struct value");
858        };
859        assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
860        assert_eq!(
861            struct_field_value(user, "name"),
862            &Value::String("unknown".into())
863        );
864    }
865
866    #[test]
867    fn test_json_settings_structured() {
868        let json = json!({
869            "name": "Eve",
870            "score": 95
871        });
872
873        let settings = JsonSettings::default();
874        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
875
876        if let Value::Struct(struct_value) = result {
877            assert_eq!(struct_value.items().len(), 2);
878        } else {
879            panic!("Expected Struct value");
880        }
881    }
882
883    #[cfg(test)]
884    mod decode_tests {
885        use ordered_float::OrderedFloat;
886        use serde_json::json;
887
888        use super::*;
889
890        #[test]
891        fn test_decode_primitive_values() {
892            let settings = JsonSettings::default();
893
894            // Test null
895            let result = settings.decode(Value::Null).unwrap();
896            assert_eq!(result, Json::Null);
897
898            // Test boolean
899            let result = settings.decode(Value::Boolean(true)).unwrap();
900            assert_eq!(result, Json::Bool(true));
901
902            // Test integer
903            let result = settings.decode(Value::Int64(42)).unwrap();
904            assert_eq!(result, Json::from(42));
905
906            // Test float
907            let result = settings.decode(Value::Float64(OrderedFloat(3.16))).unwrap();
908            assert_eq!(result, Json::from(3.16));
909
910            // Test string
911            let result = settings.decode(Value::String("hello".into())).unwrap();
912            assert_eq!(result, Json::String("hello".to_string()));
913        }
914
915        #[test]
916        fn test_decode_struct() {
917            let settings = JsonSettings::default();
918
919            let struct_value = StructValue::new(
920                vec![
921                    Value::String("Alice".into()),
922                    Value::Int64(25),
923                    Value::Boolean(true),
924                ],
925                StructType::new(Arc::new(vec![
926                    StructField::new(
927                        "name".to_string(),
928                        ConcreteDataType::string_datatype(),
929                        true,
930                    ),
931                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
932                    StructField::new(
933                        "active".to_string(),
934                        ConcreteDataType::boolean_datatype(),
935                        true,
936                    ),
937                ])),
938            );
939
940            let result = settings.decode(Value::Struct(struct_value)).unwrap();
941            let expected = json!({
942                "name": "Alice",
943                "age": 25,
944                "active": true
945            });
946            assert_eq!(result, expected);
947        }
948
949        #[test]
950        fn test_decode_list() {
951            let settings = JsonSettings::default();
952
953            let list_value = ListValue::new(
954                vec![Value::Int64(1), Value::Int64(2), Value::Int64(3)],
955                Arc::new(ConcreteDataType::int64_datatype()),
956            );
957
958            let result = settings.decode(Value::List(list_value)).unwrap();
959            let expected = json!([1, 2, 3]);
960            assert_eq!(result, expected);
961        }
962
963        #[test]
964        fn test_decode_nested_structure() {
965            let settings = JsonSettings::default();
966
967            let inner_struct = StructValue::new(
968                vec![Value::String("Alice".into()), Value::Int64(25)],
969                StructType::new(Arc::new(vec![
970                    StructField::new(
971                        "name".to_string(),
972                        ConcreteDataType::string_datatype(),
973                        true,
974                    ),
975                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
976                ])),
977            );
978
979            let score_list_item_type = Arc::new(ConcreteDataType::int64_datatype());
980            let outer_struct = StructValue::new(
981                vec![
982                    Value::Struct(inner_struct),
983                    Value::List(ListValue::new(
984                        vec![Value::Int64(95), Value::Int64(87)],
985                        score_list_item_type.clone(),
986                    )),
987                ],
988                StructType::new(Arc::new(vec![
989                    StructField::new(
990                        "user".to_string(),
991                        ConcreteDataType::Struct(StructType::new(Arc::new(vec![
992                            StructField::new(
993                                "name".to_string(),
994                                ConcreteDataType::string_datatype(),
995                                true,
996                            ),
997                            StructField::new(
998                                "age".to_string(),
999                                ConcreteDataType::int64_datatype(),
1000                                true,
1001                            ),
1002                        ]))),
1003                        true,
1004                    ),
1005                    StructField::new(
1006                        "scores".to_string(),
1007                        ConcreteDataType::List(ListType::new(score_list_item_type.clone())),
1008                        true,
1009                    ),
1010                ])),
1011            );
1012
1013            let result = settings.decode(Value::Struct(outer_struct)).unwrap();
1014            let expected = json!({
1015                "user": {
1016                    "name": "Alice",
1017                    "age": 25
1018                },
1019                "scores": [95, 87]
1020            });
1021            assert_eq!(result, expected);
1022        }
1023
1024        #[test]
1025        fn test_decode_missing_fields() {
1026            let settings = JsonSettings::default();
1027
1028            // Struct with missing field (null value)
1029            let struct_value = StructValue::new(
1030                vec![
1031                    Value::String("Bob".into()),
1032                    Value::Null, // missing age field
1033                ],
1034                StructType::new(Arc::new(vec![
1035                    StructField::new(
1036                        "name".to_string(),
1037                        ConcreteDataType::string_datatype(),
1038                        true,
1039                    ),
1040                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
1041                ])),
1042            );
1043
1044            let result = settings.decode(Value::Struct(struct_value)).unwrap();
1045            let expected = json!({
1046                "name": "Bob",
1047                "age": null
1048            });
1049            assert_eq!(result, expected);
1050        }
1051    }
1052}
datatypes/json.rs

datatypes/
json.rs
