datatypes/
json.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Data conversion between greptime's StructType and Json
16//!
17//! The idea of this module is to provide utilities to convert serde_json::Value to greptime's StructType and vice versa.
18//!
19//! The struct will carry all the fields of the Json object. We will not flatten any json object in this implementation.
20//!
21
22pub mod value;
23
24use std::collections::BTreeMap;
25use std::collections::btree_map::Entry;
26
27use serde::{Deserialize, Serialize};
28use serde_json::{Map, Value as Json};
29use snafu::ResultExt;
30
31use crate::data_type::ConcreteDataType;
32use crate::error::{self, Result};
33use crate::json::value::{JsonValue, JsonVariant};
34use crate::schema::ColumnDefaultConstraint;
35use crate::types::json_type::JsonNativeType;
36use crate::value::{ListValue, StructValue, Value};
37
38/// JSON2 settings stored in column schema metadata and represented through
39/// Arrow extension metadata.
40#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
41pub struct JsonSettings {
42    #[serde(default)]
43    pub type_hints: Vec<JsonTypeHint>,
44}
45
46/// Declares selected JSON2 subpaths as typed fields.
47///
48/// These hints let JSON2 encode frequently used subpaths in a typed layout, so
49/// queries over those subpaths can get behavior and performance closer to
50/// ordinary columns.
51#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
52pub struct JsonTypeHint {
53    /// JSON2 subpath for a typed field.
54    ///
55    /// Each item is one JSON object key. For example, `["user", "age"]`
56    /// represents `user.age`.
57    ///
58    /// Array traversal is not currently supported. For example, a hint cannot
59    /// describe `events[0].name` or fields shared by all items in `events[*]`.
60    pub path: Vec<String>,
61    #[serde(rename = "type")]
62    pub data_type: ConcreteDataType,
63    pub nullable: bool,
64    #[serde(skip_serializing_if = "Option::is_none")]
65    pub default_constraint: Option<ColumnDefaultConstraint>,
66    pub inverted_index: bool,
67}
68
69/// Context for JSON encoding/decoding that tracks the current key path.
70#[derive(Clone, Debug)]
71pub struct JsonContext<'a> {
72    /// Current key path from the JSON2 root.
73    pub path: Vec<String>,
74    /// Settings for JSON encoding/decoding.
75    pub settings: &'a JsonSettings,
76}
77
78impl JsonSettings {
79    pub fn new(type_hints: Vec<JsonTypeHint>) -> Self {
80        Self { type_hints }
81    }
82
83    /// Decode an encoded StructValue back into a serde_json::Value.
84    pub fn decode(&self, value: Value) -> Result<Json> {
85        let context = JsonContext {
86            path: Vec::new(),
87            settings: self,
88        };
89        decode_value_with_context(value, &context)
90    }
91
92    /// Encode a serde_json::Value into a Value::Json using current settings.
93    pub fn encode(&self, json: Json) -> Result<Value> {
94        let context = JsonContext {
95            path: Vec::new(),
96            settings: self,
97        };
98        encode_json_with_context(json, &context).map(|v| Value::Json(Box::new(v)))
99    }
100}
101
102impl<'a> JsonContext<'a> {
103    /// Create a new context with an updated key path
104    pub fn with_key(&self, key: &str) -> JsonContext<'a> {
105        let mut path = self.path.clone();
106        path.push(key.to_string());
107        JsonContext {
108            path,
109            settings: self.settings,
110        }
111    }
112
113    fn type_hint(&self) -> Option<&'a JsonTypeHint> {
114        self.settings
115            .type_hints
116            .iter()
117            .find(|hint| hint.path == self.path)
118    }
119}
120
121/// Main encoding function with key path tracking
122pub fn encode_json_with_context<'a>(json: Json, context: &JsonContext<'a>) -> Result<JsonValue> {
123    match json {
124        Json::Object(json_object) => encode_json_object_with_context(json_object, context),
125        Json::Array(json_array) => encode_json_array_with_context(json_array, context),
126        _ => encode_json_value_with_context(json, context),
127    }
128}
129
130fn encode_json_object_with_context<'a>(
131    json_object: Map<String, Json>,
132    context: &JsonContext<'a>,
133) -> Result<JsonValue> {
134    let mut object = BTreeMap::new();
135    for (key, value) in json_object {
136        let field_context = context.with_key(&key);
137
138        let value = if let Some(hint) = field_context.type_hint() {
139            encode_json_value_with_hint(value, hint, &field_context)?
140        } else {
141            encode_json_value_with_context(value, &field_context)?
142        };
143
144        object.insert(key, value.into_variant());
145    }
146
147    apply_missing_type_hints(&mut object, context)?;
148
149    Ok(JsonValue::new(JsonVariant::Object(object)))
150}
151
152fn apply_missing_type_hints(
153    object: &mut BTreeMap<String, JsonVariant>,
154    context: &JsonContext,
155) -> Result<()> {
156    for hint in &context.settings.type_hints {
157        if hint.path.len() > context.path.len() && hint.path.starts_with(&context.path) {
158            insert_missing_type_hint(object, context, hint, context.path.len())?;
159        }
160    }
161    Ok(())
162}
163
164fn insert_missing_type_hint(
165    object: &mut BTreeMap<String, JsonVariant>,
166    context: &JsonContext,
167    hint: &JsonTypeHint,
168    depth: usize,
169) -> Result<()> {
170    let key = &hint.path[depth];
171    let field_context = context.with_key(key);
172    let is_leaf = depth + 1 == hint.path.len();
173
174    if is_leaf {
175        if !object.contains_key(key) {
176            let value = encode_missing_type_hint_value(hint, &field_context)?;
177            object.insert(key.clone(), value.into_variant());
178        }
179        return Ok(());
180    }
181
182    match object.entry(key.clone()) {
183        Entry::Occupied(mut entry) => match entry.get_mut() {
184            JsonVariant::Object(child) => {
185                insert_missing_type_hint(child, &field_context, hint, depth + 1)
186            }
187            _ => error::InvalidJsonSnafu {
188                value: format!(
189                    "JSON2 type hint path {} expects object at {}",
190                    hint.path.join("."),
191                    field_context.path.join(".")
192                ),
193            }
194            .fail(),
195        },
196        Entry::Vacant(entry) => {
197            let mut child = BTreeMap::new();
198            insert_missing_type_hint(&mut child, &field_context, hint, depth + 1)?;
199            entry.insert(JsonVariant::Object(child));
200            Ok(())
201        }
202    }
203}
204
205fn encode_missing_type_hint_value(hint: &JsonTypeHint, context: &JsonContext) -> Result<JsonValue> {
206    if let Some(default_constraint) = &hint.default_constraint {
207        let value = default_constraint.create_default(&hint.data_type, hint.nullable)?;
208        let json = decode_primitive_value(value)?;
209        return encode_json_value_with_hint(json, hint, context);
210    }
211
212    if hint.nullable {
213        Ok(JsonValue::null())
214    } else {
215        error::InvalidJsonSnafu {
216            value: format!(
217                "missing non-null JSON2 type hint path {}",
218                hint.path.join(".")
219            ),
220        }
221        .fail()
222    }
223}
224
225fn encode_json_value_with_hint(
226    json: Json,
227    hint: &JsonTypeHint,
228    context: &JsonContext,
229) -> Result<JsonValue> {
230    if json.is_null() {
231        return if hint.nullable {
232            Ok(JsonValue::null())
233        } else {
234            error::InvalidJsonSnafu {
235                value: format!(
236                    "JSON2 type hint path {} is not nullable",
237                    context.path.join(".")
238                ),
239            }
240            .fail()
241        };
242    }
243
244    let invalid_type = || {
245        error::InvalidJsonSnafu {
246            value: format!(
247                "JSON value at {} does not match JSON2 type hint {}",
248                context.path.join("."),
249                hint.data_type
250            ),
251        }
252        .fail()
253    };
254
255    match (&hint.data_type, json) {
256        (ConcreteDataType::String(_), Json::String(v)) => Ok(v.into()),
257        (
258            ConcreteDataType::Int8(_)
259            | ConcreteDataType::Int16(_)
260            | ConcreteDataType::Int32(_)
261            | ConcreteDataType::Int64(_),
262            Json::Number(v),
263        ) => match v.as_i64() {
264            Some(v) => Ok(v.into()),
265            None => invalid_type(),
266        },
267        (
268            ConcreteDataType::UInt8(_)
269            | ConcreteDataType::UInt16(_)
270            | ConcreteDataType::UInt32(_)
271            | ConcreteDataType::UInt64(_),
272            Json::Number(v),
273        ) => match v.as_u64() {
274            Some(v) => Ok(v.into()),
275            None => invalid_type(),
276        },
277        (ConcreteDataType::Float32(_) | ConcreteDataType::Float64(_), Json::Number(v)) => {
278            match v.as_f64() {
279                Some(v) => Ok(v.into()),
280                None => invalid_type(),
281            }
282        }
283        (ConcreteDataType::Boolean(_), Json::Bool(v)) => Ok(v.into()),
284        _ => invalid_type(),
285    }
286}
287
288fn encode_json_array_with_context<'a>(
289    json_array: Vec<Json>,
290    context: &JsonContext<'a>,
291) -> Result<JsonValue> {
292    let json_array_len = json_array.len();
293    let mut items = Vec::with_capacity(json_array_len);
294
295    for (index, value) in json_array.into_iter().enumerate() {
296        let array_context = context.with_key(&index.to_string());
297        let item_value = encode_json_value_with_context(value, &array_context)?;
298        items.push(item_value);
299    }
300
301    // In specification, it's valid for a JSON array to have different types of items, for example,
302    // ["a string", 1]. However, in implementation, the `JsonValue` will be converted to Arrow list
303    // array, which requires all items have exactly the same type. So we merge out the maybe
304    // different item types to a unified type, and align all the item values to it.
305
306    let merged_item_type = if let Some((first, rests)) = items.split_first() {
307        let mut merged = first.json_type().clone();
308        for rest in rests.iter().map(|x| x.json_type()) {
309            if matches!(merged.native_type(), JsonNativeType::Variant) {
310                break;
311            }
312            merged.merge(rest)?;
313        }
314        Some(merged)
315    } else {
316        None
317    };
318    if let Some(unified_item_type) = merged_item_type {
319        for item in &mut items {
320            item.try_align(&unified_item_type)?;
321        }
322    }
323    let items = items
324        .into_iter()
325        .map(|x| x.into_variant())
326        .collect::<Vec<_>>();
327    Ok(JsonValue::new(JsonVariant::Array(items)))
328}
329
330/// Helper function to encode a JSON value to a Value and determine its ConcreteDataType with context
331fn encode_json_value_with_context<'a>(json: Json, context: &JsonContext<'a>) -> Result<JsonValue> {
332    match json {
333        Json::Null => Ok(JsonValue::null()),
334        Json::Bool(b) => Ok(b.into()),
335        Json::Number(n) => {
336            if let Some(i) = n.as_i64() {
337                Ok(i.into())
338            } else if let Some(u) = n.as_u64() {
339                if u <= i64::MAX as u64 {
340                    Ok((u as i64).into())
341                } else {
342                    Ok(u.into())
343                }
344            } else if let Some(f) = n.as_f64() {
345                Ok(f.into())
346            } else {
347                // Fallback to string representation
348                Ok(n.to_string().into())
349            }
350        }
351        Json::String(s) => Ok(s.into()),
352        Json::Array(arr) => encode_json_array_with_context(arr, context),
353        Json::Object(obj) => encode_json_object_with_context(obj, context),
354    }
355}
356
357/// Main decoding function with key path tracking
358pub fn decode_value_with_context(value: Value, context: &JsonContext) -> Result<Json> {
359    match value {
360        Value::Struct(struct_value) => decode_struct_with_context(struct_value, context),
361        Value::List(list_value) => decode_list_with_context(list_value, context),
362        _ => decode_primitive_value(value),
363    }
364}
365
366/// Decode a structured value to JSON object
367fn decode_struct_with_context<'a>(
368    struct_value: StructValue,
369    context: &JsonContext<'a>,
370) -> Result<Json> {
371    let mut json_object = Map::with_capacity(struct_value.len());
372
373    let (items, fields) = struct_value.into_parts();
374
375    for (field, field_value) in fields.fields().iter().zip(items) {
376        let field_context = context.with_key(field.name());
377        let json_value = decode_value_with_context(field_value, &field_context)?;
378        json_object.insert(field.name().to_string(), json_value);
379    }
380
381    Ok(Json::Object(json_object))
382}
383
384/// Decode a list value to JSON array
385fn decode_list_with_context(list_value: ListValue, context: &JsonContext) -> Result<Json> {
386    let mut json_array = Vec::with_capacity(list_value.len());
387
388    let data_items = list_value.take_items();
389
390    for (index, item) in data_items.into_iter().enumerate() {
391        let array_context = context.with_key(&index.to_string());
392        let json_value = decode_value_with_context(item, &array_context)?;
393        json_array.push(json_value);
394    }
395
396    Ok(Json::Array(json_array))
397}
398
399/// Decode primitive value to JSON
400fn decode_primitive_value(value: Value) -> Result<Json> {
401    match value {
402        Value::Null => Ok(Json::Null),
403        Value::Boolean(b) => Ok(Json::Bool(b)),
404        Value::UInt8(v) => Ok(Json::from(v)),
405        Value::UInt16(v) => Ok(Json::from(v)),
406        Value::UInt32(v) => Ok(Json::from(v)),
407        Value::UInt64(v) => Ok(Json::from(v)),
408        Value::Int8(v) => Ok(Json::from(v)),
409        Value::Int16(v) => Ok(Json::from(v)),
410        Value::Int32(v) => Ok(Json::from(v)),
411        Value::Int64(v) => Ok(Json::from(v)),
412        Value::Float32(v) => Ok(Json::from(v.0)),
413        Value::Float64(v) => Ok(Json::from(v.0)),
414        Value::String(s) => Ok(Json::String(s.as_utf8().to_string())),
415        Value::Binary(b) => serde_json::to_value(b.as_ref()).context(error::SerializeSnafu),
416        Value::Date(v) => Ok(Json::from(v.val())),
417        Value::Timestamp(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
418        Value::Time(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
419        Value::IntervalYearMonth(v) => {
420            serde_json::to_value(v.to_i32()).context(error::SerializeSnafu)
421        }
422        Value::IntervalDayTime(v) => {
423            serde_json::to_value(v.to_i64()).context(error::SerializeSnafu)
424        }
425        Value::IntervalMonthDayNano(v) => {
426            serde_json::to_value(v.to_i128()).context(error::SerializeSnafu)
427        }
428        Value::Duration(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
429        Value::Decimal128(v) => serde_json::to_value(v.to_string()).context(error::SerializeSnafu),
430        Value::Struct(_) | Value::List(_) | Value::Json(_) => {
431            // These should be handled by the context-aware functions
432            Err(error::InvalidJsonSnafu {
433                value: "Structured values should be handled by context-aware decoding".to_string(),
434            }
435            .build())
436        }
437    }
438}
439
440#[cfg(test)]
441mod tests {
442    use std::sync::Arc;
443
444    use serde_json::json;
445
446    use super::*;
447    use crate::data_type::ConcreteDataType;
448    use crate::types::{ListType, StructField, StructType};
449
450    fn struct_field_value<'a>(struct_value: &'a StructValue, field_name: &str) -> &'a Value {
451        let index = struct_value
452            .struct_type()
453            .fields()
454            .iter()
455            .position(|field| field.name() == field_name)
456            .expect("field exists");
457        &struct_value.items()[index]
458    }
459
460    #[test]
461    fn test_json_settings_forward_compatibility() {
462        let json_str = r#"{
463            "type_hints": [
464                {
465                    "path": ["user", "age"],
466                    "type": {
467                        "Int64": {}
468                    },
469                    "nullable": false,
470                    "default_constraint": {
471                        "Value": {
472                            "Int64": 18
473                        }
474                    },
475                    "inverted_index": true
476                },
477                {
478                    "path": ["user", "name"],
479                    "type": {
480                        "String": {
481                            "size_type": "Utf8"
482                        }
483                    },
484                    "nullable": true,
485                    "inverted_index": false
486                }
487            ]
488        }"#;
489
490        let deserialized = serde_json::from_str::<JsonSettings>(json_str).unwrap();
491
492        assert_eq!(
493            deserialized,
494            JsonSettings::new(vec![
495                JsonTypeHint {
496                    path: vec!["user".to_string(), "age".to_string()],
497                    data_type: ConcreteDataType::int64_datatype(),
498                    nullable: false,
499                    default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
500                    inverted_index: true,
501                },
502                JsonTypeHint {
503                    path: vec!["user".to_string(), "name".to_string()],
504                    data_type: ConcreteDataType::string_datatype(),
505                    nullable: true,
506                    default_constraint: None,
507                    inverted_index: false,
508                },
509            ])
510        );
511    }
512
513    #[test]
514    fn test_json_settings_ser_de() {
515        let settings = JsonSettings::new(vec![
516            JsonTypeHint {
517                path: vec!["user".to_string(), "age".to_string()],
518                data_type: ConcreteDataType::int64_datatype(),
519                nullable: false,
520                default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
521                inverted_index: true,
522            },
523            JsonTypeHint {
524                path: vec!["user".to_string(), "name".to_string()],
525                data_type: ConcreteDataType::string_datatype(),
526                nullable: true,
527                default_constraint: None,
528                inverted_index: false,
529            },
530        ]);
531
532        let serialized = serde_json::to_string(&settings).unwrap();
533        let deserialized = serde_json::from_str::<JsonSettings>(&serialized).unwrap();
534
535        assert_eq!(settings, deserialized);
536    }
537
538    #[test]
539    fn test_encode_json_null() {
540        let json = Json::Null;
541        let settings = JsonSettings::default();
542        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
543        assert_eq!(result, Value::Null);
544    }
545
546    #[test]
547    fn test_encode_json_boolean() {
548        let json = Json::Bool(true);
549        let settings = JsonSettings::default();
550        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
551        assert_eq!(result, Value::Boolean(true));
552    }
553
554    #[test]
555    fn test_encode_json_number_integer() {
556        let json = Json::from(42);
557        let settings = JsonSettings::default();
558        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
559        assert_eq!(result, Value::Int64(42));
560    }
561
562    #[test]
563    fn test_encode_json_number_float() {
564        let json = Json::from(3.15);
565        let settings = JsonSettings::default();
566        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
567        match result {
568            Value::Float64(f) => assert_eq!(f.0, 3.15),
569            _ => panic!("Expected Float64"),
570        }
571    }
572
573    #[test]
574    fn test_encode_json_string() {
575        let json = Json::String("hello".to_string());
576        let settings = JsonSettings::default();
577        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
578        assert_eq!(result, Value::String("hello".into()));
579    }
580
581    #[test]
582    fn test_encode_json_array() {
583        let json = json!([1, 2, 3]);
584        let settings = JsonSettings::default();
585        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
586
587        if let Value::List(list_value) = result {
588            assert_eq!(list_value.items().len(), 3);
589            assert_eq!(list_value.items()[0], Value::Int64(1));
590            assert_eq!(list_value.items()[1], Value::Int64(2));
591            assert_eq!(list_value.items()[2], Value::Int64(3));
592        } else {
593            panic!("Expected List value");
594        }
595    }
596
597    #[test]
598    fn test_encode_json_object() {
599        let json = json!({
600            "name": "John",
601            "age": 30,
602            "active": true
603        });
604
605        let settings = JsonSettings::default();
606        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
607        let Value::Struct(result) = result else {
608            panic!("Expected Struct value");
609        };
610        assert_eq!(result.items().len(), 3);
611
612        let items = result.items();
613        let struct_type = result.struct_type();
614
615        // Check that we have the expected fields
616        let fields = struct_type.fields();
617        let field_names: Vec<&str> = fields.iter().map(|f| f.name()).collect();
618        assert!(field_names.contains(&"name"));
619        assert!(field_names.contains(&"age"));
620        assert!(field_names.contains(&"active"));
621
622        // Find and check each field
623        for (i, field) in struct_type.fields().iter().enumerate() {
624            match field.name() {
625                "name" => {
626                    assert_eq!(items[i], Value::String("John".into()));
627                    assert_eq!(field.data_type(), &ConcreteDataType::string_datatype());
628                }
629                "age" => {
630                    assert_eq!(items[i], Value::Int64(30));
631                    assert_eq!(field.data_type(), &ConcreteDataType::int64_datatype());
632                }
633                "active" => {
634                    assert_eq!(items[i], Value::Boolean(true));
635                    assert_eq!(field.data_type(), &ConcreteDataType::boolean_datatype());
636                }
637                _ => panic!("Unexpected field: {}", field.name()),
638            }
639        }
640    }
641
642    #[test]
643    fn test_encode_json_nested_object() {
644        let json = json!({
645            "person": {
646                "name": "Alice",
647                "age": 25
648            },
649            "scores": [95, 87, 92]
650        });
651
652        let settings = JsonSettings::default();
653        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
654        let Value::Struct(result) = result else {
655            panic!("Expected Struct value");
656        };
657        assert_eq!(result.items().len(), 2);
658
659        let items = result.items();
660        let struct_type = result.struct_type();
661
662        // Check person field (nested struct)
663        let person_index = struct_type
664            .fields()
665            .iter()
666            .position(|f| f.name() == "person")
667            .unwrap();
668        if let Value::Struct(person_struct) = &items[person_index] {
669            assert_eq!(person_struct.items().len(), 2);
670            let fields = person_struct.struct_type().fields();
671            let person_fields: Vec<&str> = fields.iter().map(|f| f.name()).collect();
672            assert!(person_fields.contains(&"name"));
673            assert!(person_fields.contains(&"age"));
674        } else {
675            panic!("Expected Struct value for person field");
676        }
677
678        // Check scores field (list)
679        let scores_index = struct_type
680            .fields()
681            .iter()
682            .position(|f| f.name() == "scores")
683            .unwrap();
684        if let Value::List(scores_list) = &items[scores_index] {
685            assert_eq!(scores_list.items().len(), 3);
686            assert_eq!(scores_list.items()[0], Value::Int64(95));
687            assert_eq!(scores_list.items()[1], Value::Int64(87));
688            assert_eq!(scores_list.items()[2], Value::Int64(92));
689        } else {
690            panic!("Expected List value for scores field");
691        }
692    }
693
694    #[test]
695    fn test_encode_json_array_mixed_types() {
696        let json = json!([1, "hello", true, 3.15]);
697        let settings = JsonSettings::default();
698        let value = settings.encode(json).unwrap();
699        assert_eq!(value.data_type().to_string(), r#"Json2["<Variant>"]"#);
700    }
701
702    #[test]
703    fn test_encode_json_empty_array() {
704        let json = json!([]);
705        let settings = JsonSettings::default();
706        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
707
708        if let Value::List(list_value) = result {
709            assert_eq!(list_value.items().len(), 0);
710            // Empty arrays default to string type
711            assert_eq!(
712                list_value.datatype(),
713                Arc::new(ConcreteDataType::null_datatype())
714            );
715        } else {
716            panic!("Expected List value");
717        }
718    }
719
720    #[test]
721    fn test_encode_json_structured() {
722        let json = json!({
723            "name": "Bob",
724            "age": 35
725        });
726
727        let settings = JsonSettings::default();
728        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
729
730        if let Value::Struct(struct_value) = result {
731            assert_eq!(struct_value.items().len(), 2);
732            let fields = struct_value.struct_type().fields();
733            let field_names: Vec<&str> = fields.iter().map(|f| f.name()).collect();
734            assert!(field_names.contains(&"name"));
735            assert!(field_names.contains(&"age"));
736        } else {
737            panic!("Expected Struct value");
738        }
739    }
740
741    #[test]
742    fn test_encode_json_respects_type_hint() {
743        let settings = JsonSettings::new(vec![JsonTypeHint {
744            path: vec!["age".to_string()],
745            data_type: ConcreteDataType::int64_datatype(),
746            nullable: false,
747            default_constraint: None,
748            inverted_index: false,
749        }]);
750
751        let result = settings
752            .encode(json!({
753                "name": "Alice",
754                "age": 42
755            }))
756            .unwrap()
757            .into_json_inner()
758            .unwrap();
759
760        let Value::Struct(struct_value) = result else {
761            panic!("Expected Struct value");
762        };
763        assert_eq!(struct_field_value(&struct_value, "age"), &Value::Int64(42));
764
765        let err = settings
766            .encode(json!({
767                "age": "42"
768            }))
769            .unwrap_err();
770        assert!(err.to_string().contains("does not match JSON2 type hint"));
771    }
772
773    #[test]
774    fn test_encode_json_respects_unsigned_type_hint() {
775        let settings = JsonSettings::new(vec![JsonTypeHint {
776            path: vec!["count".to_string()],
777            data_type: ConcreteDataType::uint64_datatype(),
778            nullable: false,
779            default_constraint: None,
780            inverted_index: false,
781        }]);
782
783        let result = settings
784            .encode(json!({
785                "count": u64::MAX
786            }))
787            .unwrap()
788            .into_json_inner()
789            .unwrap();
790
791        let Value::Struct(struct_value) = result else {
792            panic!("Expected Struct value");
793        };
794        assert_eq!(
795            struct_field_value(&struct_value, "count"),
796            &Value::UInt64(u64::MAX)
797        );
798
799        let err = settings
800            .encode(json!({
801                "count": -1
802            }))
803            .unwrap_err();
804        assert!(err.to_string().contains("does not match JSON2 type hint"));
805    }
806
807    #[test]
808    fn test_encode_json_fills_missing_type_hint_with_default() {
809        let settings = JsonSettings::new(vec![JsonTypeHint {
810            path: vec!["user".to_string(), "age".to_string()],
811            data_type: ConcreteDataType::int64_datatype(),
812            nullable: false,
813            default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
814            inverted_index: false,
815        }]);
816
817        let result = settings
818            .encode(json!({}))
819            .unwrap()
820            .into_json_inner()
821            .unwrap();
822
823        let Value::Struct(root) = result else {
824            panic!("Expected Struct value");
825        };
826        let Value::Struct(user) = struct_field_value(&root, "user") else {
827            panic!("Expected user Struct value");
828        };
829        assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
830    }
831
832    #[test]
833    fn test_encode_json_fills_missing_nullable_type_hint_with_null() {
834        let settings = JsonSettings::new(vec![JsonTypeHint {
835            path: vec!["user".to_string(), "name".to_string()],
836            data_type: ConcreteDataType::string_datatype(),
837            nullable: true,
838            default_constraint: None,
839            inverted_index: false,
840        }]);
841
842        let result = settings
843            .encode(json!({ "user": {} }))
844            .unwrap()
845            .into_json_inner()
846            .unwrap();
847
848        let Value::Struct(root) = result else {
849            panic!("Expected Struct value");
850        };
851        let Value::Struct(user) = struct_field_value(&root, "user") else {
852            panic!("Expected user Struct value");
853        };
854        assert_eq!(struct_field_value(user, "name"), &Value::Null);
855    }
856
857    #[test]
858    fn test_encode_json_rejects_missing_non_null_type_hint() {
859        let settings = JsonSettings::new(vec![JsonTypeHint {
860            path: vec!["user".to_string(), "age".to_string()],
861            data_type: ConcreteDataType::int64_datatype(),
862            nullable: false,
863            default_constraint: None,
864            inverted_index: false,
865        }]);
866
867        let err = settings.encode(json!({})).unwrap_err();
868        assert!(
869            err.to_string()
870                .contains("missing non-null JSON2 type hint path user.age")
871        );
872    }
873
874    #[test]
875    fn test_encode_json_merges_missing_type_hint_prefix() {
876        let settings = JsonSettings::new(vec![
877            JsonTypeHint {
878                path: vec!["user".to_string(), "age".to_string()],
879                data_type: ConcreteDataType::int64_datatype(),
880                nullable: false,
881                default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
882                inverted_index: false,
883            },
884            JsonTypeHint {
885                path: vec!["user".to_string(), "name".to_string()],
886                data_type: ConcreteDataType::string_datatype(),
887                nullable: false,
888                default_constraint: Some(ColumnDefaultConstraint::Value(Value::String(
889                    "unknown".into(),
890                ))),
891                inverted_index: false,
892            },
893        ]);
894
895        let result = settings
896            .encode(json!({}))
897            .unwrap()
898            .into_json_inner()
899            .unwrap();
900
901        let Value::Struct(root) = result else {
902            panic!("Expected Struct value");
903        };
904        let Value::Struct(user) = struct_field_value(&root, "user") else {
905            panic!("Expected user Struct value");
906        };
907        assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
908        assert_eq!(
909            struct_field_value(user, "name"),
910            &Value::String("unknown".into())
911        );
912    }
913
914    #[test]
915    fn test_json_settings_structured() {
916        let json = json!({
917            "name": "Eve",
918            "score": 95
919        });
920
921        let settings = JsonSettings::default();
922        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
923
924        if let Value::Struct(struct_value) = result {
925            assert_eq!(struct_value.items().len(), 2);
926        } else {
927            panic!("Expected Struct value");
928        }
929    }
930
931    #[cfg(test)]
932    mod decode_tests {
933        use ordered_float::OrderedFloat;
934        use serde_json::json;
935
936        use super::*;
937
938        #[test]
939        fn test_decode_primitive_values() {
940            let settings = JsonSettings::default();
941
942            // Test null
943            let result = settings.decode(Value::Null).unwrap();
944            assert_eq!(result, Json::Null);
945
946            // Test boolean
947            let result = settings.decode(Value::Boolean(true)).unwrap();
948            assert_eq!(result, Json::Bool(true));
949
950            // Test integer
951            let result = settings.decode(Value::Int64(42)).unwrap();
952            assert_eq!(result, Json::from(42));
953
954            // Test float
955            let result = settings.decode(Value::Float64(OrderedFloat(3.16))).unwrap();
956            assert_eq!(result, Json::from(3.16));
957
958            // Test string
959            let result = settings.decode(Value::String("hello".into())).unwrap();
960            assert_eq!(result, Json::String("hello".to_string()));
961        }
962
963        #[test]
964        fn test_decode_struct() {
965            let settings = JsonSettings::default();
966
967            let struct_value = StructValue::new(
968                vec![
969                    Value::String("Alice".into()),
970                    Value::Int64(25),
971                    Value::Boolean(true),
972                ],
973                StructType::new(Arc::new(vec![
974                    StructField::new(
975                        "name".to_string(),
976                        ConcreteDataType::string_datatype(),
977                        true,
978                    ),
979                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
980                    StructField::new(
981                        "active".to_string(),
982                        ConcreteDataType::boolean_datatype(),
983                        true,
984                    ),
985                ])),
986            );
987
988            let result = settings.decode(Value::Struct(struct_value)).unwrap();
989            let expected = json!({
990                "name": "Alice",
991                "age": 25,
992                "active": true
993            });
994            assert_eq!(result, expected);
995        }
996
997        #[test]
998        fn test_decode_list() {
999            let settings = JsonSettings::default();
1000
1001            let list_value = ListValue::new(
1002                vec![Value::Int64(1), Value::Int64(2), Value::Int64(3)],
1003                Arc::new(ConcreteDataType::int64_datatype()),
1004            );
1005
1006            let result = settings.decode(Value::List(list_value)).unwrap();
1007            let expected = json!([1, 2, 3]);
1008            assert_eq!(result, expected);
1009        }
1010
1011        #[test]
1012        fn test_decode_nested_structure() {
1013            let settings = JsonSettings::default();
1014
1015            let inner_struct = StructValue::new(
1016                vec![Value::String("Alice".into()), Value::Int64(25)],
1017                StructType::new(Arc::new(vec![
1018                    StructField::new(
1019                        "name".to_string(),
1020                        ConcreteDataType::string_datatype(),
1021                        true,
1022                    ),
1023                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
1024                ])),
1025            );
1026
1027            let score_list_item_type = Arc::new(ConcreteDataType::int64_datatype());
1028            let outer_struct = StructValue::new(
1029                vec![
1030                    Value::Struct(inner_struct),
1031                    Value::List(ListValue::new(
1032                        vec![Value::Int64(95), Value::Int64(87)],
1033                        score_list_item_type.clone(),
1034                    )),
1035                ],
1036                StructType::new(Arc::new(vec![
1037                    StructField::new(
1038                        "user".to_string(),
1039                        ConcreteDataType::Struct(StructType::new(Arc::new(vec![
1040                            StructField::new(
1041                                "name".to_string(),
1042                                ConcreteDataType::string_datatype(),
1043                                true,
1044                            ),
1045                            StructField::new(
1046                                "age".to_string(),
1047                                ConcreteDataType::int64_datatype(),
1048                                true,
1049                            ),
1050                        ]))),
1051                        true,
1052                    ),
1053                    StructField::new(
1054                        "scores".to_string(),
1055                        ConcreteDataType::List(ListType::new(score_list_item_type.clone())),
1056                        true,
1057                    ),
1058                ])),
1059            );
1060
1061            let result = settings.decode(Value::Struct(outer_struct)).unwrap();
1062            let expected = json!({
1063                "user": {
1064                    "name": "Alice",
1065                    "age": 25
1066                },
1067                "scores": [95, 87]
1068            });
1069            assert_eq!(result, expected);
1070        }
1071
1072        #[test]
1073        fn test_decode_missing_fields() {
1074            let settings = JsonSettings::default();
1075
1076            // Struct with missing field (null value)
1077            let struct_value = StructValue::new(
1078                vec![
1079                    Value::String("Bob".into()),
1080                    Value::Null, // missing age field
1081                ],
1082                StructType::new(Arc::new(vec![
1083                    StructField::new(
1084                        "name".to_string(),
1085                        ConcreteDataType::string_datatype(),
1086                        true,
1087                    ),
1088                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
1089                ])),
1090            );
1091
1092            let result = settings.decode(Value::Struct(struct_value)).unwrap();
1093            let expected = json!({
1094                "name": "Bob",
1095                "age": null
1096            });
1097            assert_eq!(result, expected);
1098        }
1099    }
1100
1101    #[test]
1102    fn test_encode_json_large_unsigned_integer() {
1103        // Test unsigned integer that fits in i64
1104        let json = Json::from(u64::MAX / 2);
1105        let settings = JsonSettings::default();
1106        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
1107        assert_eq!(result, Value::Int64((u64::MAX / 2) as i64));
1108
1109        // Test unsigned integer that exceeds i64 range
1110        let json = Json::from(u64::MAX);
1111        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
1112        assert_eq!(result, Value::UInt64(u64::MAX));
1113    }
1114}
datatypes/json.rs

datatypes/
json.rs
