Skip to main content

datatypes/
json.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Data conversion between greptime's StructType and Json
16//!
17//! The idea of this module is to provide utilities to convert serde_json::Value to greptime's StructType and vice versa.
18//!
19//! The struct will carry all the fields of the Json object. We will not flatten any json object in this implementation.
20//!
21
22pub mod value;
23
24use std::collections::BTreeMap;
25use std::collections::btree_map::Entry;
26
27use serde::{Deserialize, Serialize};
28use serde_json::{Map, Value as Json};
29use snafu::ResultExt;
30
31use crate::data_type::ConcreteDataType;
32use crate::error::{self, InvalidJson2LayoutSnafu, Result, UnsupportedJsonTypeSnafu};
33use crate::json::value::{JsonValue, JsonVariant, encode_serde_json_as_jsonb};
34use crate::schema::ColumnDefaultConstraint;
35use crate::types::json_type::{JsonNativeType, JsonObjectType};
36use crate::value::{ListValue, StructValue, Value};
37
38/// Maximum number of JSON container levels represented as nested Arrow types.
39pub const JSON2_MAX_STRUCTURED_DEPTH: usize = 50;
40/// Reserved physical field containing unexpanded JSON2 paths.
41pub const JSON2_REMAINDER_FIELD_NAME: &str = "!__remainder__!";
42
43/// JSON2 settings stored in column schema metadata and represented through
44/// Arrow extension metadata.
45#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
46pub struct JsonSettings {
47    #[serde(default)]
48    type_hints: Vec<JsonTypeHint>,
49    /// Maximum number of unhinted JSON paths expanded into Arrow fields.
50    ///
51    /// `None` preserves the legacy unlimited layout.
52    #[serde(default, skip_serializing_if = "Option::is_none")]
53    max_auto_expanded_paths: Option<u32>,
54}
55
56#[derive(Deserialize)]
57struct JsonSettingsSerde {
58    #[serde(default)]
59    type_hints: Vec<JsonTypeHint>,
60    #[serde(default)]
61    max_auto_expanded_paths: Option<u32>,
62}
63
64impl<'de> Deserialize<'de> for JsonSettings {
65    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
66    where
67        D: serde::Deserializer<'de>,
68    {
69        let settings = JsonSettingsSerde::deserialize(deserializer)?;
70        Self::try_new(settings.type_hints, settings.max_auto_expanded_paths)
71            .map_err(serde::de::Error::custom)
72    }
73}
74
75/// Declares selected JSON2 subpaths as typed fields.
76///
77/// These hints let JSON2 encode frequently used subpaths in a typed layout, so
78/// queries over those subpaths can get behavior and performance closer to
79/// ordinary columns.
80#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
81pub struct JsonTypeHint {
82    /// JSON2 subpath for a typed field.
83    ///
84    /// Each item is one JSON object key. For example, `["user", "age"]`
85    /// represents `user.age`.
86    ///
87    /// Array traversal is not currently supported. For example, a hint cannot
88    /// describe `events[0].name` or fields shared by all items in `events[*]`.
89    pub path: Vec<String>,
90    #[serde(rename = "type")]
91    pub data_type: ConcreteDataType,
92    pub nullable: bool,
93    #[serde(skip_serializing_if = "Option::is_none")]
94    pub default_constraint: Option<ColumnDefaultConstraint>,
95    pub inverted_index: bool,
96}
97
98/// Context for JSON encoding/decoding that tracks the current key path.
99#[derive(Clone, Debug)]
100pub struct JsonContext<'a> {
101    /// Current key path from the JSON2 root.
102    pub path: Vec<String>,
103    /// Settings for JSON encoding/decoding.
104    pub settings: &'a JsonSettings,
105}
106
107impl JsonSettings {
108    /// Creates and validates JSON2 settings.
109    pub fn try_new(
110        type_hints: Vec<JsonTypeHint>,
111        max_auto_expanded_paths: Option<u32>,
112    ) -> Result<Self> {
113        validate_type_hints(&type_hints)?;
114        Ok(Self {
115            type_hints,
116            max_auto_expanded_paths,
117        })
118    }
119
120    pub fn type_hints(&self) -> &[JsonTypeHint] {
121        &self.type_hints
122    }
123
124    pub fn max_auto_expanded_paths(&self) -> Option<u32> {
125        self.max_auto_expanded_paths
126    }
127
128    pub fn into_parts(self) -> (Vec<JsonTypeHint>, Option<u32>) {
129        (self.type_hints, self.max_auto_expanded_paths)
130    }
131
132    /// Decode an encoded StructValue back into a serde_json::Value.
133    pub fn decode(&self, value: Value) -> Result<Json> {
134        let mut context = JsonContext {
135            path: Vec::new(),
136            settings: self,
137        };
138        decode_value_with_context(value, &mut context)
139    }
140
141    /// Encode a serde_json::Value into a Value::Json using current settings.
142    pub fn encode(&self, json: Json) -> Result<Value> {
143        let mut context = JsonContext {
144            path: Vec::new(),
145            settings: self,
146        };
147        encode_json_with_context(json, &mut context).map(|v| Value::Json(Box::new(v)))
148    }
149}
150
151fn validate_type_hints(type_hints: &[JsonTypeHint]) -> Result<()> {
152    let mut object = JsonObjectType::new();
153    for hint in type_hints {
154        if hint.path.len() > JSON2_MAX_STRUCTURED_DEPTH {
155            return InvalidJson2LayoutSnafu {
156                reason: format!(
157                    "JSON2 type hint path cannot exceed {JSON2_MAX_STRUCTURED_DEPTH} segments"
158                ),
159            }
160            .fail();
161        }
162        if hint
163            .path
164            .first()
165            .is_some_and(|x| x == JSON2_REMAINDER_FIELD_NAME)
166        {
167            return InvalidJson2LayoutSnafu {
168                reason: format!(
169                    "JSON2 type hint path cannot be rooted at reserved field '{JSON2_REMAINDER_FIELD_NAME}'"
170                ),
171            }
172            .fail();
173        }
174        let data_type = match &hint.data_type {
175            ConcreteDataType::Boolean(_)
176            | ConcreteDataType::UInt8(_)
177            | ConcreteDataType::UInt16(_)
178            | ConcreteDataType::UInt32(_)
179            | ConcreteDataType::UInt64(_)
180            | ConcreteDataType::Int8(_)
181            | ConcreteDataType::Int16(_)
182            | ConcreteDataType::Int32(_)
183            | ConcreteDataType::Int64(_)
184            | ConcreteDataType::Float32(_)
185            | ConcreteDataType::Float64(_)
186            | ConcreteDataType::String(_) => (&hint.data_type).into(),
187            data_type => {
188                return InvalidJson2LayoutSnafu {
189                    reason: format!("unsupported JSON2 type hint data type: {data_type}"),
190                }
191                .fail();
192            }
193        };
194        validate_type_hint(&mut object, &hint.path, data_type)?;
195    }
196    Ok(())
197}
198
199fn validate_type_hint(
200    object: &mut JsonObjectType,
201    path: &[String],
202    data_type: JsonNativeType,
203) -> Result<()> {
204    let Some((name, path)) = path.split_first() else {
205        return InvalidJson2LayoutSnafu {
206            reason: "JSON2 type hint path must not be empty".to_string(),
207        }
208        .fail();
209    };
210    if path.is_empty() {
211        if object.insert(name.clone(), data_type).is_some() {
212            return InvalidJson2LayoutSnafu {
213                reason: format!("duplicate JSON2 type hint path '{name}'"),
214            }
215            .fail();
216        }
217        return Ok(());
218    }
219
220    let child = object
221        .entry(name.clone())
222        .or_insert_with(|| JsonNativeType::Object(JsonObjectType::new()));
223    let JsonNativeType::Object(child) = child else {
224        return InvalidJson2LayoutSnafu {
225            reason: format!("conflicting JSON2 type hint path at '{name}'"),
226        }
227        .fail();
228    };
229    validate_type_hint(child, path, data_type)
230}
231
232impl<'a> JsonContext<'a> {
233    fn type_hint(&self) -> Option<&'a JsonTypeHint> {
234        self.settings
235            .type_hints
236            .iter()
237            .find(|hint| hint.path == self.path)
238    }
239}
240
241fn with_key_context<T>(
242    context: &mut JsonContext,
243    key: &str,
244    f: impl FnOnce(&mut JsonContext) -> Result<T>,
245) -> Result<T> {
246    context.path.push(key.to_string());
247    let result = f(context);
248    context.path.pop();
249    result
250}
251
252/// Main encoding function with key path tracking
253fn encode_json_with_context(json: Json, context: &mut JsonContext) -> Result<JsonValue> {
254    if context.path.is_empty() && !matches!(json, Json::Object(_)) {
255        return UnsupportedJsonTypeSnafu.fail();
256    }
257
258    match json {
259        Json::Object(json_object) => encode_json_object_with_context(json_object, context),
260        Json::Array(json_array) => encode_json_array_with_context(json_array, context),
261        _ => encode_json_value_with_context(json, context),
262    }
263}
264
265fn encode_json_object_with_context<'a>(
266    json_object: Map<String, Json>,
267    context: &mut JsonContext<'a>,
268) -> Result<JsonValue> {
269    let mut object = BTreeMap::new();
270    for (key, value) in json_object {
271        let value = with_key_context(context, &key, |context| {
272            if let Some(hint) = context.type_hint() {
273                encode_json_value_with_hint(value, hint, context)
274            } else {
275                encode_json_value_with_context(value, context)
276            }
277        })?;
278
279        object.insert(key, value.into_variant());
280    }
281
282    apply_missing_type_hints(&mut object, context)?;
283
284    Ok(JsonValue::new(JsonVariant::Object(object)))
285}
286
287fn apply_missing_type_hints(
288    object: &mut BTreeMap<String, JsonVariant>,
289    context: &mut JsonContext,
290) -> Result<()> {
291    for hint in &context.settings.type_hints {
292        if hint.path.len() > context.path.len() && hint.path.starts_with(&context.path) {
293            let depth = context.path.len();
294            let key = &hint.path[depth];
295            with_key_context(context, key, |context| {
296                insert_missing_type_hint(object, context, hint, depth)
297            })?;
298        }
299    }
300    Ok(())
301}
302
303fn insert_missing_type_hint(
304    object: &mut BTreeMap<String, JsonVariant>,
305    field_context: &mut JsonContext,
306    hint: &JsonTypeHint,
307    depth: usize,
308) -> Result<()> {
309    let key = &hint.path[depth];
310    let is_leaf = depth + 1 == hint.path.len();
311
312    if is_leaf {
313        if !object.contains_key(key) {
314            let value = encode_missing_type_hint_value(hint, field_context)?;
315            object.insert(key.clone(), value.into_variant());
316        }
317        return Ok(());
318    }
319
320    match object.entry(key.clone()) {
321        Entry::Occupied(mut entry) => match entry.get_mut() {
322            JsonVariant::Object(child) => {
323                insert_missing_type_hint(child, field_context, hint, depth + 1)
324            }
325            _ => error::InvalidJsonSnafu {
326                value: format!(
327                    "JSON2 type hint path {} expects object at {}",
328                    hint.path.join("."),
329                    field_context.path.join(".")
330                ),
331            }
332            .fail(),
333        },
334        Entry::Vacant(entry) => {
335            let mut child = BTreeMap::new();
336            insert_missing_type_hint(&mut child, field_context, hint, depth + 1)?;
337            entry.insert(JsonVariant::Object(child));
338            Ok(())
339        }
340    }
341}
342
343fn encode_missing_type_hint_value(
344    hint: &JsonTypeHint,
345    context: &mut JsonContext,
346) -> Result<JsonValue> {
347    if let Some(default_constraint) = &hint.default_constraint {
348        let value = default_constraint.create_default(&hint.data_type, hint.nullable)?;
349        let json = decode_primitive_value(value)?;
350        return encode_json_value_with_hint(json, hint, context);
351    }
352
353    if hint.nullable {
354        Ok(JsonValue::null())
355    } else {
356        error::InvalidJsonSnafu {
357            value: format!(
358                "missing non-null JSON2 type hint path {}",
359                hint.path.join(".")
360            ),
361        }
362        .fail()
363    }
364}
365
366fn encode_json_value_with_hint(
367    json: Json,
368    hint: &JsonTypeHint,
369    context: &mut JsonContext,
370) -> Result<JsonValue> {
371    if json.is_null() {
372        return if hint.nullable {
373            Ok(JsonValue::null())
374        } else {
375            error::InvalidJsonSnafu {
376                value: format!(
377                    "JSON2 type hint path {} is not nullable",
378                    context.path.join(".")
379                ),
380            }
381            .fail()
382        };
383    }
384
385    let invalid_type = || {
386        error::InvalidJsonSnafu {
387            value: format!(
388                "JSON value at {} does not match JSON2 type hint {}",
389                context.path.join("."),
390                hint.data_type
391            ),
392        }
393        .fail()
394    };
395
396    match (&hint.data_type, json) {
397        (ConcreteDataType::String(_), Json::String(v)) => Ok(v.into()),
398        (
399            ConcreteDataType::Int8(_)
400            | ConcreteDataType::Int16(_)
401            | ConcreteDataType::Int32(_)
402            | ConcreteDataType::Int64(_),
403            Json::Number(v),
404        ) => match v.as_i64() {
405            Some(v) => Ok(v.into()),
406            None => invalid_type(),
407        },
408        (
409            ConcreteDataType::UInt8(_)
410            | ConcreteDataType::UInt16(_)
411            | ConcreteDataType::UInt32(_)
412            | ConcreteDataType::UInt64(_),
413            Json::Number(v),
414        ) => match v.as_u64() {
415            Some(v) => Ok(v.into()),
416            None => invalid_type(),
417        },
418        (ConcreteDataType::Float32(_) | ConcreteDataType::Float64(_), Json::Number(v)) => {
419            match v.as_f64() {
420                Some(v) => Ok(v.into()),
421                None => invalid_type(),
422            }
423        }
424        (ConcreteDataType::Boolean(_), Json::Bool(v)) => Ok(v.into()),
425        _ => invalid_type(),
426    }
427}
428
429fn encode_json_array_with_context<'a>(
430    json_array: Vec<Json>,
431    context: &mut JsonContext<'a>,
432) -> Result<JsonValue> {
433    let json_array_len = json_array.len();
434    let mut items = Vec::with_capacity(json_array_len);
435
436    for (index, value) in json_array.into_iter().enumerate() {
437        let item_value = with_key_context(context, &index.to_string(), |context| {
438            encode_json_value_with_context(value, context)
439        })?;
440        items.push(item_value);
441    }
442
443    // In specification, it's valid for a JSON array to have different types of items, for example,
444    // ["a string", 1]. However, in implementation, the `JsonValue` will be converted to Arrow list
445    // array, which requires all items have exactly the same type. So we merge out the maybe
446    // different item types to a unified type, and align all the item values to it.
447
448    let merged_item_type = if let Some((first, rests)) = items.split_first() {
449        let mut merged = first.json_type().clone();
450        for rest in rests.iter().map(|x| x.json_type()) {
451            if matches!(merged, JsonNativeType::Variant) {
452                break;
453            }
454            merged.merge(rest);
455        }
456        Some(merged)
457    } else {
458        None
459    };
460    if let Some(unified_item_type) = merged_item_type {
461        for item in &mut items {
462            item.try_align(&unified_item_type)?;
463        }
464    }
465    let items = items
466        .into_iter()
467        .map(|x| x.into_variant())
468        .collect::<Vec<_>>();
469    Ok(JsonValue::new(JsonVariant::Array(items)))
470}
471
472/// Helper function to encode a JSON value to a Value and determine its ConcreteDataType with context
473fn encode_json_value_with_context(json: Json, context: &mut JsonContext) -> Result<JsonValue> {
474    if context.path.len() >= JSON2_MAX_STRUCTURED_DEPTH
475        && matches!(&json, Json::Object(_) | Json::Array(_))
476    {
477        return Ok(JsonValue::new(JsonVariant::Variant(
478            encode_serde_json_as_jsonb(json),
479        )));
480    }
481
482    match json {
483        Json::Null => Ok(JsonValue::null()),
484        Json::Bool(b) => Ok(b.into()),
485        Json::Number(n) => {
486            if let Some(i) = n.as_i64() {
487                Ok(i.into())
488            } else if let Some(u) = n.as_u64() {
489                if u <= i64::MAX as u64 {
490                    Ok((u as i64).into())
491                } else {
492                    Ok(u.into())
493                }
494            } else if let Some(f) = n.as_f64() {
495                Ok(f.into())
496            } else {
497                // Fallback to string representation
498                Ok(n.to_string().into())
499            }
500        }
501        Json::String(s) => Ok(s.into()),
502        Json::Array(arr) => encode_json_array_with_context(arr, context),
503        Json::Object(obj) => encode_json_object_with_context(obj, context),
504    }
505}
506
507/// Main decoding function with key path tracking
508fn decode_value_with_context(value: Value, context: &mut JsonContext) -> Result<Json> {
509    match value {
510        Value::Struct(struct_value) => decode_struct_with_context(struct_value, context),
511        Value::List(list_value) => decode_list_with_context(list_value, context),
512        _ => decode_primitive_value(value),
513    }
514}
515
516/// Decode a structured value to JSON object
517fn decode_struct_with_context<'a>(
518    struct_value: StructValue,
519    context: &mut JsonContext<'a>,
520) -> Result<Json> {
521    let mut json_object = Map::with_capacity(struct_value.len());
522
523    let (items, fields) = struct_value.into_parts();
524
525    for (field, field_value) in fields.fields().iter().zip(items) {
526        let json_value = with_key_context(context, field.name(), |context| {
527            decode_value_with_context(field_value, context)
528        })?;
529        json_object.insert(field.name().to_string(), json_value);
530    }
531
532    Ok(Json::Object(json_object))
533}
534
535/// Decode a list value to JSON array
536fn decode_list_with_context(list_value: ListValue, context: &mut JsonContext) -> Result<Json> {
537    let mut json_array = Vec::with_capacity(list_value.len());
538
539    let data_items = list_value.take_items();
540
541    for (index, item) in data_items.into_iter().enumerate() {
542        let json_value = with_key_context(context, &index.to_string(), |context| {
543            decode_value_with_context(item, context)
544        })?;
545        json_array.push(json_value);
546    }
547
548    Ok(Json::Array(json_array))
549}
550
551/// Decode primitive value to JSON
552fn decode_primitive_value(value: Value) -> Result<Json> {
553    match value {
554        Value::Null => Ok(Json::Null),
555        Value::Boolean(b) => Ok(Json::Bool(b)),
556        Value::UInt8(v) => Ok(Json::from(v)),
557        Value::UInt16(v) => Ok(Json::from(v)),
558        Value::UInt32(v) => Ok(Json::from(v)),
559        Value::UInt64(v) => Ok(Json::from(v)),
560        Value::Int8(v) => Ok(Json::from(v)),
561        Value::Int16(v) => Ok(Json::from(v)),
562        Value::Int32(v) => Ok(Json::from(v)),
563        Value::Int64(v) => Ok(Json::from(v)),
564        Value::Float32(v) => Ok(Json::from(v.0)),
565        Value::Float64(v) => Ok(Json::from(v.0)),
566        Value::String(s) => Ok(Json::String(s.as_utf8().to_string())),
567        Value::Binary(b) => serde_json::to_value(b.as_ref()).context(error::SerializeSnafu),
568        Value::Date(v) => Ok(Json::from(v.val())),
569        Value::Timestamp(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
570        Value::Time(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
571        Value::IntervalYearMonth(v) => {
572            serde_json::to_value(v.to_i32()).context(error::SerializeSnafu)
573        }
574        Value::IntervalDayTime(v) => {
575            serde_json::to_value(v.to_i64()).context(error::SerializeSnafu)
576        }
577        Value::IntervalMonthDayNano(v) => {
578            serde_json::to_value(v.to_i128()).context(error::SerializeSnafu)
579        }
580        Value::Duration(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
581        Value::Decimal128(v) => serde_json::to_value(v.to_string()).context(error::SerializeSnafu),
582        Value::Struct(_) | Value::List(_) | Value::Json(_) => {
583            // These should be handled by the context-aware functions
584            Err(error::InvalidJsonSnafu {
585                value: "Structured values should be handled by context-aware decoding".to_string(),
586            }
587            .build())
588        }
589    }
590}
591
592#[cfg(test)]
593mod tests {
594    use std::sync::Arc;
595
596    use serde_json::json;
597
598    use super::*;
599    use crate::data_type::ConcreteDataType;
600    use crate::types::{ListType, StructField, StructType};
601
602    fn struct_field_value<'a>(struct_value: &'a StructValue, field_name: &str) -> &'a Value {
603        let index = struct_value
604            .struct_type()
605            .fields()
606            .iter()
607            .position(|field| field.name() == field_name)
608            .expect("field exists");
609        &struct_value.items()[index]
610    }
611
612    #[test]
613    fn test_json_settings_forward_compatibility()
614    -> std::result::Result<(), Box<dyn std::error::Error>> {
615        let json_str = r#"{
616            "type_hints": [
617                {
618                    "path": ["user", "age"],
619                    "type": {
620                        "Int64": {}
621                    },
622                    "nullable": false,
623                    "default_constraint": {
624                        "Value": {
625                            "Int64": 18
626                        }
627                    },
628                    "inverted_index": true
629                },
630                {
631                    "path": ["user", "name"],
632                    "type": {
633                        "String": {
634                            "size_type": "Utf8"
635                        }
636                    },
637                    "nullable": true,
638                    "inverted_index": false
639                }
640            ]
641        }"#;
642
643        let deserialized = serde_json::from_str::<JsonSettings>(json_str)?;
644
645        let expected = JsonSettings::try_new(
646            vec![
647                JsonTypeHint {
648                    path: vec!["user".to_string(), "age".to_string()],
649                    data_type: ConcreteDataType::int64_datatype(),
650                    nullable: false,
651                    default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
652                    inverted_index: true,
653                },
654                JsonTypeHint {
655                    path: vec!["user".to_string(), "name".to_string()],
656                    data_type: ConcreteDataType::string_datatype(),
657                    nullable: true,
658                    default_constraint: None,
659                    inverted_index: false,
660                },
661            ],
662            None,
663        )?;
664        assert_eq!(deserialized, expected);
665        Ok(())
666    }
667
668    #[test]
669    fn test_json_settings_ser_de() -> std::result::Result<(), Box<dyn std::error::Error>> {
670        let type_hints = vec![
671            JsonTypeHint {
672                path: vec!["user".to_string(), "age".to_string()],
673                data_type: ConcreteDataType::int64_datatype(),
674                nullable: false,
675                default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
676                inverted_index: true,
677            },
678            JsonTypeHint {
679                path: vec!["user".to_string(), "name".to_string()],
680                data_type: ConcreteDataType::string_datatype(),
681                nullable: true,
682                default_constraint: None,
683                inverted_index: false,
684            },
685        ];
686
687        for settings in [
688            JsonSettings {
689                type_hints: type_hints.clone(),
690                max_auto_expanded_paths: None,
691            },
692            JsonSettings {
693                type_hints,
694                max_auto_expanded_paths: Some(1),
695            },
696        ] {
697            let serialized = serde_json::to_string(&settings)?;
698            let deserialized = serde_json::from_str::<JsonSettings>(&serialized)?;
699            assert_eq!(settings, deserialized);
700        }
701        Ok(())
702    }
703
704    #[test]
705    fn test_json_settings_reject_invalid_type_hint_layout() {
706        for type_hints in [
707            json!([{"path": [], "type": {"Int64": {}}, "nullable": true, "inverted_index": false}]),
708            json!([
709                {"path": ["a"], "type": {"Int64": {}}, "nullable": true, "inverted_index": false},
710                {"path": ["a", "b"], "type": {"Int64": {}}, "nullable": true, "inverted_index": false}
711            ]),
712            json!([{"path": [JSON2_REMAINDER_FIELD_NAME], "type": {"Int64": {}}, "nullable": true, "inverted_index": false}]),
713        ] {
714            let settings = json!({"type_hints": type_hints});
715            assert!(serde_json::from_value::<JsonSettings>(settings).is_err());
716        }
717
718        let hint = |path, data_type| JsonTypeHint {
719            path,
720            data_type,
721            nullable: true,
722            default_constraint: None,
723            inverted_index: false,
724        };
725        assert!(
726            JsonSettings::try_new(
727                vec![hint(
728                    vec!["nested".to_string(), JSON2_REMAINDER_FIELD_NAME.to_string()],
729                    ConcreteDataType::string_datatype(),
730                )],
731                None,
732            )
733            .is_ok()
734        );
735        assert!(
736            JsonSettings::try_new(
737                vec![hint(
738                    vec!["nested".to_string(); JSON2_MAX_STRUCTURED_DEPTH + 1],
739                    ConcreteDataType::string_datatype(),
740                )],
741                None,
742            )
743            .is_err()
744        );
745        assert!(
746            JsonSettings::try_new(
747                vec![hint(
748                    vec!["date".to_string()],
749                    ConcreteDataType::date_datatype(),
750                )],
751                None,
752            )
753            .is_err()
754        );
755    }
756
757    #[test]
758    fn test_encode_root_non_object_json() {
759        let settings = JsonSettings::default();
760        let cases = [
761            ("null", Json::Null),
762            ("boolean", Json::Bool(true)),
763            ("integer", Json::from(42)),
764            ("float", Json::from(3.15)),
765            ("string", Json::String("hello".to_string())),
766            ("array", json!([1, 2, 3])),
767            ("mixed array", json!([1, "hello", true, 3.15])),
768            ("empty array", json!([])),
769        ];
770
771        for (name, json) in cases {
772            let err = settings.encode(json).unwrap_err();
773            assert!(
774                matches!(err, crate::error::Error::UnsupportedJsonType { .. }),
775                "{name}: {err:?}"
776            );
777        }
778    }
779
780    #[test]
781    fn test_encode_json_object() {
782        let json = json!({
783            "name": "John",
784            "age": 30,
785            "active": true
786        });
787
788        let settings = JsonSettings::default();
789        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
790        let Value::Struct(result) = result else {
791            panic!("Expected Struct value");
792        };
793        assert_eq!(result.items().len(), 3);
794
795        let items = result.items();
796        let struct_type = result.struct_type();
797
798        // Check that we have the expected fields
799        let fields = struct_type.fields();
800        let field_names: Vec<&str> = fields.iter().map(|f| f.name()).collect();
801        assert!(field_names.contains(&"name"));
802        assert!(field_names.contains(&"age"));
803        assert!(field_names.contains(&"active"));
804
805        // Find and check each field
806        for (i, field) in struct_type.fields().iter().enumerate() {
807            match field.name() {
808                "name" => {
809                    assert_eq!(items[i], Value::String("John".into()));
810                    assert_eq!(field.data_type(), &ConcreteDataType::string_datatype());
811                }
812                "age" => {
813                    assert_eq!(items[i], Value::Int64(30));
814                    assert_eq!(field.data_type(), &ConcreteDataType::int64_datatype());
815                }
816                "active" => {
817                    assert_eq!(items[i], Value::Boolean(true));
818                    assert_eq!(field.data_type(), &ConcreteDataType::boolean_datatype());
819                }
820                _ => panic!("Unexpected field: {}", field.name()),
821            }
822        }
823    }
824
825    #[test]
826    fn test_encode_json_nested_object() {
827        let json = json!({
828            "person": {
829                "name": "Alice",
830                "age": 25
831            },
832            "scores": [95, 87, 92]
833        });
834
835        let settings = JsonSettings::default();
836        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
837        let Value::Struct(result) = result else {
838            panic!("Expected Struct value");
839        };
840        assert_eq!(result.items().len(), 2);
841
842        let items = result.items();
843        let struct_type = result.struct_type();
844
845        // Check person field (nested struct)
846        let person_index = struct_type
847            .fields()
848            .iter()
849            .position(|f| f.name() == "person")
850            .unwrap();
851        if let Value::Struct(person_struct) = &items[person_index] {
852            assert_eq!(person_struct.items().len(), 2);
853            let fields = person_struct.struct_type().fields();
854            let person_fields: Vec<&str> = fields.iter().map(|f| f.name()).collect();
855            assert!(person_fields.contains(&"name"));
856            assert!(person_fields.contains(&"age"));
857        } else {
858            panic!("Expected Struct value for person field");
859        }
860
861        // Check scores field (list)
862        let scores_index = struct_type
863            .fields()
864            .iter()
865            .position(|f| f.name() == "scores")
866            .unwrap();
867        if let Value::List(scores_list) = &items[scores_index] {
868            assert_eq!(scores_list.items().len(), 3);
869            assert_eq!(scores_list.items()[0], Value::Int64(95));
870            assert_eq!(scores_list.items()[1], Value::Int64(87));
871            assert_eq!(scores_list.items()[2], Value::Int64(92));
872        } else {
873            panic!("Expected List value for scores field");
874        }
875    }
876
877    #[test]
878    fn test_encode_json_structured() {
879        let json = json!({
880            "name": "Bob",
881            "age": 35
882        });
883
884        let settings = JsonSettings::default();
885        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
886
887        if let Value::Struct(struct_value) = result {
888            assert_eq!(struct_value.items().len(), 2);
889            let fields = struct_value.struct_type().fields();
890            let field_names: Vec<&str> = fields.iter().map(|f| f.name()).collect();
891            assert!(field_names.contains(&"name"));
892            assert!(field_names.contains(&"age"));
893        } else {
894            panic!("Expected Struct value");
895        }
896    }
897
898    #[test]
899    fn test_encode_json_respects_type_hint() -> Result<()> {
900        let type_hints = vec![JsonTypeHint {
901            path: vec!["age".to_string()],
902            data_type: ConcreteDataType::int64_datatype(),
903            nullable: false,
904            default_constraint: None,
905            inverted_index: false,
906        }];
907        let settings = JsonSettings::try_new(type_hints, None)?;
908
909        let result = settings
910            .encode(json!({
911                "name": "Alice",
912                "age": 42
913            }))
914            .unwrap()
915            .into_json_inner()
916            .unwrap();
917
918        let Value::Struct(struct_value) = result else {
919            panic!("Expected Struct value");
920        };
921        assert_eq!(struct_field_value(&struct_value, "age"), &Value::Int64(42));
922
923        let err = settings
924            .encode(json!({
925                "age": "42"
926            }))
927            .unwrap_err();
928        assert!(err.to_string().contains("does not match JSON2 type hint"));
929        Ok(())
930    }
931
932    #[test]
933    fn test_encode_json_respects_unsigned_type_hint() -> Result<()> {
934        let type_hints = vec![JsonTypeHint {
935            path: vec!["count".to_string()],
936            data_type: ConcreteDataType::uint64_datatype(),
937            nullable: false,
938            default_constraint: None,
939            inverted_index: false,
940        }];
941        let settings = JsonSettings::try_new(type_hints, None)?;
942
943        let result = settings
944            .encode(json!({
945                "count": u64::MAX
946            }))
947            .unwrap()
948            .into_json_inner()
949            .unwrap();
950
951        let Value::Struct(struct_value) = result else {
952            panic!("Expected Struct value");
953        };
954        assert_eq!(
955            struct_field_value(&struct_value, "count"),
956            &Value::UInt64(u64::MAX)
957        );
958
959        let err = settings
960            .encode(json!({
961                "count": -1
962            }))
963            .unwrap_err();
964        assert!(err.to_string().contains("does not match JSON2 type hint"));
965        Ok(())
966    }
967
968    #[test]
969    fn test_encode_json_fills_missing_type_hint_with_default() -> Result<()> {
970        let type_hints = vec![JsonTypeHint {
971            path: vec!["user".to_string(), "age".to_string()],
972            data_type: ConcreteDataType::int64_datatype(),
973            nullable: false,
974            default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
975            inverted_index: false,
976        }];
977        let settings = JsonSettings::try_new(type_hints, None)?;
978
979        let result = settings
980            .encode(json!({}))
981            .unwrap()
982            .into_json_inner()
983            .unwrap();
984
985        let Value::Struct(root) = result else {
986            panic!("Expected Struct value");
987        };
988        let Value::Struct(user) = struct_field_value(&root, "user") else {
989            panic!("Expected user Struct value");
990        };
991        assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
992        Ok(())
993    }
994
995    #[test]
996    fn test_encode_json_fills_missing_nullable_type_hint_with_null() -> Result<()> {
997        let type_hints = vec![JsonTypeHint {
998            path: vec!["user".to_string(), "name".to_string()],
999            data_type: ConcreteDataType::string_datatype(),
1000            nullable: true,
1001            default_constraint: None,
1002            inverted_index: false,
1003        }];
1004        let settings = JsonSettings::try_new(type_hints, None)?;
1005
1006        let result = settings
1007            .encode(json!({ "user": {} }))
1008            .unwrap()
1009            .into_json_inner()
1010            .unwrap();
1011
1012        let Value::Struct(root) = result else {
1013            panic!("Expected Struct value");
1014        };
1015        let Value::Struct(user) = struct_field_value(&root, "user") else {
1016            panic!("Expected user Struct value");
1017        };
1018        assert_eq!(struct_field_value(user, "name"), &Value::Null);
1019        Ok(())
1020    }
1021
1022    #[test]
1023    fn test_encode_json_rejects_missing_non_null_type_hint() -> Result<()> {
1024        let type_hints = vec![JsonTypeHint {
1025            path: vec!["user".to_string(), "age".to_string()],
1026            data_type: ConcreteDataType::int64_datatype(),
1027            nullable: false,
1028            default_constraint: None,
1029            inverted_index: false,
1030        }];
1031        let settings = JsonSettings::try_new(type_hints, None)?;
1032
1033        let err = settings.encode(json!({})).unwrap_err();
1034        assert!(
1035            err.to_string()
1036                .contains("missing non-null JSON2 type hint path user.age")
1037        );
1038        Ok(())
1039    }
1040
1041    #[test]
1042    fn test_encode_json_merges_missing_type_hint_prefix() -> Result<()> {
1043        let type_hints = vec![
1044            JsonTypeHint {
1045                path: vec!["user".to_string(), "age".to_string()],
1046                data_type: ConcreteDataType::int64_datatype(),
1047                nullable: false,
1048                default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
1049                inverted_index: false,
1050            },
1051            JsonTypeHint {
1052                path: vec!["user".to_string(), "name".to_string()],
1053                data_type: ConcreteDataType::string_datatype(),
1054                nullable: false,
1055                default_constraint: Some(ColumnDefaultConstraint::Value(Value::String(
1056                    "unknown".into(),
1057                ))),
1058                inverted_index: false,
1059            },
1060        ];
1061        let settings = JsonSettings::try_new(type_hints, None)?;
1062
1063        let result = settings.encode(json!({}))?.into_json_inner().unwrap();
1064
1065        let Value::Struct(root) = result else {
1066            panic!("Expected Struct value");
1067        };
1068        let Value::Struct(user) = struct_field_value(&root, "user") else {
1069            panic!("Expected user Struct value");
1070        };
1071        assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
1072        assert_eq!(
1073            struct_field_value(user, "name"),
1074            &Value::String("unknown".into())
1075        );
1076        Ok(())
1077    }
1078
1079    #[test]
1080    fn test_json_settings_structured() {
1081        let json = json!({
1082            "name": "Eve",
1083            "score": 95
1084        });
1085
1086        let settings = JsonSettings::default();
1087        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
1088
1089        if let Value::Struct(struct_value) = result {
1090            assert_eq!(struct_value.items().len(), 2);
1091        } else {
1092            panic!("Expected Struct value");
1093        }
1094    }
1095
1096    #[cfg(test)]
1097    mod decode_tests {
1098        use ordered_float::OrderedFloat;
1099        use serde_json::json;
1100
1101        use super::*;
1102
1103        #[test]
1104        fn test_decode_primitive_values() {
1105            let settings = JsonSettings::default();
1106
1107            // Test null
1108            let result = settings.decode(Value::Null).unwrap();
1109            assert_eq!(result, Json::Null);
1110
1111            // Test boolean
1112            let result = settings.decode(Value::Boolean(true)).unwrap();
1113            assert_eq!(result, Json::Bool(true));
1114
1115            // Test integer
1116            let result = settings.decode(Value::Int64(42)).unwrap();
1117            assert_eq!(result, Json::from(42));
1118
1119            // Test float
1120            let result = settings.decode(Value::Float64(OrderedFloat(3.16))).unwrap();
1121            assert_eq!(result, Json::from(3.16));
1122
1123            // Test string
1124            let result = settings.decode(Value::String("hello".into())).unwrap();
1125            assert_eq!(result, Json::String("hello".to_string()));
1126        }
1127
1128        #[test]
1129        fn test_decode_struct() {
1130            let settings = JsonSettings::default();
1131
1132            let struct_value = StructValue::new(
1133                vec![
1134                    Value::String("Alice".into()),
1135                    Value::Int64(25),
1136                    Value::Boolean(true),
1137                ],
1138                StructType::new(Arc::new(vec![
1139                    StructField::new(
1140                        "name".to_string(),
1141                        ConcreteDataType::string_datatype(),
1142                        true,
1143                    ),
1144                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
1145                    StructField::new(
1146                        "active".to_string(),
1147                        ConcreteDataType::boolean_datatype(),
1148                        true,
1149                    ),
1150                ])),
1151            );
1152
1153            let result = settings.decode(Value::Struct(struct_value)).unwrap();
1154            let expected = json!({
1155                "name": "Alice",
1156                "age": 25,
1157                "active": true
1158            });
1159            assert_eq!(result, expected);
1160        }
1161
1162        #[test]
1163        fn test_decode_list() {
1164            let settings = JsonSettings::default();
1165
1166            let list_value = ListValue::new(
1167                vec![Value::Int64(1), Value::Int64(2), Value::Int64(3)],
1168                Arc::new(ConcreteDataType::int64_datatype()),
1169            );
1170
1171            let result = settings.decode(Value::List(list_value)).unwrap();
1172            let expected = json!([1, 2, 3]);
1173            assert_eq!(result, expected);
1174        }
1175
1176        #[test]
1177        fn test_decode_nested_structure() {
1178            let settings = JsonSettings::default();
1179
1180            let inner_struct = StructValue::new(
1181                vec![Value::String("Alice".into()), Value::Int64(25)],
1182                StructType::new(Arc::new(vec![
1183                    StructField::new(
1184                        "name".to_string(),
1185                        ConcreteDataType::string_datatype(),
1186                        true,
1187                    ),
1188                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
1189                ])),
1190            );
1191
1192            let score_list_item_type = Arc::new(ConcreteDataType::int64_datatype());
1193            let outer_struct = StructValue::new(
1194                vec![
1195                    Value::Struct(inner_struct),
1196                    Value::List(ListValue::new(
1197                        vec![Value::Int64(95), Value::Int64(87)],
1198                        score_list_item_type.clone(),
1199                    )),
1200                ],
1201                StructType::new(Arc::new(vec![
1202                    StructField::new(
1203                        "user".to_string(),
1204                        ConcreteDataType::Struct(StructType::new(Arc::new(vec![
1205                            StructField::new(
1206                                "name".to_string(),
1207                                ConcreteDataType::string_datatype(),
1208                                true,
1209                            ),
1210                            StructField::new(
1211                                "age".to_string(),
1212                                ConcreteDataType::int64_datatype(),
1213                                true,
1214                            ),
1215                        ]))),
1216                        true,
1217                    ),
1218                    StructField::new(
1219                        "scores".to_string(),
1220                        ConcreteDataType::List(ListType::new(score_list_item_type.clone())),
1221                        true,
1222                    ),
1223                ])),
1224            );
1225
1226            let result = settings.decode(Value::Struct(outer_struct)).unwrap();
1227            let expected = json!({
1228                "user": {
1229                    "name": "Alice",
1230                    "age": 25
1231                },
1232                "scores": [95, 87]
1233            });
1234            assert_eq!(result, expected);
1235        }
1236
1237        #[test]
1238        fn test_decode_missing_fields() {
1239            let settings = JsonSettings::default();
1240
1241            // Struct with missing field (null value)
1242            let struct_value = StructValue::new(
1243                vec![
1244                    Value::String("Bob".into()),
1245                    Value::Null, // missing age field
1246                ],
1247                StructType::new(Arc::new(vec![
1248                    StructField::new(
1249                        "name".to_string(),
1250                        ConcreteDataType::string_datatype(),
1251                        true,
1252                    ),
1253                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
1254                ])),
1255            );
1256
1257            let result = settings.decode(Value::Struct(struct_value)).unwrap();
1258            let expected = json!({
1259                "name": "Bob",
1260                "age": null
1261            });
1262            assert_eq!(result, expected);
1263        }
1264    }
1265}