datatypes/
json.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Data conversion between greptime's StructType and Json
16//!
17//! The idea of this module is to provide utilities to convert serde_json::Value to greptime's StructType and vice versa.
18//!
19//! The struct will carry all the fields of the Json object. We will not flatten any json object in this implementation.
20//!
21
22pub mod value;
23
24use std::collections::BTreeMap;
25use std::collections::btree_map::Entry;
26
27use serde::{Deserialize, Serialize};
28use serde_json::{Map, Value as Json};
29use snafu::ResultExt;
30
31use crate::data_type::ConcreteDataType;
32use crate::error::{self, InvalidJson2LayoutSnafu, Result, UnsupportedJsonTypeSnafu};
33use crate::json::value::{JsonValue, JsonVariant, encode_serde_json_as_jsonb};
34use crate::schema::ColumnDefaultConstraint;
35use crate::types::json_type::{JsonNativeType, JsonObjectType};
36use crate::value::{ListValue, StructValue, Value};
37
38/// Maximum number of JSON container levels represented as nested Arrow types.
39pub const JSON2_MAX_STRUCTURED_DEPTH: usize = 50;
40/// Reserved physical field containing unexpanded JSON2 paths.
41pub const JSON2_REMAINDER_FIELD_NAME: &str = "!__remainder__!";
42
43/// JSON2 settings stored in column schema metadata and represented through
44/// Arrow extension metadata.
45#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
46pub struct JsonSettings {
47    #[serde(default)]
48    type_hints: Vec<JsonTypeHint>,
49    /// Maximum number of unhinted JSON paths expanded into Arrow fields.
50    ///
51    /// `None` preserves the legacy unlimited layout.
52    #[serde(default, skip_serializing_if = "Option::is_none")]
53    max_auto_expanded_paths: Option<u32>,
54}
55
56#[derive(Deserialize)]
57struct JsonSettingsSerde {
58    #[serde(default)]
59    type_hints: Vec<JsonTypeHint>,
60    #[serde(default)]
61    max_auto_expanded_paths: Option<u32>,
62}
63
64impl<'de> Deserialize<'de> for JsonSettings {
65    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
66    where
67        D: serde::Deserializer<'de>,
68    {
69        let settings = JsonSettingsSerde::deserialize(deserializer)?;
70        Self::try_new(settings.type_hints, settings.max_auto_expanded_paths)
71            .map_err(serde::de::Error::custom)
72    }
73}
74
75/// Declares selected JSON2 subpaths as typed fields.
76///
77/// These hints let JSON2 encode frequently used subpaths in a typed layout, so
78/// queries over those subpaths can get behavior and performance closer to
79/// ordinary columns.
80#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
81pub struct JsonTypeHint {
82    /// JSON2 subpath for a typed field.
83    ///
84    /// Each item is one JSON object key. For example, `["user", "age"]`
85    /// represents `user.age`.
86    ///
87    /// Array traversal is not currently supported. For example, a hint cannot
88    /// describe `events[0].name` or fields shared by all items in `events[*]`.
89    pub path: Vec<String>,
90    #[serde(rename = "type")]
91    pub data_type: ConcreteDataType,
92    pub nullable: bool,
93    #[serde(skip_serializing_if = "Option::is_none")]
94    pub default_constraint: Option<ColumnDefaultConstraint>,
95    pub inverted_index: bool,
96}
97
98/// Context for JSON encoding/decoding that tracks the current key path.
99#[derive(Clone, Debug)]
100pub struct JsonContext<'a> {
101    /// Current key path from the JSON2 root.
102    pub path: Vec<String>,
103    /// Settings for JSON encoding/decoding.
104    pub settings: &'a JsonSettings,
105}
106
107impl JsonSettings {
108    /// Creates and validates JSON2 settings.
109    pub fn try_new(
110        type_hints: Vec<JsonTypeHint>,
111        max_auto_expanded_paths: Option<u32>,
112    ) -> Result<Self> {
113        validate_type_hints(&type_hints)?;
114        Ok(Self {
115            type_hints,
116            max_auto_expanded_paths,
117        })
118    }
119
120    pub fn type_hints(&self) -> &[JsonTypeHint] {
121        &self.type_hints
122    }
123
124    pub fn max_auto_expanded_paths(&self) -> Option<u32> {
125        self.max_auto_expanded_paths
126    }
127
128    pub fn into_parts(self) -> (Vec<JsonTypeHint>, Option<u32>) {
129        (self.type_hints, self.max_auto_expanded_paths)
130    }
131
132    /// Decode an encoded StructValue back into a serde_json::Value.
133    pub fn decode(&self, value: Value) -> Result<Json> {
134        let mut context = JsonContext {
135            path: Vec::new(),
136            settings: self,
137        };
138        decode_value_with_context(value, &mut context)
139    }
140
141    /// Encode a serde_json::Value into a Value::Json using current settings.
142    pub fn encode(&self, json: Json) -> Result<Value> {
143        if let Json::Object(object) = &json
144            && object.contains_key(JSON2_REMAINDER_FIELD_NAME)
145        {
146            return error::InvalidJsonSnafu {
147                value: format!(
148                    "root object cannot contain reserved field '{JSON2_REMAINDER_FIELD_NAME}'"
149                ),
150            }
151            .fail();
152        }
153        let mut context = JsonContext {
154            path: Vec::new(),
155            settings: self,
156        };
157        encode_json_with_context(json, &mut context).map(|v| Value::Json(Box::new(v)))
158    }
159}
160
161fn validate_type_hints(type_hints: &[JsonTypeHint]) -> Result<()> {
162    let mut object = JsonObjectType::new();
163    for hint in type_hints {
164        if hint.path.len() > JSON2_MAX_STRUCTURED_DEPTH {
165            return InvalidJson2LayoutSnafu {
166                reason: format!(
167                    "JSON2 type hint path cannot exceed {JSON2_MAX_STRUCTURED_DEPTH} segments"
168                ),
169            }
170            .fail();
171        }
172        if hint
173            .path
174            .first()
175            .is_some_and(|x| x == JSON2_REMAINDER_FIELD_NAME)
176        {
177            return InvalidJson2LayoutSnafu {
178                reason: format!(
179                    "JSON2 type hint path cannot be rooted at reserved field '{JSON2_REMAINDER_FIELD_NAME}'"
180                ),
181            }
182            .fail();
183        }
184        let data_type = match &hint.data_type {
185            ConcreteDataType::Boolean(_)
186            | ConcreteDataType::UInt8(_)
187            | ConcreteDataType::UInt16(_)
188            | ConcreteDataType::UInt32(_)
189            | ConcreteDataType::UInt64(_)
190            | ConcreteDataType::Int8(_)
191            | ConcreteDataType::Int16(_)
192            | ConcreteDataType::Int32(_)
193            | ConcreteDataType::Int64(_)
194            | ConcreteDataType::Float32(_)
195            | ConcreteDataType::Float64(_)
196            | ConcreteDataType::String(_) => (&hint.data_type).into(),
197            data_type => {
198                return InvalidJson2LayoutSnafu {
199                    reason: format!("unsupported JSON2 type hint data type: {data_type}"),
200                }
201                .fail();
202            }
203        };
204        validate_type_hint(&mut object, &hint.path, data_type)?;
205    }
206    Ok(())
207}
208
209fn validate_type_hint(
210    object: &mut JsonObjectType,
211    path: &[String],
212    data_type: JsonNativeType,
213) -> Result<()> {
214    let Some((name, path)) = path.split_first() else {
215        return InvalidJson2LayoutSnafu {
216            reason: "JSON2 type hint path must not be empty".to_string(),
217        }
218        .fail();
219    };
220    if path.is_empty() {
221        if object.insert(name.clone(), data_type).is_some() {
222            return InvalidJson2LayoutSnafu {
223                reason: format!("duplicate JSON2 type hint path '{name}'"),
224            }
225            .fail();
226        }
227        return Ok(());
228    }
229
230    let child = object
231        .entry(name.clone())
232        .or_insert_with(|| JsonNativeType::Object(JsonObjectType::new()));
233    let JsonNativeType::Object(child) = child else {
234        return InvalidJson2LayoutSnafu {
235            reason: format!("conflicting JSON2 type hint path at '{name}'"),
236        }
237        .fail();
238    };
239    validate_type_hint(child, path, data_type)
240}
241
242impl<'a> JsonContext<'a> {
243    fn type_hint(&self) -> Option<&'a JsonTypeHint> {
244        self.settings
245            .type_hints
246            .iter()
247            .find(|hint| hint.path == self.path)
248    }
249}
250
251fn with_key_context<T>(
252    context: &mut JsonContext,
253    key: &str,
254    f: impl FnOnce(&mut JsonContext) -> Result<T>,
255) -> Result<T> {
256    context.path.push(key.to_string());
257    let result = f(context);
258    context.path.pop();
259    result
260}
261
262/// Main encoding function with key path tracking
263fn encode_json_with_context(json: Json, context: &mut JsonContext) -> Result<JsonValue> {
264    if context.path.is_empty() && !matches!(json, Json::Object(_)) {
265        return UnsupportedJsonTypeSnafu.fail();
266    }
267
268    match json {
269        Json::Object(json_object) => encode_json_object_with_context(json_object, context),
270        Json::Array(json_array) => encode_json_array_with_context(json_array, context),
271        _ => encode_json_value_with_context(json, context),
272    }
273}
274
275fn encode_json_object_with_context<'a>(
276    json_object: Map<String, Json>,
277    context: &mut JsonContext<'a>,
278) -> Result<JsonValue> {
279    let mut object = BTreeMap::new();
280    for (key, value) in json_object {
281        let value = with_key_context(context, &key, |context| {
282            if let Some(hint) = context.type_hint() {
283                encode_json_value_with_hint(value, hint, context)
284            } else {
285                encode_json_value_with_context(value, context)
286            }
287        })?;
288
289        object.insert(key, value.into_variant());
290    }
291
292    apply_missing_type_hints(&mut object, context)?;
293
294    Ok(JsonValue::new(JsonVariant::Object(object)))
295}
296
297fn apply_missing_type_hints(
298    object: &mut BTreeMap<String, JsonVariant>,
299    context: &mut JsonContext,
300) -> Result<()> {
301    for hint in &context.settings.type_hints {
302        if hint.path.len() > context.path.len() && hint.path.starts_with(&context.path) {
303            let depth = context.path.len();
304            let key = &hint.path[depth];
305            with_key_context(context, key, |context| {
306                insert_missing_type_hint(object, context, hint, depth)
307            })?;
308        }
309    }
310    Ok(())
311}
312
313fn insert_missing_type_hint(
314    object: &mut BTreeMap<String, JsonVariant>,
315    field_context: &mut JsonContext,
316    hint: &JsonTypeHint,
317    depth: usize,
318) -> Result<()> {
319    let key = &hint.path[depth];
320    let is_leaf = depth + 1 == hint.path.len();
321
322    if is_leaf {
323        if !object.contains_key(key) {
324            let value = encode_missing_type_hint_value(hint, field_context)?;
325            object.insert(key.clone(), value.into_variant());
326        }
327        return Ok(());
328    }
329
330    match object.entry(key.clone()) {
331        Entry::Occupied(mut entry) => match entry.get_mut() {
332            JsonVariant::Object(child) => {
333                insert_missing_type_hint(child, field_context, hint, depth + 1)
334            }
335            _ => error::InvalidJsonSnafu {
336                value: format!(
337                    "JSON2 type hint path {} expects object at {}",
338                    hint.path.join("."),
339                    field_context.path.join(".")
340                ),
341            }
342            .fail(),
343        },
344        Entry::Vacant(entry) => {
345            let mut child = BTreeMap::new();
346            insert_missing_type_hint(&mut child, field_context, hint, depth + 1)?;
347            entry.insert(JsonVariant::Object(child));
348            Ok(())
349        }
350    }
351}
352
353fn encode_missing_type_hint_value(
354    hint: &JsonTypeHint,
355    context: &mut JsonContext,
356) -> Result<JsonValue> {
357    if let Some(default_constraint) = &hint.default_constraint {
358        let value = default_constraint.create_default(&hint.data_type, hint.nullable)?;
359        let json = decode_primitive_value(value)?;
360        return encode_json_value_with_hint(json, hint, context);
361    }
362
363    if hint.nullable {
364        Ok(JsonValue::null())
365    } else {
366        error::InvalidJsonSnafu {
367            value: format!(
368                "missing non-null JSON2 type hint path {}",
369                hint.path.join(".")
370            ),
371        }
372        .fail()
373    }
374}
375
376fn encode_json_value_with_hint(
377    json: Json,
378    hint: &JsonTypeHint,
379    context: &mut JsonContext,
380) -> Result<JsonValue> {
381    if json.is_null() {
382        return if hint.nullable {
383            Ok(JsonValue::null())
384        } else {
385            error::InvalidJsonSnafu {
386                value: format!(
387                    "JSON2 type hint path {} is not nullable",
388                    context.path.join(".")
389                ),
390            }
391            .fail()
392        };
393    }
394
395    let invalid_type = || {
396        error::InvalidJsonSnafu {
397            value: format!(
398                "JSON value at {} does not match JSON2 type hint {}",
399                context.path.join("."),
400                hint.data_type
401            ),
402        }
403        .fail()
404    };
405
406    match (&hint.data_type, json) {
407        (ConcreteDataType::String(_), Json::String(v)) => Ok(v.into()),
408        (
409            ConcreteDataType::Int8(_)
410            | ConcreteDataType::Int16(_)
411            | ConcreteDataType::Int32(_)
412            | ConcreteDataType::Int64(_),
413            Json::Number(v),
414        ) => match v.as_i64() {
415            Some(v) => Ok(v.into()),
416            None => invalid_type(),
417        },
418        (
419            ConcreteDataType::UInt8(_)
420            | ConcreteDataType::UInt16(_)
421            | ConcreteDataType::UInt32(_)
422            | ConcreteDataType::UInt64(_),
423            Json::Number(v),
424        ) => match v.as_u64() {
425            Some(v) => Ok(v.into()),
426            None => invalid_type(),
427        },
428        (ConcreteDataType::Float32(_) | ConcreteDataType::Float64(_), Json::Number(v)) => {
429            match v.as_f64() {
430                Some(v) => Ok(v.into()),
431                None => invalid_type(),
432            }
433        }
434        (ConcreteDataType::Boolean(_), Json::Bool(v)) => Ok(v.into()),
435        _ => invalid_type(),
436    }
437}
438
439fn encode_json_array_with_context<'a>(
440    json_array: Vec<Json>,
441    context: &mut JsonContext<'a>,
442) -> Result<JsonValue> {
443    let json_array_len = json_array.len();
444    let mut items = Vec::with_capacity(json_array_len);
445
446    for (index, value) in json_array.into_iter().enumerate() {
447        let item_value = with_key_context(context, &index.to_string(), |context| {
448            encode_json_value_with_context(value, context)
449        })?;
450        items.push(item_value);
451    }
452
453    // In specification, it's valid for a JSON array to have different types of items, for example,
454    // ["a string", 1]. However, in implementation, the `JsonValue` will be converted to Arrow list
455    // array, which requires all items have exactly the same type. So we merge out the maybe
456    // different item types to a unified type, and align all the item values to it.
457
458    let merged_item_type = if let Some((first, rests)) = items.split_first() {
459        let mut merged = first.json_type().clone();
460        for rest in rests.iter().map(|x| x.json_type()) {
461            if matches!(merged, JsonNativeType::Variant) {
462                break;
463            }
464            merged.merge(rest);
465        }
466        Some(merged)
467    } else {
468        None
469    };
470    if let Some(unified_item_type) = merged_item_type {
471        for item in &mut items {
472            item.try_align(&unified_item_type)?;
473        }
474    }
475    let items = items
476        .into_iter()
477        .map(|x| x.into_variant())
478        .collect::<Vec<_>>();
479    Ok(JsonValue::new(JsonVariant::Array(items)))
480}
481
482/// Helper function to encode a JSON value to a Value and determine its ConcreteDataType with context
483fn encode_json_value_with_context(json: Json, context: &mut JsonContext) -> Result<JsonValue> {
484    if context.path.len() >= JSON2_MAX_STRUCTURED_DEPTH
485        && matches!(&json, Json::Object(_) | Json::Array(_))
486    {
487        return Ok(JsonValue::new(JsonVariant::Variant(
488            encode_serde_json_as_jsonb(json),
489        )));
490    }
491
492    match json {
493        Json::Null => Ok(JsonValue::null()),
494        Json::Bool(b) => Ok(b.into()),
495        Json::Number(n) => {
496            if let Some(i) = n.as_i64() {
497                Ok(i.into())
498            } else if let Some(u) = n.as_u64() {
499                if u <= i64::MAX as u64 {
500                    Ok((u as i64).into())
501                } else {
502                    Ok(u.into())
503                }
504            } else if let Some(f) = n.as_f64() {
505                Ok(f.into())
506            } else {
507                // Fallback to string representation
508                Ok(n.to_string().into())
509            }
510        }
511        Json::String(s) => Ok(s.into()),
512        Json::Array(arr) => encode_json_array_with_context(arr, context),
513        Json::Object(obj) => encode_json_object_with_context(obj, context),
514    }
515}
516
517/// Main decoding function with key path tracking
518fn decode_value_with_context(value: Value, context: &mut JsonContext) -> Result<Json> {
519    match value {
520        Value::Struct(struct_value) => decode_struct_with_context(struct_value, context),
521        Value::List(list_value) => decode_list_with_context(list_value, context),
522        _ => decode_primitive_value(value),
523    }
524}
525
526/// Decode a structured value to JSON object
527fn decode_struct_with_context<'a>(
528    struct_value: StructValue,
529    context: &mut JsonContext<'a>,
530) -> Result<Json> {
531    let mut json_object = Map::with_capacity(struct_value.len());
532
533    let (items, fields) = struct_value.into_parts();
534
535    for (field, field_value) in fields.fields().iter().zip(items) {
536        let json_value = with_key_context(context, field.name(), |context| {
537            decode_value_with_context(field_value, context)
538        })?;
539        json_object.insert(field.name().to_string(), json_value);
540    }
541
542    Ok(Json::Object(json_object))
543}
544
545/// Decode a list value to JSON array
546fn decode_list_with_context(list_value: ListValue, context: &mut JsonContext) -> Result<Json> {
547    let mut json_array = Vec::with_capacity(list_value.len());
548
549    let data_items = list_value.take_items();
550
551    for (index, item) in data_items.into_iter().enumerate() {
552        let json_value = with_key_context(context, &index.to_string(), |context| {
553            decode_value_with_context(item, context)
554        })?;
555        json_array.push(json_value);
556    }
557
558    Ok(Json::Array(json_array))
559}
560
561/// Decode primitive value to JSON
562fn decode_primitive_value(value: Value) -> Result<Json> {
563    match value {
564        Value::Null => Ok(Json::Null),
565        Value::Boolean(b) => Ok(Json::Bool(b)),
566        Value::UInt8(v) => Ok(Json::from(v)),
567        Value::UInt16(v) => Ok(Json::from(v)),
568        Value::UInt32(v) => Ok(Json::from(v)),
569        Value::UInt64(v) => Ok(Json::from(v)),
570        Value::Int8(v) => Ok(Json::from(v)),
571        Value::Int16(v) => Ok(Json::from(v)),
572        Value::Int32(v) => Ok(Json::from(v)),
573        Value::Int64(v) => Ok(Json::from(v)),
574        Value::Float32(v) => Ok(Json::from(v.0)),
575        Value::Float64(v) => Ok(Json::from(v.0)),
576        Value::String(s) => Ok(Json::String(s.as_utf8().to_string())),
577        Value::Binary(b) => serde_json::to_value(b.as_ref()).context(error::SerializeSnafu),
578        Value::Date(v) => Ok(Json::from(v.val())),
579        Value::Timestamp(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
580        Value::Time(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
581        Value::IntervalYearMonth(v) => {
582            serde_json::to_value(v.to_i32()).context(error::SerializeSnafu)
583        }
584        Value::IntervalDayTime(v) => {
585            serde_json::to_value(v.to_i64()).context(error::SerializeSnafu)
586        }
587        Value::IntervalMonthDayNano(v) => {
588            serde_json::to_value(v.to_i128()).context(error::SerializeSnafu)
589        }
590        Value::Duration(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
591        Value::Decimal128(v) => serde_json::to_value(v.to_string()).context(error::SerializeSnafu),
592        Value::Struct(_) | Value::List(_) | Value::Json(_) => {
593            // These should be handled by the context-aware functions
594            Err(error::InvalidJsonSnafu {
595                value: "Structured values should be handled by context-aware decoding".to_string(),
596            }
597            .build())
598        }
599    }
600}
601
602#[cfg(test)]
603mod tests {
604    use std::sync::Arc;
605
606    use serde_json::json;
607
608    use super::*;
609    use crate::data_type::ConcreteDataType;
610    use crate::types::{ListType, StructField, StructType};
611
612    fn struct_field_value<'a>(struct_value: &'a StructValue, field_name: &str) -> &'a Value {
613        let index = struct_value
614            .struct_type()
615            .fields()
616            .iter()
617            .position(|field| field.name() == field_name)
618            .expect("field exists");
619        &struct_value.items()[index]
620    }
621
622    #[test]
623    fn test_json_settings_forward_compatibility()
624    -> std::result::Result<(), Box<dyn std::error::Error>> {
625        let json_str = r#"{
626            "type_hints": [
627                {
628                    "path": ["user", "age"],
629                    "type": {
630                        "Int64": {}
631                    },
632                    "nullable": false,
633                    "default_constraint": {
634                        "Value": {
635                            "Int64": 18
636                        }
637                    },
638                    "inverted_index": true
639                },
640                {
641                    "path": ["user", "name"],
642                    "type": {
643                        "String": {
644                            "size_type": "Utf8"
645                        }
646                    },
647                    "nullable": true,
648                    "inverted_index": false
649                }
650            ]
651        }"#;
652
653        let deserialized = serde_json::from_str::<JsonSettings>(json_str)?;
654
655        let expected = JsonSettings::try_new(
656            vec![
657                JsonTypeHint {
658                    path: vec!["user".to_string(), "age".to_string()],
659                    data_type: ConcreteDataType::int64_datatype(),
660                    nullable: false,
661                    default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
662                    inverted_index: true,
663                },
664                JsonTypeHint {
665                    path: vec!["user".to_string(), "name".to_string()],
666                    data_type: ConcreteDataType::string_datatype(),
667                    nullable: true,
668                    default_constraint: None,
669                    inverted_index: false,
670                },
671            ],
672            None,
673        )?;
674        assert_eq!(deserialized, expected);
675        Ok(())
676    }
677
678    #[test]
679    fn test_json_settings_ser_de() -> std::result::Result<(), Box<dyn std::error::Error>> {
680        let type_hints = vec![
681            JsonTypeHint {
682                path: vec!["user".to_string(), "age".to_string()],
683                data_type: ConcreteDataType::int64_datatype(),
684                nullable: false,
685                default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
686                inverted_index: true,
687            },
688            JsonTypeHint {
689                path: vec!["user".to_string(), "name".to_string()],
690                data_type: ConcreteDataType::string_datatype(),
691                nullable: true,
692                default_constraint: None,
693                inverted_index: false,
694            },
695        ];
696
697        for settings in [
698            JsonSettings {
699                type_hints: type_hints.clone(),
700                max_auto_expanded_paths: None,
701            },
702            JsonSettings {
703                type_hints,
704                max_auto_expanded_paths: Some(1),
705            },
706        ] {
707            let serialized = serde_json::to_string(&settings)?;
708            let deserialized = serde_json::from_str::<JsonSettings>(&serialized)?;
709            assert_eq!(settings, deserialized);
710        }
711        Ok(())
712    }
713
714    #[test]
715    fn test_json_settings_reject_invalid_type_hint_layout() {
716        for type_hints in [
717            json!([{"path": [], "type": {"Int64": {}}, "nullable": true, "inverted_index": false}]),
718            json!([
719                {"path": ["a"], "type": {"Int64": {}}, "nullable": true, "inverted_index": false},
720                {"path": ["a", "b"], "type": {"Int64": {}}, "nullable": true, "inverted_index": false}
721            ]),
722            json!([{"path": [JSON2_REMAINDER_FIELD_NAME], "type": {"Int64": {}}, "nullable": true, "inverted_index": false}]),
723        ] {
724            let settings = json!({"type_hints": type_hints});
725            assert!(serde_json::from_value::<JsonSettings>(settings).is_err());
726        }
727
728        let hint = |path, data_type| JsonTypeHint {
729            path,
730            data_type,
731            nullable: true,
732            default_constraint: None,
733            inverted_index: false,
734        };
735        assert!(
736            JsonSettings::try_new(
737                vec![hint(
738                    vec!["nested".to_string(), JSON2_REMAINDER_FIELD_NAME.to_string()],
739                    ConcreteDataType::string_datatype(),
740                )],
741                None,
742            )
743            .is_ok()
744        );
745        assert!(
746            JsonSettings::try_new(
747                vec![hint(
748                    vec!["nested".to_string(); JSON2_MAX_STRUCTURED_DEPTH + 1],
749                    ConcreteDataType::string_datatype(),
750                )],
751                None,
752            )
753            .is_err()
754        );
755        assert!(
756            JsonSettings::try_new(
757                vec![hint(
758                    vec!["date".to_string()],
759                    ConcreteDataType::date_datatype(),
760                )],
761                None,
762            )
763            .is_err()
764        );
765    }
766
767    #[test]
768    fn test_encode_root_non_object_json() {
769        let settings = JsonSettings::default();
770        let cases = [
771            ("null", Json::Null),
772            ("boolean", Json::Bool(true)),
773            ("integer", Json::from(42)),
774            ("float", Json::from(3.15)),
775            ("string", Json::String("hello".to_string())),
776            ("array", json!([1, 2, 3])),
777            ("mixed array", json!([1, "hello", true, 3.15])),
778            ("empty array", json!([])),
779        ];
780
781        for (name, json) in cases {
782            let err = settings.encode(json).unwrap_err();
783            assert!(
784                matches!(err, crate::error::Error::UnsupportedJsonType { .. }),
785                "{name}: {err:?}"
786            );
787        }
788    }
789
790    #[test]
791    fn test_encode_rejects_reserved_remainder_field() -> Result<()> {
792        let settings = JsonSettings::default();
793        let err = settings
794            .encode(json!({"!__remainder__!": "user-value"}))
795            .unwrap_err();
796        assert!(
797            err.to_string()
798                .contains("root object cannot contain reserved field")
799        );
800
801        settings.encode(json!({"nested": {"!__remainder__!": "user-value"}}))?;
802        Ok(())
803    }
804
805    #[test]
806    fn test_encode_json_object() {
807        let json = json!({
808            "name": "John",
809            "age": 30,
810            "active": true
811        });
812
813        let settings = JsonSettings::default();
814        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
815        let Value::Struct(result) = result else {
816            panic!("Expected Struct value");
817        };
818        assert_eq!(result.items().len(), 3);
819
820        let items = result.items();
821        let struct_type = result.struct_type();
822
823        // Check that we have the expected fields
824        let fields = struct_type.fields();
825        let field_names: Vec<&str> = fields.iter().map(|f| f.name()).collect();
826        assert!(field_names.contains(&"name"));
827        assert!(field_names.contains(&"age"));
828        assert!(field_names.contains(&"active"));
829
830        // Find and check each field
831        for (i, field) in struct_type.fields().iter().enumerate() {
832            match field.name() {
833                "name" => {
834                    assert_eq!(items[i], Value::String("John".into()));
835                    assert_eq!(field.data_type(), &ConcreteDataType::string_datatype());
836                }
837                "age" => {
838                    assert_eq!(items[i], Value::Int64(30));
839                    assert_eq!(field.data_type(), &ConcreteDataType::int64_datatype());
840                }
841                "active" => {
842                    assert_eq!(items[i], Value::Boolean(true));
843                    assert_eq!(field.data_type(), &ConcreteDataType::boolean_datatype());
844                }
845                _ => panic!("Unexpected field: {}", field.name()),
846            }
847        }
848    }
849
850    #[test]
851    fn test_encode_json_nested_object() {
852        let json = json!({
853            "person": {
854                "name": "Alice",
855                "age": 25
856            },
857            "scores": [95, 87, 92]
858        });
859
860        let settings = JsonSettings::default();
861        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
862        let Value::Struct(result) = result else {
863            panic!("Expected Struct value");
864        };
865        assert_eq!(result.items().len(), 2);
866
867        let items = result.items();
868        let struct_type = result.struct_type();
869
870        // Check person field (nested struct)
871        let person_index = struct_type
872            .fields()
873            .iter()
874            .position(|f| f.name() == "person")
875            .unwrap();
876        if let Value::Struct(person_struct) = &items[person_index] {
877            assert_eq!(person_struct.items().len(), 2);
878            let fields = person_struct.struct_type().fields();
879            let person_fields: Vec<&str> = fields.iter().map(|f| f.name()).collect();
880            assert!(person_fields.contains(&"name"));
881            assert!(person_fields.contains(&"age"));
882        } else {
883            panic!("Expected Struct value for person field");
884        }
885
886        // Check scores field (list)
887        let scores_index = struct_type
888            .fields()
889            .iter()
890            .position(|f| f.name() == "scores")
891            .unwrap();
892        if let Value::List(scores_list) = &items[scores_index] {
893            assert_eq!(scores_list.items().len(), 3);
894            assert_eq!(scores_list.items()[0], Value::Int64(95));
895            assert_eq!(scores_list.items()[1], Value::Int64(87));
896            assert_eq!(scores_list.items()[2], Value::Int64(92));
897        } else {
898            panic!("Expected List value for scores field");
899        }
900    }
901
902    #[test]
903    fn test_encode_json_structured() {
904        let json = json!({
905            "name": "Bob",
906            "age": 35
907        });
908
909        let settings = JsonSettings::default();
910        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
911
912        if let Value::Struct(struct_value) = result {
913            assert_eq!(struct_value.items().len(), 2);
914            let fields = struct_value.struct_type().fields();
915            let field_names: Vec<&str> = fields.iter().map(|f| f.name()).collect();
916            assert!(field_names.contains(&"name"));
917            assert!(field_names.contains(&"age"));
918        } else {
919            panic!("Expected Struct value");
920        }
921    }
922
923    #[test]
924    fn test_encode_json_respects_type_hint() -> Result<()> {
925        let type_hints = vec![JsonTypeHint {
926            path: vec!["age".to_string()],
927            data_type: ConcreteDataType::int64_datatype(),
928            nullable: false,
929            default_constraint: None,
930            inverted_index: false,
931        }];
932        let settings = JsonSettings::try_new(type_hints, None)?;
933
934        let result = settings
935            .encode(json!({
936                "name": "Alice",
937                "age": 42
938            }))
939            .unwrap()
940            .into_json_inner()
941            .unwrap();
942
943        let Value::Struct(struct_value) = result else {
944            panic!("Expected Struct value");
945        };
946        assert_eq!(struct_field_value(&struct_value, "age"), &Value::Int64(42));
947
948        let err = settings
949            .encode(json!({
950                "age": "42"
951            }))
952            .unwrap_err();
953        assert!(err.to_string().contains("does not match JSON2 type hint"));
954        Ok(())
955    }
956
957    #[test]
958    fn test_encode_json_respects_unsigned_type_hint() -> Result<()> {
959        let type_hints = vec![JsonTypeHint {
960            path: vec!["count".to_string()],
961            data_type: ConcreteDataType::uint64_datatype(),
962            nullable: false,
963            default_constraint: None,
964            inverted_index: false,
965        }];
966        let settings = JsonSettings::try_new(type_hints, None)?;
967
968        let result = settings
969            .encode(json!({
970                "count": u64::MAX
971            }))
972            .unwrap()
973            .into_json_inner()
974            .unwrap();
975
976        let Value::Struct(struct_value) = result else {
977            panic!("Expected Struct value");
978        };
979        assert_eq!(
980            struct_field_value(&struct_value, "count"),
981            &Value::UInt64(u64::MAX)
982        );
983
984        let err = settings
985            .encode(json!({
986                "count": -1
987            }))
988            .unwrap_err();
989        assert!(err.to_string().contains("does not match JSON2 type hint"));
990        Ok(())
991    }
992
993    #[test]
994    fn test_encode_json_fills_missing_type_hint_with_default() -> Result<()> {
995        let type_hints = vec![JsonTypeHint {
996            path: vec!["user".to_string(), "age".to_string()],
997            data_type: ConcreteDataType::int64_datatype(),
998            nullable: false,
999            default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
1000            inverted_index: false,
1001        }];
1002        let settings = JsonSettings::try_new(type_hints, None)?;
1003
1004        let result = settings
1005            .encode(json!({}))
1006            .unwrap()
1007            .into_json_inner()
1008            .unwrap();
1009
1010        let Value::Struct(root) = result else {
1011            panic!("Expected Struct value");
1012        };
1013        let Value::Struct(user) = struct_field_value(&root, "user") else {
1014            panic!("Expected user Struct value");
1015        };
1016        assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
1017        Ok(())
1018    }
1019
1020    #[test]
1021    fn test_encode_json_fills_missing_nullable_type_hint_with_null() -> Result<()> {
1022        let type_hints = vec![JsonTypeHint {
1023            path: vec!["user".to_string(), "name".to_string()],
1024            data_type: ConcreteDataType::string_datatype(),
1025            nullable: true,
1026            default_constraint: None,
1027            inverted_index: false,
1028        }];
1029        let settings = JsonSettings::try_new(type_hints, None)?;
1030
1031        let result = settings
1032            .encode(json!({ "user": {} }))
1033            .unwrap()
1034            .into_json_inner()
1035            .unwrap();
1036
1037        let Value::Struct(root) = result else {
1038            panic!("Expected Struct value");
1039        };
1040        let Value::Struct(user) = struct_field_value(&root, "user") else {
1041            panic!("Expected user Struct value");
1042        };
1043        assert_eq!(struct_field_value(user, "name"), &Value::Null);
1044        Ok(())
1045    }
1046
1047    #[test]
1048    fn test_encode_json_rejects_missing_non_null_type_hint() -> Result<()> {
1049        let type_hints = vec![JsonTypeHint {
1050            path: vec!["user".to_string(), "age".to_string()],
1051            data_type: ConcreteDataType::int64_datatype(),
1052            nullable: false,
1053            default_constraint: None,
1054            inverted_index: false,
1055        }];
1056        let settings = JsonSettings::try_new(type_hints, None)?;
1057
1058        let err = settings.encode(json!({})).unwrap_err();
1059        assert!(
1060            err.to_string()
1061                .contains("missing non-null JSON2 type hint path user.age")
1062        );
1063        Ok(())
1064    }
1065
1066    #[test]
1067    fn test_encode_json_merges_missing_type_hint_prefix() -> Result<()> {
1068        let type_hints = vec![
1069            JsonTypeHint {
1070                path: vec!["user".to_string(), "age".to_string()],
1071                data_type: ConcreteDataType::int64_datatype(),
1072                nullable: false,
1073                default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
1074                inverted_index: false,
1075            },
1076            JsonTypeHint {
1077                path: vec!["user".to_string(), "name".to_string()],
1078                data_type: ConcreteDataType::string_datatype(),
1079                nullable: false,
1080                default_constraint: Some(ColumnDefaultConstraint::Value(Value::String(
1081                    "unknown".into(),
1082                ))),
1083                inverted_index: false,
1084            },
1085        ];
1086        let settings = JsonSettings::try_new(type_hints, None)?;
1087
1088        let result = settings.encode(json!({}))?.into_json_inner().unwrap();
1089
1090        let Value::Struct(root) = result else {
1091            panic!("Expected Struct value");
1092        };
1093        let Value::Struct(user) = struct_field_value(&root, "user") else {
1094            panic!("Expected user Struct value");
1095        };
1096        assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
1097        assert_eq!(
1098            struct_field_value(user, "name"),
1099            &Value::String("unknown".into())
1100        );
1101        Ok(())
1102    }
1103
1104    #[test]
1105    fn test_json_settings_structured() {
1106        let json = json!({
1107            "name": "Eve",
1108            "score": 95
1109        });
1110
1111        let settings = JsonSettings::default();
1112        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
1113
1114        if let Value::Struct(struct_value) = result {
1115            assert_eq!(struct_value.items().len(), 2);
1116        } else {
1117            panic!("Expected Struct value");
1118        }
1119    }
1120
1121    #[cfg(test)]
1122    mod decode_tests {
1123        use ordered_float::OrderedFloat;
1124        use serde_json::json;
1125
1126        use super::*;
1127
1128        #[test]
1129        fn test_decode_primitive_values() {
1130            let settings = JsonSettings::default();
1131
1132            // Test null
1133            let result = settings.decode(Value::Null).unwrap();
1134            assert_eq!(result, Json::Null);
1135
1136            // Test boolean
1137            let result = settings.decode(Value::Boolean(true)).unwrap();
1138            assert_eq!(result, Json::Bool(true));
1139
1140            // Test integer
1141            let result = settings.decode(Value::Int64(42)).unwrap();
1142            assert_eq!(result, Json::from(42));
1143
1144            // Test float
1145            let result = settings.decode(Value::Float64(OrderedFloat(3.16))).unwrap();
1146            assert_eq!(result, Json::from(3.16));
1147
1148            // Test string
1149            let result = settings.decode(Value::String("hello".into())).unwrap();
1150            assert_eq!(result, Json::String("hello".to_string()));
1151        }
1152
1153        #[test]
1154        fn test_decode_struct() {
1155            let settings = JsonSettings::default();
1156
1157            let struct_value = StructValue::new(
1158                vec![
1159                    Value::String("Alice".into()),
1160                    Value::Int64(25),
1161                    Value::Boolean(true),
1162                ],
1163                StructType::new(Arc::new(vec![
1164                    StructField::new(
1165                        "name".to_string(),
1166                        ConcreteDataType::string_datatype(),
1167                        true,
1168                    ),
1169                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
1170                    StructField::new(
1171                        "active".to_string(),
1172                        ConcreteDataType::boolean_datatype(),
1173                        true,
1174                    ),
1175                ])),
1176            );
1177
1178            let result = settings.decode(Value::Struct(struct_value)).unwrap();
1179            let expected = json!({
1180                "name": "Alice",
1181                "age": 25,
1182                "active": true
1183            });
1184            assert_eq!(result, expected);
1185        }
1186
1187        #[test]
1188        fn test_decode_list() {
1189            let settings = JsonSettings::default();
1190
1191            let list_value = ListValue::new(
1192                vec![Value::Int64(1), Value::Int64(2), Value::Int64(3)],
1193                Arc::new(ConcreteDataType::int64_datatype()),
1194            );
1195
1196            let result = settings.decode(Value::List(list_value)).unwrap();
1197            let expected = json!([1, 2, 3]);
1198            assert_eq!(result, expected);
1199        }
1200
1201        #[test]
1202        fn test_decode_nested_structure() {
1203            let settings = JsonSettings::default();
1204
1205            let inner_struct = StructValue::new(
1206                vec![Value::String("Alice".into()), Value::Int64(25)],
1207                StructType::new(Arc::new(vec![
1208                    StructField::new(
1209                        "name".to_string(),
1210                        ConcreteDataType::string_datatype(),
1211                        true,
1212                    ),
1213                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
1214                ])),
1215            );
1216
1217            let score_list_item_type = Arc::new(ConcreteDataType::int64_datatype());
1218            let outer_struct = StructValue::new(
1219                vec![
1220                    Value::Struct(inner_struct),
1221                    Value::List(ListValue::new(
1222                        vec![Value::Int64(95), Value::Int64(87)],
1223                        score_list_item_type.clone(),
1224                    )),
1225                ],
1226                StructType::new(Arc::new(vec![
1227                    StructField::new(
1228                        "user".to_string(),
1229                        ConcreteDataType::Struct(StructType::new(Arc::new(vec![
1230                            StructField::new(
1231                                "name".to_string(),
1232                                ConcreteDataType::string_datatype(),
1233                                true,
1234                            ),
1235                            StructField::new(
1236                                "age".to_string(),
1237                                ConcreteDataType::int64_datatype(),
1238                                true,
1239                            ),
1240                        ]))),
1241                        true,
1242                    ),
1243                    StructField::new(
1244                        "scores".to_string(),
1245                        ConcreteDataType::List(ListType::new(score_list_item_type.clone())),
1246                        true,
1247                    ),
1248                ])),
1249            );
1250
1251            let result = settings.decode(Value::Struct(outer_struct)).unwrap();
1252            let expected = json!({
1253                "user": {
1254                    "name": "Alice",
1255                    "age": 25
1256                },
1257                "scores": [95, 87]
1258            });
1259            assert_eq!(result, expected);
1260        }
1261
1262        #[test]
1263        fn test_decode_missing_fields() {
1264            let settings = JsonSettings::default();
1265
1266            // Struct with missing field (null value)
1267            let struct_value = StructValue::new(
1268                vec![
1269                    Value::String("Bob".into()),
1270                    Value::Null, // missing age field
1271                ],
1272                StructType::new(Arc::new(vec![
1273                    StructField::new(
1274                        "name".to_string(),
1275                        ConcreteDataType::string_datatype(),
1276                        true,
1277                    ),
1278                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
1279                ])),
1280            );
1281
1282            let result = settings.decode(Value::Struct(struct_value)).unwrap();
1283            let expected = json!({
1284                "name": "Bob",
1285                "age": null
1286            });
1287            assert_eq!(result, expected);
1288        }
1289    }
1290}
datatypes/json.rs

datatypes/
json.rs
