fix(json2): keep empty structs in remainder (#9027)

Signed-off-by: luofucong <luofc@foxmail.com>

src/datatypes/src/json/value.rs

@@ -162,7 +162,8 @@ impl JsonVariant {
         }
     }
 
-    fn contains_empty_object(&self) -> bool {
+    /// Returns whether this value recursively contains an empty object.
+    pub(crate) fn contains_empty_object(&self) -> bool {
         match self {
             JsonVariant::Array(array) => array.iter().any(JsonVariant::contains_empty_object),
             JsonVariant::Object(object) => {

src/datatypes/src/vectors/json/builder.rs

@@ -399,6 +399,9 @@ fn infer_expanded_type(
         // Explicit paths are already in the output schema and do not consume the dynamic
         // expansion budget.
         .filter(|(_, stat)| !stat.is_explicit && stat.is_leaf)
+        // Parquet cannot store empty structs, while widening an empty object to a non-empty struct
+        // loses its shape. Keep paths containing empty objects in the Variant remainder.
+        .filter(|(_, stat)| !stat.contains_empty_object)
         // A leaf is eligible only when both itself and every object prefix have one stable
         // role and type across all observed values.
         .filter(|(path, _)| {
@@ -446,6 +449,8 @@ struct PathStats {
     expected_leaf_type: JsonNativeType,
     /// Number of compatible observations used to rank dynamic leaves.
     seen_count: usize,
+    /// Whether any observed value contains an empty object that requires lossless Variant storage.
+    contains_empty_object: bool,
     /// Whether the path has ever had inconsistent roles or leaf types.
     conflicts: bool,
 }
@@ -474,6 +479,7 @@ fn init_explicit_path_stats<'a>(
                 is_leaf,
                 expected_leaf_type,
                 seen_count: 0,
+                contains_empty_object: false,
                 conflicts: false,
             },
         );
@@ -494,7 +500,9 @@ fn count_dynamic_paths<'a>(
 
     if !path.is_empty() {
         let is_leaf = !matches!(value, JsonVariant::Object(_));
+        let contains_empty_object = is_leaf && value.contains_empty_object();
         let conflicts = if let Some(stats) = stats.get_mut(path.as_slice()) {
+            stats.contains_empty_object |= contains_empty_object;
             if !stats.conflicts {
                 let role_conflict = stats.is_leaf != is_leaf;
                 let type_conflict = || match (&stats.expected_leaf_type, value) {
@@ -524,6 +532,7 @@ fn count_dynamic_paths<'a>(
                     is_leaf,
                     expected_leaf_type,
                     seen_count: 1,
+                    contains_empty_object,
                     conflicts: false,
                 },
             );

src/mito2/src/compaction/json2.rs

@@ -247,6 +247,9 @@ fn select_dynamic_hints(
         .iter()
         .filter(|(path, stat)| {
             !stat.is_type_conflicted
+                // TODO(LFC): Instead of "primitive only", consider retaining stable compound types
+                // that are safe to write to Parquet, as flush does. Or better, unite the two 
+                // selection process.
                 && stat.data_type.is_primitive()
                 && !has_ancestor_path(path)
                 && !has_descendant_path(path)

tests/cases/standalone/common/types/json/json2_empty.result

@@ -179,6 +179,96 @@ drop table json_empty_nested;
 
 Affected Rows: 0
 
+-- Lists containing empty objects cannot be materialized as Parquet list fields because that
+-- would produce an empty Struct child. They must stay in the remainder and preserve the empty
+-- objects through flush and compaction.
+create table json_empty_in_lists (
+    ts timestamp time index,
+    j json2
+) with (
+    'append_mode' = 'true',
+    'sst_format' = 'flat'
+);
+
+Affected Rows: 0
+
+insert into json_empty_in_lists values
+    (1, '{"items":[{}],"mixed":[{"id":1},{}],"nested":[{"meta":{"value":1}},{"meta":{}}]}');
+
+Affected Rows: 1
+
+admin flush_table('json_empty_in_lists');
+
++------------------------------------------+
+| ADMIN flush_table('json_empty_in_lists') |
++------------------------------------------+
+| 0                                        |
++------------------------------------------+
+
+select ts, j, j.items[0] as item0, j.items[1] as item1,
+       j.mixed[1] as mixed, j.nested[1].meta as meta
+from json_empty_in_lists
+order by ts;
+
++-------------------------+----------------------------------------------------------------------------------+-------+-------+-------+------+
+| ts                      | j                                                                                | item0 | item1 | mixed | meta |
++-------------------------+----------------------------------------------------------------------------------+-------+-------+-------+------+
+| 1970-01-01T00:00:00.001 | {"items":[{}],"mixed":[{"id":1},{}],"nested":[{"meta":{"value":1}},{"meta":{}}]} | {}    |       | {}    | {}   |
++-------------------------+----------------------------------------------------------------------------------+-------+-------+-------+------+
+
+insert into json_empty_in_lists values
+    (2, '{"items":[{}],"mixed":[{"id":2},{}],"nested":[{"meta":{"value":2}},{"meta":{}}],"name":"second"}');
+
+Affected Rows: 1
+
+admin flush_table('json_empty_in_lists');
+
++------------------------------------------+
+| ADMIN flush_table('json_empty_in_lists') |
++------------------------------------------+
+| 0                                        |
++------------------------------------------+
+
+-- Empty objects mixed with non-empty objects in the same list must also stay in
+-- the remainder. Otherwise the first item may be reconstructed as {"id":null}.
+insert into json_empty_in_lists values
+    (3, '{"items":[{},{"id":1}]}');
+
+Affected Rows: 1
+
+admin flush_table('json_empty_in_lists');
+
++------------------------------------------+
+| ADMIN flush_table('json_empty_in_lists') |
++------------------------------------------+
+| 0                                        |
++------------------------------------------+
+
+admin compact_table('json_empty_in_lists');
+
++--------------------------------------------+
+| ADMIN compact_table('json_empty_in_lists') |
++--------------------------------------------+
+| 0                                          |
++--------------------------------------------+
+
+select ts, j, j.items[0] as item0, j.items[1] as item1,
+       j.mixed[1] as mixed, j.nested[1].meta as meta
+from json_empty_in_lists
+order by ts;
+
++-------------------------+--------------------------------------------------------------------------------------------------+-------+----------+-------+------+
+| ts                      | j                                                                                                | item0 | item1    | mixed | meta |
++-------------------------+--------------------------------------------------------------------------------------------------+-------+----------+-------+------+
+| 1970-01-01T00:00:00.001 | {"items":[{}],"mixed":[{"id":1},{}],"nested":[{"meta":{"value":1}},{"meta":{}}]}                 | {}    |          | {}    | {}   |
+| 1970-01-01T00:00:00.002 | {"items":[{}],"mixed":[{"id":2},{}],"name":"second","nested":[{"meta":{"value":2}},{"meta":{}}]} | {}    |          | {}    | {}   |
+| 1970-01-01T00:00:00.003 | {"items":[{},{"id":1}]}                                                                          | {}    | {"id":1} |       |      |
++-------------------------+--------------------------------------------------------------------------------------------------+-------+----------+-------+------+
+
+drop table json_empty_in_lists;
+
+Affected Rows: 0
+
 -- Empty objects, SQL NULL values, and JSON null values must remain distinct
 -- through memtable flush and compaction.
 create table json_empty_null_mixed (

tests/cases/standalone/common/types/json/json2_empty.sql

@@ -66,6 +66,48 @@ select ts, j from json_empty_nested order by ts;
 
 drop table json_empty_nested;
 
+-- Lists containing empty objects cannot be materialized as Parquet list fields because that
+-- would produce an empty Struct child. They must stay in the remainder and preserve the empty
+-- objects through flush and compaction.
+create table json_empty_in_lists (
+    ts timestamp time index,
+    j json2
+) with (
+    'append_mode' = 'true',
+    'sst_format' = 'flat'
+);
+
+insert into json_empty_in_lists values
+    (1, '{"items":[{}],"mixed":[{"id":1},{}],"nested":[{"meta":{"value":1}},{"meta":{}}]}');
+
+admin flush_table('json_empty_in_lists');
+
+select ts, j, j.items[0] as item0, j.items[1] as item1,
+       j.mixed[1] as mixed, j.nested[1].meta as meta
+from json_empty_in_lists
+order by ts;
+
+insert into json_empty_in_lists values
+    (2, '{"items":[{}],"mixed":[{"id":2},{}],"nested":[{"meta":{"value":2}},{"meta":{}}],"name":"second"}');
+
+admin flush_table('json_empty_in_lists');
+
+-- Empty objects mixed with non-empty objects in the same list must also stay in
+-- the remainder. Otherwise the first item may be reconstructed as {"id":null}.
+insert into json_empty_in_lists values
+    (3, '{"items":[{},{"id":1}]}');
+
+admin flush_table('json_empty_in_lists');
+
+admin compact_table('json_empty_in_lists');
+
+select ts, j, j.items[0] as item0, j.items[1] as item1,
+       j.mixed[1] as mixed, j.nested[1].meta as meta
+from json_empty_in_lists
+order by ts;
+
+drop table json_empty_in_lists;
+
 -- Empty objects, SQL NULL values, and JSON null values must remain distinct
 -- through memtable flush and compaction.
 create table json_empty_null_mixed (