From d67d3501a9b55da2cb20be302920bb332bafe514 Mon Sep 17 00:00:00 2001 From: LFC <990479+MichaelScofield@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:16:28 +0000 Subject: [PATCH] fix(json2): keep empty structs in remainder (#9027) Signed-off-by: luofucong --- src/datatypes/src/json/value.rs | 3 +- src/datatypes/src/vectors/json/builder.rs | 9 ++ src/mito2/src/compaction/json2.rs | 3 + .../common/types/json/json2_empty.result | 90 +++++++++++++++++++ .../common/types/json/json2_empty.sql | 42 +++++++++ 5 files changed, 146 insertions(+), 1 deletion(-) diff --git a/src/datatypes/src/json/value.rs b/src/datatypes/src/json/value.rs index aef53a76c6..7bcb1ecc2b 100644 --- a/src/datatypes/src/json/value.rs +++ b/src/datatypes/src/json/value.rs @@ -162,7 +162,8 @@ impl JsonVariant { } } - fn contains_empty_object(&self) -> bool { + /// Returns whether this value recursively contains an empty object. + pub(crate) fn contains_empty_object(&self) -> bool { match self { JsonVariant::Array(array) => array.iter().any(JsonVariant::contains_empty_object), JsonVariant::Object(object) => { diff --git a/src/datatypes/src/vectors/json/builder.rs b/src/datatypes/src/vectors/json/builder.rs index 1ec45c2066..3a9c158796 100644 --- a/src/datatypes/src/vectors/json/builder.rs +++ b/src/datatypes/src/vectors/json/builder.rs @@ -399,6 +399,9 @@ fn infer_expanded_type( // Explicit paths are already in the output schema and do not consume the dynamic // expansion budget. .filter(|(_, stat)| !stat.is_explicit && stat.is_leaf) + // Parquet cannot store empty structs, while widening an empty object to a non-empty struct + // loses its shape. Keep paths containing empty objects in the Variant remainder. + .filter(|(_, stat)| !stat.contains_empty_object) // A leaf is eligible only when both itself and every object prefix have one stable // role and type across all observed values. .filter(|(path, _)| { @@ -446,6 +449,8 @@ struct PathStats { expected_leaf_type: JsonNativeType, /// Number of compatible observations used to rank dynamic leaves. seen_count: usize, + /// Whether any observed value contains an empty object that requires lossless Variant storage. + contains_empty_object: bool, /// Whether the path has ever had inconsistent roles or leaf types. conflicts: bool, } @@ -474,6 +479,7 @@ fn init_explicit_path_stats<'a>( is_leaf, expected_leaf_type, seen_count: 0, + contains_empty_object: false, conflicts: false, }, ); @@ -494,7 +500,9 @@ fn count_dynamic_paths<'a>( if !path.is_empty() { let is_leaf = !matches!(value, JsonVariant::Object(_)); + let contains_empty_object = is_leaf && value.contains_empty_object(); let conflicts = if let Some(stats) = stats.get_mut(path.as_slice()) { + stats.contains_empty_object |= contains_empty_object; if !stats.conflicts { let role_conflict = stats.is_leaf != is_leaf; let type_conflict = || match (&stats.expected_leaf_type, value) { @@ -524,6 +532,7 @@ fn count_dynamic_paths<'a>( is_leaf, expected_leaf_type, seen_count: 1, + contains_empty_object, conflicts: false, }, ); diff --git a/src/mito2/src/compaction/json2.rs b/src/mito2/src/compaction/json2.rs index 02782b6bf4..71b571c586 100644 --- a/src/mito2/src/compaction/json2.rs +++ b/src/mito2/src/compaction/json2.rs @@ -247,6 +247,9 @@ fn select_dynamic_hints( .iter() .filter(|(path, stat)| { !stat.is_type_conflicted + // TODO(LFC): Instead of "primitive only", consider retaining stable compound types + // that are safe to write to Parquet, as flush does. Or better, unite the two + // selection process. && stat.data_type.is_primitive() && !has_ancestor_path(path) && !has_descendant_path(path) diff --git a/tests/cases/standalone/common/types/json/json2_empty.result b/tests/cases/standalone/common/types/json/json2_empty.result index d30758e250..c9fec51396 100644 --- a/tests/cases/standalone/common/types/json/json2_empty.result +++ b/tests/cases/standalone/common/types/json/json2_empty.result @@ -179,6 +179,96 @@ drop table json_empty_nested; Affected Rows: 0 +-- Lists containing empty objects cannot be materialized as Parquet list fields because that +-- would produce an empty Struct child. They must stay in the remainder and preserve the empty +-- objects through flush and compaction. +create table json_empty_in_lists ( + ts timestamp time index, + j json2 +) with ( + 'append_mode' = 'true', + 'sst_format' = 'flat' +); + +Affected Rows: 0 + +insert into json_empty_in_lists values + (1, '{"items":[{}],"mixed":[{"id":1},{}],"nested":[{"meta":{"value":1}},{"meta":{}}]}'); + +Affected Rows: 1 + +admin flush_table('json_empty_in_lists'); + ++------------------------------------------+ +| ADMIN flush_table('json_empty_in_lists') | ++------------------------------------------+ +| 0 | ++------------------------------------------+ + +select ts, j, j.items[0] as item0, j.items[1] as item1, + j.mixed[1] as mixed, j.nested[1].meta as meta +from json_empty_in_lists +order by ts; + ++-------------------------+----------------------------------------------------------------------------------+-------+-------+-------+------+ +| ts | j | item0 | item1 | mixed | meta | ++-------------------------+----------------------------------------------------------------------------------+-------+-------+-------+------+ +| 1970-01-01T00:00:00.001 | {"items":[{}],"mixed":[{"id":1},{}],"nested":[{"meta":{"value":1}},{"meta":{}}]} | {} | | {} | {} | ++-------------------------+----------------------------------------------------------------------------------+-------+-------+-------+------+ + +insert into json_empty_in_lists values + (2, '{"items":[{}],"mixed":[{"id":2},{}],"nested":[{"meta":{"value":2}},{"meta":{}}],"name":"second"}'); + +Affected Rows: 1 + +admin flush_table('json_empty_in_lists'); + ++------------------------------------------+ +| ADMIN flush_table('json_empty_in_lists') | ++------------------------------------------+ +| 0 | ++------------------------------------------+ + +-- Empty objects mixed with non-empty objects in the same list must also stay in +-- the remainder. Otherwise the first item may be reconstructed as {"id":null}. +insert into json_empty_in_lists values + (3, '{"items":[{},{"id":1}]}'); + +Affected Rows: 1 + +admin flush_table('json_empty_in_lists'); + ++------------------------------------------+ +| ADMIN flush_table('json_empty_in_lists') | ++------------------------------------------+ +| 0 | ++------------------------------------------+ + +admin compact_table('json_empty_in_lists'); + ++--------------------------------------------+ +| ADMIN compact_table('json_empty_in_lists') | ++--------------------------------------------+ +| 0 | ++--------------------------------------------+ + +select ts, j, j.items[0] as item0, j.items[1] as item1, + j.mixed[1] as mixed, j.nested[1].meta as meta +from json_empty_in_lists +order by ts; + ++-------------------------+--------------------------------------------------------------------------------------------------+-------+----------+-------+------+ +| ts | j | item0 | item1 | mixed | meta | ++-------------------------+--------------------------------------------------------------------------------------------------+-------+----------+-------+------+ +| 1970-01-01T00:00:00.001 | {"items":[{}],"mixed":[{"id":1},{}],"nested":[{"meta":{"value":1}},{"meta":{}}]} | {} | | {} | {} | +| 1970-01-01T00:00:00.002 | {"items":[{}],"mixed":[{"id":2},{}],"name":"second","nested":[{"meta":{"value":2}},{"meta":{}}]} | {} | | {} | {} | +| 1970-01-01T00:00:00.003 | {"items":[{},{"id":1}]} | {} | {"id":1} | | | ++-------------------------+--------------------------------------------------------------------------------------------------+-------+----------+-------+------+ + +drop table json_empty_in_lists; + +Affected Rows: 0 + -- Empty objects, SQL NULL values, and JSON null values must remain distinct -- through memtable flush and compaction. create table json_empty_null_mixed ( diff --git a/tests/cases/standalone/common/types/json/json2_empty.sql b/tests/cases/standalone/common/types/json/json2_empty.sql index 3cc9b8fb77..4374df3ab7 100644 --- a/tests/cases/standalone/common/types/json/json2_empty.sql +++ b/tests/cases/standalone/common/types/json/json2_empty.sql @@ -66,6 +66,48 @@ select ts, j from json_empty_nested order by ts; drop table json_empty_nested; +-- Lists containing empty objects cannot be materialized as Parquet list fields because that +-- would produce an empty Struct child. They must stay in the remainder and preserve the empty +-- objects through flush and compaction. +create table json_empty_in_lists ( + ts timestamp time index, + j json2 +) with ( + 'append_mode' = 'true', + 'sst_format' = 'flat' +); + +insert into json_empty_in_lists values + (1, '{"items":[{}],"mixed":[{"id":1},{}],"nested":[{"meta":{"value":1}},{"meta":{}}]}'); + +admin flush_table('json_empty_in_lists'); + +select ts, j, j.items[0] as item0, j.items[1] as item1, + j.mixed[1] as mixed, j.nested[1].meta as meta +from json_empty_in_lists +order by ts; + +insert into json_empty_in_lists values + (2, '{"items":[{}],"mixed":[{"id":2},{}],"nested":[{"meta":{"value":2}},{"meta":{}}],"name":"second"}'); + +admin flush_table('json_empty_in_lists'); + +-- Empty objects mixed with non-empty objects in the same list must also stay in +-- the remainder. Otherwise the first item may be reconstructed as {"id":null}. +insert into json_empty_in_lists values + (3, '{"items":[{},{"id":1}]}'); + +admin flush_table('json_empty_in_lists'); + +admin compact_table('json_empty_in_lists'); + +select ts, j, j.items[0] as item0, j.items[1] as item1, + j.mixed[1] as mixed, j.nested[1].meta as meta +from json_empty_in_lists +order by ts; + +drop table json_empty_in_lists; + -- Empty objects, SQL NULL values, and JSON null values must remain distinct -- through memtable flush and compaction. create table json_empty_null_mixed (