From 932f87f7a82d6d7af270b93e3bc2420f0f5cc7ab Mon Sep 17 00:00:00 2001 From: LFC <990479+MichaelScofield@users.noreply.github.com> Date: Tue, 25 Aug 2026 11:11:06 +0000 Subject: [PATCH] refactor(json2): support querying v2 storage layout (#8940) * feat(json2): support querying v2 storage layout - route missing JSON2 paths to the v2 remainder field - reconstruct complete values from explicit fields and remainder data - preserve root JSON2 columns across projections and filters - support nested JSON values in json_get string results - add and reorganize JSON2 sqlness coverage Signed-off-by: luofucong * resolve PR comments Signed-off-by: luofucong --------- Signed-off-by: luofucong --- .../function/src/scalars/json/json_get.rs | 70 ++++++-- src/mito2/src/read/flat_projection.rs | 4 +- src/mito2/src/sst/parquet/read_columns.rs | 167 +++++++++++++++++- .../src/optimizer/json_type_concretize.rs | 59 +++++-- .../standalone/common/types/json/json2.result | 25 +++ .../standalone/common/types/json/json2.sql | 4 + .../common/types/json/json2_join.result | 68 +++++++ .../common/types/json/json2_join.sql | 37 ++++ .../common/types/json/json2_limit.result | 77 -------- .../common/types/json/json2_limit.sql | 43 ----- 10 files changed, 392 insertions(+), 162 deletions(-) create mode 100644 tests/cases/standalone/common/types/json/json2_join.result create mode 100644 tests/cases/standalone/common/types/json/json2_join.sql diff --git a/src/common/function/src/scalars/json/json_get.rs b/src/common/function/src/scalars/json/json_get.rs index ce49f9d59f..2357bc88f9 100644 --- a/src/common/function/src/scalars/json/json_get.rs +++ b/src/common/function/src/scalars/json/json_get.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::borrow::Cow; use std::sync::Arc; use arrow::array::{ArrayRef, BinaryViewArray, new_null_array}; @@ -54,12 +55,17 @@ trait JsonGetResultBuilder { fn build(&mut self) -> ArrayRef; } -fn result_builder(len: usize, with_type: &DataType) -> Result> { +fn result_builder( + len: usize, + with_type: &DataType, + is_json2: bool, +) -> Result> { let builder = match with_type { - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => { - Box::new(StringResultBuilder(StringViewBuilder::with_capacity(len))) - as Box - } + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Box::new(StringResultBuilder { + inner: StringViewBuilder::with_capacity(len), + is_json2, + }) + as Box, DataType::Int64 => Box::new(IntResultBuilder(Int64Builder::with_capacity(len))), DataType::Float64 => Box::new(FloatResultBuilder(Float64Builder::with_capacity(len))), DataType::Boolean => Box::new(BoolResultBuilder(BooleanBuilder::with_capacity(len))), @@ -71,20 +77,30 @@ fn result_builder(len: usize, with_type: &DataType) -> Result Result<()> { - self.0.append_option(jsonb::to_str(value).ok()); + // Scalar casts stay unquoted and map JSON null to SQL NULL; only containers + // use `to_string` to preserve their JSON representation. + let value = if self.is_json2 && (jsonb::is_array(value) || jsonb::is_object(value)) { + Some(jsonb::to_string(value)) + } else { + jsonb::to_str(value).ok() + }; + self.inner.append_option(value); Ok(()) } fn append_null(&mut self) { - self.0.append_null(); + self.inner.append_null(); } fn build(&mut self) -> ArrayRef { - Arc::new(self.0.finish()) + Arc::new(self.inner.finish()) } } @@ -404,18 +420,21 @@ impl Function for JsonGetWithType { let result = match arg0.data_type() { DataType::Binary | DataType::LargeBinary | DataType::BinaryView => { let arg0 = compute::cast(&arg0, &DataType::BinaryView)?; + let is_json2 = args.arg_fields.first().is_some_and(is_json2_extension_type); - if args.arg_fields.first().is_some_and(is_json2_extension_type) { - // Query concretization projects nested JSON2 paths as Struct arrays. A binary - // JSON2 argument is therefore an already-selected scalar or root value that - // only needs conversion from its JSONB representation to the requested type. + if is_json2 && path.trim_start_matches('$').split('.').all(str::is_empty) { JsonArray::from(&arg0) .project_to(&with_type) .map_err(|e| exec_datafusion_err!("{e:?}"))? } else { let jsons = arg0.as_binary_view(); - let mut builder = result_builder(len, &with_type)?; - jsonb_get(jsons, path, builder.as_mut())?; + let path = if is_json2 && !path.starts_with('$') { + Cow::Owned(format!("$.{path}")) + } else { + Cow::Borrowed(path) + }; + let mut builder = result_builder(len, &with_type, is_json2)?; + jsonb_get(jsons, &path, builder.as_mut())?; builder.build() } } @@ -511,6 +530,7 @@ mod tests { use datafusion_common::ScalarValue; use datafusion_common::arrow::array::{BinaryArray, BinaryViewArray, StringArray}; use datafusion_common::arrow::datatypes::{Float64Type, Int64Type}; + use datatypes::extension::json::Json2ExtensionType; use datatypes::types::parse_string_to_jsonb; use serde_json::json; @@ -566,6 +586,15 @@ mod tests { )) } + fn test_json_field(json: &ArrayRef, is_json2: bool) -> Arc { + let field = Field::new("json", json.data_type().clone(), true); + Arc::new(if is_json2 { + field.with_extension_type(Json2ExtensionType::default()) + } else { + field + }) + } + #[test] fn test_json_get_int() { let json_get_int = JsonGetInt::default(); @@ -850,7 +879,10 @@ mod tests { ColumnarValue::Array(json.clone()), ColumnarValue::Scalar(path.into()), ], - arg_fields: vec![], + arg_fields: vec![ + test_json_field(json, i >= json_strings.len()), + Arc::new(Field::new("path", DataType::Utf8, false)), + ], number_rows: 1, return_field: Arc::new(Field::new("x", DataType::Utf8View, false)), config_options: Arc::new(Default::default()), @@ -984,7 +1016,11 @@ mod tests { ColumnarValue::Scalar(path.into()), ColumnarValue::Scalar(ScalarValue::Utf8View(None)), ], - arg_fields: vec![], + arg_fields: vec![ + test_json_field(json, i >= json_strings.len()), + Arc::new(Field::new("path", DataType::Utf8, false)), + Arc::new(Field::new("with_type", DataType::Utf8View, true)), + ], number_rows: 1, return_field: Arc::new(Field::new("x", DataType::Utf8View, false)), config_options: Arc::new(Default::default()), diff --git a/src/mito2/src/read/flat_projection.rs b/src/mito2/src/read/flat_projection.rs index e631cf45db..5178778ba9 100644 --- a/src/mito2/src/read/flat_projection.rs +++ b/src/mito2/src/read/flat_projection.rs @@ -348,9 +348,9 @@ impl FlatProjectionMapper { } let field = &self.output_schema.arrow_schema().fields()[output_idx]; - if is_json2_extension_type(field) { + if is_json2_extension_type(field) && array.data_type() != field.data_type() { array = JsonArray::from(&array) - .project_to(field.data_type()) + .project_to_v2(batch.schema_ref().field(*index), field.data_type()) .context(DataTypesSnafu)?; } diff --git a/src/mito2/src/sst/parquet/read_columns.rs b/src/mito2/src/sst/parquet/read_columns.rs index 6f632aee30..68d7cae619 100644 --- a/src/mito2/src/sst/parquet/read_columns.rs +++ b/src/mito2/src/sst/parquet/read_columns.rs @@ -14,6 +14,7 @@ use std::collections::{HashMap, HashSet}; +use datatypes::extension::json::JSON2_REMAINDER_FIELD_NAME; use parquet::arrow::ProjectionMask; use parquet::basic::{ConvertedType, Type as PhysicalType}; use parquet::schema::types::{ColumnDescriptor, SchemaDescriptor}; @@ -179,7 +180,7 @@ pub struct ProjectionMaskPlan { /// returned plan keeps `k` in the projection mask and marks `j` as /// not present in the output, so it can be synthesized during /// post-processing. -pub fn build_projection_plan( +pub(crate) fn build_projection_plan( parquet_read_cols: &ParquetReadColumns, parquet_schema_desc: &SchemaDescriptor, ) -> ProjectionMaskPlan { @@ -261,13 +262,30 @@ fn build_parquet_leaves_indices( } } - // Then fallback prefix misses to their nearest variant parent. + // Then include v2 remainder leaves or fallback prefix misses to their nearest variant parent. // TODO(fys): Gate fallback planning on the root being JSON2. A raw Binary // leaf is a JSONB variant only under a JSON2 root; plain struct Binary // children should not enter this fallback path. for col in &projection.cols { - for (path_idx, nested_path) in col.nested_paths.iter().enumerate() { - if prefix_matched[&col.root_index][path_idx] { + let path_matches = &prefix_matched[&col.root_index]; + let needs_remainder = col + .nested_paths + .iter() + .zip(path_matches) + .any(|(path, matched)| { + !*matched || path_points_to_struct(parquet_schema_desc, col.root_index, path) + }); + if needs_remainder { + let remainder_leaves = find_remainder_leaves(parquet_schema_desc, col.root_index); + if !remainder_leaves.is_empty() { + matched_leaves.extend(remainder_leaves); + matched_roots.insert(col.root_index); + continue; + } + } + + for (matched, nested_path) in path_matches.iter().zip(&col.nested_paths) { + if *matched { continue; } @@ -287,6 +305,49 @@ fn build_parquet_leaves_indices( (matched_leaves, matched_roots) } +/// Returns whether a nested path points to an explicitly materialized object. +/// +/// JSON2 v2 can split an object's children between its Struct field and the remainder, +/// so reading the Struct leaves alone may produce an incomplete object. +fn path_points_to_struct( + parquet_schema_desc: &SchemaDescriptor, + root_idx: usize, + path: &[String], +) -> bool { + let Some(mut field) = parquet_schema_desc.root_schema().get_fields().get(root_idx) else { + return false; + }; + for name in path.iter().skip(1) { + if !field.is_group() { + return false; + } + let Some(child) = field.get_fields().iter().find(|field| field.name() == name) else { + return false; + }; + field = child; + } + field.is_group() +} + +/// Finds the Parquet leaves backing a JSON2 v2 remainder field. +/// +/// The remainder is a sibling of explicitly materialized fields, so prefix matching a +/// requested path cannot find it. These leaves are needed when an explicit path is absent +/// or an explicitly materialized object may have additional children in the remainder. +fn find_remainder_leaves(parquet_schema_desc: &SchemaDescriptor, root_idx: usize) -> Vec { + parquet_schema_desc + .columns() + .iter() + .enumerate() + .filter_map(|(i, column)| { + let path = column.path().parts(); + (parquet_schema_desc.get_column_root_idx(i) == root_idx + && path.get(1).is_some_and(|x| x == JSON2_REMAINDER_FIELD_NAME)) + .then_some(i) + }) + .collect::>() +} + fn find_nearest_variant_parent( parquet_schema_desc: &SchemaDescriptor, root_idx: usize, @@ -329,6 +390,7 @@ mod tests { use std::sync::Arc; use parquet::basic::{ConvertedType, LogicalType, Repetition}; + use parquet::errors::ParquetError; use parquet::schema::types::Type; use super::*; @@ -441,6 +503,54 @@ mod tests { ); } + #[test] + fn test_v2_routes_missing_path_to_remainder() -> Result<(), ParquetError> { + let parquet = build_test_v2_schema()?; + let projection = + ParquetReadColumns::from_deduped(vec![ParquetReadColumn::new(0).with_nested_paths( + vec![ + vec!["j".to_string(), "cold".to_string()], + vec!["j".to_string(), "another".to_string()], + ], + )]); + + let plan = build_projection_plan(&projection, &parquet); + + assert_eq!(vec![true], plan.projected_root_presence); + assert_eq!(ProjectionMask::leaves(&parquet, [0, 1]), plan.mask); + Ok(()) + } + + #[test] + fn test_v2_explicit_path_does_not_read_remainder() -> Result<(), ParquetError> { + let parquet = build_test_v2_schema()?; + let projection = ParquetReadColumns::from_deduped(vec![ + ParquetReadColumn::new(0) + .with_nested_paths(vec![vec!["j".to_string(), "hot".to_string()]]), + ]); + + let plan = build_projection_plan(&projection, &parquet); + + assert_eq!(vec![true], plan.projected_root_presence); + assert_eq!(ProjectionMask::leaves(&parquet, [3]), plan.mask); + Ok(()) + } + + #[test] + fn test_v2_container_path_reads_remainder() -> Result<(), ParquetError> { + let parquet = build_test_v2_schema()?; + let projection = ParquetReadColumns::from_deduped(vec![ + ParquetReadColumn::new(0) + .with_nested_paths(vec![vec!["j".to_string(), "commit".to_string()]]), + ]); + + let plan = build_projection_plan(&projection, &parquet); + + assert_eq!(vec![true], plan.projected_root_presence); + assert_eq!(ProjectionMask::leaves(&parquet, [0, 1, 2]), plan.mask); + Ok(()) + } + #[test] fn test_merges_mixed_paths() { let parquet_schema_desc = build_test_nested_parquet_schema(); @@ -673,6 +783,55 @@ mod tests { SchemaDescriptor::new(schema) } + fn build_test_v2_schema() -> Result { + let metadata = Arc::new( + Type::primitive_type_builder("metadata", parquet::basic::Type::BYTE_ARRAY) + .with_repetition(Repetition::REQUIRED) + .build()?, + ); + let value = Arc::new( + Type::primitive_type_builder("value", parquet::basic::Type::BYTE_ARRAY) + .with_repetition(Repetition::REQUIRED) + .build()?, + ); + let remainder = Arc::new( + Type::group_type_builder(JSON2_REMAINDER_FIELD_NAME) + .with_repetition(Repetition::OPTIONAL) + .with_logical_type(Some(LogicalType::Variant { + specification_version: None, + })) + .with_fields(vec![metadata, value]) + .build()?, + ); + let operation = Arc::new( + Type::primitive_type_builder("operation", parquet::basic::Type::INT64) + .with_repetition(Repetition::OPTIONAL) + .build()?, + ); + let commit = Arc::new( + Type::group_type_builder("commit") + .with_repetition(Repetition::OPTIONAL) + .with_fields(vec![operation]) + .build()?, + ); + let hot = Arc::new( + Type::primitive_type_builder("hot", parquet::basic::Type::INT64) + .with_repetition(Repetition::OPTIONAL) + .build()?, + ); + let root = Arc::new( + Type::group_type_builder("j") + .with_repetition(Repetition::OPTIONAL) + .with_fields(vec![remainder, commit, hot]) + .build()?, + ); + Ok(SchemaDescriptor::new(Arc::new( + Type::group_type_builder("schema") + .with_fields(vec![root]) + .build()?, + ))) + } + // Test schema: // schema // `- j diff --git a/src/query/src/optimizer/json_type_concretize.rs b/src/query/src/optimizer/json_type_concretize.rs index 4625b61978..c9964e119d 100644 --- a/src/query/src/optimizer/json_type_concretize.rs +++ b/src/query/src/optimizer/json_type_concretize.rs @@ -111,13 +111,26 @@ fn apply_json_type_hint( fn deduce_json_types(plan: &LogicalPlan) -> Result> { let mut json_types = HashMap::::new(); + // JSON2 columns in the final output must retain their complete values even when + // predicates or other expressions access only specific paths. + // For example, `SELECT j FROM t WHERE json_get(j, 'a') = 1`. + plan.schema() + .fields() + .iter() + .filter(|field| is_json2_extension_type(field)) + .for_each(|field| { + json_types.insert(field.name().clone(), JsonNativeType::Variant); + }); + plan.apply(|plan| { for expr in plan.expressions() { expr.apply(|expr| { if let Some((column, json_type)) = deduce_json_type(expr)? { json_types.entry(column).or_default().merge(&json_type); + Ok(TreeNodeRecursion::Jump) + } else { + Ok(TreeNodeRecursion::Continue) } - Ok(TreeNodeRecursion::Continue) })?; } Ok(TreeNodeRecursion::Continue) @@ -128,6 +141,7 @@ fn deduce_json_types(plan: &LogicalPlan) -> Result Result> { let f = match expr { Expr::ScalarFunction(f) if f.name().eq_ignore_ascii_case(JsonGetWithType::NAME) => f, + Expr::Column(c) => return Ok(Some((c.name.clone(), JsonNativeType::Variant))), _ => return Ok(None), }; @@ -360,25 +374,13 @@ mod tests { .rewrite(plan, &OptimizerContext::default())? .transformed ); - assert!(provider.scan_request().json_type_hint.contains_key("j")); - Ok(()) - } - - #[test] - fn test_allow_json2_passthrough_for_later_projection() -> Result<()> { - let json_get = json_get_expr(col("j"), path_expr("a"), Some(DataType::Int64))?; - let (provider, plan) = build_json2_scan()?; - let plan = plan - .project(vec![json_get.alias("__common_expr"), col("j")])? - .aggregate(Vec::::new(), vec![count(lit(1))])? - .build()?; - - assert!( - JsonTypeConcretizeRule - .rewrite(plan, &OptimizerContext::default())? - .transformed + assert_eq!( + Some(&JsonNativeType::Object(JsonObjectType::from([( + "a".to_string(), + JsonNativeType::i64(), + )]))), + provider.scan_request().json_type_hint.get("j") ); - assert!(provider.scan_request().json_type_hint.contains_key("j")); Ok(()) } @@ -402,6 +404,25 @@ mod tests { Ok(()) } + #[test] + fn test_allow_json2_filter_with_root_projection() -> Result<()> { + let predicate = + json_get_expr(col("j"), path_expr("a"), Some(DataType::Int64))?.eq(lit(1_i64)); + let (provider, plan) = build_json2_scan()?; + let plan = plan.filter(predicate)?.build()?; + + assert!( + JsonTypeConcretizeRule + .rewrite(plan, &OptimizerContext::default())? + .transformed + ); + assert_eq!( + Some(&JsonNativeType::Variant), + provider.scan_request().json_type_hint.get("j") + ); + Ok(()) + } + #[test] fn test_deduce_json_type_with_non_column_base() -> Result<()> { let expr = json_get_expr( diff --git a/tests/cases/standalone/common/types/json/json2.result b/tests/cases/standalone/common/types/json/json2.result index 2e9c9c8ee8..6afa0a759c 100644 --- a/tests/cases/standalone/common/types/json/json2.result +++ b/tests/cases/standalone/common/types/json/json2.result @@ -134,6 +134,31 @@ select j.a, j.a.x from json2_table order by ts; | {"b":10,"x":null} | | +-----------------------------------+-------------------------------------+ +select j, j.a from json2_table order by ts; + ++--------------------------------------------------------------------+-----------------------------------+ +| j | json_get(json2_table.j,Utf8("a")) | ++--------------------------------------------------------------------+-----------------------------------+ +| {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]} | {"b":1} | +| {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]} | {"b":-2} | +| {"a":{"b":3},"c":"s3","d":null} | {"b":3} | +| {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]} | {"b":-4} | +| {"a":null,"c":"s5","d":null} | | +| {"a":null,"c":"s6","d":null} | | +| {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]} | {"b":"s7"} | +| {"a":{"b":8},"c":"s8","d":null} | {"b":8} | +| {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} | {"b":null,"x":true} | +| {"a":{"b":10,"x":null},"c":null,"d":null,"y":false} | {"b":10,"x":null} | ++--------------------------------------------------------------------+-----------------------------------+ + +select j from json2_table where j.a.b = 1; + ++-------------------------------------------------------+ +| j | ++-------------------------------------------------------+ +| {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]} | ++-------------------------------------------------------+ + select j.c, j.y from json2_table order by ts; +-----------------------------------+-----------------------------------+ diff --git a/tests/cases/standalone/common/types/json/json2.sql b/tests/cases/standalone/common/types/json/json2.sql index f0bb5286b4..406cc4af54 100644 --- a/tests/cases/standalone/common/types/json/json2.sql +++ b/tests/cases/standalone/common/types/json/json2.sql @@ -44,6 +44,10 @@ select j.a.b from json2_table order by ts; select j.a, j.a.x from json2_table order by ts; +select j, j.a from json2_table order by ts; + +select j from json2_table where j.a.b = 1; + select j.c, j.y from json2_table order by ts; select j from json2_table order by ts; diff --git a/tests/cases/standalone/common/types/json/json2_join.result b/tests/cases/standalone/common/types/json/json2_join.result new file mode 100644 index 0000000000..cd5b215a98 --- /dev/null +++ b/tests/cases/standalone/common/types/json/json2_join.result @@ -0,0 +1,68 @@ +create table json2_join_same_name_left ( + ts timestamp time index, + k string, + j json2 +) +with ( + 'append_mode' = 'true' +); + +Affected Rows: 0 + +create table json2_join_same_name_right ( + ts timestamp time index, + k string, + j json2 +) +with ( + 'append_mode' = 'true' +); + +Affected Rows: 0 + +insert into json2_join_same_name_left values + (1, 'a', '{"a": 1, "left_only": "kept"}'); + +Affected Rows: 1 + +insert into json2_join_same_name_right values + (1, 'a', '{"a": "right", "right_only": "should be kept"}'); + +Affected Rows: 1 + +admin flush_table('json2_join_same_name_left'); + ++------------------------------------------------+ +| ADMIN flush_table('json2_join_same_name_left') | ++------------------------------------------------+ +| 0 | ++------------------------------------------------+ + +admin flush_table('json2_join_same_name_right'); + ++-------------------------------------------------+ +| ADMIN flush_table('json2_join_same_name_right') | ++-------------------------------------------------+ +| 0 | ++-------------------------------------------------+ + +-- Conflicting hints for same-named JSON2 columns preserve both values as Variant. +select json_get(r.j, 'a')::string, json_get(l.j, 'a')::int64 +from json2_join_same_name_left l +join json2_join_same_name_right r +on l.k = r.k; + ++-------------------------+---------------------------------------------------+ +| json_get(r.j,Utf8("a")) | arrow_cast(json_get(l.j,Utf8("a")),Utf8("Int64")) | ++-------------------------+---------------------------------------------------+ +| right | 1 | ++-------------------------+---------------------------------------------------+ + +drop table json2_join_same_name_left; + +Affected Rows: 0 + +drop table json2_join_same_name_right; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/types/json/json2_join.sql b/tests/cases/standalone/common/types/json/json2_join.sql new file mode 100644 index 0000000000..3d49d740f6 --- /dev/null +++ b/tests/cases/standalone/common/types/json/json2_join.sql @@ -0,0 +1,37 @@ +create table json2_join_same_name_left ( + ts timestamp time index, + k string, + j json2 +) +with ( + 'append_mode' = 'true' +); + +create table json2_join_same_name_right ( + ts timestamp time index, + k string, + j json2 +) +with ( + 'append_mode' = 'true' +); + +insert into json2_join_same_name_left values + (1, 'a', '{"a": 1, "left_only": "kept"}'); + +insert into json2_join_same_name_right values + (1, 'a', '{"a": "right", "right_only": "should be kept"}'); + +admin flush_table('json2_join_same_name_left'); + +admin flush_table('json2_join_same_name_right'); + +-- Conflicting hints for same-named JSON2 columns preserve both values as Variant. +select json_get(r.j, 'a')::string, json_get(l.j, 'a')::int64 +from json2_join_same_name_left l +join json2_join_same_name_right r +on l.k = r.k; + +drop table json2_join_same_name_left; + +drop table json2_join_same_name_right; diff --git a/tests/cases/standalone/common/types/json/json2_limit.result b/tests/cases/standalone/common/types/json/json2_limit.result index 035c4ed07b..0394bfca66 100644 --- a/tests/cases/standalone/common/types/json/json2_limit.result +++ b/tests/cases/standalone/common/types/json/json2_limit.result @@ -65,87 +65,10 @@ order by json_get(j, 'a.b'); | 2 | 1 | +---------------------------------------------------+----------+ -select j, j.a from json2_whole_and_path_read; - -Error: 3001(EngineExecuteQuery), Invalid argument error: column types must match schema types, expected Binary but found Struct("a": Utf8View) at column index 0 - -select j from json2_whole_and_path_read where j.a.b = 1; - -Error: 3001(EngineExecuteQuery), Invalid argument error: column types must match schema types, expected Binary but found Struct("a": Struct("b": Int64)) at column index 0 - drop table json2_whole_and_path_read; Affected Rows: 0 -create table json2_join_same_name_left ( - ts timestamp time index, - k string, - j json2 -) -with ( - 'append_mode' = 'true' -); - -Affected Rows: 0 - -create table json2_join_same_name_right ( - ts timestamp time index, - k string, - j json2 -) -with ( - 'append_mode' = 'true' -); - -Affected Rows: 0 - -insert into json2_join_same_name_left values - (1, 'a', '{"a": 1, "left_only": "kept"}'); - -Affected Rows: 1 - -insert into json2_join_same_name_right values - (1, 'a', '{"a": "right", "right_only": "should be kept"}'); - -Affected Rows: 1 - -admin flush_table('json2_join_same_name_left'); - -+------------------------------------------------+ -| ADMIN flush_table('json2_join_same_name_left') | -+------------------------------------------------+ -| 0 | -+------------------------------------------------+ - -admin flush_table('json2_join_same_name_right'); - -+-------------------------------------------------+ -| ADMIN flush_table('json2_join_same_name_right') | -+-------------------------------------------------+ -| 0 | -+-------------------------------------------------+ - --- FIXME: This should return `right` and `1`. The current NULL values are caused --- by JSON type hints losing the table qualifier in joins. -select json_get(r.j, 'a')::string, json_get(l.j, 'a')::int64 -from json2_join_same_name_left l -join json2_join_same_name_right r -on l.k = r.k; - -+---------------------------------------------+---------------------------------------------------+ -| json_get(r.j,Utf8("a")) | arrow_cast(json_get(l.j,Utf8("a")),Utf8("Int64")) | -+---------------------------------------------+---------------------------------------------------+ -| {"a":"right","right_only":"should be kept"} | | -+---------------------------------------------+---------------------------------------------------+ - -drop table json2_join_same_name_left; - -Affected Rows: 0 - -drop table json2_join_same_name_right; - -Affected Rows: 0 - create table json2_without_append_mode ( ts timestamp time index, j json2 diff --git a/tests/cases/standalone/common/types/json/json2_limit.sql b/tests/cases/standalone/common/types/json/json2_limit.sql index 986ed58eb2..fc80e97cd0 100644 --- a/tests/cases/standalone/common/types/json/json2_limit.sql +++ b/tests/cases/standalone/common/types/json/json2_limit.sql @@ -38,51 +38,8 @@ from json2_whole_and_path_read group by json_get(j, 'a.b') order by json_get(j, 'a.b'); -select j, j.a from json2_whole_and_path_read; - -select j from json2_whole_and_path_read where j.a.b = 1; - drop table json2_whole_and_path_read; -create table json2_join_same_name_left ( - ts timestamp time index, - k string, - j json2 -) -with ( - 'append_mode' = 'true' -); - -create table json2_join_same_name_right ( - ts timestamp time index, - k string, - j json2 -) -with ( - 'append_mode' = 'true' -); - -insert into json2_join_same_name_left values - (1, 'a', '{"a": 1, "left_only": "kept"}'); - -insert into json2_join_same_name_right values - (1, 'a', '{"a": "right", "right_only": "should be kept"}'); - -admin flush_table('json2_join_same_name_left'); - -admin flush_table('json2_join_same_name_right'); - --- FIXME: This should return `right` and `1`. The current NULL values are caused --- by JSON type hints losing the table qualifier in joins. -select json_get(r.j, 'a')::string, json_get(l.j, 'a')::int64 -from json2_join_same_name_left l -join json2_join_same_name_right r -on l.k = r.k; - -drop table json2_join_same_name_left; - -drop table json2_join_same_name_right; - create table json2_without_append_mode ( ts timestamp time index, j json2