mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-09-05 13:08:58 +00:00
refactor(json2): support querying v2 storage layout (#8940)
* feat(json2): support querying v2 storage layout - route missing JSON2 paths to the v2 remainder field - reconstruct complete values from explicit fields and remainder data - preserve root JSON2 columns across projections and filters - support nested JSON values in json_get string results - add and reorganize JSON2 sqlness coverage Signed-off-by: luofucong <luofc@foxmail.com> * resolve PR comments Signed-off-by: luofucong <luofc@foxmail.com> --------- Signed-off-by: luofucong <luofc@foxmail.com>
This commit is contained in:
LFC
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow::array::{ArrayRef, BinaryViewArray, new_null_array};
|
||||
@@ -54,12 +55,17 @@ trait JsonGetResultBuilder {
|
||||
fn build(&mut self) -> ArrayRef;
|
||||
}
|
||||
|
||||
fn result_builder(len: usize, with_type: &DataType) -> Result<Box<dyn JsonGetResultBuilder>> {
|
||||
fn result_builder(
|
||||
len: usize,
|
||||
with_type: &DataType,
|
||||
is_json2: bool,
|
||||
) -> Result<Box<dyn JsonGetResultBuilder>> {
|
||||
let builder = match with_type {
|
||||
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {
|
||||
Box::new(StringResultBuilder(StringViewBuilder::with_capacity(len)))
|
||||
as Box<dyn JsonGetResultBuilder>
|
||||
}
|
||||
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Box::new(StringResultBuilder {
|
||||
inner: StringViewBuilder::with_capacity(len),
|
||||
is_json2,
|
||||
})
|
||||
as Box<dyn JsonGetResultBuilder>,
|
||||
DataType::Int64 => Box::new(IntResultBuilder(Int64Builder::with_capacity(len))),
|
||||
DataType::Float64 => Box::new(FloatResultBuilder(Float64Builder::with_capacity(len))),
|
||||
DataType::Boolean => Box::new(BoolResultBuilder(BooleanBuilder::with_capacity(len))),
|
||||
@@ -71,20 +77,30 @@ fn result_builder(len: usize, with_type: &DataType) -> Result<Box<dyn JsonGetRes
|
||||
}
|
||||
|
||||
// TODO: refactor this to StringLikeArrayBuilder from Arrow 57
|
||||
struct StringResultBuilder(StringViewBuilder);
|
||||
struct StringResultBuilder {
|
||||
inner: StringViewBuilder,
|
||||
is_json2: bool,
|
||||
}
|
||||
|
||||
impl JsonGetResultBuilder for StringResultBuilder {
|
||||
fn append_value(&mut self, value: &[u8]) -> Result<()> {
|
||||
self.0.append_option(jsonb::to_str(value).ok());
|
||||
// Scalar casts stay unquoted and map JSON null to SQL NULL; only containers
|
||||
// use `to_string` to preserve their JSON representation.
|
||||
let value = if self.is_json2 && (jsonb::is_array(value) || jsonb::is_object(value)) {
|
||||
Some(jsonb::to_string(value))
|
||||
} else {
|
||||
jsonb::to_str(value).ok()
|
||||
};
|
||||
self.inner.append_option(value);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn append_null(&mut self) {
|
||||
self.0.append_null();
|
||||
self.inner.append_null();
|
||||
}
|
||||
|
||||
fn build(&mut self) -> ArrayRef {
|
||||
Arc::new(self.0.finish())
|
||||
Arc::new(self.inner.finish())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -404,18 +420,21 @@ impl Function for JsonGetWithType {
|
||||
let result = match arg0.data_type() {
|
||||
DataType::Binary | DataType::LargeBinary | DataType::BinaryView => {
|
||||
let arg0 = compute::cast(&arg0, &DataType::BinaryView)?;
|
||||
let is_json2 = args.arg_fields.first().is_some_and(is_json2_extension_type);
|
||||
|
||||
if args.arg_fields.first().is_some_and(is_json2_extension_type) {
|
||||
// Query concretization projects nested JSON2 paths as Struct arrays. A binary
|
||||
// JSON2 argument is therefore an already-selected scalar or root value that
|
||||
// only needs conversion from its JSONB representation to the requested type.
|
||||
if is_json2 && path.trim_start_matches('$').split('.').all(str::is_empty) {
|
||||
JsonArray::from(&arg0)
|
||||
.project_to(&with_type)
|
||||
.map_err(|e| exec_datafusion_err!("{e:?}"))?
|
||||
} else {
|
||||
let jsons = arg0.as_binary_view();
|
||||
let mut builder = result_builder(len, &with_type)?;
|
||||
jsonb_get(jsons, path, builder.as_mut())?;
|
||||
let path = if is_json2 && !path.starts_with('$') {
|
||||
Cow::Owned(format!("$.{path}"))
|
||||
} else {
|
||||
Cow::Borrowed(path)
|
||||
};
|
||||
let mut builder = result_builder(len, &with_type, is_json2)?;
|
||||
jsonb_get(jsons, &path, builder.as_mut())?;
|
||||
builder.build()
|
||||
}
|
||||
}
|
||||
@@ -511,6 +530,7 @@ mod tests {
|
||||
use datafusion_common::ScalarValue;
|
||||
use datafusion_common::arrow::array::{BinaryArray, BinaryViewArray, StringArray};
|
||||
use datafusion_common::arrow::datatypes::{Float64Type, Int64Type};
|
||||
use datatypes::extension::json::Json2ExtensionType;
|
||||
use datatypes::types::parse_string_to_jsonb;
|
||||
use serde_json::json;
|
||||
|
||||
@@ -566,6 +586,15 @@ mod tests {
|
||||
))
|
||||
}
|
||||
|
||||
fn test_json_field(json: &ArrayRef, is_json2: bool) -> Arc<Field> {
|
||||
let field = Field::new("json", json.data_type().clone(), true);
|
||||
Arc::new(if is_json2 {
|
||||
field.with_extension_type(Json2ExtensionType::default())
|
||||
} else {
|
||||
field
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_json_get_int() {
|
||||
let json_get_int = JsonGetInt::default();
|
||||
@@ -850,7 +879,10 @@ mod tests {
|
||||
ColumnarValue::Array(json.clone()),
|
||||
ColumnarValue::Scalar(path.into()),
|
||||
],
|
||||
arg_fields: vec![],
|
||||
arg_fields: vec![
|
||||
test_json_field(json, i >= json_strings.len()),
|
||||
Arc::new(Field::new("path", DataType::Utf8, false)),
|
||||
],
|
||||
number_rows: 1,
|
||||
return_field: Arc::new(Field::new("x", DataType::Utf8View, false)),
|
||||
config_options: Arc::new(Default::default()),
|
||||
@@ -984,7 +1016,11 @@ mod tests {
|
||||
ColumnarValue::Scalar(path.into()),
|
||||
ColumnarValue::Scalar(ScalarValue::Utf8View(None)),
|
||||
],
|
||||
arg_fields: vec![],
|
||||
arg_fields: vec![
|
||||
test_json_field(json, i >= json_strings.len()),
|
||||
Arc::new(Field::new("path", DataType::Utf8, false)),
|
||||
Arc::new(Field::new("with_type", DataType::Utf8View, true)),
|
||||
],
|
||||
number_rows: 1,
|
||||
return_field: Arc::new(Field::new("x", DataType::Utf8View, false)),
|
||||
config_options: Arc::new(Default::default()),
|
||||
|
||||
@@ -348,9 +348,9 @@ impl FlatProjectionMapper {
|
||||
}
|
||||
|
||||
let field = &self.output_schema.arrow_schema().fields()[output_idx];
|
||||
if is_json2_extension_type(field) {
|
||||
if is_json2_extension_type(field) && array.data_type() != field.data_type() {
|
||||
array = JsonArray::from(&array)
|
||||
.project_to(field.data_type())
|
||||
.project_to_v2(batch.schema_ref().field(*index), field.data_type())
|
||||
.context(DataTypesSnafu)?;
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use datatypes::extension::json::JSON2_REMAINDER_FIELD_NAME;
|
||||
use parquet::arrow::ProjectionMask;
|
||||
use parquet::basic::{ConvertedType, Type as PhysicalType};
|
||||
use parquet::schema::types::{ColumnDescriptor, SchemaDescriptor};
|
||||
@@ -179,7 +180,7 @@ pub struct ProjectionMaskPlan {
|
||||
/// returned plan keeps `k` in the projection mask and marks `j` as
|
||||
/// not present in the output, so it can be synthesized during
|
||||
/// post-processing.
|
||||
pub fn build_projection_plan(
|
||||
pub(crate) fn build_projection_plan(
|
||||
parquet_read_cols: &ParquetReadColumns,
|
||||
parquet_schema_desc: &SchemaDescriptor,
|
||||
) -> ProjectionMaskPlan {
|
||||
@@ -261,13 +262,30 @@ fn build_parquet_leaves_indices(
|
||||
}
|
||||
}
|
||||
|
||||
// Then fallback prefix misses to their nearest variant parent.
|
||||
// Then include v2 remainder leaves or fallback prefix misses to their nearest variant parent.
|
||||
// TODO(fys): Gate fallback planning on the root being JSON2. A raw Binary
|
||||
// leaf is a JSONB variant only under a JSON2 root; plain struct Binary
|
||||
// children should not enter this fallback path.
|
||||
for col in &projection.cols {
|
||||
for (path_idx, nested_path) in col.nested_paths.iter().enumerate() {
|
||||
if prefix_matched[&col.root_index][path_idx] {
|
||||
let path_matches = &prefix_matched[&col.root_index];
|
||||
let needs_remainder = col
|
||||
.nested_paths
|
||||
.iter()
|
||||
.zip(path_matches)
|
||||
.any(|(path, matched)| {
|
||||
!*matched || path_points_to_struct(parquet_schema_desc, col.root_index, path)
|
||||
});
|
||||
if needs_remainder {
|
||||
let remainder_leaves = find_remainder_leaves(parquet_schema_desc, col.root_index);
|
||||
if !remainder_leaves.is_empty() {
|
||||
matched_leaves.extend(remainder_leaves);
|
||||
matched_roots.insert(col.root_index);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
for (matched, nested_path) in path_matches.iter().zip(&col.nested_paths) {
|
||||
if *matched {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -287,6 +305,49 @@ fn build_parquet_leaves_indices(
|
||||
(matched_leaves, matched_roots)
|
||||
}
|
||||
|
||||
/// Returns whether a nested path points to an explicitly materialized object.
|
||||
///
|
||||
/// JSON2 v2 can split an object's children between its Struct field and the remainder,
|
||||
/// so reading the Struct leaves alone may produce an incomplete object.
|
||||
fn path_points_to_struct(
|
||||
parquet_schema_desc: &SchemaDescriptor,
|
||||
root_idx: usize,
|
||||
path: &[String],
|
||||
) -> bool {
|
||||
let Some(mut field) = parquet_schema_desc.root_schema().get_fields().get(root_idx) else {
|
||||
return false;
|
||||
};
|
||||
for name in path.iter().skip(1) {
|
||||
if !field.is_group() {
|
||||
return false;
|
||||
}
|
||||
let Some(child) = field.get_fields().iter().find(|field| field.name() == name) else {
|
||||
return false;
|
||||
};
|
||||
field = child;
|
||||
}
|
||||
field.is_group()
|
||||
}
|
||||
|
||||
/// Finds the Parquet leaves backing a JSON2 v2 remainder field.
|
||||
///
|
||||
/// The remainder is a sibling of explicitly materialized fields, so prefix matching a
|
||||
/// requested path cannot find it. These leaves are needed when an explicit path is absent
|
||||
/// or an explicitly materialized object may have additional children in the remainder.
|
||||
fn find_remainder_leaves(parquet_schema_desc: &SchemaDescriptor, root_idx: usize) -> Vec<usize> {
|
||||
parquet_schema_desc
|
||||
.columns()
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, column)| {
|
||||
let path = column.path().parts();
|
||||
(parquet_schema_desc.get_column_root_idx(i) == root_idx
|
||||
&& path.get(1).is_some_and(|x| x == JSON2_REMAINDER_FIELD_NAME))
|
||||
.then_some(i)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
|
||||
fn find_nearest_variant_parent(
|
||||
parquet_schema_desc: &SchemaDescriptor,
|
||||
root_idx: usize,
|
||||
@@ -329,6 +390,7 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use parquet::basic::{ConvertedType, LogicalType, Repetition};
|
||||
use parquet::errors::ParquetError;
|
||||
use parquet::schema::types::Type;
|
||||
|
||||
use super::*;
|
||||
@@ -441,6 +503,54 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_v2_routes_missing_path_to_remainder() -> Result<(), ParquetError> {
|
||||
let parquet = build_test_v2_schema()?;
|
||||
let projection =
|
||||
ParquetReadColumns::from_deduped(vec![ParquetReadColumn::new(0).with_nested_paths(
|
||||
vec![
|
||||
vec!["j".to_string(), "cold".to_string()],
|
||||
vec!["j".to_string(), "another".to_string()],
|
||||
],
|
||||
)]);
|
||||
|
||||
let plan = build_projection_plan(&projection, &parquet);
|
||||
|
||||
assert_eq!(vec![true], plan.projected_root_presence);
|
||||
assert_eq!(ProjectionMask::leaves(&parquet, [0, 1]), plan.mask);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_v2_explicit_path_does_not_read_remainder() -> Result<(), ParquetError> {
|
||||
let parquet = build_test_v2_schema()?;
|
||||
let projection = ParquetReadColumns::from_deduped(vec![
|
||||
ParquetReadColumn::new(0)
|
||||
.with_nested_paths(vec![vec!["j".to_string(), "hot".to_string()]]),
|
||||
]);
|
||||
|
||||
let plan = build_projection_plan(&projection, &parquet);
|
||||
|
||||
assert_eq!(vec![true], plan.projected_root_presence);
|
||||
assert_eq!(ProjectionMask::leaves(&parquet, [3]), plan.mask);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_v2_container_path_reads_remainder() -> Result<(), ParquetError> {
|
||||
let parquet = build_test_v2_schema()?;
|
||||
let projection = ParquetReadColumns::from_deduped(vec![
|
||||
ParquetReadColumn::new(0)
|
||||
.with_nested_paths(vec![vec!["j".to_string(), "commit".to_string()]]),
|
||||
]);
|
||||
|
||||
let plan = build_projection_plan(&projection, &parquet);
|
||||
|
||||
assert_eq!(vec![true], plan.projected_root_presence);
|
||||
assert_eq!(ProjectionMask::leaves(&parquet, [0, 1, 2]), plan.mask);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_merges_mixed_paths() {
|
||||
let parquet_schema_desc = build_test_nested_parquet_schema();
|
||||
@@ -673,6 +783,55 @@ mod tests {
|
||||
SchemaDescriptor::new(schema)
|
||||
}
|
||||
|
||||
fn build_test_v2_schema() -> Result<SchemaDescriptor, ParquetError> {
|
||||
let metadata = Arc::new(
|
||||
Type::primitive_type_builder("metadata", parquet::basic::Type::BYTE_ARRAY)
|
||||
.with_repetition(Repetition::REQUIRED)
|
||||
.build()?,
|
||||
);
|
||||
let value = Arc::new(
|
||||
Type::primitive_type_builder("value", parquet::basic::Type::BYTE_ARRAY)
|
||||
.with_repetition(Repetition::REQUIRED)
|
||||
.build()?,
|
||||
);
|
||||
let remainder = Arc::new(
|
||||
Type::group_type_builder(JSON2_REMAINDER_FIELD_NAME)
|
||||
.with_repetition(Repetition::OPTIONAL)
|
||||
.with_logical_type(Some(LogicalType::Variant {
|
||||
specification_version: None,
|
||||
}))
|
||||
.with_fields(vec![metadata, value])
|
||||
.build()?,
|
||||
);
|
||||
let operation = Arc::new(
|
||||
Type::primitive_type_builder("operation", parquet::basic::Type::INT64)
|
||||
.with_repetition(Repetition::OPTIONAL)
|
||||
.build()?,
|
||||
);
|
||||
let commit = Arc::new(
|
||||
Type::group_type_builder("commit")
|
||||
.with_repetition(Repetition::OPTIONAL)
|
||||
.with_fields(vec![operation])
|
||||
.build()?,
|
||||
);
|
||||
let hot = Arc::new(
|
||||
Type::primitive_type_builder("hot", parquet::basic::Type::INT64)
|
||||
.with_repetition(Repetition::OPTIONAL)
|
||||
.build()?,
|
||||
);
|
||||
let root = Arc::new(
|
||||
Type::group_type_builder("j")
|
||||
.with_repetition(Repetition::OPTIONAL)
|
||||
.with_fields(vec![remainder, commit, hot])
|
||||
.build()?,
|
||||
);
|
||||
Ok(SchemaDescriptor::new(Arc::new(
|
||||
Type::group_type_builder("schema")
|
||||
.with_fields(vec![root])
|
||||
.build()?,
|
||||
)))
|
||||
}
|
||||
|
||||
// Test schema:
|
||||
// schema
|
||||
// `- j
|
||||
|
||||
@@ -111,13 +111,26 @@ fn apply_json_type_hint(
|
||||
fn deduce_json_types(plan: &LogicalPlan) -> Result<HashMap<String, JsonNativeType>> {
|
||||
let mut json_types = HashMap::<String, JsonNativeType>::new();
|
||||
|
||||
// JSON2 columns in the final output must retain their complete values even when
|
||||
// predicates or other expressions access only specific paths.
|
||||
// For example, `SELECT j FROM t WHERE json_get(j, 'a') = 1`.
|
||||
plan.schema()
|
||||
.fields()
|
||||
.iter()
|
||||
.filter(|field| is_json2_extension_type(field))
|
||||
.for_each(|field| {
|
||||
json_types.insert(field.name().clone(), JsonNativeType::Variant);
|
||||
});
|
||||
|
||||
plan.apply(|plan| {
|
||||
for expr in plan.expressions() {
|
||||
expr.apply(|expr| {
|
||||
if let Some((column, json_type)) = deduce_json_type(expr)? {
|
||||
json_types.entry(column).or_default().merge(&json_type);
|
||||
Ok(TreeNodeRecursion::Jump)
|
||||
} else {
|
||||
Ok(TreeNodeRecursion::Continue)
|
||||
}
|
||||
Ok(TreeNodeRecursion::Continue)
|
||||
})?;
|
||||
}
|
||||
Ok(TreeNodeRecursion::Continue)
|
||||
@@ -128,6 +141,7 @@ fn deduce_json_types(plan: &LogicalPlan) -> Result<HashMap<String, JsonNativeTyp
|
||||
fn deduce_json_type(expr: &Expr) -> Result<Option<(String, JsonNativeType)>> {
|
||||
let f = match expr {
|
||||
Expr::ScalarFunction(f) if f.name().eq_ignore_ascii_case(JsonGetWithType::NAME) => f,
|
||||
Expr::Column(c) => return Ok(Some((c.name.clone(), JsonNativeType::Variant))),
|
||||
_ => return Ok(None),
|
||||
};
|
||||
|
||||
@@ -360,25 +374,13 @@ mod tests {
|
||||
.rewrite(plan, &OptimizerContext::default())?
|
||||
.transformed
|
||||
);
|
||||
assert!(provider.scan_request().json_type_hint.contains_key("j"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_allow_json2_passthrough_for_later_projection() -> Result<()> {
|
||||
let json_get = json_get_expr(col("j"), path_expr("a"), Some(DataType::Int64))?;
|
||||
let (provider, plan) = build_json2_scan()?;
|
||||
let plan = plan
|
||||
.project(vec![json_get.alias("__common_expr"), col("j")])?
|
||||
.aggregate(Vec::<Expr>::new(), vec![count(lit(1))])?
|
||||
.build()?;
|
||||
|
||||
assert!(
|
||||
JsonTypeConcretizeRule
|
||||
.rewrite(plan, &OptimizerContext::default())?
|
||||
.transformed
|
||||
assert_eq!(
|
||||
Some(&JsonNativeType::Object(JsonObjectType::from([(
|
||||
"a".to_string(),
|
||||
JsonNativeType::i64(),
|
||||
)]))),
|
||||
provider.scan_request().json_type_hint.get("j")
|
||||
);
|
||||
assert!(provider.scan_request().json_type_hint.contains_key("j"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -402,6 +404,25 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_allow_json2_filter_with_root_projection() -> Result<()> {
|
||||
let predicate =
|
||||
json_get_expr(col("j"), path_expr("a"), Some(DataType::Int64))?.eq(lit(1_i64));
|
||||
let (provider, plan) = build_json2_scan()?;
|
||||
let plan = plan.filter(predicate)?.build()?;
|
||||
|
||||
assert!(
|
||||
JsonTypeConcretizeRule
|
||||
.rewrite(plan, &OptimizerContext::default())?
|
||||
.transformed
|
||||
);
|
||||
assert_eq!(
|
||||
Some(&JsonNativeType::Variant),
|
||||
provider.scan_request().json_type_hint.get("j")
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deduce_json_type_with_non_column_base() -> Result<()> {
|
||||
let expr = json_get_expr(
|
||||
|
||||
@@ -134,6 +134,31 @@ select j.a, j.a.x from json2_table order by ts;
|
||||
| {"b":10,"x":null} | |
|
||||
+-----------------------------------+-------------------------------------+
|
||||
|
||||
select j, j.a from json2_table order by ts;
|
||||
|
||||
+--------------------------------------------------------------------+-----------------------------------+
|
||||
| j | json_get(json2_table.j,Utf8("a")) |
|
||||
+--------------------------------------------------------------------+-----------------------------------+
|
||||
| {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]} | {"b":1} |
|
||||
| {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]} | {"b":-2} |
|
||||
| {"a":{"b":3},"c":"s3","d":null} | {"b":3} |
|
||||
| {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]} | {"b":-4} |
|
||||
| {"a":null,"c":"s5","d":null} | |
|
||||
| {"a":null,"c":"s6","d":null} | |
|
||||
| {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]} | {"b":"s7"} |
|
||||
| {"a":{"b":8},"c":"s8","d":null} | {"b":8} |
|
||||
| {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} | {"b":null,"x":true} |
|
||||
| {"a":{"b":10,"x":null},"c":null,"d":null,"y":false} | {"b":10,"x":null} |
|
||||
+--------------------------------------------------------------------+-----------------------------------+
|
||||
|
||||
select j from json2_table where j.a.b = 1;
|
||||
|
||||
+-------------------------------------------------------+
|
||||
| j |
|
||||
+-------------------------------------------------------+
|
||||
| {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]} |
|
||||
+-------------------------------------------------------+
|
||||
|
||||
select j.c, j.y from json2_table order by ts;
|
||||
|
||||
+-----------------------------------+-----------------------------------+
|
||||
|
||||
@@ -44,6 +44,10 @@ select j.a.b from json2_table order by ts;
|
||||
|
||||
select j.a, j.a.x from json2_table order by ts;
|
||||
|
||||
select j, j.a from json2_table order by ts;
|
||||
|
||||
select j from json2_table where j.a.b = 1;
|
||||
|
||||
select j.c, j.y from json2_table order by ts;
|
||||
|
||||
select j from json2_table order by ts;
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
create table json2_join_same_name_left (
|
||||
ts timestamp time index,
|
||||
k string,
|
||||
j json2
|
||||
)
|
||||
with (
|
||||
'append_mode' = 'true'
|
||||
);
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
create table json2_join_same_name_right (
|
||||
ts timestamp time index,
|
||||
k string,
|
||||
j json2
|
||||
)
|
||||
with (
|
||||
'append_mode' = 'true'
|
||||
);
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
insert into json2_join_same_name_left values
|
||||
(1, 'a', '{"a": 1, "left_only": "kept"}');
|
||||
|
||||
Affected Rows: 1
|
||||
|
||||
insert into json2_join_same_name_right values
|
||||
(1, 'a', '{"a": "right", "right_only": "should be kept"}');
|
||||
|
||||
Affected Rows: 1
|
||||
|
||||
admin flush_table('json2_join_same_name_left');
|
||||
|
||||
+------------------------------------------------+
|
||||
| ADMIN flush_table('json2_join_same_name_left') |
|
||||
+------------------------------------------------+
|
||||
| 0 |
|
||||
+------------------------------------------------+
|
||||
|
||||
admin flush_table('json2_join_same_name_right');
|
||||
|
||||
+-------------------------------------------------+
|
||||
| ADMIN flush_table('json2_join_same_name_right') |
|
||||
+-------------------------------------------------+
|
||||
| 0 |
|
||||
+-------------------------------------------------+
|
||||
|
||||
-- Conflicting hints for same-named JSON2 columns preserve both values as Variant.
|
||||
select json_get(r.j, 'a')::string, json_get(l.j, 'a')::int64
|
||||
from json2_join_same_name_left l
|
||||
join json2_join_same_name_right r
|
||||
on l.k = r.k;
|
||||
|
||||
+-------------------------+---------------------------------------------------+
|
||||
| json_get(r.j,Utf8("a")) | arrow_cast(json_get(l.j,Utf8("a")),Utf8("Int64")) |
|
||||
+-------------------------+---------------------------------------------------+
|
||||
| right | 1 |
|
||||
+-------------------------+---------------------------------------------------+
|
||||
|
||||
drop table json2_join_same_name_left;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
drop table json2_join_same_name_right;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
create table json2_join_same_name_left (
|
||||
ts timestamp time index,
|
||||
k string,
|
||||
j json2
|
||||
)
|
||||
with (
|
||||
'append_mode' = 'true'
|
||||
);
|
||||
|
||||
create table json2_join_same_name_right (
|
||||
ts timestamp time index,
|
||||
k string,
|
||||
j json2
|
||||
)
|
||||
with (
|
||||
'append_mode' = 'true'
|
||||
);
|
||||
|
||||
insert into json2_join_same_name_left values
|
||||
(1, 'a', '{"a": 1, "left_only": "kept"}');
|
||||
|
||||
insert into json2_join_same_name_right values
|
||||
(1, 'a', '{"a": "right", "right_only": "should be kept"}');
|
||||
|
||||
admin flush_table('json2_join_same_name_left');
|
||||
|
||||
admin flush_table('json2_join_same_name_right');
|
||||
|
||||
-- Conflicting hints for same-named JSON2 columns preserve both values as Variant.
|
||||
select json_get(r.j, 'a')::string, json_get(l.j, 'a')::int64
|
||||
from json2_join_same_name_left l
|
||||
join json2_join_same_name_right r
|
||||
on l.k = r.k;
|
||||
|
||||
drop table json2_join_same_name_left;
|
||||
|
||||
drop table json2_join_same_name_right;
|
||||
@@ -65,87 +65,10 @@ order by json_get(j, 'a.b');
|
||||
| 2 | 1 |
|
||||
+---------------------------------------------------+----------+
|
||||
|
||||
select j, j.a from json2_whole_and_path_read;
|
||||
|
||||
Error: 3001(EngineExecuteQuery), Invalid argument error: column types must match schema types, expected Binary but found Struct("a": Utf8View) at column index 0
|
||||
|
||||
select j from json2_whole_and_path_read where j.a.b = 1;
|
||||
|
||||
Error: 3001(EngineExecuteQuery), Invalid argument error: column types must match schema types, expected Binary but found Struct("a": Struct("b": Int64)) at column index 0
|
||||
|
||||
drop table json2_whole_and_path_read;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
create table json2_join_same_name_left (
|
||||
ts timestamp time index,
|
||||
k string,
|
||||
j json2
|
||||
)
|
||||
with (
|
||||
'append_mode' = 'true'
|
||||
);
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
create table json2_join_same_name_right (
|
||||
ts timestamp time index,
|
||||
k string,
|
||||
j json2
|
||||
)
|
||||
with (
|
||||
'append_mode' = 'true'
|
||||
);
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
insert into json2_join_same_name_left values
|
||||
(1, 'a', '{"a": 1, "left_only": "kept"}');
|
||||
|
||||
Affected Rows: 1
|
||||
|
||||
insert into json2_join_same_name_right values
|
||||
(1, 'a', '{"a": "right", "right_only": "should be kept"}');
|
||||
|
||||
Affected Rows: 1
|
||||
|
||||
admin flush_table('json2_join_same_name_left');
|
||||
|
||||
+------------------------------------------------+
|
||||
| ADMIN flush_table('json2_join_same_name_left') |
|
||||
+------------------------------------------------+
|
||||
| 0 |
|
||||
+------------------------------------------------+
|
||||
|
||||
admin flush_table('json2_join_same_name_right');
|
||||
|
||||
+-------------------------------------------------+
|
||||
| ADMIN flush_table('json2_join_same_name_right') |
|
||||
+-------------------------------------------------+
|
||||
| 0 |
|
||||
+-------------------------------------------------+
|
||||
|
||||
-- FIXME: This should return `right` and `1`. The current NULL values are caused
|
||||
-- by JSON type hints losing the table qualifier in joins.
|
||||
select json_get(r.j, 'a')::string, json_get(l.j, 'a')::int64
|
||||
from json2_join_same_name_left l
|
||||
join json2_join_same_name_right r
|
||||
on l.k = r.k;
|
||||
|
||||
+---------------------------------------------+---------------------------------------------------+
|
||||
| json_get(r.j,Utf8("a")) | arrow_cast(json_get(l.j,Utf8("a")),Utf8("Int64")) |
|
||||
+---------------------------------------------+---------------------------------------------------+
|
||||
| {"a":"right","right_only":"should be kept"} | |
|
||||
+---------------------------------------------+---------------------------------------------------+
|
||||
|
||||
drop table json2_join_same_name_left;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
drop table json2_join_same_name_right;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
create table json2_without_append_mode (
|
||||
ts timestamp time index,
|
||||
j json2
|
||||
|
||||
@@ -38,51 +38,8 @@ from json2_whole_and_path_read
|
||||
group by json_get(j, 'a.b')
|
||||
order by json_get(j, 'a.b');
|
||||
|
||||
select j, j.a from json2_whole_and_path_read;
|
||||
|
||||
select j from json2_whole_and_path_read where j.a.b = 1;
|
||||
|
||||
drop table json2_whole_and_path_read;
|
||||
|
||||
create table json2_join_same_name_left (
|
||||
ts timestamp time index,
|
||||
k string,
|
||||
j json2
|
||||
)
|
||||
with (
|
||||
'append_mode' = 'true'
|
||||
);
|
||||
|
||||
create table json2_join_same_name_right (
|
||||
ts timestamp time index,
|
||||
k string,
|
||||
j json2
|
||||
)
|
||||
with (
|
||||
'append_mode' = 'true'
|
||||
);
|
||||
|
||||
insert into json2_join_same_name_left values
|
||||
(1, 'a', '{"a": 1, "left_only": "kept"}');
|
||||
|
||||
insert into json2_join_same_name_right values
|
||||
(1, 'a', '{"a": "right", "right_only": "should be kept"}');
|
||||
|
||||
admin flush_table('json2_join_same_name_left');
|
||||
|
||||
admin flush_table('json2_join_same_name_right');
|
||||
|
||||
-- FIXME: This should return `right` and `1`. The current NULL values are caused
|
||||
-- by JSON type hints losing the table qualifier in joins.
|
||||
select json_get(r.j, 'a')::string, json_get(l.j, 'a')::int64
|
||||
from json2_join_same_name_left l
|
||||
join json2_join_same_name_right r
|
||||
on l.k = r.k;
|
||||
|
||||
drop table json2_join_same_name_left;
|
||||
|
||||
drop table json2_join_same_name_right;
|
||||
|
||||
create table json2_without_append_mode (
|
||||
ts timestamp time index,
|
||||
j json2
|
||||
|
||||
Reference in New Issue
Block a user