diff --git a/src/common/function/src/scalars/json/json_get.rs b/src/common/function/src/scalars/json/json_get.rs index 15e8542bfe..aabe6d08c8 100644 --- a/src/common/function/src/scalars/json/json_get.rs +++ b/src/common/function/src/scalars/json/json_get.rs @@ -24,6 +24,7 @@ use datafusion_common::arrow::array::{ use datafusion_common::arrow::datatypes::DataType; use datafusion_common::{DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err}; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, Volatility}; +use datatypes::extension::json::is_json2_extension_type; use datatypes::vectors::json::array::JsonArray; use derive_more::Display; @@ -277,6 +278,7 @@ fn json_struct_get(array: &ArrayRef, path: &str, with_type: &DataType) -> Result .split('.') .filter(|segment| !segment.is_empty()) .collect::>(); + let mut curr = array.clone(); for (idx, segment) in segments.iter().enumerate() { @@ -404,9 +406,18 @@ impl Function for JsonGetWithType { let arg0 = compute::cast(&arg0, &DataType::BinaryView)?; let jsons = arg0.as_binary_view(); - let mut builder = result_builder(len, &with_type)?; - jsonb_get(jsons, path, builder.as_mut())?; - builder.build() + if args.arg_fields.first().is_some_and(is_json2_extension_type) { + // Query concretization projects nested JSON2 paths as Struct arrays. A binary + // JSON2 argument is therefore an already-selected scalar or root value that + // only needs conversion from its JSONB representation to the requested type. + JsonArray::from(&arg0) + .project_to(&with_type) + .map_err(|e| exec_datafusion_err!("{e:?}"))? + } else { + let mut builder = result_builder(len, &with_type)?; + jsonb_get(jsons, path, builder.as_mut())?; + builder.build() + } } DataType::Struct(_) => json_struct_get(&arg0, path, &with_type)?, _ => { diff --git a/src/common/recordbatch/src/lib.rs b/src/common/recordbatch/src/lib.rs index faa3495c55..ed3bda72cc 100644 --- a/src/common/recordbatch/src/lib.rs +++ b/src/common/recordbatch/src/lib.rs @@ -22,7 +22,7 @@ pub mod filter; pub mod recordbatch; pub mod util; -use std::fmt; +use std::fmt::{self, Write}; use std::future::Future; use std::pin::Pin; use std::sync::Arc; @@ -36,10 +36,16 @@ use common_memory_manager::{ }; use common_telemetry::tracing::Span; pub use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream; -use datatypes::arrow::array::{ArrayRef, AsArray, StringBuilder}; +use datatypes::arrow::array::{Array, ArrayRef, AsArray, StringBuilder}; use datatypes::arrow::compute::SortOptions; +use datatypes::arrow::datatypes::{DataType as ArrowDataType, Field}; +use datatypes::arrow::error::ArrowError; pub use datatypes::arrow::record_batch::RecordBatch as DfRecordBatch; -use datatypes::arrow::util::pretty; +use datatypes::arrow::util::display::{ + ArrayFormatter, ArrayFormatterFactory, DisplayIndex, FormatOptions, FormatResult, +}; +use datatypes::arrow::util::pretty::pretty_format_batches_with_options; +use datatypes::extension::json::is_json_extension_type; use datatypes::prelude::{ConcreteDataType, DataType, VectorRef}; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; use datatypes::types::{JsonFormat, StructField, StructType, jsonb_to_string}; @@ -390,7 +396,10 @@ impl RecordBatches { .iter() .map(|x| x.df_record_batch().clone()) .collect::>(); - let result = pretty::pretty_format_batches(df_batches).context(error::FormatSnafu)?; + let options = + FormatOptions::default().with_formatter_factory(Some(&BinaryFormatterFactory)); + let result = + pretty_format_batches_with_options(df_batches, &options).context(error::FormatSnafu)?; Ok(result.to_string()) } @@ -429,6 +438,63 @@ impl RecordBatches { } } +#[derive(Debug)] +struct BinaryFormatterFactory; + +impl ArrayFormatterFactory for BinaryFormatterFactory { + fn create_array_formatter<'a>( + &self, + array: &'a dyn Array, + options: &FormatOptions<'a>, + field: Option<&'a Field>, + ) -> std::result::Result>, ArrowError> { + if !array.data_type().is_binary() { + return Ok(None); + } + + Ok(Some(ArrayFormatter::new( + Box::new(BinaryFormatter { + array, + is_json: field.is_some_and(is_json_extension_type), + default: ArrayFormatter::try_new(array, options)?, + null: options.null(), + }), + options.safe(), + ))) + } +} + +struct BinaryFormatter<'a> { + array: &'a dyn Array, + is_json: bool, + default: ArrayFormatter<'a>, + null: &'a str, +} + +impl DisplayIndex for BinaryFormatter<'_> { + fn write(&self, idx: usize, f: &mut dyn Write) -> FormatResult { + if !self.is_json { + self.default.value(idx).write(f)?; + return Ok(()); + } + + if self.array.is_null(idx) { + write!(f, "{}", self.null)?; + } else { + let bytes = match self.array.data_type() { + ArrowDataType::Binary => self.array.as_binary::().value(idx), + ArrowDataType::LargeBinary => self.array.as_binary::().value(idx), + ArrowDataType::BinaryView => self.array.as_binary_view().value(idx), + _ => unreachable!(), + }; + let value = + jsonb_to_string(bytes).map_err(|e| ArrowError::ExternalError(Box::new(e)))?; + write!(f, "{value}")?; + } + Ok(()) + } +} + impl IntoIterator for RecordBatches { type Item = RecordBatch; type IntoIter = std::vec::IntoIter; diff --git a/src/datatypes/src/data_type.rs b/src/datatypes/src/data_type.rs index a630827019..fb5a0af85a 100644 --- a/src/datatypes/src/data_type.rs +++ b/src/datatypes/src/data_type.rs @@ -228,6 +228,11 @@ impl ConcreteDataType { matches!(self, ConcreteDataType::Json(_)) } + /// Returns whether this is a JSON2 data type. + pub fn is_json2(&self) -> bool { + self.as_json().is_some_and(|json_type| json_type.is_json2()) + } + pub fn is_vector(&self) -> bool { matches!(self, ConcreteDataType::Vector(_)) } diff --git a/src/datatypes/src/extension/json.rs b/src/datatypes/src/extension/json.rs index 6418b97131..b78a9df49b 100644 --- a/src/datatypes/src/extension/json.rs +++ b/src/datatypes/src/extension/json.rs @@ -15,7 +15,7 @@ use std::sync::Arc; use arrow_schema::extension::ExtensionType; -use arrow_schema::{ArrowError, DataType, FieldRef}; +use arrow_schema::{ArrowError, DataType, Field, FieldRef}; use serde::{Deserialize, Serialize}; use crate::json::JsonSettings; @@ -103,8 +103,22 @@ impl ExtensionType for JsonExtensionType { } /// Check if this field is to be treated as json extension type. -pub fn is_json_extension_type(field: &FieldRef) -> bool { - field.extension_type_name() == Some(JsonExtensionType::NAME) +pub fn is_json_extension_type>(field: T) -> bool { + field.as_ref().extension_type_name() == Some(JsonExtensionType::NAME) +} + +/// Check if this field is a JSON2 extension type. +/// +/// Legacy JSONB and JSON2 share the same JSON extension name. The column schema construction +/// invariant is that JSON2 always stores its settings as `Some`, including default settings, +/// while legacy JSONB stores no JSON settings. Therefore, after checking the extension name, +/// the presence of JSON settings distinguishes JSON2 from legacy JSONB. +pub fn is_json2_extension_type>(field: T) -> bool { + let field = field.as_ref(); + is_json_extension_type(field) + && field + .try_extension_type::() + .is_ok_and(|x| x.metadata().json_settings.is_some()) } /// Check if this field is a structured JSON field. diff --git a/src/datatypes/src/types/json_type.rs b/src/datatypes/src/types/json_type.rs index 4bab0da52f..7905fce76b 100644 --- a/src/datatypes/src/types/json_type.rs +++ b/src/datatypes/src/types/json_type.rs @@ -282,7 +282,8 @@ impl JsonType { matches!(self.format, JsonFormat::Json2(_)) } - pub(crate) fn native_type(&self) -> &JsonNativeType { + /// Returns the native JSON type represented by this data type. + pub fn native_type(&self) -> &JsonNativeType { match &self.format { JsonFormat::Jsonb => &JsonNativeType::String, JsonFormat::Json2(x) => x.as_ref(), diff --git a/src/mito2/src/read/flat_projection.rs b/src/mito2/src/read/flat_projection.rs index df674d0389..ed5e0fa36a 100644 --- a/src/mito2/src/read/flat_projection.rs +++ b/src/mito2/src/read/flat_projection.rs @@ -25,9 +25,10 @@ use common_recordbatch::error::{ use common_recordbatch::{DfRecordBatch, RecordBatch}; use datatypes::arrow::array::Array; use datatypes::arrow::datatypes::{DataType as ArrowDataType, Field}; -use datatypes::extension::json::is_structured_json_field; +use datatypes::extension::json::{is_json_extension_type, is_structured_json_field}; use datatypes::prelude::{ConcreteDataType, DataType}; -use datatypes::schema::{Schema, SchemaRef}; +use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; +use datatypes::types::JsonType; use datatypes::types::json_type::JsonNativeType; use datatypes::value::Value; use datatypes::vectors::Helper; @@ -114,17 +115,7 @@ impl FlatProjectionMapper { output_col_ids.push(col.column_id); let mut schema = col.column_schema.clone(); - if let Some(concretized) = json_type_hint - .and_then(|x| x.get(&schema.name)) - .cloned() - .map(ConcreteDataType::json2) - && schema - .data_type - .as_json() - .is_some_and(|json_type| json_type.is_json2()) - { - schema.data_type = concretized; - } + maybe_concretize_json2_datatype(&mut schema, json_type_hint); col_schemas.push(schema); } @@ -141,19 +132,21 @@ impl FlatProjectionMapper { let mut batch_schema = flat_projected_columns(metadata, &format_projection); - if let Some(json_type_hint) = json_type_hint - && !json_type_hint.is_empty() - { - for (column_id, data_type) in batch_schema.iter_mut() { - if data_type - .as_json() - .is_some_and(|json_type| json_type.is_json2()) - && let Some(concretized) = metadata - .column_by_id(*column_id) - .and_then(|x| json_type_hint.get(&x.column_schema.name).cloned()) - .map(ConcreteDataType::json2) + for (column_id, data_type) in batch_schema.iter_mut() { + if let Some(json_type) = data_type.as_json() + && json_type.is_json2() + { + if let Some(concretized) = metadata + .column_by_id(*column_id) + .and_then(|metadata| { + json_type_hint.and_then(|x| x.get(&metadata.column_schema.name).cloned()) + }) + .map(ConcreteDataType::json2) { *data_type = concretized; + } else if is_empty_json2_type(json_type) { + // see `merge_scan::maybe_amend_json2_field` + *data_type = ConcreteDataType::json2(JsonNativeType::Variant); } } } @@ -365,7 +358,7 @@ impl FlatProjectionMapper { } let field = &self.output_schema.arrow_schema().fields()[output_idx]; - if is_structured_json_field(field) { + if is_json_extension_type(field) { array = JsonArray::from(&array) .project_to(field.data_type()) .context(DataTypesSnafu)?; @@ -408,6 +401,34 @@ impl FlatProjectionMapper { } } +fn maybe_concretize_json2_datatype( + schema: &mut ColumnSchema, + json_type_hint: Option<&HashMap>, +) { + if let Some(json_type) = schema.data_type.as_json() + && json_type.is_json2() + { + if let Some(concretized) = json_type_hint + .and_then(|x| x.get(&schema.name)) + .cloned() + .map(ConcreteDataType::json2) + { + schema.data_type = concretized; + } else if is_empty_json2_type(json_type) { + // see `merge_scan::maybe_amend_json2_field` + schema.data_type = ConcreteDataType::json2(JsonNativeType::Variant); + } + } +} + +fn is_empty_json2_type(json_type: &JsonType) -> bool { + match json_type.native_type() { + JsonNativeType::Null => true, + JsonNativeType::Object(fields) if fields.is_empty() => true, + _ => false, + } +} + fn single_value_string_dictionary<'a>( array: &'a Arc, output_type: &ConcreteDataType, diff --git a/src/operator/src/statement/ddl.rs b/src/operator/src/statement/ddl.rs index 25a82280a4..4502ebe36b 100644 --- a/src/operator/src/statement/ddl.rs +++ b/src/operator/src/statement/ddl.rs @@ -2419,11 +2419,7 @@ fn validate_json2_columns_append_mode(schema: &Schema, table_options: &TableOpti .is_some_and(|value| value == "true"); for column in schema.column_schemas() { - if column - .data_type - .as_json() - .is_some_and(|json_type| json_type.is_json2()) - { + if column.data_type.is_json2() { ensure!( append_mode, InvalidSqlSnafu { diff --git a/src/query/src/datafusion/error.rs b/src/query/src/datafusion/error.rs index 5b85d7619d..9a726b52db 100644 --- a/src/query/src/datafusion/error.rs +++ b/src/query/src/datafusion/error.rs @@ -17,7 +17,6 @@ use std::any::Any; use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use common_macro::stack_trace_debug; -use datafusion::error::DataFusionError; use snafu::{Location, Snafu}; /// Inner error of datafusion based query engine. @@ -25,14 +24,6 @@ use snafu::{Location, Snafu}; #[snafu(visibility(pub))] #[stack_trace_debug] pub enum InnerError { - #[snafu(transparent)] - Datafusion { - #[snafu(source)] - error: DataFusionError, - #[snafu(implicit)] - location: Location, - }, - #[snafu(display("Failed to convert DataFusion's recordbatch stream"))] ConvertDfRecordBatchStream { #[snafu(implicit)] @@ -46,8 +37,6 @@ impl ErrorExt for InnerError { use InnerError::*; match self { - // TODO(yingwen): Further categorize datafusion error. - Datafusion { .. } => StatusCode::EngineExecuteQuery, ConvertDfRecordBatchStream { source, .. } => source.status_code(), } } diff --git a/src/query/src/dist_plan/merge_scan.rs b/src/query/src/dist_plan/merge_scan.rs index 07dfe3de7d..83f9dad01f 100644 --- a/src/query/src/dist_plan/merge_scan.rs +++ b/src/query/src/dist_plan/merge_scan.rs @@ -17,7 +17,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use ahash::{HashMap, HashSet}; -use arrow_schema::{Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, SortOptions}; +use arrow_schema::{DataType, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, SortOptions}; use async_stream::stream; use common_catalog::parse_catalog_and_schema_from_db_string; use common_plugins::GREPTIME_EXEC_READ_COST; @@ -41,6 +41,7 @@ use datafusion_common::{Column as ColumnExpr, DataFusionError, Result}; use datafusion_expr::{Expr, Extension, LogicalPlan, UserDefinedLogicalNodeCore}; use datafusion_physical_expr::expressions::Column; use datafusion_physical_expr::{Distribution, EquivalenceProperties, PhysicalSortExpr}; +use datatypes::extension::json::is_json_extension_type; use futures_util::StreamExt; use greptime_proto::v1::region::RegionRequestHeader; use meter_core::data::ReadItem; @@ -253,10 +254,7 @@ impl MergeScanExec { remote_dyn_filter_producer_id: Option, enable_per_region_metrics: bool, ) -> Result { - // TODO(CookiePieWw): Initially we removed the metadata from the schema in #2000, but we have to - // keep it for #4619 to identify json type in src/datatypes/src/schema/column_schema.rs. - // Reconsider if it's possible to remove it. - let arrow_schema = Arc::new(arrow_schema.clone()); + let arrow_schema = maybe_amend_json2_field(arrow_schema); // States the output ordering of the plan. // @@ -705,6 +703,37 @@ impl MergeScanExec { } } +// If the schema has JSON2 field, AND the field is of empty Struct datatype, amend it with Binary +// datatype. +// This is a very hacky way to make it possible to query the whole JSON2 column. Because when +// querying a whole JSON2 column, like in the SQL `select * from ...`, we can't concretize the JSON2 +// datatype from the query. Hence, the JSON2 datatype remains what in the column schema, i.e., empty +// Struct. An empty Struct is not alignable like any other concretized JSON2 datatypes, so to make +// the query work, we amend(rewrite) it to Binary datatype. +// Why the Binary datatype? Because underlying the scan and projection stage, the JSON2 data are +// variant shape, will be all converted to bytes. +// Anyway, this is not clean nor elegant. TODO(LFC) Maybe make it into some plan analyzer rule? +fn maybe_amend_json2_field(schema: &ArrowSchema) -> ArrowSchemaRef { + let schema = schema.clone(); + let mut new_fields = Vec::with_capacity(schema.fields().len()); + for field in schema.fields().iter() { + let new_field = if is_json_extension_type(field) + && matches!(field.data_type(), DataType::Struct(fields) if fields.is_empty()) + { + let mut new_field = field.as_ref().clone(); + new_field.set_data_type(DataType::Binary); + Arc::new(new_field) + } else { + field.clone() + }; + new_fields.push(new_field); + } + Arc::new(ArrowSchema::new_with_metadata( + new_fields, + schema.metadata().clone(), + )) +} + #[cfg(test)] impl MergeScanExec { fn remote_dyn_filter_producer_id(&self) -> Option { diff --git a/src/query/src/optimizer/json_type_concretize.rs b/src/query/src/optimizer/json_type_concretize.rs index 809c85adaa..2687402583 100644 --- a/src/query/src/optimizer/json_type_concretize.rs +++ b/src/query/src/optimizer/json_type_concretize.rs @@ -18,11 +18,9 @@ use arrow_schema::DataType; use common_function::scalars::json::json_get::JsonGetWithType; use datafusion::datasource::DefaultTableSource; use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; -use datafusion_common::{ExprSchema, Result, plan_datafusion_err, plan_err}; -use datafusion_expr::utils::merge_schema; +use datafusion_common::{Result, plan_datafusion_err, plan_err}; use datafusion_expr::{Expr, LogicalPlan}; use datafusion_optimizer::{OptimizerConfig, OptimizerRule}; -use datatypes::extension::json::is_structured_json_field; use datatypes::types::json_type::{JsonNativeType, JsonObjectType}; use crate::dummy_catalog::DummyTableProvider; @@ -43,8 +41,6 @@ impl OptimizerRule for JsonTypeConcretizeRule { plan: LogicalPlan, _config: &dyn OptimizerConfig, ) -> Result> { - ensure_no_whole_json2_read(&plan)?; - let json_types = deduce_json_types(&plan)?; if json_types.is_empty() { return Ok(Transformed::no(plan)); @@ -76,102 +72,6 @@ impl OptimizerRule for JsonTypeConcretizeRule { } } -/// Rejects unsupported whole-column JSON2 reads in a logical plan. -fn ensure_no_whole_json2_read(plan: &LogicalPlan) -> Result<()> { - // Reject whole JSON2 columns in the final query output, including `SELECT *`. - for field in plan.schema().fields() { - if is_structured_json_field(field) { - return plan_err!( - "Querying the whole JSON2 column '{}' is currently not supported; use json_get to select its fields", - field.name() - ); - } - } - - // Reject whole JSON2 columns consumed by intermediate expressions, for example: - // `SELECT count(*) FROM (SELECT j FROM t GROUP BY j)`. - plan.apply(|plan| { - let input_schema = merge_schema(&plan.inputs()); - for expr in plan.expressions() { - // A bare column in an intermediate projection is only passed through, not consumed. - if matches!(plan, LogicalPlan::Projection(_)) && is_passthrough_column(&expr) { - continue; - } - - expr.apply(|expr| { - // For JSON2, `json_get` is allowed only with a non-empty path; skip its arguments - // after validation. - if let Expr::ScalarFunction(function) = expr - && function.name().eq_ignore_ascii_case(JsonGetWithType::NAME) - { - let Some(Expr::Column(col)) = function.args.first() else { - return Ok(TreeNodeRecursion::Jump); - }; - let Some(path) = function - .args - .get(1) - .and_then(Expr::as_literal) - .and_then(|value| value.try_as_str()) - .flatten() - else { - return Ok(TreeNodeRecursion::Jump); - }; - let reads_whole_column = path - .trim_start_matches('$') - .split('.') - .all(str::is_empty); - if !reads_whole_column { - return Ok(TreeNodeRecursion::Jump); - } - - let Ok(field) = input_schema - .field_from_column(col) - .or_else(|_| plan.schema().field_from_column(col)) - else { - return Ok(TreeNodeRecursion::Jump); - }; - if is_structured_json_field(field) { - return plan_err!( - "Querying the whole JSON2 column '{}' is currently not supported; use json_get to select its fields", - col.name - ); - } - return Ok(TreeNodeRecursion::Jump); - } - - // Any remaining JSON2 column reference is a whole-column read. - let Expr::Column(col) = expr else { - return Ok(TreeNodeRecursion::Continue); - }; - let Ok(field) = input_schema - .field_from_column(col) - .or_else(|_| plan.schema().field_from_column(col)) - else { - return Ok(TreeNodeRecursion::Continue); - }; - if is_structured_json_field(field) { - return plan_err!( - "Querying the whole JSON2 column '{}' is currently not supported; use json_get to select its fields", - col.name - ); - } - Ok(TreeNodeRecursion::Continue) - })?; - } - Ok(TreeNodeRecursion::Continue) - })?; - - Ok(()) -} - -fn is_passthrough_column(expr: &Expr) -> bool { - match expr { - Expr::Column(_) => true, - Expr::Alias(alias) => is_passthrough_column(&alias.expr), - _ => false, - } -} - fn deduce_json_types(plan: &LogicalPlan) -> Result> { let mut json_types = HashMap::::new(); @@ -227,7 +127,7 @@ fn deduce_json_type(expr: &Expr) -> Result> { JsonNativeType::try_from(&with_type).map_err(|e| plan_datafusion_err!("{e:?}"))?; let mut split = path.rsplit("."); - let Some(leaf) = split.next() else { + let Some(leaf) = split.next().filter(|&x| !x.is_empty() && x != "$") else { return Ok(Some((column.name.clone(), JsonNativeType::String))); }; @@ -397,56 +297,6 @@ mod tests { Ok(()) } - #[test] - fn test_reject_whole_json2_projection() -> Result<()> { - for (exprs, output_name) in [ - (vec![col("j")], "j"), - (vec![col("j").alias("json"), col("ts")], "json"), - ] { - let (_, plan) = build_json2_plan(exprs)?; - let err = JsonTypeConcretizeRule - .rewrite(plan, &OptimizerContext::default()) - .unwrap_err(); - assert!(err.to_string().contains(&format!( - "Querying the whole JSON2 column '{output_name}' is currently not supported" - ))); - } - Ok(()) - } - - #[test] - fn test_reject_whole_json2_output_without_projection() -> Result<()> { - let (_, plan) = build_json2_scan()?; - let plan = plan.sort(vec![col("ts").sort(true, false)])?.build()?; - - let err = JsonTypeConcretizeRule - .rewrite(plan, &OptimizerContext::default()) - .unwrap_err(); - assert!( - err.to_string() - .contains("Querying the whole JSON2 column 'j' is currently not supported") - ); - Ok(()) - } - - #[test] - fn test_reject_whole_json2_use_in_intermediate_plan() -> Result<()> { - let (_, plan) = build_json2_scan()?; - let plan = plan - .aggregate(vec![col("j")], Vec::::new())? - .aggregate(Vec::::new(), vec![count(lit(1))])? - .build()?; - - let err = JsonTypeConcretizeRule - .rewrite(plan, &OptimizerContext::default()) - .unwrap_err(); - assert!( - err.to_string() - .contains("Querying the whole JSON2 column 'j' is currently not supported") - ); - Ok(()) - } - #[test] fn test_allow_json2_path_use_in_intermediate_plan() -> Result<()> { let json_get = json_get_expr(col("j"), path_expr("a"), Some(DataType::Int64))?; @@ -503,23 +353,6 @@ mod tests { Ok(()) } - #[test] - fn test_reject_json2_projection_with_empty_path() -> Result<()> { - for path in ["", "$", ".", "$."] { - let expr = json_get_expr(col("j"), path_expr(path), Some(DataType::Utf8View))?; - let (_, plan) = build_json2_plan(vec![expr])?; - - let err = JsonTypeConcretizeRule - .rewrite(plan, &OptimizerContext::default()) - .unwrap_err(); - assert!( - err.to_string() - .contains("Querying the whole JSON2 column 'j' is currently not supported") - ); - } - Ok(()) - } - #[test] fn test_deduce_json_type_with_non_column_base() -> Result<()> { let expr = json_get_expr( diff --git a/tests/cases/standalone/common/types/json/json2.result b/tests/cases/standalone/common/types/json/json2.result index 10b288fc58..2e9c9c8ee8 100644 --- a/tests/cases/standalone/common/types/json/json2.result +++ b/tests/cases/standalone/common/types/json/json2.result @@ -151,6 +151,141 @@ select j.c, j.y from json2_table order by ts; | | false | +-----------------------------------+-----------------------------------+ +select j from json2_table order by ts; + ++--------------------------------------------------------------------+ +| j | ++--------------------------------------------------------------------+ +| {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]} | +| {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]} | +| {"a":{"b":3},"c":"s3","d":null} | +| {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]} | +| {"a":null,"c":"s5","d":null} | +| {"a":null,"c":"s6","d":null} | +| {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]} | +| {"a":{"b":8},"c":"s8","d":null} | +| {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} | +| {"a":{"b":10,"x":null},"c":null,"d":null,"y":false} | ++--------------------------------------------------------------------+ + +select * from json2_table order by ts; + ++-------------------------+--------------------------------------------------------------------+ +| ts | j | ++-------------------------+--------------------------------------------------------------------+ +| 1970-01-01T00:00:00.001 | {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]} | +| 1970-01-01T00:00:00.002 | {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]} | +| 1970-01-01T00:00:00.003 | {"a":{"b":3},"c":"s3","d":null} | +| 1970-01-01T00:00:00.004 | {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]} | +| 1970-01-01T00:00:00.005 | {"a":null,"c":"s5","d":null} | +| 1970-01-01T00:00:00.006 | {"a":null,"c":"s6","d":null} | +| 1970-01-01T00:00:00.007 | {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]} | +| 1970-01-01T00:00:00.008 | {"a":{"b":8},"c":"s8","d":null} | +| 1970-01-01T00:00:00.009 | {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} | +| 1970-01-01T00:00:00.010 | {"a":{"b":10,"x":null},"c":null,"d":null,"y":false} | ++-------------------------+--------------------------------------------------------------------+ + +select count(*) from (select j from json2_table group by j); + ++----------+ +| count(*) | ++----------+ +| 10 | ++----------+ + +select count(*) from (select distinct j from json2_table); + ++----------+ +| count(*) | ++----------+ +| 10 | ++----------+ + +select ts, j from (select ts, j from json2_table) order by ts; + ++-------------------------+--------------------------------------------------------------------+ +| ts | j | ++-------------------------+--------------------------------------------------------------------+ +| 1970-01-01T00:00:00.001 | {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]} | +| 1970-01-01T00:00:00.002 | {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]} | +| 1970-01-01T00:00:00.003 | {"a":{"b":3},"c":"s3","d":null} | +| 1970-01-01T00:00:00.004 | {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]} | +| 1970-01-01T00:00:00.005 | {"a":null,"c":"s5","d":null} | +| 1970-01-01T00:00:00.006 | {"a":null,"c":"s6","d":null} | +| 1970-01-01T00:00:00.007 | {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]} | +| 1970-01-01T00:00:00.008 | {"a":{"b":8},"c":"s8","d":null} | +| 1970-01-01T00:00:00.009 | {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} | +| 1970-01-01T00:00:00.010 | {"a":{"b":10,"x":null},"c":null,"d":null,"y":false} | ++-------------------------+--------------------------------------------------------------------+ + +select json_get(j, '') from json2_table order by ts; + ++--------------------------------------------------------------------+ +| json_get(json2_table.j,Utf8("")) | ++--------------------------------------------------------------------+ +| {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]} | +| {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]} | +| {"a":{"b":3},"c":"s3","d":null} | +| {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]} | +| {"a":null,"c":"s5","d":null} | +| {"a":null,"c":"s6","d":null} | +| {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]} | +| {"a":{"b":8},"c":"s8","d":null} | +| {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} | +| {"a":{"b":10,"x":null},"c":null,"d":null,"y":false} | ++--------------------------------------------------------------------+ + +select json_get(j, '$') from json2_table order by ts; + ++--------------------------------------------------------------------+ +| json_get(json2_table.j,Utf8("$")) | ++--------------------------------------------------------------------+ +| {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]} | +| {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]} | +| {"a":{"b":3},"c":"s3","d":null} | +| {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]} | +| {"a":null,"c":"s5","d":null} | +| {"a":null,"c":"s6","d":null} | +| {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]} | +| {"a":{"b":8},"c":"s8","d":null} | +| {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} | +| {"a":{"b":10,"x":null},"c":null,"d":null,"y":false} | ++--------------------------------------------------------------------+ + +select json_get(j, '.') from json2_table order by ts; + ++--------------------------------------------------------------------+ +| json_get(json2_table.j,Utf8(".")) | ++--------------------------------------------------------------------+ +| {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]} | +| {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]} | +| {"a":{"b":3},"c":"s3","d":null} | +| {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]} | +| {"a":null,"c":"s5","d":null} | +| {"a":null,"c":"s6","d":null} | +| {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]} | +| {"a":{"b":8},"c":"s8","d":null} | +| {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} | +| {"a":{"b":10,"x":null},"c":null,"d":null,"y":false} | ++--------------------------------------------------------------------+ + +select json_get(j, '$.') from json2_table order by ts; + ++--------------------------------------------------------------------+ +| json_get(json2_table.j,Utf8("$.")) | ++--------------------------------------------------------------------+ +| {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]} | +| {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]} | +| {"a":{"b":3},"c":"s3","d":null} | +| {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]} | +| {"a":null,"c":"s5","d":null} | +| {"a":null,"c":"s6","d":null} | +| {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]} | +| {"a":{"b":8},"c":"s8","d":null} | +| {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} | +| {"a":{"b":10,"x":null},"c":null,"d":null,"y":false} | ++--------------------------------------------------------------------+ + select j.a.b + 1 from json2_table order by ts; +------------------------------------------------------------+ diff --git a/tests/cases/standalone/common/types/json/json2.sql b/tests/cases/standalone/common/types/json/json2.sql index d6d8cb4df1..f0bb5286b4 100644 --- a/tests/cases/standalone/common/types/json/json2.sql +++ b/tests/cases/standalone/common/types/json/json2.sql @@ -46,6 +46,24 @@ select j.a, j.a.x from json2_table order by ts; select j.c, j.y from json2_table order by ts; +select j from json2_table order by ts; + +select * from json2_table order by ts; + +select count(*) from (select j from json2_table group by j); + +select count(*) from (select distinct j from json2_table); + +select ts, j from (select ts, j from json2_table) order by ts; + +select json_get(j, '') from json2_table order by ts; + +select json_get(j, '$') from json2_table order by ts; + +select json_get(j, '.') from json2_table order by ts; + +select json_get(j, '$.') from json2_table order by ts; + select j.a.b + 1 from json2_table order by ts; select abs(j.a.b) from json2_table order by ts; diff --git a/tests/cases/standalone/common/types/json/json2_limit.result b/tests/cases/standalone/common/types/json/json2_limit.result index 9302db37f6..838717ba2d 100644 --- a/tests/cases/standalone/common/types/json/json2_limit.result +++ b/tests/cases/standalone/common/types/json/json2_limit.result @@ -52,43 +52,6 @@ insert into json2_disable_whole_column_read values Affected Rows: 2 --- Whole JSON2 uses are unsupported (case 1): direct projection. -select j from json2_disable_whole_column_read order by ts; - -Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields - --- Whole JSON2 uses are unsupported (case 2): wildcard projection. -select * from json2_disable_whole_column_read order by ts; - -Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields - --- Whole JSON2 uses are unsupported (case 3): json_get with an empty path. -select json_get(j, '') from json2_disable_whole_column_read; - -Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields - -select json_get(j, '$') from json2_disable_whole_column_read; - -Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields - -select json_get(j, '.') from json2_disable_whole_column_read; - -Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields - -select json_get(j, '$.') from json2_disable_whole_column_read; - -Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields - --- Whole JSON2 uses are unsupported (case 4): use in an intermediate plan node. -select count(*) -from ( - select j - from json2_disable_whole_column_read - group by j -); - -Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields - -- JSON2 field projection remains supported (case 5): use in an intermediate plan node. select json_get(j, 'a.b'), count(*) from json2_disable_whole_column_read @@ -102,24 +65,13 @@ order by json_get(j, 'a.b'); | 2 | 1 | +---------------------------------------------------------+----------+ --- Whole JSON2 uses are unsupported (case 6): output after an intermediate projection. -select ts, j -from ( - select ts, j - from json2_disable_whole_column_read -) -order by ts; +select j, j.a from json2_disable_whole_column_read; -Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields +Error: 3001(EngineExecuteQuery), Invalid argument error: column types must match schema types, expected Binary but found Struct("a": Utf8View) at column index 0 --- Whole JSON2 uses are unsupported (case 7): DISTINCT in an intermediate plan node. -select count(*) -from ( - select distinct j - from json2_disable_whole_column_read -); +select j from json2_disable_whole_column_read where j.a.b = 1; -Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields +Error: 3001(EngineExecuteQuery), Invalid argument error: column types must match schema types, expected Binary but found Struct("a": Struct("b": Int64)) at column index 0 drop table json2_disable_whole_column_read; diff --git a/tests/cases/standalone/common/types/json/json2_limit.sql b/tests/cases/standalone/common/types/json/json2_limit.sql index d792d335f5..42fac25362 100644 --- a/tests/cases/standalone/common/types/json/json2_limit.sql +++ b/tests/cases/standalone/common/types/json/json2_limit.sql @@ -32,49 +32,15 @@ insert into json2_disable_whole_column_read values (1, '{"a": {"b": 1}}'), (2, '{"a": {"b": 2}}'); --- Whole JSON2 uses are unsupported (case 1): direct projection. -select j from json2_disable_whole_column_read order by ts; - --- Whole JSON2 uses are unsupported (case 2): wildcard projection. -select * from json2_disable_whole_column_read order by ts; - --- Whole JSON2 uses are unsupported (case 3): json_get with an empty path. -select json_get(j, '') from json2_disable_whole_column_read; - -select json_get(j, '$') from json2_disable_whole_column_read; - -select json_get(j, '.') from json2_disable_whole_column_read; - -select json_get(j, '$.') from json2_disable_whole_column_read; - --- Whole JSON2 uses are unsupported (case 4): use in an intermediate plan node. -select count(*) -from ( - select j - from json2_disable_whole_column_read - group by j -); - -- JSON2 field projection remains supported (case 5): use in an intermediate plan node. select json_get(j, 'a.b'), count(*) from json2_disable_whole_column_read group by json_get(j, 'a.b') order by json_get(j, 'a.b'); --- Whole JSON2 uses are unsupported (case 6): output after an intermediate projection. -select ts, j -from ( - select ts, j - from json2_disable_whole_column_read -) -order by ts; +select j, j.a from json2_disable_whole_column_read; --- Whole JSON2 uses are unsupported (case 7): DISTINCT in an intermediate plan node. -select count(*) -from ( - select distinct j - from json2_disable_whole_column_read -); +select j from json2_disable_whole_column_read where j.a.b = 1; drop table json2_disable_whole_column_read;