fix: make select whole json2 column worked (#8683)

* fix: make select whole json2 column worked

Signed-off-by: luofucong <luofc@foxmail.com>

* resolve PR comments

Signed-off-by: luofucong <luofc@foxmail.com>

* x

Signed-off-by: luofucong <luofc@foxmail.com>

---------

Signed-off-by: luofucong <luofc@foxmail.com>

src/common/function/src/scalars/json/json_get.rs

@@ -24,6 +24,7 @@ use datafusion_common::arrow::array::{
 use datafusion_common::arrow::datatypes::DataType;
 use datafusion_common::{DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err};
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, Volatility};
+use datatypes::extension::json::is_json2_extension_type;
 use datatypes::vectors::json::array::JsonArray;
 use derive_more::Display;
 
@@ -277,6 +278,7 @@ fn json_struct_get(array: &ArrayRef, path: &str, with_type: &DataType) -> Result
         .split('.')
         .filter(|segment| !segment.is_empty())
         .collect::<Vec<_>>();
+
     let mut curr = array.clone();
 
     for (idx, segment) in segments.iter().enumerate() {
@@ -404,9 +406,18 @@ impl Function for JsonGetWithType {
                 let arg0 = compute::cast(&arg0, &DataType::BinaryView)?;
                 let jsons = arg0.as_binary_view();
 
-                let mut builder = result_builder(len, &with_type)?;
-                jsonb_get(jsons, path, builder.as_mut())?;
-                builder.build()
+                if args.arg_fields.first().is_some_and(is_json2_extension_type) {
+                    // Query concretization projects nested JSON2 paths as Struct arrays. A binary
+                    // JSON2 argument is therefore an already-selected scalar or root value that
+                    // only needs conversion from its JSONB representation to the requested type.
+                    JsonArray::from(&arg0)
+                        .project_to(&with_type)
+                        .map_err(|e| exec_datafusion_err!("{e:?}"))?
+                } else {
+                    let mut builder = result_builder(len, &with_type)?;
+                    jsonb_get(jsons, path, builder.as_mut())?;
+                    builder.build()
+                }
             }
             DataType::Struct(_) => json_struct_get(&arg0, path, &with_type)?,
             _ => {

src/common/recordbatch/src/lib.rs

@@ -22,7 +22,7 @@ pub mod filter;
 pub mod recordbatch;
 pub mod util;
 
-use std::fmt;
+use std::fmt::{self, Write};
 use std::future::Future;
 use std::pin::Pin;
 use std::sync::Arc;
@@ -36,10 +36,16 @@ use common_memory_manager::{
 };
 use common_telemetry::tracing::Span;
 pub use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
-use datatypes::arrow::array::{ArrayRef, AsArray, StringBuilder};
+use datatypes::arrow::array::{Array, ArrayRef, AsArray, StringBuilder};
 use datatypes::arrow::compute::SortOptions;
+use datatypes::arrow::datatypes::{DataType as ArrowDataType, Field};
+use datatypes::arrow::error::ArrowError;
 pub use datatypes::arrow::record_batch::RecordBatch as DfRecordBatch;
-use datatypes::arrow::util::pretty;
+use datatypes::arrow::util::display::{
+    ArrayFormatter, ArrayFormatterFactory, DisplayIndex, FormatOptions, FormatResult,
+};
+use datatypes::arrow::util::pretty::pretty_format_batches_with_options;
+use datatypes::extension::json::is_json_extension_type;
 use datatypes::prelude::{ConcreteDataType, DataType, VectorRef};
 use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
 use datatypes::types::{JsonFormat, StructField, StructType, jsonb_to_string};
@@ -390,7 +396,10 @@ impl RecordBatches {
             .iter()
             .map(|x| x.df_record_batch().clone())
             .collect::<Vec<_>>();
-        let result = pretty::pretty_format_batches(df_batches).context(error::FormatSnafu)?;
+        let options =
+            FormatOptions::default().with_formatter_factory(Some(&BinaryFormatterFactory));
+        let result =
+            pretty_format_batches_with_options(df_batches, &options).context(error::FormatSnafu)?;
 
         Ok(result.to_string())
     }
@@ -429,6 +438,63 @@ impl RecordBatches {
     }
 }
 
+#[derive(Debug)]
+struct BinaryFormatterFactory;
+
+impl ArrayFormatterFactory for BinaryFormatterFactory {
+    fn create_array_formatter<'a>(
+        &self,
+        array: &'a dyn Array,
+        options: &FormatOptions<'a>,
+        field: Option<&'a Field>,
+    ) -> std::result::Result<Option<ArrayFormatter<'a>>, ArrowError> {
+        if !array.data_type().is_binary() {
+            return Ok(None);
+        }
+
+        Ok(Some(ArrayFormatter::new(
+            Box::new(BinaryFormatter {
+                array,
+                is_json: field.is_some_and(is_json_extension_type),
+                default: ArrayFormatter::try_new(array, options)?,
+                null: options.null(),
+            }),
+            options.safe(),
+        )))
+    }
+}
+
+struct BinaryFormatter<'a> {
+    array: &'a dyn Array,
+    is_json: bool,
+    default: ArrayFormatter<'a>,
+    null: &'a str,
+}
+
+impl DisplayIndex for BinaryFormatter<'_> {
+    fn write(&self, idx: usize, f: &mut dyn Write) -> FormatResult {
+        if !self.is_json {
+            self.default.value(idx).write(f)?;
+            return Ok(());
+        }
+
+        if self.array.is_null(idx) {
+            write!(f, "{}", self.null)?;
+        } else {
+            let bytes = match self.array.data_type() {
+                ArrowDataType::Binary => self.array.as_binary::<i32>().value(idx),
+                ArrowDataType::LargeBinary => self.array.as_binary::<i64>().value(idx),
+                ArrowDataType::BinaryView => self.array.as_binary_view().value(idx),
+                _ => unreachable!(),
+            };
+            let value =
+                jsonb_to_string(bytes).map_err(|e| ArrowError::ExternalError(Box::new(e)))?;
+            write!(f, "{value}")?;
+        }
+        Ok(())
+    }
+}
+
 impl IntoIterator for RecordBatches {
     type Item = RecordBatch;
     type IntoIter = std::vec::IntoIter<Self::Item>;

src/datatypes/src/data_type.rs

@@ -228,6 +228,11 @@ impl ConcreteDataType {
         matches!(self, ConcreteDataType::Json(_))
     }
 
+    /// Returns whether this is a JSON2 data type.
+    pub fn is_json2(&self) -> bool {
+        self.as_json().is_some_and(|json_type| json_type.is_json2())
+    }
+
     pub fn is_vector(&self) -> bool {
         matches!(self, ConcreteDataType::Vector(_))
     }

src/datatypes/src/extension/json.rs

@@ -15,7 +15,7 @@
 use std::sync::Arc;
 
 use arrow_schema::extension::ExtensionType;
-use arrow_schema::{ArrowError, DataType, FieldRef};
+use arrow_schema::{ArrowError, DataType, Field, FieldRef};
 use serde::{Deserialize, Serialize};
 
 use crate::json::JsonSettings;
@@ -103,8 +103,22 @@ impl ExtensionType for JsonExtensionType {
 }
 
 /// Check if this field is to be treated as json extension type.
-pub fn is_json_extension_type(field: &FieldRef) -> bool {
-    field.extension_type_name() == Some(JsonExtensionType::NAME)
+pub fn is_json_extension_type<T: AsRef<Field>>(field: T) -> bool {
+    field.as_ref().extension_type_name() == Some(JsonExtensionType::NAME)
+}
+
+/// Check if this field is a JSON2 extension type.
+///
+/// Legacy JSONB and JSON2 share the same JSON extension name. The column schema construction
+/// invariant is that JSON2 always stores its settings as `Some`, including default settings,
+/// while legacy JSONB stores no JSON settings. Therefore, after checking the extension name,
+/// the presence of JSON settings distinguishes JSON2 from legacy JSONB.
+pub fn is_json2_extension_type<T: AsRef<Field>>(field: T) -> bool {
+    let field = field.as_ref();
+    is_json_extension_type(field)
+        && field
+            .try_extension_type::<JsonExtensionType>()
+            .is_ok_and(|x| x.metadata().json_settings.is_some())
 }
 
 /// Check if this field is a structured JSON field.

src/datatypes/src/types/json_type.rs

@@ -282,7 +282,8 @@ impl JsonType {
         matches!(self.format, JsonFormat::Json2(_))
     }
 
-    pub(crate) fn native_type(&self) -> &JsonNativeType {
+    /// Returns the native JSON type represented by this data type.
+    pub fn native_type(&self) -> &JsonNativeType {
         match &self.format {
             JsonFormat::Jsonb => &JsonNativeType::String,
             JsonFormat::Json2(x) => x.as_ref(),

src/mito2/src/read/flat_projection.rs

@@ -25,9 +25,10 @@ use common_recordbatch::error::{
 use common_recordbatch::{DfRecordBatch, RecordBatch};
 use datatypes::arrow::array::Array;
 use datatypes::arrow::datatypes::{DataType as ArrowDataType, Field};
-use datatypes::extension::json::is_structured_json_field;
+use datatypes::extension::json::{is_json_extension_type, is_structured_json_field};
 use datatypes::prelude::{ConcreteDataType, DataType};
-use datatypes::schema::{Schema, SchemaRef};
+use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
+use datatypes::types::JsonType;
 use datatypes::types::json_type::JsonNativeType;
 use datatypes::value::Value;
 use datatypes::vectors::Helper;
@@ -114,17 +115,7 @@ impl FlatProjectionMapper {
             output_col_ids.push(col.column_id);
 
             let mut schema = col.column_schema.clone();
-            if let Some(concretized) = json_type_hint
-                .and_then(|x| x.get(&schema.name))
-                .cloned()
-                .map(ConcreteDataType::json2)
-                && schema
-                    .data_type
-                    .as_json()
-                    .is_some_and(|json_type| json_type.is_json2())
-            {
-                schema.data_type = concretized;
-            }
+            maybe_concretize_json2_datatype(&mut schema, json_type_hint);
             col_schemas.push(schema);
         }
 
@@ -141,19 +132,21 @@ impl FlatProjectionMapper {
 
         let mut batch_schema = flat_projected_columns(metadata, &format_projection);
 
-        if let Some(json_type_hint) = json_type_hint
-            && !json_type_hint.is_empty()
-        {
-            for (column_id, data_type) in batch_schema.iter_mut() {
-                if data_type
-                    .as_json()
-                    .is_some_and(|json_type| json_type.is_json2())
-                    && let Some(concretized) = metadata
-                        .column_by_id(*column_id)
-                        .and_then(|x| json_type_hint.get(&x.column_schema.name).cloned())
-                        .map(ConcreteDataType::json2)
+        for (column_id, data_type) in batch_schema.iter_mut() {
+            if let Some(json_type) = data_type.as_json()
+                && json_type.is_json2()
+            {
+                if let Some(concretized) = metadata
+                    .column_by_id(*column_id)
+                    .and_then(|metadata| {
+                        json_type_hint.and_then(|x| x.get(&metadata.column_schema.name).cloned())
+                    })
+                    .map(ConcreteDataType::json2)
                 {
                     *data_type = concretized;
+                } else if is_empty_json2_type(json_type) {
+                    // see `merge_scan::maybe_amend_json2_field`
+                    *data_type = ConcreteDataType::json2(JsonNativeType::Variant);
                 }
             }
         }
@@ -365,7 +358,7 @@ impl FlatProjectionMapper {
             }
 
             let field = &self.output_schema.arrow_schema().fields()[output_idx];
-            if is_structured_json_field(field) {
+            if is_json_extension_type(field) {
                 array = JsonArray::from(&array)
                     .project_to(field.data_type())
                     .context(DataTypesSnafu)?;
@@ -408,6 +401,34 @@ impl FlatProjectionMapper {
     }
 }
 
+fn maybe_concretize_json2_datatype(
+    schema: &mut ColumnSchema,
+    json_type_hint: Option<&HashMap<String, JsonNativeType>>,
+) {
+    if let Some(json_type) = schema.data_type.as_json()
+        && json_type.is_json2()
+    {
+        if let Some(concretized) = json_type_hint
+            .and_then(|x| x.get(&schema.name))
+            .cloned()
+            .map(ConcreteDataType::json2)
+        {
+            schema.data_type = concretized;
+        } else if is_empty_json2_type(json_type) {
+            // see `merge_scan::maybe_amend_json2_field`
+            schema.data_type = ConcreteDataType::json2(JsonNativeType::Variant);
+        }
+    }
+}
+
+fn is_empty_json2_type(json_type: &JsonType) -> bool {
+    match json_type.native_type() {
+        JsonNativeType::Null => true,
+        JsonNativeType::Object(fields) if fields.is_empty() => true,
+        _ => false,
+    }
+}
+
 fn single_value_string_dictionary<'a>(
     array: &'a Arc<dyn Array>,
     output_type: &ConcreteDataType,

src/operator/src/statement/ddl.rs

@@ -2419,11 +2419,7 @@ fn validate_json2_columns_append_mode(schema: &Schema, table_options: &TableOpti
         .is_some_and(|value| value == "true");
 
     for column in schema.column_schemas() {
-        if column
-            .data_type
-            .as_json()
-            .is_some_and(|json_type| json_type.is_json2())
-        {
+        if column.data_type.is_json2() {
             ensure!(
                 append_mode,
                 InvalidSqlSnafu {

src/query/src/datafusion/error.rs

@@ -17,7 +17,6 @@ use std::any::Any;
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use common_macro::stack_trace_debug;
-use datafusion::error::DataFusionError;
 use snafu::{Location, Snafu};
 
 /// Inner error of datafusion based query engine.
@@ -25,14 +24,6 @@ use snafu::{Location, Snafu};
 #[snafu(visibility(pub))]
 #[stack_trace_debug]
 pub enum InnerError {
-    #[snafu(transparent)]
-    Datafusion {
-        #[snafu(source)]
-        error: DataFusionError,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
     #[snafu(display("Failed to convert DataFusion's recordbatch stream"))]
     ConvertDfRecordBatchStream {
         #[snafu(implicit)]
@@ -46,8 +37,6 @@ impl ErrorExt for InnerError {
         use InnerError::*;
 
         match self {
-            // TODO(yingwen): Further categorize datafusion error.
-            Datafusion { .. } => StatusCode::EngineExecuteQuery,
             ConvertDfRecordBatchStream { source, .. } => source.status_code(),
         }
     }

src/query/src/dist_plan/merge_scan.rs

@@ -17,7 +17,7 @@ use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use ahash::{HashMap, HashSet};
-use arrow_schema::{Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, SortOptions};
+use arrow_schema::{DataType, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, SortOptions};
 use async_stream::stream;
 use common_catalog::parse_catalog_and_schema_from_db_string;
 use common_plugins::GREPTIME_EXEC_READ_COST;
@@ -41,6 +41,7 @@ use datafusion_common::{Column as ColumnExpr, DataFusionError, Result};
 use datafusion_expr::{Expr, Extension, LogicalPlan, UserDefinedLogicalNodeCore};
 use datafusion_physical_expr::expressions::Column;
 use datafusion_physical_expr::{Distribution, EquivalenceProperties, PhysicalSortExpr};
+use datatypes::extension::json::is_json_extension_type;
 use futures_util::StreamExt;
 use greptime_proto::v1::region::RegionRequestHeader;
 use meter_core::data::ReadItem;
@@ -253,10 +254,7 @@ impl MergeScanExec {
         remote_dyn_filter_producer_id: Option<RemoteDynFilterProducerId>,
         enable_per_region_metrics: bool,
     ) -> Result<Self> {
-        // TODO(CookiePieWw): Initially we removed the metadata from the schema in #2000, but we have to
-        // keep it for #4619 to identify json type in src/datatypes/src/schema/column_schema.rs.
-        // Reconsider if it's possible to remove it.
-        let arrow_schema = Arc::new(arrow_schema.clone());
+        let arrow_schema = maybe_amend_json2_field(arrow_schema);
 
         // States the output ordering of the plan.
         //
@@ -705,6 +703,37 @@ impl MergeScanExec {
     }
 }
 
+// If the schema has JSON2 field, AND the field is of empty Struct datatype, amend it with Binary
+// datatype.
+// This is a very hacky way to make it possible to query the whole JSON2 column. Because when
+// querying a whole JSON2 column, like in the SQL `select * from ...`, we can't concretize the JSON2
+// datatype from the query. Hence, the JSON2 datatype remains what in the column schema, i.e., empty
+// Struct. An empty Struct is not alignable like any other concretized JSON2 datatypes, so to make
+// the query work, we amend(rewrite) it to Binary datatype.
+// Why the Binary datatype? Because underlying the scan and projection stage, the JSON2 data are
+// variant shape, will be all converted to bytes.
+// Anyway, this is not clean nor elegant. TODO(LFC) Maybe make it into some plan analyzer rule?
+fn maybe_amend_json2_field(schema: &ArrowSchema) -> ArrowSchemaRef {
+    let schema = schema.clone();
+    let mut new_fields = Vec::with_capacity(schema.fields().len());
+    for field in schema.fields().iter() {
+        let new_field = if is_json_extension_type(field)
+            && matches!(field.data_type(), DataType::Struct(fields) if fields.is_empty())
+        {
+            let mut new_field = field.as_ref().clone();
+            new_field.set_data_type(DataType::Binary);
+            Arc::new(new_field)
+        } else {
+            field.clone()
+        };
+        new_fields.push(new_field);
+    }
+    Arc::new(ArrowSchema::new_with_metadata(
+        new_fields,
+        schema.metadata().clone(),
+    ))
+}
+
 #[cfg(test)]
 impl MergeScanExec {
     fn remote_dyn_filter_producer_id(&self) -> Option<RemoteDynFilterProducerId> {

src/query/src/optimizer/json_type_concretize.rs

@@ -18,11 +18,9 @@ use arrow_schema::DataType;
 use common_function::scalars::json::json_get::JsonGetWithType;
 use datafusion::datasource::DefaultTableSource;
 use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
-use datafusion_common::{ExprSchema, Result, plan_datafusion_err, plan_err};
-use datafusion_expr::utils::merge_schema;
+use datafusion_common::{Result, plan_datafusion_err, plan_err};
 use datafusion_expr::{Expr, LogicalPlan};
 use datafusion_optimizer::{OptimizerConfig, OptimizerRule};
-use datatypes::extension::json::is_structured_json_field;
 use datatypes::types::json_type::{JsonNativeType, JsonObjectType};
 
 use crate::dummy_catalog::DummyTableProvider;
@@ -43,8 +41,6 @@ impl OptimizerRule for JsonTypeConcretizeRule {
         plan: LogicalPlan,
         _config: &dyn OptimizerConfig,
     ) -> Result<Transformed<LogicalPlan>> {
-        ensure_no_whole_json2_read(&plan)?;
-
         let json_types = deduce_json_types(&plan)?;
         if json_types.is_empty() {
             return Ok(Transformed::no(plan));
@@ -76,102 +72,6 @@ impl OptimizerRule for JsonTypeConcretizeRule {
     }
 }
 
-/// Rejects unsupported whole-column JSON2 reads in a logical plan.
-fn ensure_no_whole_json2_read(plan: &LogicalPlan) -> Result<()> {
-    // Reject whole JSON2 columns in the final query output, including `SELECT *`.
-    for field in plan.schema().fields() {
-        if is_structured_json_field(field) {
-            return plan_err!(
-                "Querying the whole JSON2 column '{}' is currently not supported; use json_get to select its fields",
-                field.name()
-            );
-        }
-    }
-
-    // Reject whole JSON2 columns consumed by intermediate expressions, for example:
-    // `SELECT count(*) FROM (SELECT j FROM t GROUP BY j)`.
-    plan.apply(|plan| {
-        let input_schema = merge_schema(&plan.inputs());
-        for expr in plan.expressions() {
-            // A bare column in an intermediate projection is only passed through, not consumed.
-            if matches!(plan, LogicalPlan::Projection(_)) && is_passthrough_column(&expr) {
-                continue;
-            }
-
-            expr.apply(|expr| {
-                // For JSON2, `json_get` is allowed only with a non-empty path; skip its arguments
-                // after validation.
-                if let Expr::ScalarFunction(function) = expr
-                    && function.name().eq_ignore_ascii_case(JsonGetWithType::NAME)
-                {
-                    let Some(Expr::Column(col)) = function.args.first() else {
-                        return Ok(TreeNodeRecursion::Jump);
-                    };
-                    let Some(path) = function
-                        .args
-                        .get(1)
-                        .and_then(Expr::as_literal)
-                        .and_then(|value| value.try_as_str())
-                        .flatten()
-                    else {
-                        return Ok(TreeNodeRecursion::Jump);
-                    };
-                    let reads_whole_column = path
-                        .trim_start_matches('$')
-                        .split('.')
-                        .all(str::is_empty);
-                    if !reads_whole_column {
-                        return Ok(TreeNodeRecursion::Jump);
-                    }
-
-                    let Ok(field) = input_schema
-                        .field_from_column(col)
-                        .or_else(|_| plan.schema().field_from_column(col))
-                    else {
-                        return Ok(TreeNodeRecursion::Jump);
-                    };
-                    if is_structured_json_field(field) {
-                        return plan_err!(
-                            "Querying the whole JSON2 column '{}' is currently not supported; use json_get to select its fields",
-                            col.name
-                        );
-                    }
-                    return Ok(TreeNodeRecursion::Jump);
-                }
-
-                // Any remaining JSON2 column reference is a whole-column read.
-                let Expr::Column(col) = expr else {
-                    return Ok(TreeNodeRecursion::Continue);
-                };
-                let Ok(field) = input_schema
-                    .field_from_column(col)
-                    .or_else(|_| plan.schema().field_from_column(col))
-                else {
-                    return Ok(TreeNodeRecursion::Continue);
-                };
-                if is_structured_json_field(field) {
-                    return plan_err!(
-                        "Querying the whole JSON2 column '{}' is currently not supported; use json_get to select its fields",
-                        col.name
-                    );
-                }
-                Ok(TreeNodeRecursion::Continue)
-            })?;
-        }
-        Ok(TreeNodeRecursion::Continue)
-    })?;
-
-    Ok(())
-}
-
-fn is_passthrough_column(expr: &Expr) -> bool {
-    match expr {
-        Expr::Column(_) => true,
-        Expr::Alias(alias) => is_passthrough_column(&alias.expr),
-        _ => false,
-    }
-}
-
 fn deduce_json_types(plan: &LogicalPlan) -> Result<HashMap<String, JsonNativeType>> {
     let mut json_types = HashMap::<String, JsonNativeType>::new();
 
@@ -227,7 +127,7 @@ fn deduce_json_type(expr: &Expr) -> Result<Option<(String, JsonNativeType)>> {
         JsonNativeType::try_from(&with_type).map_err(|e| plan_datafusion_err!("{e:?}"))?;
 
     let mut split = path.rsplit(".");
-    let Some(leaf) = split.next() else {
+    let Some(leaf) = split.next().filter(|&x| !x.is_empty() && x != "$") else {
         return Ok(Some((column.name.clone(), JsonNativeType::String)));
     };
 
@@ -397,56 +297,6 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn test_reject_whole_json2_projection() -> Result<()> {
-        for (exprs, output_name) in [
-            (vec![col("j")], "j"),
-            (vec![col("j").alias("json"), col("ts")], "json"),
-        ] {
-            let (_, plan) = build_json2_plan(exprs)?;
-            let err = JsonTypeConcretizeRule
-                .rewrite(plan, &OptimizerContext::default())
-                .unwrap_err();
-            assert!(err.to_string().contains(&format!(
-                "Querying the whole JSON2 column '{output_name}' is currently not supported"
-            )));
-        }
-        Ok(())
-    }
-
-    #[test]
-    fn test_reject_whole_json2_output_without_projection() -> Result<()> {
-        let (_, plan) = build_json2_scan()?;
-        let plan = plan.sort(vec![col("ts").sort(true, false)])?.build()?;
-
-        let err = JsonTypeConcretizeRule
-            .rewrite(plan, &OptimizerContext::default())
-            .unwrap_err();
-        assert!(
-            err.to_string()
-                .contains("Querying the whole JSON2 column 'j' is currently not supported")
-        );
-        Ok(())
-    }
-
-    #[test]
-    fn test_reject_whole_json2_use_in_intermediate_plan() -> Result<()> {
-        let (_, plan) = build_json2_scan()?;
-        let plan = plan
-            .aggregate(vec![col("j")], Vec::<Expr>::new())?
-            .aggregate(Vec::<Expr>::new(), vec![count(lit(1))])?
-            .build()?;
-
-        let err = JsonTypeConcretizeRule
-            .rewrite(plan, &OptimizerContext::default())
-            .unwrap_err();
-        assert!(
-            err.to_string()
-                .contains("Querying the whole JSON2 column 'j' is currently not supported")
-        );
-        Ok(())
-    }
-
     #[test]
     fn test_allow_json2_path_use_in_intermediate_plan() -> Result<()> {
         let json_get = json_get_expr(col("j"), path_expr("a"), Some(DataType::Int64))?;
@@ -503,23 +353,6 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn test_reject_json2_projection_with_empty_path() -> Result<()> {
-        for path in ["", "$", ".", "$."] {
-            let expr = json_get_expr(col("j"), path_expr(path), Some(DataType::Utf8View))?;
-            let (_, plan) = build_json2_plan(vec![expr])?;
-
-            let err = JsonTypeConcretizeRule
-                .rewrite(plan, &OptimizerContext::default())
-                .unwrap_err();
-            assert!(
-                err.to_string()
-                    .contains("Querying the whole JSON2 column 'j' is currently not supported")
-            );
-        }
-        Ok(())
-    }
-
     #[test]
     fn test_deduce_json_type_with_non_column_base() -> Result<()> {
         let expr = json_get_expr(

tests/cases/standalone/common/types/json/json2.result

@@ -151,6 +151,141 @@ select j.c, j.y from json2_table order by ts;
 |                                   | false                             |
 +-----------------------------------+-----------------------------------+
 
+select j from json2_table order by ts;
+
++--------------------------------------------------------------------+
+| j                                                                  |
++--------------------------------------------------------------------+
+| {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]}              |
+| {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]}             |
+| {"a":{"b":3},"c":"s3","d":null}                                    |
+| {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]}            |
+| {"a":null,"c":"s5","d":null}                                       |
+| {"a":null,"c":"s6","d":null}                                       |
+| {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]}                    |
+| {"a":{"b":8},"c":"s8","d":null}                                    |
+| {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} |
+| {"a":{"b":10,"x":null},"c":null,"d":null,"y":false}                |
++--------------------------------------------------------------------+
+
+select * from json2_table order by ts;
+
++-------------------------+--------------------------------------------------------------------+
+| ts                      | j                                                                  |
++-------------------------+--------------------------------------------------------------------+
+| 1970-01-01T00:00:00.001 | {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]}              |
+| 1970-01-01T00:00:00.002 | {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]}             |
+| 1970-01-01T00:00:00.003 | {"a":{"b":3},"c":"s3","d":null}                                    |
+| 1970-01-01T00:00:00.004 | {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]}            |
+| 1970-01-01T00:00:00.005 | {"a":null,"c":"s5","d":null}                                       |
+| 1970-01-01T00:00:00.006 | {"a":null,"c":"s6","d":null}                                       |
+| 1970-01-01T00:00:00.007 | {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]}                    |
+| 1970-01-01T00:00:00.008 | {"a":{"b":8},"c":"s8","d":null}                                    |
+| 1970-01-01T00:00:00.009 | {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} |
+| 1970-01-01T00:00:00.010 | {"a":{"b":10,"x":null},"c":null,"d":null,"y":false}                |
++-------------------------+--------------------------------------------------------------------+
+
+select count(*) from (select j from json2_table group by j);
+
++----------+
+| count(*) |
++----------+
+| 10       |
++----------+
+
+select count(*) from (select distinct j from json2_table);
+
++----------+
+| count(*) |
++----------+
+| 10       |
++----------+
+
+select ts, j from (select ts, j from json2_table) order by ts;
+
++-------------------------+--------------------------------------------------------------------+
+| ts                      | j                                                                  |
++-------------------------+--------------------------------------------------------------------+
+| 1970-01-01T00:00:00.001 | {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]}              |
+| 1970-01-01T00:00:00.002 | {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]}             |
+| 1970-01-01T00:00:00.003 | {"a":{"b":3},"c":"s3","d":null}                                    |
+| 1970-01-01T00:00:00.004 | {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]}            |
+| 1970-01-01T00:00:00.005 | {"a":null,"c":"s5","d":null}                                       |
+| 1970-01-01T00:00:00.006 | {"a":null,"c":"s6","d":null}                                       |
+| 1970-01-01T00:00:00.007 | {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]}                    |
+| 1970-01-01T00:00:00.008 | {"a":{"b":8},"c":"s8","d":null}                                    |
+| 1970-01-01T00:00:00.009 | {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} |
+| 1970-01-01T00:00:00.010 | {"a":{"b":10,"x":null},"c":null,"d":null,"y":false}                |
++-------------------------+--------------------------------------------------------------------+
+
+select json_get(j, '') from json2_table order by ts;
+
++--------------------------------------------------------------------+
+| json_get(json2_table.j,Utf8(""))                                   |
++--------------------------------------------------------------------+
+| {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]}              |
+| {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]}             |
+| {"a":{"b":3},"c":"s3","d":null}                                    |
+| {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]}            |
+| {"a":null,"c":"s5","d":null}                                       |
+| {"a":null,"c":"s6","d":null}                                       |
+| {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]}                    |
+| {"a":{"b":8},"c":"s8","d":null}                                    |
+| {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} |
+| {"a":{"b":10,"x":null},"c":null,"d":null,"y":false}                |
++--------------------------------------------------------------------+
+
+select json_get(j, '$') from json2_table order by ts;
+
++--------------------------------------------------------------------+
+| json_get(json2_table.j,Utf8("$"))                                  |
++--------------------------------------------------------------------+
+| {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]}              |
+| {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]}             |
+| {"a":{"b":3},"c":"s3","d":null}                                    |
+| {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]}            |
+| {"a":null,"c":"s5","d":null}                                       |
+| {"a":null,"c":"s6","d":null}                                       |
+| {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]}                    |
+| {"a":{"b":8},"c":"s8","d":null}                                    |
+| {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} |
+| {"a":{"b":10,"x":null},"c":null,"d":null,"y":false}                |
++--------------------------------------------------------------------+
+
+select json_get(j, '.') from json2_table order by ts;
+
++--------------------------------------------------------------------+
+| json_get(json2_table.j,Utf8("."))                                  |
++--------------------------------------------------------------------+
+| {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]}              |
+| {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]}             |
+| {"a":{"b":3},"c":"s3","d":null}                                    |
+| {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]}            |
+| {"a":null,"c":"s5","d":null}                                       |
+| {"a":null,"c":"s6","d":null}                                       |
+| {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]}                    |
+| {"a":{"b":8},"c":"s8","d":null}                                    |
+| {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} |
+| {"a":{"b":10,"x":null},"c":null,"d":null,"y":false}                |
++--------------------------------------------------------------------+
+
+select json_get(j, '$.') from json2_table order by ts;
+
++--------------------------------------------------------------------+
+| json_get(json2_table.j,Utf8("$."))                                 |
++--------------------------------------------------------------------+
+| {"a":{"b":1},"c":"s1","d":[{"e":{"f":0.1,"g":null}}]}              |
+| {"a":{"b":-2},"c":"s2","d":[{"e":{"f":0.2,"g":null}}]}             |
+| {"a":{"b":3},"c":"s3","d":null}                                    |
+| {"a":{"b":-4},"c":null,"d":[{"e":{"f":null,"g":-0.4}}]}            |
+| {"a":null,"c":"s5","d":null}                                       |
+| {"a":null,"c":"s6","d":null}                                       |
+| {"a":{"b":"s7"},"c":[1],"d":[{"e":{"g":-0.7}}]}                    |
+| {"a":{"b":8},"c":"s8","d":null}                                    |
+| {"a":{"b":null,"x":true},"c":"s9","d":[{"e":{"g":-0.9}}],"y":null} |
+| {"a":{"b":10,"x":null},"c":null,"d":null,"y":false}                |
++--------------------------------------------------------------------+
+
 select j.a.b + 1 from json2_table order by ts;
 
 +------------------------------------------------------------+

tests/cases/standalone/common/types/json/json2.sql

@@ -46,6 +46,24 @@ select j.a, j.a.x from json2_table order by ts;
 
 select j.c, j.y from json2_table order by ts;
 
+select j from json2_table order by ts;
+
+select * from json2_table order by ts;
+
+select count(*) from (select j from json2_table group by j);
+
+select count(*) from (select distinct j from json2_table);
+
+select ts, j from (select ts, j from json2_table) order by ts;
+
+select json_get(j, '') from json2_table order by ts;
+
+select json_get(j, '$') from json2_table order by ts;
+
+select json_get(j, '.') from json2_table order by ts;
+
+select json_get(j, '$.') from json2_table order by ts;
+
 select j.a.b + 1 from json2_table order by ts;
 
 select abs(j.a.b) from json2_table order by ts;

tests/cases/standalone/common/types/json/json2_limit.result

@@ -52,43 +52,6 @@ insert into json2_disable_whole_column_read values
 
 Affected Rows: 2
 
--- Whole JSON2 uses are unsupported (case 1): direct projection.
-select j from json2_disable_whole_column_read order by ts;
-
-Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields
-
--- Whole JSON2 uses are unsupported (case 2): wildcard projection.
-select * from json2_disable_whole_column_read order by ts;
-
-Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields
-
--- Whole JSON2 uses are unsupported (case 3): json_get with an empty path.
-select json_get(j, '') from json2_disable_whole_column_read;
-
-Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields
-
-select json_get(j, '$') from json2_disable_whole_column_read;
-
-Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields
-
-select json_get(j, '.') from json2_disable_whole_column_read;
-
-Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields
-
-select json_get(j, '$.') from json2_disable_whole_column_read;
-
-Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields
-
--- Whole JSON2 uses are unsupported (case 4): use in an intermediate plan node.
-select count(*)
-from (
-    select j
-    from json2_disable_whole_column_read
-    group by j
-);
-
-Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields
-
 -- JSON2 field projection remains supported (case 5): use in an intermediate plan node.
 select json_get(j, 'a.b'), count(*)
 from json2_disable_whole_column_read
@@ -102,24 +65,13 @@ order by json_get(j, 'a.b');
 | 2                                                       | 1        |
 +---------------------------------------------------------+----------+
 
--- Whole JSON2 uses are unsupported (case 6): output after an intermediate projection.
-select ts, j
-from (
-    select ts, j
-    from json2_disable_whole_column_read
-)
-order by ts;
+select j, j.a from json2_disable_whole_column_read;
 
-Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields
+Error: 3001(EngineExecuteQuery), Invalid argument error: column types must match schema types, expected Binary but found Struct("a": Utf8View) at column index 0
 
--- Whole JSON2 uses are unsupported (case 7): DISTINCT in an intermediate plan node.
-select count(*)
-from (
-    select distinct j
-    from json2_disable_whole_column_read
-);
+select j from json2_disable_whole_column_read where j.a.b = 1;
 
-Error: 3001(EngineExecuteQuery), Error during planning: Querying the whole JSON2 column 'j' is currently not supported; use json_get to select its fields
+Error: 3001(EngineExecuteQuery), Invalid argument error: column types must match schema types, expected Binary but found Struct("a": Struct("b": Int64)) at column index 0
 
 drop table json2_disable_whole_column_read;
 

tests/cases/standalone/common/types/json/json2_limit.sql

@@ -32,49 +32,15 @@ insert into json2_disable_whole_column_read values
     (1, '{"a": {"b": 1}}'),
     (2, '{"a": {"b": 2}}');
 
--- Whole JSON2 uses are unsupported (case 1): direct projection.
-select j from json2_disable_whole_column_read order by ts;
-
--- Whole JSON2 uses are unsupported (case 2): wildcard projection.
-select * from json2_disable_whole_column_read order by ts;
-
--- Whole JSON2 uses are unsupported (case 3): json_get with an empty path.
-select json_get(j, '') from json2_disable_whole_column_read;
-
-select json_get(j, '$') from json2_disable_whole_column_read;
-
-select json_get(j, '.') from json2_disable_whole_column_read;
-
-select json_get(j, '$.') from json2_disable_whole_column_read;
-
--- Whole JSON2 uses are unsupported (case 4): use in an intermediate plan node.
-select count(*)
-from (
-    select j
-    from json2_disable_whole_column_read
-    group by j
-);
-
 -- JSON2 field projection remains supported (case 5): use in an intermediate plan node.
 select json_get(j, 'a.b'), count(*)
 from json2_disable_whole_column_read
 group by json_get(j, 'a.b')
 order by json_get(j, 'a.b');
 
--- Whole JSON2 uses are unsupported (case 6): output after an intermediate projection.
-select ts, j
-from (
-    select ts, j
-    from json2_disable_whole_column_read
-)
-order by ts;
+select j, j.a from json2_disable_whole_column_read;
 
--- Whole JSON2 uses are unsupported (case 7): DISTINCT in an intermediate plan node.
-select count(*)
-from (
-    select distinct j
-    from json2_disable_whole_column_read
-);
+select j from json2_disable_whole_column_read where j.a.b = 1;
 
 drop table json2_disable_whole_column_read;