mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-05-20 23:10:37 +00:00
@@ -129,7 +129,7 @@ impl From<ColumnDataTypeWrapper> for ConcreteDataType {
|
||||
};
|
||||
ConcreteDataType::json_native_datatype(inner_type.into())
|
||||
}
|
||||
None => ConcreteDataType::Json(JsonType::null()),
|
||||
None => ConcreteDataType::Json(JsonType::new(JsonFormat::Json2)),
|
||||
_ => {
|
||||
// invalid state, type extension is missing or invalid
|
||||
ConcreteDataType::null_datatype()
|
||||
@@ -461,6 +461,7 @@ impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
|
||||
})
|
||||
}
|
||||
}
|
||||
JsonFormat::Json2 => Some(ColumnDataTypeExtension { type_ext: None }),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
|
||||
@@ -306,7 +306,7 @@ pub(crate) fn parse_string_to_value(
|
||||
let v = parse_string_to_jsonb(&s).context(DatatypeSnafu)?;
|
||||
Ok(Value::Binary(v.into()))
|
||||
}
|
||||
JsonFormat::Native(_) => {
|
||||
JsonFormat::Native(_) | JsonFormat::Json2 => {
|
||||
let extension_type: Option<JsonExtensionType> =
|
||||
column_schema.extension_type().context(DatatypeSnafu)?;
|
||||
let json_structure_settings = extension_type
|
||||
|
||||
@@ -18,6 +18,7 @@ use std::str::FromStr;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
|
||||
use arrow::datatypes::DataType as ArrowDataType;
|
||||
use arrow_schema::Fields;
|
||||
use common_base::bytes::Bytes;
|
||||
use regex::{Captures, Regex};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -33,6 +34,7 @@ use crate::type_id::LogicalTypeId;
|
||||
use crate::types::{ListType, StructField, StructType};
|
||||
use crate::value::Value;
|
||||
use crate::vectors::json::builder::JsonVectorBuilder;
|
||||
use crate::vectors::json::builder2::Json2VectorBuilder;
|
||||
use crate::vectors::{BinaryVectorBuilder, MutableVector};
|
||||
|
||||
pub const JSON_TYPE_NAME: &str = "Json";
|
||||
@@ -164,6 +166,7 @@ pub enum JsonFormat {
|
||||
#[default]
|
||||
Jsonb,
|
||||
Native(Box<JsonNativeType>),
|
||||
Json2,
|
||||
}
|
||||
|
||||
/// JsonType is a data type for JSON data. It is stored as binary data of jsonb format.
|
||||
@@ -192,6 +195,7 @@ impl JsonType {
|
||||
match &self.format {
|
||||
JsonFormat::Jsonb => &JsonNativeType::String,
|
||||
JsonFormat::Native(x) => x.as_ref(),
|
||||
JsonFormat::Json2 => unimplemented!(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,15 +216,24 @@ impl JsonType {
|
||||
ConcreteDataType::Struct(t) => t.clone(),
|
||||
x => plain_json_struct_type(x),
|
||||
},
|
||||
JsonFormat::Json2 => unimplemented!(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to merge this json type with others, error on datatype conflict.
|
||||
pub fn merge(&mut self, other: &JsonType) -> Result<()> {
|
||||
self.merge_with(other, false)
|
||||
}
|
||||
|
||||
pub fn merge_with_lifting(&mut self, other: &JsonType) -> Result<()> {
|
||||
self.merge_with(other, true)
|
||||
}
|
||||
|
||||
fn merge_with(&mut self, other: &JsonType, lift: bool) -> Result<()> {
|
||||
match (&self.format, &other.format) {
|
||||
(JsonFormat::Jsonb, JsonFormat::Jsonb) => Ok(()),
|
||||
(JsonFormat::Native(this), JsonFormat::Native(that)) => {
|
||||
let merged = merge(this.as_ref(), that.as_ref())?;
|
||||
let merged = merge(this.as_ref(), that.as_ref(), lift)?;
|
||||
self.format = JsonFormat::Native(Box::new(merged));
|
||||
Ok(())
|
||||
}
|
||||
@@ -313,13 +326,17 @@ fn is_mergeable(this: &JsonNativeType, that: &JsonNativeType) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
fn merge(this: &JsonNativeType, that: &JsonNativeType) -> Result<JsonNativeType> {
|
||||
fn merge_object(this: &JsonObjectType, that: &JsonObjectType) -> Result<JsonObjectType> {
|
||||
fn merge(this: &JsonNativeType, that: &JsonNativeType, lift: bool) -> Result<JsonNativeType> {
|
||||
fn merge_object(
|
||||
this: &JsonObjectType,
|
||||
that: &JsonObjectType,
|
||||
lift: bool,
|
||||
) -> Result<JsonObjectType> {
|
||||
let mut this = this.clone();
|
||||
// merge "that" into "this" directly:
|
||||
for (type_name, that_type) in that {
|
||||
if let Some(this_type) = this.get_mut(type_name) {
|
||||
let merged_type = merge(this_type, that_type)?;
|
||||
let merged_type = merge(this_type, that_type, lift)?;
|
||||
*this_type = merged_type;
|
||||
} else {
|
||||
this.insert(type_name.clone(), that_type.clone());
|
||||
@@ -331,16 +348,22 @@ fn merge(this: &JsonNativeType, that: &JsonNativeType) -> Result<JsonNativeType>
|
||||
match (this, that) {
|
||||
(this, that) if this == that => Ok(this.clone()),
|
||||
(JsonNativeType::Array(this), JsonNativeType::Array(that)) => {
|
||||
merge(this.as_ref(), that.as_ref()).map(|x| JsonNativeType::Array(Box::new(x)))
|
||||
merge(this.as_ref(), that.as_ref(), lift).map(|x| JsonNativeType::Array(Box::new(x)))
|
||||
}
|
||||
(JsonNativeType::Object(this), JsonNativeType::Object(that)) => {
|
||||
merge_object(this, that).map(JsonNativeType::Object)
|
||||
merge_object(this, that, lift).map(JsonNativeType::Object)
|
||||
}
|
||||
(JsonNativeType::Null, x) | (x, JsonNativeType::Null) => Ok(x.clone()),
|
||||
_ => MergeJsonDatatypeSnafu {
|
||||
reason: format!("datatypes have conflict, this: {this}, that: {that}"),
|
||||
_ => {
|
||||
if lift {
|
||||
Ok(JsonNativeType::String)
|
||||
} else {
|
||||
MergeJsonDatatypeSnafu {
|
||||
reason: format!("datatypes have conflict, this: {this}, that: {that}"),
|
||||
}
|
||||
.fail()
|
||||
}
|
||||
}
|
||||
.fail(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -349,6 +372,7 @@ impl DataType for JsonType {
|
||||
match &self.format {
|
||||
JsonFormat::Jsonb => JSON_TYPE_NAME.to_string(),
|
||||
JsonFormat::Native(x) => format!("Json<{x}>"),
|
||||
JsonFormat::Json2 => "JSON2".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -364,6 +388,7 @@ impl DataType for JsonType {
|
||||
match self.format {
|
||||
JsonFormat::Jsonb => ArrowDataType::Binary,
|
||||
JsonFormat::Native(_) => self.as_struct_type().as_arrow_type(),
|
||||
JsonFormat::Json2 => ArrowDataType::Struct(Fields::empty()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -371,6 +396,7 @@ impl DataType for JsonType {
|
||||
match &self.format {
|
||||
JsonFormat::Jsonb => Box::new(BinaryVectorBuilder::with_capacity(capacity)),
|
||||
JsonFormat::Native(x) => Box::new(JsonVectorBuilder::new(*x.clone(), capacity)),
|
||||
JsonFormat::Json2 => Box::new(Json2VectorBuilder::new(JsonNativeType::Null, capacity)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,3 +13,4 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod builder;
|
||||
pub(crate) mod builder2;
|
||||
|
||||
163
src/datatypes/src/vectors/json/builder2.rs
Normal file
163
src/datatypes/src/vectors/json/builder2.rs
Normal file
@@ -0,0 +1,163 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::any::Any;
|
||||
use std::borrow::Cow;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use crate::data_type::ConcreteDataType;
|
||||
use crate::error::{Result, TryFromValueSnafu, UnsupportedOperationSnafu};
|
||||
use crate::json::value::{JsonValue, JsonValueRef, JsonVariant};
|
||||
use crate::prelude::{ValueRef, Vector, VectorRef};
|
||||
use crate::types::JsonType;
|
||||
use crate::types::json_type::JsonNativeType;
|
||||
use crate::vectors::{MutableVector, StructVectorBuilder};
|
||||
|
||||
pub(crate) struct Json2VectorBuilder {
|
||||
merged_type: JsonType,
|
||||
capacity: usize,
|
||||
values: Vec<JsonValue>,
|
||||
}
|
||||
|
||||
impl Json2VectorBuilder {
|
||||
pub(crate) fn new(json_type: JsonNativeType, capacity: usize) -> Self {
|
||||
Self {
|
||||
merged_type: JsonType::new_native(json_type),
|
||||
capacity,
|
||||
values: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
fn build(&self) -> VectorRef {
|
||||
let mut builder = StructVectorBuilder::with_type_and_capacity(
|
||||
self.merged_type.as_struct_type(),
|
||||
self.capacity,
|
||||
);
|
||||
for value in self.values.iter() {
|
||||
let value = align_json_value_with_type(&self.merged_type, value);
|
||||
builder
|
||||
.try_push_value_ref(&(*value).as_ref().as_value_ref())
|
||||
// Safety: after the `align_json_value_with_type`, the values to push must have
|
||||
// the same types with the builder, so it's not expected to meet any errors here.
|
||||
.unwrap_or_else(|e| panic!("Failed to push JSON value {value}: {e:?}"));
|
||||
}
|
||||
builder.to_vector()
|
||||
}
|
||||
}
|
||||
|
||||
impl MutableVector for Json2VectorBuilder {
|
||||
fn data_type(&self) -> ConcreteDataType {
|
||||
ConcreteDataType::Json(self.merged_type.clone())
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
self.values.len()
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn as_mut_any(&mut self) -> &mut dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn to_vector(&mut self) -> VectorRef {
|
||||
self.build()
|
||||
}
|
||||
|
||||
fn to_vector_cloned(&self) -> VectorRef {
|
||||
self.build()
|
||||
}
|
||||
|
||||
fn try_push_value_ref(&mut self, value: &ValueRef) -> Result<()> {
|
||||
let ValueRef::Json(value) = value else {
|
||||
return TryFromValueSnafu {
|
||||
reason: format!("expected json value, got {value:?}"),
|
||||
}
|
||||
.fail();
|
||||
};
|
||||
let json_type = value.json_type();
|
||||
self.merged_type.merge_with_lifting(json_type)?;
|
||||
|
||||
let value = JsonValue::from(value.clone().into_variant());
|
||||
self.values.push(value);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn push_null(&mut self) {
|
||||
static NULL_JSON: LazyLock<ValueRef> =
|
||||
LazyLock::new(|| ValueRef::Json(Box::new(JsonValueRef::null())));
|
||||
self.try_push_value_ref(&NULL_JSON)
|
||||
// Safety: learning from the method "try_push_value_ref", a null json value should be
|
||||
// always able to push into any json vectors.
|
||||
.unwrap_or_else(|e| panic!("failed to push null json value, error: {e}"));
|
||||
}
|
||||
|
||||
fn extend_slice_of(&mut self, _: &dyn Vector, _: usize, _: usize) -> Result<()> {
|
||||
UnsupportedOperationSnafu {
|
||||
op: "extend_slice_of",
|
||||
vector_type: "JsonVector",
|
||||
}
|
||||
.fail()
|
||||
}
|
||||
}
|
||||
|
||||
fn align_json_value_with_type<'a>(
|
||||
expected_type: &JsonType,
|
||||
value: &'a JsonValue,
|
||||
) -> Cow<'a, JsonValue> {
|
||||
if value.json_type() == expected_type {
|
||||
return Cow::Borrowed(value);
|
||||
}
|
||||
|
||||
fn helper(expected_type: &JsonNativeType, value: JsonVariant) -> JsonVariant {
|
||||
match (expected_type, value) {
|
||||
(_, JsonVariant::Null) | (JsonNativeType::Null, _) => JsonVariant::Null,
|
||||
(JsonNativeType::Bool, JsonVariant::Bool(v)) => JsonVariant::Bool(v),
|
||||
(JsonNativeType::Number(_), JsonVariant::Number(v)) => JsonVariant::Number(v),
|
||||
(JsonNativeType::String, JsonVariant::String(v)) => JsonVariant::String(v),
|
||||
|
||||
(JsonNativeType::Array(item_type), JsonVariant::Array(items)) => JsonVariant::Array(
|
||||
items
|
||||
.into_iter()
|
||||
.map(|item| helper(item_type.as_ref(), item))
|
||||
.collect(),
|
||||
),
|
||||
|
||||
(JsonNativeType::Object(expected_fields), JsonVariant::Object(object)) => {
|
||||
JsonVariant::Object(
|
||||
expected_fields
|
||||
.iter()
|
||||
.map(|(field_name, expected_field_type)| {
|
||||
let value =
|
||||
object.get(field_name).cloned().unwrap_or(JsonVariant::Null);
|
||||
(field_name.clone(), helper(expected_field_type, value))
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
(JsonNativeType::String, v) => {
|
||||
let json: serde_json::Value = JsonValue::from(v).into();
|
||||
JsonVariant::String(json.to_string())
|
||||
}
|
||||
|
||||
(t, v) => panic!("unsupported json alignment cast from {v} to {t}"),
|
||||
}
|
||||
}
|
||||
|
||||
let value = helper(expected_type.native_type(), value.clone().into_variant());
|
||||
Cow::Owned(JsonValue::from(value))
|
||||
}
|
||||
@@ -39,6 +39,7 @@ use datatypes::arrow::datatypes::{
|
||||
};
|
||||
use datatypes::arrow_array::BinaryArray;
|
||||
use datatypes::data_type::DataType;
|
||||
use datatypes::extension::json::is_json_extension_type;
|
||||
use datatypes::prelude::{MutableVector, ScalarVectorBuilder, Vector};
|
||||
use datatypes::value::{Value, ValueRef};
|
||||
use datatypes::vectors::Helper;
|
||||
@@ -477,6 +478,10 @@ impl UnorderedPart {
|
||||
self.max_timestamp = i64::MIN;
|
||||
self.max_sequence = 0;
|
||||
}
|
||||
|
||||
pub(crate) fn parts(&self) -> &[BulkPart] {
|
||||
&self.parts
|
||||
}
|
||||
}
|
||||
|
||||
/// More accurate estimation of the size of a record batch.
|
||||
@@ -693,7 +698,8 @@ impl BulkPartConverter {
|
||||
columns.push(values.sequence.to_arrow_array());
|
||||
columns.push(values.op_type.to_arrow_array());
|
||||
|
||||
let batch = RecordBatch::try_new(self.schema, columns).context(NewRecordBatchSnafu)?;
|
||||
let schema = align_schema_with_json_array(self.schema, &columns);
|
||||
let batch = RecordBatch::try_new(schema, columns).context(NewRecordBatchSnafu)?;
|
||||
// Sorts the record batch.
|
||||
let batch = sort_primary_key_record_batch(&batch)?;
|
||||
|
||||
@@ -708,6 +714,26 @@ impl BulkPartConverter {
|
||||
}
|
||||
}
|
||||
|
||||
fn align_schema_with_json_array(schema: SchemaRef, columns: &[ArrayRef]) -> SchemaRef {
|
||||
if schema.fields().iter().all(|f| !is_json_extension_type(f)) {
|
||||
return schema;
|
||||
}
|
||||
|
||||
let mut fields = Vec::with_capacity(schema.fields().len());
|
||||
for (field, array) in schema.fields().iter().zip(columns) {
|
||||
if !is_json_extension_type(field) {
|
||||
fields.push(field.clone());
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut field = field.as_ref().clone();
|
||||
field.set_data_type(array.data_type().clone());
|
||||
fields.push(Arc::new(field));
|
||||
}
|
||||
|
||||
Arc::new(Schema::new_with_metadata(fields, schema.metadata().clone()))
|
||||
}
|
||||
|
||||
fn new_primary_key_column_builders(
|
||||
metadata: &RegionMetadata,
|
||||
capacity: usize,
|
||||
|
||||
@@ -917,7 +917,9 @@ impl ValueBuilder {
|
||||
size += field_value.data_size();
|
||||
if !field_value.is_null() || self.fields[idx].is_some() {
|
||||
if let Some(field) = self.fields[idx].as_mut() {
|
||||
let _ = field.push(field_value);
|
||||
field
|
||||
.push(field_value)
|
||||
.unwrap_or_else(|e| panic!("Failed to push field value: {e:?}"));
|
||||
} else {
|
||||
let mut mutable_vector =
|
||||
if let ConcreteDataType::String(_) = &self.field_types[idx] {
|
||||
|
||||
@@ -301,12 +301,22 @@ impl<'a, 'b> JsonColumnTypeUpdater<'a, 'b> {
|
||||
.or_insert_with(|| value_type.clone());
|
||||
|
||||
if !merged_type.is_include(&value_type) {
|
||||
merged_type.merge(&value_type).map_err(|e| {
|
||||
if column_schema
|
||||
.data_type
|
||||
.as_json()
|
||||
.map(|x| x.is_native_type())
|
||||
.unwrap_or(false)
|
||||
{
|
||||
merged_type.merge(&value_type)
|
||||
} else {
|
||||
merged_type.merge_with_lifting(&value_type)
|
||||
}
|
||||
.map_err(|e| {
|
||||
InvalidInsertRequestSnafu {
|
||||
reason: format!(r#"cannot merge "{value_type}" into "{merged_type}": {e}"#),
|
||||
}
|
||||
.build()
|
||||
})?;
|
||||
})?
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -323,7 +333,17 @@ impl<'a, 'b> JsonColumnTypeUpdater<'a, 'b> {
|
||||
for (column_name, merged_type) in self.merged_value_types.iter() {
|
||||
let Some(column_type) = insert_columns
|
||||
.iter()
|
||||
.find_map(|x| (&x.name == column_name).then(|| x.data_type.as_json()))
|
||||
.find_map(|x| {
|
||||
(&x.name == column_name).then(|| {
|
||||
if let ConcreteDataType::Json(t) = &x.data_type
|
||||
&& t.is_native_type()
|
||||
{
|
||||
Some(t)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
})
|
||||
.flatten()
|
||||
else {
|
||||
continue;
|
||||
|
||||
@@ -153,7 +153,16 @@ pub fn column_to_schema(
|
||||
|
||||
column_schema.set_inverted_index(column.extensions.inverted_index_options.is_some());
|
||||
|
||||
if matches!(column.data_type(), SqlDataType::JSON) {
|
||||
let is_json2_column = if let SqlDataType::Custom(object_name, _) = column.data_type() {
|
||||
object_name
|
||||
.0
|
||||
.first()
|
||||
.map(|x| x.to_string_unquoted().eq_ignore_ascii_case("JSON2"))
|
||||
.unwrap_or_default()
|
||||
} else {
|
||||
false
|
||||
};
|
||||
if is_json2_column || matches!(column.data_type(), SqlDataType::JSON) {
|
||||
let settings = column
|
||||
.extensions
|
||||
.build_json_structure_settings()?
|
||||
@@ -290,22 +299,25 @@ pub fn sql_data_type_to_concrete_data_type(
|
||||
};
|
||||
Ok(ConcreteDataType::Json(JsonType::new(format)))
|
||||
}
|
||||
// Vector type
|
||||
SqlDataType::Custom(name, d)
|
||||
if name.0.as_slice().len() == 1
|
||||
&& name.0.as_slice()[0]
|
||||
.to_string_unquoted()
|
||||
.to_ascii_uppercase()
|
||||
== VECTOR_TYPE_NAME
|
||||
&& d.len() == 1 =>
|
||||
{
|
||||
let dim = d[0].parse().map_err(|e| {
|
||||
error::ParseSqlValueSnafu {
|
||||
msg: format!("Failed to parse vector dimension: {}", e),
|
||||
// Vector type and JSON2 type
|
||||
SqlDataType::Custom(name, d) if name.0.len() == 1 => {
|
||||
let name = name.0[0].to_string_unquoted().to_ascii_uppercase();
|
||||
match name.as_str() {
|
||||
VECTOR_TYPE_NAME if d.len() == 1 => {
|
||||
let dim = d[0].parse().map_err(|e| {
|
||||
error::ParseSqlValueSnafu {
|
||||
msg: format!(r#"Failed to parse vector dimension "{}": {}"#, d[0], e),
|
||||
}
|
||||
.build()
|
||||
})?;
|
||||
Ok(ConcreteDataType::vector_datatype(dim))
|
||||
}
|
||||
.build()
|
||||
})?;
|
||||
Ok(ConcreteDataType::vector_datatype(dim))
|
||||
"JSON2" => Ok(ConcreteDataType::Json(JsonType::new(JsonFormat::Json2))),
|
||||
_ => error::SqlTypeNotSupportedSnafu {
|
||||
t: data_type.clone(),
|
||||
}
|
||||
.fail(),
|
||||
}
|
||||
}
|
||||
_ => error::SqlTypeNotSupportedSnafu {
|
||||
t: data_type.clone(),
|
||||
|
||||
@@ -376,32 +376,35 @@ impl ColumnExtensions {
|
||||
None
|
||||
};
|
||||
|
||||
options
|
||||
let format = options
|
||||
.get(JSON_OPT_FORMAT)
|
||||
.map(|format| match format {
|
||||
JSON_FORMAT_FULL_STRUCTURED => Ok(JsonStructureSettings::Structured(fields)),
|
||||
JSON_FORMAT_PARTIAL => {
|
||||
let fields = fields.map(|fields| {
|
||||
let mut fields = Arc::unwrap_or_clone(fields.fields());
|
||||
fields.push(datatypes::types::StructField::new(
|
||||
JsonStructureSettings::RAW_FIELD.to_string(),
|
||||
ConcreteDataType::string_datatype(),
|
||||
true,
|
||||
));
|
||||
StructType::new(Arc::new(fields))
|
||||
});
|
||||
Ok(JsonStructureSettings::PartialUnstructuredByKey {
|
||||
fields,
|
||||
unstructured_keys,
|
||||
})
|
||||
.unwrap_or(JSON_FORMAT_FULL_STRUCTURED);
|
||||
let settings = match format {
|
||||
JSON_FORMAT_FULL_STRUCTURED => JsonStructureSettings::Structured(fields),
|
||||
JSON_FORMAT_PARTIAL => {
|
||||
let fields = fields.map(|fields| {
|
||||
let mut fields = Arc::unwrap_or_clone(fields.fields());
|
||||
fields.push(datatypes::types::StructField::new(
|
||||
JsonStructureSettings::RAW_FIELD.to_string(),
|
||||
ConcreteDataType::string_datatype(),
|
||||
true,
|
||||
));
|
||||
StructType::new(Arc::new(fields))
|
||||
});
|
||||
JsonStructureSettings::PartialUnstructuredByKey {
|
||||
fields,
|
||||
unstructured_keys,
|
||||
}
|
||||
JSON_FORMAT_RAW => Ok(JsonStructureSettings::UnstructuredRaw),
|
||||
_ => InvalidSqlSnafu {
|
||||
}
|
||||
JSON_FORMAT_RAW => JsonStructureSettings::UnstructuredRaw,
|
||||
_ => {
|
||||
return InvalidSqlSnafu {
|
||||
msg: format!("unknown JSON datatype 'format': {format}"),
|
||||
}
|
||||
.fail(),
|
||||
})
|
||||
.transpose()
|
||||
.fail();
|
||||
}
|
||||
};
|
||||
Ok(Some(settings))
|
||||
}
|
||||
|
||||
pub fn set_json_structure_settings(&mut self, settings: JsonStructureSettings) {
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
CREATE TABLE t (ts TIMESTAMP TIME INDEX, j JSON(format = "structured") DEFAULT '{"foo": "bar"}');
|
||||
|
||||
Error: 1001(Unsupported), Unsupported default constraint for column: 'j', reason: json column cannot have a default value
|
||||
|
||||
CREATE TABLE t (ts TIMESTAMP TIME INDEX, j JSON(format = "structured"));
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
DESC TABLE t;
|
||||
|
||||
+--------+----------------------+-----+------+---------+---------------+
|
||||
| Column | Type | Key | Null | Default | Semantic Type |
|
||||
+--------+----------------------+-----+------+---------+---------------+
|
||||
| ts | TimestampMillisecond | PRI | NO | | TIMESTAMP |
|
||||
| j | Json<"<Null>"> | | YES | | FIELD |
|
||||
+--------+----------------------+-----+------+---------+---------------+
|
||||
|
||||
INSERT INTO t VALUES
|
||||
(1762128001000, '{"int": 1}'),
|
||||
(1762128002000, '{"int": 2, "list": [0.1, 0.2, 0.3]}'),
|
||||
(1762128003000, '{"int": 3, "list": [0.4, 0.5, 0.6], "nested": {"a": {"x": "hello"}, "b": {"y": -1}}}');
|
||||
|
||||
Affected Rows: 3
|
||||
|
||||
DESC TABLE t;
|
||||
|
||||
+--------+---------------------------------------------------------------------------------------------------+-----+------+---------+---------------+
|
||||
| Column | Type | Key | Null | Default | Semantic Type |
|
||||
+--------+---------------------------------------------------------------------------------------------------+-----+------+---------+---------------+
|
||||
| ts | TimestampMillisecond | PRI | NO | | TIMESTAMP |
|
||||
| j | Json<{"int":"<Number>","list":["<Number>"],"nested":{"a":{"x":"<String>"},"b":{"y":"<Number>"}}}> | | YES | | FIELD |
|
||||
+--------+---------------------------------------------------------------------------------------------------+-----+------+---------+---------------+
|
||||
|
||||
INSERT INTO t VALUES
|
||||
(1762128004000, '{"int": 4, "bool": true, "nested": {"a": {"y": 1}}}'),
|
||||
(1762128005000, '{"int": 5, "bool": false, "nested": {"b": {"x": "world"}}}');
|
||||
|
||||
Affected Rows: 2
|
||||
|
||||
DESC TABLE t;
|
||||
|
||||
+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+-----+------+---------+---------------+
|
||||
| Column | Type | Key | Null | Default | Semantic Type |
|
||||
+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+-----+------+---------+---------------+
|
||||
| ts | TimestampMillisecond | PRI | NO | | TIMESTAMP |
|
||||
| j | Json<{"bool":"<Bool>","int":"<Number>","list":["<Number>"],"nested":{"a":{"x":"<String>","y":"<Number>"},"b":{"x":"<String>","y":"<Number>"}}}> | | YES | | FIELD |
|
||||
+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+-----+------+---------+---------------+
|
||||
|
||||
INSERT INTO t VALUES (1762128006000, '{"int": 6, "list": [-6.0], "bool": true, "nested": {"a": {"x": "ax", "y": 66}, "b": {"y": -66, "x": "bx"}}}');
|
||||
|
||||
Affected Rows: 1
|
||||
|
||||
DESC TABLE t;
|
||||
|
||||
+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+-----+------+---------+---------------+
|
||||
| Column | Type | Key | Null | Default | Semantic Type |
|
||||
+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+-----+------+---------+---------------+
|
||||
| ts | TimestampMillisecond | PRI | NO | | TIMESTAMP |
|
||||
| j | Json<{"bool":"<Bool>","int":"<Number>","list":["<Number>"],"nested":{"a":{"x":"<String>","y":"<Number>"},"b":{"x":"<String>","y":"<Number>"}}}> | | YES | | FIELD |
|
||||
+--------+-------------------------------------------------------------------------------------------------------------------------------------------------+-----+------+---------+---------------+
|
||||
|
||||
INSERT INTO t VALUES (1762128011000, '{}');
|
||||
|
||||
Error: 1004(InvalidArguments), Invalid InsertRequest, reason: empty json object is not supported, consider adding a dummy field
|
||||
|
||||
SELECT ts, j FROM t order by ts;
|
||||
|
||||
+---------------------+----------------------------------------------------------------------------------------+
|
||||
| ts | j |
|
||||
+---------------------+----------------------------------------------------------------------------------------+
|
||||
| 2025-11-03T00:00:01 | {bool: , int: 1, list: , nested: } |
|
||||
| 2025-11-03T00:00:02 | {bool: , int: 2, list: [0.1, 0.2, 0.3], nested: } |
|
||||
| 2025-11-03T00:00:03 | {bool: , int: 3, list: [0.4, 0.5, 0.6], nested: {a: {x: hello, y: }, b: {x: , y: -1}}} |
|
||||
| 2025-11-03T00:00:04 | {bool: true, int: 4, list: , nested: {a: {x: , y: 1}, b: }} |
|
||||
| 2025-11-03T00:00:05 | {bool: false, int: 5, list: , nested: {a: , b: {x: world, y: }}} |
|
||||
| 2025-11-03T00:00:06 | {bool: true, int: 6, list: [-6.0], nested: {a: {x: ax, y: 66}, b: {x: bx, y: -66}}} |
|
||||
+---------------------+----------------------------------------------------------------------------------------+
|
||||
|
||||
DROP table t;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
CREATE TABLE t (ts TIMESTAMP TIME INDEX, j JSON(format = "structured") DEFAULT '{"foo": "bar"}');
|
||||
|
||||
CREATE TABLE t (ts TIMESTAMP TIME INDEX, j JSON(format = "structured"));
|
||||
|
||||
DESC TABLE t;
|
||||
|
||||
INSERT INTO t VALUES
|
||||
(1762128001000, '{"int": 1}'),
|
||||
(1762128002000, '{"int": 2, "list": [0.1, 0.2, 0.3]}'),
|
||||
(1762128003000, '{"int": 3, "list": [0.4, 0.5, 0.6], "nested": {"a": {"x": "hello"}, "b": {"y": -1}}}');
|
||||
|
||||
DESC TABLE t;
|
||||
|
||||
INSERT INTO t VALUES
|
||||
(1762128004000, '{"int": 4, "bool": true, "nested": {"a": {"y": 1}}}'),
|
||||
(1762128005000, '{"int": 5, "bool": false, "nested": {"b": {"x": "world"}}}');
|
||||
|
||||
DESC TABLE t;
|
||||
|
||||
INSERT INTO t VALUES (1762128006000, '{"int": 6, "list": [-6.0], "bool": true, "nested": {"a": {"x": "ax", "y": 66}, "b": {"y": -66, "x": "bx"}}}');
|
||||
|
||||
DESC TABLE t;
|
||||
|
||||
INSERT INTO t VALUES (1762128011000, '{}');
|
||||
|
||||
SELECT ts, j FROM t order by ts;
|
||||
|
||||
DROP table t;
|
||||
34
tests/cases/standalone/common/types/json/json2.result
Normal file
34
tests/cases/standalone/common/types/json/json2.result
Normal file
@@ -0,0 +1,34 @@
|
||||
create table foo
|
||||
(
|
||||
ts timestamp time index,
|
||||
j json2
|
||||
);
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
insert into foo (ts, j)
|
||||
values (1, '{"a": {"b": 1}, "c": "s1"}'),
|
||||
(2, '{"a": {"b": 2}, "c": "s2"}'),
|
||||
(3, '{"a": {"b": 3}, "c": "s3"}');
|
||||
|
||||
Affected Rows: 3
|
||||
|
||||
insert into foo
|
||||
values (4, '{"a": {"b": 4}}'),
|
||||
(5, '{"a": {}, "c": "s5"}'),
|
||||
(6, '{"c": "s6"}');
|
||||
|
||||
Affected Rows: 3
|
||||
|
||||
insert into foo
|
||||
values (7, '{"a": {"b": "s7"}, "c": [1]}'),
|
||||
(8, '{"a": {"b": 8}, "c": "s8"}');
|
||||
|
||||
Affected Rows: 2
|
||||
|
||||
insert into foo
|
||||
values (9, '{"a": {"x": true}, "c": "s9"}'),
|
||||
(10, '{"a": {"b": 10}, "y": false}');
|
||||
|
||||
Affected Rows: 2
|
||||
|
||||
23
tests/cases/standalone/common/types/json/json2.sql
Normal file
23
tests/cases/standalone/common/types/json/json2.sql
Normal file
@@ -0,0 +1,23 @@
|
||||
create table foo
|
||||
(
|
||||
ts timestamp time index,
|
||||
j json2
|
||||
);
|
||||
|
||||
insert into foo (ts, j)
|
||||
values (1, '{"a": {"b": 1}, "c": "s1"}'),
|
||||
(2, '{"a": {"b": 2}, "c": "s2"}'),
|
||||
(3, '{"a": {"b": 3}, "c": "s3"}');
|
||||
|
||||
insert into foo
|
||||
values (4, '{"a": {"b": 4}}'),
|
||||
(5, '{"a": {}, "c": "s5"}'),
|
||||
(6, '{"c": "s6"}');
|
||||
|
||||
insert into foo
|
||||
values (7, '{"a": {"b": "s7"}, "c": [1]}'),
|
||||
(8, '{"a": {"b": 8}, "c": "s8"}');
|
||||
|
||||
insert into foo
|
||||
values (9, '{"a": {"x": true}, "c": "s9"}'),
|
||||
(10, '{"a": {"b": 10}, "y": false}');
|
||||
Reference in New Issue
Block a user