rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-07-04 13:00:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(json2): type hint (#8247)

Browse Source 
* feat(json2): type hint

* test(datatypes): add JsonSettings serde round-trip test

* minor refactor

This reverts commit 7ff5a5249a09be5396536284fe822b5761ef4e6a.

* fix: code review
This commit is contained in:
fys
2026-06-30 15:37:31 +08:00
committed by GitHub
parent d3cc1b1888
commit aa563628fb
26 changed files with 1756 additions and 2216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/api/src/helper.rs
View File

@@ -451,10 +451,10 @@ impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
if native_type.is_null() {
None
} else {
let native_type =
let concrete_type =
ConcreteDataType::from_arrow_type(&native_type.as_arrow_type());
let (datatype, datatype_extension) =
ColumnDataTypeWrapper::try_from(native_type)?.into_parts();
ColumnDataTypeWrapper::try_from(concrete_type)?.into_parts();
Some(ColumnDataTypeExtension {
type_ext: Some(TypeExt::JsonNativeType(Box::new(
JsonNativeTypeExtension {

6
src/common/sql/src/convert.rs
View File

@@ -309,11 +309,11 @@ pub(crate) fn parse_string_to_value(
JsonFormat::Json2(_) => {
let extension_type: Option<JsonExtensionType> =
column_schema.extension_type().context(DatatypeSnafu)?;
let json_structure_settings = extension_type
.and_then(|x| x.metadata().json_structure_settings.clone())
let json_settings = extension_type
.and_then(|x| x.metadata().json_settings.clone())
.unwrap_or_default();
let v = serde_json::from_str(&s).context(DeserializeSnafu { json: s })?;
json_structure_settings.encode(v).context(DatatypeSnafu)
json_settings.encode(v).context(DatatypeSnafu)
}
},
ConcreteDataType::Vector(d) => {

9
src/datatypes/src/extension/json.rs
View File

@@ -18,14 +18,13 @@ use arrow_schema::extension::ExtensionType;
use arrow_schema::{ArrowError, DataType, FieldRef};
use serde::{Deserialize, Serialize};
use crate::json::JsonStructureSettings;
use crate::json::JsonSettings;
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct JsonMetadata {
/// Indicates how to handle JSON is stored in underlying data type
///
/// This field can be `None` for data is converted to complete structured in-memory form.
pub json_structure_settings: Option<JsonStructureSettings>,
/// JSON2 settings stored in column schema metadata and represented through
/// Arrow extension metadata.
pub json_settings: Option<JsonSettings>,
}
#[derive(Debug, Clone)]

2275
src/datatypes/src/json.rs
View File

File diff suppressed because it is too large Load Diff

9
src/datatypes/src/json/value.rs
View File

@@ -144,13 +144,6 @@ impl JsonVariant {
}
}
pub(crate) fn as_f64(&self) -> Option<f64> {
match self {
JsonVariant::Number(n) => Some(n.as_f64()),
_ => None,
}
}
pub(crate) fn native_type(&self) -> JsonNativeType {
match self {
JsonVariant::Null => JsonNativeType::Null,
@@ -513,6 +506,7 @@ impl JsonValue {
}
let x = std::mem::take(&mut self.json_variant);
self.json_variant = helper(x, expected.native_type())?;
self.json_type = OnceLock::new();
Ok(())
@@ -650,6 +644,7 @@ where
Some(t) => t,
None => return JsonNativeType::Array(Box::new(JsonNativeType::Null)),
};
for x in iter {
if matches!(item_type, JsonNativeType::Variant) {
break;

38
src/datatypes/src/types/json_type.rs
View File

@@ -168,7 +168,7 @@ impl From<&ConcreteDataType> for JsonNativeType {
ConcreteDataType::Float64(_) | ConcreteDataType::Float32(_) => JsonNativeType::f64(),
ConcreteDataType::String(_) => JsonNativeType::String,
ConcreteDataType::List(list_type) => {
JsonNativeType::Array(Box::new(list_type.item_type().into()))
JsonNativeType::Array(Box::new(JsonNativeType::from(list_type.item_type())))
}
ConcreteDataType::Struct(struct_type) => JsonNativeType::Object(
struct_type
@@ -243,9 +243,7 @@ impl Display for JsonNativeType {
write!(f, r#""<Number>""#)
}
JsonNativeType::String => write!(f, r#""<String>""#),
JsonNativeType::Array(item_type) => {
write!(f, "[{}]", item_type)
}
JsonNativeType::Array(item_type) => write!(f, "[{}]", item_type),
JsonNativeType::Object(object) => {
write!(
f,
@@ -342,9 +340,7 @@ impl JsonType {
pub fn is_include(&self, other: &JsonType) -> bool {
match (&self.format, &other.format) {
(JsonFormat::Jsonb, JsonFormat::Jsonb) => true,
(JsonFormat::Json2(this), JsonFormat::Json2(that)) => {
is_include(this.as_ref(), that.as_ref())
}
(JsonFormat::Json2(this), JsonFormat::Json2(that)) => is_include(this, that),
_ => false,
}
}
@@ -365,9 +361,7 @@ fn is_include(this: &JsonNativeType, that: &JsonNativeType) -> bool {
match (this, that) {
(this, that) if this == that => true,
(JsonNativeType::Array(this), JsonNativeType::Array(that)) => {
is_include(this.as_ref(), that.as_ref())
}
(JsonNativeType::Array(this), JsonNativeType::Array(that)) => is_include(this, that),
(JsonNativeType::Object(this), JsonNativeType::Object(that)) => {
is_include_object(this, that)
}
@@ -398,14 +392,9 @@ impl DataType for JsonType {
fn name(&self) -> String {
match &self.format {
JsonFormat::Jsonb => JSON_TYPE_NAME.to_string(),
JsonFormat::Json2(x) => format!(
"{JSON2_TYPE_NAME}{}",
if x.is_null() {
"".to_string()
} else {
x.to_string()
}
),
JsonFormat::Json2(ty) => {
format!("{JSON2_TYPE_NAME}{}", ty)
}
}
}
@@ -427,7 +416,7 @@ impl DataType for JsonType {
fn create_mutable_vector(&self, capacity: usize) -> Box<dyn MutableVector> {
match &self.format {
JsonFormat::Jsonb => Box::new(BinaryVectorBuilder::with_capacity(capacity)),
JsonFormat::Json2(x) => Box::new(JsonVectorBuilder::new(*x.clone(), capacity)),
JsonFormat::Json2(x) => Box::new(JsonVectorBuilder::new(x.as_ref().clone(), capacity)),
}
}
@@ -528,7 +517,7 @@ pub fn parse_string_to_jsonb(s: &str) -> Result<Vec<u8>> {
#[cfg(test)]
mod tests {
use super::*;
use crate::json::JsonStructureSettings;
use crate::json::JsonSettings;
#[test]
fn test_fix_unicode_point() -> Result<()> {
@@ -585,13 +574,13 @@ mod tests {
#[test]
fn test_json_type_include() {
fn test(this: &JsonNativeType, that: &JsonNativeType, expected: bool) {
assert_eq!(is_include(this, that), expected);
assert_eq!(is_include(this, that), expected, "this={this}, that={that}");
}
test(&JsonNativeType::Null, &JsonNativeType::Null, true);
test(&JsonNativeType::Null, &JsonNativeType::Bool, false);
test(&JsonNativeType::Bool, &JsonNativeType::Null, true);
test(&JsonNativeType::Bool, &JsonNativeType::Bool, true);
test(&JsonNativeType::Bool, &JsonNativeType::u64(), false);
@@ -637,7 +626,6 @@ mod tests {
"foo".to_string(),
JsonNativeType::String,
)]));
test(simple_json_object, &JsonNativeType::Null, true);
test(simple_json_object, simple_json_object, true);
test(simple_json_object, &JsonNativeType::i64(), false);
test(
@@ -665,7 +653,7 @@ mod tests {
),
("bar".to_string(), JsonNativeType::i64()),
]));
test(complex_json_object, &JsonNativeType::Null, true);
test(simple_json_object, &JsonNativeType::Null, true);
test(complex_json_object, &JsonNativeType::String, false);
test(complex_json_object, complex_json_object, true);
test(
@@ -789,7 +777,7 @@ mod tests {
) -> Result<()> {
let json: serde_json::Value = serde_json::from_str(json).unwrap();
let settings = JsonStructureSettings::Structured(None);
let settings = JsonSettings::default();
let value = settings.encode(json)?;
let value_type = value.data_type();
let Some(other) = value_type.as_json() else {

31
src/datatypes/src/vectors/json/builder.rs
View File

@@ -29,9 +29,9 @@ pub(crate) struct JsonVectorBuilder {
}
impl JsonVectorBuilder {
pub(crate) fn new(json_type: JsonNativeType, capacity: usize) -> Self {
pub(crate) fn new(initial_native_type: JsonNativeType, capacity: usize) -> Self {
Self {
merged_type: JsonType::new_json2(json_type),
merged_type: JsonType::new_json2(initial_native_type),
values: Vec::with_capacity(capacity),
}
}
@@ -173,6 +173,33 @@ mod tests {
))
);
// A Null initial type represents an unknown JSON2 runtime type. The first
// non-null value should set the concrete type instead of aligning all rows to Null.
let mut inferred_builder = JsonVectorBuilder::new(JsonNativeType::Null, 2);
let inferred_value = parse_json_value(r#"{"id":3}"#);
inferred_builder.push_null();
inferred_builder.try_push_value_ref(&inferred_value.as_value_ref())?;
let inferred_type = JsonType::new_json2(JsonNativeType::Object(JsonObjectType::from([(
"id".to_string(),
JsonNativeType::i64(),
)])));
assert_eq!(
inferred_builder.data_type(),
ConcreteDataType::Json(inferred_type.clone())
);
let inferred_struct_type = inferred_type.as_struct_type();
let vector = inferred_builder.to_vector();
assert_eq!(vector.get(0), Value::Null);
assert_eq!(
vector.get(1),
Value::Struct(StructValue::new(
vec![Value::Int64(3)],
inferred_struct_type,
))
);
// Root-level conflicts should be lifted to a plain Variant field that preserves
// each original JSON payload.
let mut variant_builder = JsonVectorBuilder::new(JsonNativeType::Bool, 2);

3
src/operator/src/expr_helper.rs
View File

@@ -806,8 +806,7 @@ pub(crate) fn to_alter_table_expr(
target_type,
} => {
let target_type =
sql_data_type_to_concrete_data_type(&target_type, &Default::default())
.context(ParseSqlSnafu)?;
sql_data_type_to_concrete_data_type(&target_type).context(ParseSqlSnafu)?;
let (target_type, target_type_extension) = ColumnDataTypeWrapper::try_from(target_type)
.map(|w| w.to_parts())
.context(ColumnDataTypeSnafu)?;

2
src/pipeline/src/etl/transform/transformer/greptime.rs
View File

@@ -707,7 +707,7 @@ fn resolve_value(
None
};
let settings = json_extension_type
.and_then(|x| x.metadata().json_structure_settings.clone())
.and_then(|x| x.metadata().json_settings.clone())
.unwrap_or_default();
let value: serde_json::Value = value.try_into().map_err(|e: StdError| {
CoerceIncompatibleTypesSnafu { msg: e.to_string() }.build()

24
src/query/src/sql.rs
View File

@@ -1082,11 +1082,18 @@ fn describe_column_types(columns_schemas: &[ColumnSchema]) -> VectorRef {
Arc::new(StringVector::from(
columns_schemas
.iter()
.map(|cs| cs.data_type.name())
.map(|cs| describe_column_type_name(&cs.data_type))
.collect::<Vec<_>>(),
))
}
fn describe_column_type_name(data_type: &ConcreteDataType) -> String {
match data_type {
ConcreteDataType::Json(json_type) if json_type.is_json2() => "Json2".to_string(),
data_type => data_type.name(),
}
}
fn describe_column_keys(
columns_schemas: &[ColumnSchema],
primary_key_indices: &[usize],
@@ -1340,6 +1347,7 @@ mod test {
use common_time::timestamp::TimeUnit;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema, Schema, SchemaRef};
use datatypes::types::json_type::JsonNativeType;
use datatypes::vectors::{StringVector, TimestampMillisecondVector, UInt32Vector, VectorRef};
use session::context::QueryContextBuilder;
use snafu::ResultExt;
@@ -1348,7 +1356,7 @@ mod test {
use table::TableRef;
use table::test_util::MemTable;
use super::show_variable;
use super::{describe_column_type_name, show_variable};
use crate::error;
use crate::error::Result;
use crate::sql::{
@@ -1391,6 +1399,18 @@ mod test {
describe_table_test_by_schema(table_name, schema, data, expected_columns)
}
#[test]
fn test_describe_column_type_name_json2() {
assert_eq!(
describe_column_type_name(&ConcreteDataType::json2(JsonNativeType::Null)),
"Json2"
);
assert_eq!(
describe_column_type_name(&ConcreteDataType::uint32_datatype()),
"UInt32"
);
}
fn describe_table_test_by_schema(
table_name: &str,
schema: Vec<ColumnSchema>,

38
src/query/src/sql/show_create_table.rs
View File

@@ -29,12 +29,13 @@ use datatypes::schema::{
COLUMN_VECTOR_INDEX_OPT_KEY_METRIC, COMMENT_KEY, ColumnDefaultConstraint, ColumnSchema,
FulltextBackend, SchemaRef,
};
use datatypes::types::JsonFormat;
use snafu::ResultExt;
use sql::ast::{ColumnDef, ColumnOption, ColumnOptionDef, Expr, Ident, ObjectName};
use sql::ast::{ColumnDef, ColumnOption, ColumnOptionDef, DataType, Expr, Ident, ObjectName};
use sql::dialect::GreptimeDbDialect;
use sql::parser::ParserContext;
use sql::statements::create::{Column, ColumnExtensions, CreateTable, TableConstraint};
use sql::statements::{self, OptionMap};
use sql::statements::{self, OptionMap, concrete_data_type_to_sql_data_type};
use store_api::metric_engine_consts::{is_metric_engine, is_metric_engine_internal_column};
use table::metadata::{TableInfoRef, TableMeta};
use table::requests::{
@@ -197,27 +198,32 @@ fn create_column(column_schema: &ColumnSchema, quote_style: char) -> Result<Colu
extensions.inverted_index_options = Some(HashMap::new().into());
}
if column_schema
.data_type
.as_json()
.is_some_and(|json_type| json_type.is_json2())
&& let Some(json_extension) = column_schema.extension_type::<JsonExtensionType>()?
{
let mut data_type = concrete_data_type_to_sql_data_type(&column_schema.data_type)
.with_context(|_| ConvertSqlTypeSnafu {
datatype: column_schema.data_type.clone(),
})?;
if matches!(
&column_schema.data_type,
datatypes::data_type::ConcreteDataType::Json(json_type)
if matches!(json_type.format, JsonFormat::Json2(_))
) {
data_type = DataType::Custom(ObjectName::from(vec![Ident::new("JSON2")]), vec![]);
}
if let Some(json_extension) = column_schema.extension_type::<JsonExtensionType>()? {
let settings = json_extension
.metadata()
.json_structure_settings
.json_settings
.clone()
.unwrap_or_default();
extensions.set_json_structure_settings(settings);
extensions.set_json_settings(settings).context(SqlSnafu)?;
}
Ok(Column {
column_def: ColumnDef {
name: Ident::with_quote(quote_style, name),
data_type: statements::concrete_data_type_to_sql_data_type(&column_schema.data_type)
.with_context(|_| ConvertSqlTypeSnafu {
datatype: column_schema.data_type.clone(),
})?,
data_type,
options,
},
extensions,
@@ -429,9 +435,7 @@ WITH(
let mut json_column = ColumnSchema::new("j", ConcreteDataType::json_datatype(), true);
json_column
.with_extension_type(&JsonExtensionType::new(Arc::new(
datatypes::extension::json::JsonMetadata {
json_structure_settings: Some(datatypes::json::JsonStructureSettings::default()),
},
datatypes::extension::json::JsonMetadata::default(),
)))
.unwrap();

6
src/servers/src/postgres/types.rs
View File

@@ -30,7 +30,7 @@ use common_time::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth};
use datafusion_common::ScalarValue;
use datafusion_expr::LogicalPlan;
use datatypes::arrow::datatypes::DataType as ArrowDataType;
use datatypes::json::JsonStructureSettings;
use datatypes::json::JsonSettings;
use datatypes::prelude::{ConcreteDataType, Value};
use datatypes::schema::{Schema, SchemaRef};
use datatypes::types::{Decimal128Type, IntervalType, TimestampType, jsonb_to_string};
@@ -81,7 +81,7 @@ pub(super) fn schema_to_pg(
/// this function will encode greptime's `StructValue` into PostgreSQL jsonb type
///
/// Note that greptimedb has different types of StructValue for storing json data,
/// based on policy defined in `JsonStructureSettings`. But here the `StructValue`
/// based on policy defined in `JsonSettings`. But here the `StructValue`
/// should be fully structured.
///
/// there are alternatives like records, arrays, etc. but there are also limitations:
@@ -93,7 +93,7 @@ fn encode_struct<S: Encoder>(
builder: &mut S,
pg_field: &FieldInfo,
) -> PgWireResult<()> {
let encoding_setting = JsonStructureSettings::Structured(None);
let encoding_setting = JsonSettings::default();
let json_value = encoding_setting
.decode(Value::Struct(struct_value))
.map_err(|e| PgWireError::ApiError(Box::new(e)))?;

15
src/sql/src/error.rs
View File

@@ -215,13 +215,6 @@ pub enum Error {
location: Location,
},
#[snafu(display("Invalid JSON structure setting, reason: {reason}"))]
InvalidJsonStructureSetting {
reason: String,
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Failed to serialize column default constraint"))]
SerializeColumnDefaultConstraint {
#[snafu(implicit)]
@@ -348,7 +341,7 @@ pub enum Error {
},
#[snafu(display("Failed to set JSON structure settings: {value}"))]
SetJsonStructureSettings {
SetJsonSettings {
value: String,
source: datatypes::error::Error,
#[snafu(implicit)]
@@ -381,7 +374,6 @@ impl ErrorExt for Error {
InvalidColumnOption { .. }
| InvalidExprAsOptionValue { .. }
| InvalidJsonStructureSetting { .. }
| InvalidDatabaseName { .. }
| InvalidDatabaseOption { .. }
| ColumnTypeMismatch { .. }
@@ -400,8 +392,9 @@ impl ErrorExt for Error {
#[cfg(feature = "enterprise")]
InvalidTriggerWebhookOption { .. } => StatusCode::InvalidArguments,
SerializeColumnDefaultConstraint { source, .. }
| SetJsonStructureSettings { source, .. } => source.status_code(),
SerializeColumnDefaultConstraint { source, .. } | SetJsonSettings { source, .. } => {
source.status_code()
}
ConvertToGrpcDataType { source, .. } => source.status_code(),
SqlCommon { source, .. } => source.status_code(),

3
src/sql/src/parser.rs
View File

@@ -372,8 +372,7 @@ mod tests {
let ts_col = columns.first().unwrap();
assert_eq!(
expected_type,
sql_data_type_to_concrete_data_type(ts_col.data_type(), &Default::default())
.unwrap()
sql_data_type_to_concrete_data_type(ts_col.data_type()).unwrap()
);
}
_ => unreachable!(),

17
src/sql/src/parsers/create_parser.rs
View File

@@ -710,12 +710,13 @@ impl<'a> ParserContext<'a> {
let mut extensions = ColumnExtensions::default();
let data_type = parser.parse_data_type().context(SyntaxSnafu)?;
// Must immediately parse the JSON datatype format because it is closely after the "JSON"
// datatype, like this: "JSON(format = ...)".
if matches!(data_type, DataType::JSON) {
extensions.json_datatype_options = json::parse_json_datatype_options(parser)?;
}
let data_type =
if let Some((data_type, type_hints)) = json::parse_json2_type_and_hints(parser)? {
extensions.json_type_hints = type_hints;
data_type
} else {
parser.parse_data_type().context(SyntaxSnafu)?
};
let mut options = vec![];
loop {
@@ -910,7 +911,7 @@ impl<'a> ParserContext<'a> {
);
let column_type = get_unalias_type(column_type);
let data_type = sql_data_type_to_concrete_data_type(&column_type, column_extensions)?;
let data_type = sql_data_type_to_concrete_data_type(&column_type)?;
ensure!(
data_type == ConcreteDataType::string_datatype(),
InvalidColumnOptionSnafu {
@@ -1007,7 +1008,7 @@ impl<'a> ParserContext<'a> {
// Check that column is a vector type
let column_type = get_unalias_type(column_type);
let data_type = sql_data_type_to_concrete_data_type(&column_type, column_extensions)?;
let data_type = sql_data_type_to_concrete_data_type(&column_type)?;
ensure!(
matches!(data_type, ConcreteDataType::Vector(_)),
InvalidColumnOptionSnafu {

636
src/sql/src/parsers/create_parser/json.rs
View File

@@ -12,163 +12,523 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use snafu::ResultExt;
use snafu::{ResultExt, ensure};
use sqlparser::ast::{DataType, ExactNumberInfo, Expr, ObjectName, UnaryOperator};
use sqlparser::dialect::keywords::Keyword;
use sqlparser::parser::Parser;
use sqlparser::tokenizer::Token;
use crate::error::{Result, SyntaxSnafu};
use crate::statements::OptionMap;
use crate::util;
use crate::ast::Ident;
use crate::error::{InvalidSqlSnafu, Result, SyntaxSnafu};
use crate::parsers::create_parser::{INVERTED, SKIPPING};
use crate::statements::create::JsonTypeHint;
use crate::statements::transform::type_alias::get_type_by_alias;
pub(super) fn parse_json_datatype_options(parser: &mut Parser<'_>) -> Result<Option<OptionMap>> {
if parser.consume_token(&Token::LParen) {
let result = parser
.parse_comma_separated0(Parser::parse_sql_option, Token::RParen)
.context(SyntaxSnafu)
.and_then(|options| {
options
.into_iter()
.map(util::parse_option_string)
.collect::<Result<Vec<_>>>()
})?;
parser.expect_token(&Token::RParen).context(SyntaxSnafu)?;
Ok(Some(OptionMap::new(result)))
} else {
Ok(None)
const JSON2_TYPE_NAME: &str = "JSON2";
pub(super) fn parse_json2_type_and_hints(
parser: &mut Parser<'_>,
) -> Result<Option<(DataType, Vec<JsonTypeHint>)>> {
let token = parser.peek_token();
let Token::Word(word) = &token.token else {
return Ok(None);
};
if !word.value.eq_ignore_ascii_case(JSON2_TYPE_NAME) || word.quote_style.is_some() {
return Ok(None);
}
parser.next_token();
let data_type = DataType::Custom(ObjectName::from(vec![Ident::new(JSON2_TYPE_NAME)]), vec![]);
let type_hints = if parser.consume_token(&Token::LParen) {
parse_json2_type_hints(parser)?
} else {
vec![]
};
Ok(Some((data_type, type_hints)))
}
fn parse_json2_type_hints(parser: &mut Parser<'_>) -> Result<Vec<JsonTypeHint>> {
let mut hints = Vec::new();
if parser.consume_token(&Token::RParen) {
return Ok(hints);
}
loop {
let hint = parse_json2_type_hint(parser)?;
ensure_no_path_conflict(&hints, &hint.path)?;
hints.push(hint);
if parser.consume_token(&Token::Comma) {
if parser.consume_token(&Token::RParen) {
break;
}
} else {
parser.expect_token(&Token::RParen).context(SyntaxSnafu)?;
break;
}
}
Ok(hints)
}
fn parse_json2_type_hint(parser: &mut Parser<'_>) -> Result<JsonTypeHint> {
let path = parse_json2_path(parser)?;
let data_type = parser.parse_data_type().context(SyntaxSnafu)?;
let data_type = normalize_json2_type_hint_type(data_type)?;
let mut nullable = true;
let mut nullable_set = false;
let mut default = None;
let mut inverted_index = false;
loop {
if parser.parse_keywords(&[Keyword::NOT, Keyword::NULL]) {
ensure!(
!nullable_set,
InvalidSqlSnafu {
msg: format!(
"NULL/NOT NULL option already specified for JSON2 type hint '{}'",
path.join(".")
)
}
);
nullable = false;
nullable_set = true;
} else if parser.parse_keyword(Keyword::NULL) {
ensure!(
!nullable_set,
InvalidSqlSnafu {
msg: format!(
"NULL/NOT NULL option already specified for JSON2 type hint '{}'",
path.join(".")
)
}
);
nullable = true;
nullable_set = true;
} else if parser.parse_keyword(Keyword::DEFAULT) {
ensure!(
default.is_none(),
InvalidSqlSnafu {
msg: format!(
"duplicated DEFAULT option for JSON2 type hint '{}'",
path.join(".")
)
}
);
let expr = parser.parse_expr().context(SyntaxSnafu)?;
ensure_json2_default_expr_is_literal(&expr)?;
default = Some(expr);
} else if let Token::Word(word) = parser.peek_token().token
&& word.value.eq_ignore_ascii_case(INVERTED)
{
parser.next_token();
ensure!(
parser.parse_keyword(Keyword::INDEX),
InvalidSqlSnafu {
msg: format!(
"expect INDEX after INVERTED keyword for JSON2 type hint '{}'",
path.join(".")
)
}
);
ensure!(
!inverted_index,
InvalidSqlSnafu {
msg: format!(
"duplicated INVERTED INDEX option for JSON2 type hint '{}'",
path.join(".")
)
}
);
inverted_index = true;
} else if let Token::Word(word) = parser.peek_token().token
&& word.value.eq_ignore_ascii_case(SKIPPING)
{
return InvalidSqlSnafu {
msg: "JSON2 type hint SKIPPING INDEX is not supported yet".to_string(),
}
.fail();
} else if matches!(parser.peek_token().token, Token::Comma | Token::RParen) {
break;
} else {
return parser
.expected("JSON2 type hint option", parser.peek_token())
.context(SyntaxSnafu);
}
}
Ok(JsonTypeHint {
path,
data_type,
nullable,
default,
inverted_index,
})
}
fn parse_json2_path(parser: &mut Parser<'_>) -> Result<Vec<String>> {
let first = parser.parse_identifier().context(SyntaxSnafu)?;
let mut path = vec![first.value];
while parser.consume_token(&Token::Period) {
let segment = parser.parse_identifier().context(SyntaxSnafu)?;
path.push(segment.value);
}
ensure!(
!path.iter().any(|segment| segment.is_empty()),
InvalidSqlSnafu {
msg: "JSON2 type hint path segment cannot be empty".to_string(),
}
);
Ok(path)
}
fn normalize_json2_type_hint_type(data_type: DataType) -> Result<DataType> {
let data_type = get_type_by_alias(&data_type).unwrap_or(data_type);
let normalized = match data_type {
DataType::String(_) | DataType::Text | DataType::Varchar(_) | DataType::Char(_) => {
DataType::String(None)
}
DataType::TinyInt(_)
| DataType::SmallInt(_)
| DataType::Int(_)
| DataType::Integer(_)
| DataType::BigInt(_) => DataType::BigInt(None),
DataType::TinyIntUnsigned(_)
| DataType::SmallIntUnsigned(_)
| DataType::IntUnsigned(_)
| DataType::UnsignedInteger
| DataType::BigIntUnsigned(_) => DataType::BigIntUnsigned(None),
DataType::Float(_) | DataType::Real | DataType::Double(_) => {
DataType::Double(ExactNumberInfo::None)
}
DataType::Boolean => DataType::Boolean,
_ => {
return InvalidSqlSnafu {
msg: format!("unsupported JSON2 type hint data type: {data_type}"),
}
.fail();
}
};
Ok(normalized)
}
fn ensure_json2_default_expr_is_literal(expr: &Expr) -> Result<()> {
let is_literal = match expr {
Expr::Value(_) => true,
Expr::UnaryOp { op, expr } => {
matches!(op, UnaryOperator::Plus | UnaryOperator::Minus)
&& matches!(expr.as_ref(), Expr::Value(_))
}
_ => false,
};
ensure!(
is_literal,
InvalidSqlSnafu {
msg: "JSON2 type hint DEFAULT only supports literal values",
}
);
Ok(())
}
fn ensure_no_path_conflict(hints: &[JsonTypeHint], path: &[String]) -> Result<()> {
for hint in hints {
ensure!(
hint.path != path,
InvalidSqlSnafu {
msg: format!("duplicated JSON2 type hint path '{}'", path.join("."))
}
);
ensure!(
!hint.path.starts_with(path) && !path.starts_with(&hint.path),
InvalidSqlSnafu {
msg: format!(
"JSON2 type hint path '{}' conflicts with '{}'",
path.join("."),
hint.path.join(".")
)
}
);
}
Ok(())
}
#[cfg(test)]
mod tests {
use sqlparser::ast::{DataType, Expr, Ident, StructField};
use sqlparser::ast::{DataType, ExactNumberInfo};
use crate::dialect::GreptimeDbDialect;
use crate::parser::{ParseOptions, ParserContext};
use crate::statements::OptionMap;
use crate::statements::create::{
Column, JSON_FORMAT_FULL_STRUCTURED, JSON_FORMAT_PARTIAL, JSON_FORMAT_RAW, JSON_OPT_FIELDS,
JSON_OPT_FORMAT, JSON_OPT_UNSTRUCTURED_KEYS,
};
use crate::statements::create::Column;
use crate::statements::statement::Statement;
use crate::util::OptionValue;
fn parse_json2_column(sql: &str) -> Column {
let Statement::CreateTable(mut create_table) =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap()
.remove(0)
else {
unreachable!()
};
create_table.columns.remove(0)
}
#[test]
fn test_parse_json_datatype_options() {
fn parse(sql: &str) -> Option<OptionMap> {
let Statement::CreateTable(mut create_table) = ParserContext::create_with_dialect(
fn test_parse_json2_type_hints() {
let column = parse_json2_column(
r#"
CREATE TABLE traces (
log_json_data JSON2 (
"service.name" STRING NOT NULL DEFAULT 'null' INVERTED INDEX,
http.method STRING NOT NULL,
status_code INT64 NOT NULL,
comment STRING NULL,
),
ts TIMESTAMP TIME INDEX,
)"#,
);
assert!(matches!(
column.column_def.data_type,
DataType::Custom(_, _)
));
let hints = column.extensions.json_type_hints;
assert_eq!(hints.len(), 4);
assert_eq!(hints[0].path, vec!["service.name"]);
assert_eq!(hints[0].data_type, DataType::String(None));
assert!(!hints[0].nullable);
assert_eq!(
hints[0]
.default
.as_ref()
.map(|expr| expr.to_string())
.as_deref(),
Some("'null'")
);
assert!(hints[0].inverted_index);
assert_eq!(hints[1].path, vec!["http", "method"]);
assert_eq!(hints[1].data_type, DataType::String(None));
assert!(!hints[1].nullable);
assert!(!hints[1].inverted_index);
assert_eq!(hints[2].path, vec!["status_code"]);
assert_eq!(hints[2].data_type, DataType::BigInt(None));
assert!(!hints[2].nullable);
assert_eq!(hints[3].path, vec!["comment"]);
assert_eq!(hints[3].data_type, DataType::String(None));
assert!(hints[3].nullable);
}
#[test]
fn test_parse_json2_type_hint_default_nullable() {
let column = parse_json2_column(
r#"
CREATE TABLE traces (
log_json_data JSON2 (http.method STRING),
ts TIMESTAMP TIME INDEX,
)"#,
);
let hints = column.extensions.json_type_hints;
assert_eq!(hints.len(), 1);
assert!(hints[0].nullable);
}
#[test]
fn test_parse_json2_type_hint_quoted_path_segments() {
let column = parse_json2_column(
r#"
CREATE TABLE traces (
log_json_data JSON2 (
"a".b STRING,
"x"."y" STRING,
"a.b"."c" STRING,
a."b.c" STRING
),
ts TIMESTAMP TIME INDEX,
)"#,
);
let hints = column.extensions.json_type_hints;
assert_eq!(hints.len(), 4);
assert_eq!(hints[0].path, vec!["a", "b"]);
assert_eq!(hints[1].path, vec!["x", "y"]);
assert_eq!(hints[2].path, vec!["a.b", "c"]);
assert_eq!(hints[3].path, vec!["a", "b.c"]);
}
#[test]
fn test_parse_json2_type_hint_normalizes_numeric_types() {
let column = parse_json2_column(
r#"
CREATE TABLE traces (
log_json_data JSON2 (
tinyint_value TINYINT,
smallint_value SMALLINT,
int_value INT,
integer_value INTEGER,
bigint_value BIGINT,
int64_value INT64,
tinyuint_value TINYINT UNSIGNED,
smalluint_value SMALLINT UNSIGNED,
uint_value INT UNSIGNED,
uint64_value UINT64,
float_value FLOAT,
real_value REAL,
double_value DOUBLE,
float64_value FLOAT64
),
ts TIMESTAMP TIME INDEX,
)"#,
);
let hints = column.extensions.json_type_hints;
assert_eq!(hints.len(), 14);
for hint in hints.iter().take(6) {
assert_eq!(hint.data_type, DataType::BigInt(None));
}
for hint in hints.iter().skip(6).take(4) {
assert_eq!(hint.data_type, DataType::BigIntUnsigned(None));
}
for hint in hints.iter().skip(10) {
assert_eq!(hint.data_type, DataType::Double(ExactNumberInfo::None));
}
}
#[test]
fn test_parse_json2_type_hint_default_accepts_signed_literals() {
let column = parse_json2_column(
r#"
CREATE TABLE traces (
log_json_data JSON2 (
negative_int INT64 DEFAULT -5,
positive_float FLOAT64 DEFAULT +1.5
),
ts TIMESTAMP TIME INDEX,
)"#,
);
let hints = column.extensions.json_type_hints;
assert_eq!(hints.len(), 2);
assert_eq!(
hints[0]
.default
.as_ref()
.map(|expr| expr.to_string())
.as_deref(),
Some("-5")
);
assert_eq!(
hints[1]
.default
.as_ref()
.map(|expr| expr.to_string())
.as_deref(),
Some("+1.5")
);
}
#[test]
fn test_parse_json2_type_hint_default_rejects_function() {
let result = ParserContext::create_with_dialect(
r#"
CREATE TABLE traces (
log_json_data JSON2 (status_code INT64 DEFAULT abs(-1)),
ts TIMESTAMP TIME INDEX,
)"#,
&GreptimeDbDialect {},
ParseOptions::default(),
);
assert!(result.is_err());
assert!(
result
.unwrap_err()
.to_string()
.contains("DEFAULT only supports literal values")
);
}
#[test]
fn test_parse_json2_type_hint_rejects_duplicate_path() {
let result = ParserContext::create_with_dialect(
r#"
CREATE TABLE traces (
log_json_data JSON2 (a.b STRING, a.b INT64),
ts TIMESTAMP TIME INDEX,
)"#,
&GreptimeDbDialect {},
ParseOptions::default(),
);
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("duplicated"));
}
#[test]
fn test_parse_json2_type_hint_rejects_parent_child_path() {
let result = ParserContext::create_with_dialect(
r#"
CREATE TABLE traces (
log_json_data JSON2 (a STRING, a.b INT64),
ts TIMESTAMP TIME INDEX,
)"#,
&GreptimeDbDialect {},
ParseOptions::default(),
);
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("conflicts"));
}
#[test]
fn test_parse_json2_type_hint_rejects_duplicated_nullability() {
for sql in [
r#"
CREATE TABLE traces (
log_json_data JSON2 (a STRING NULL NULL),
ts TIMESTAMP TIME INDEX,
)"#,
r#"
CREATE TABLE traces (
log_json_data JSON2 (a STRING NOT NULL NOT NULL),
ts TIMESTAMP TIME INDEX,
)"#,
r#"
CREATE TABLE traces (
log_json_data JSON2 (a STRING NOT NULL NULL),
ts TIMESTAMP TIME INDEX,
)"#,
r#"
CREATE TABLE traces (
log_json_data JSON2 (a STRING NULL NOT NULL),
ts TIMESTAMP TIME INDEX,
)"#,
] {
let result = ParserContext::create_with_dialect(
sql,
&GreptimeDbDialect {},
ParseOptions::default(),
)
.unwrap()
.remove(0) else {
unreachable!()
};
);
let Column {
column_def,
extensions,
} = create_table.columns.remove(0);
assert_eq!(column_def.name.to_string(), "my_json");
assert_eq!(column_def.data_type, DataType::JSON);
assert!(column_def.options.is_empty());
extensions.json_datatype_options
assert!(result.is_err());
assert!(
result
.unwrap_err()
.to_string()
.contains("NULL/NOT NULL option already specified")
);
}
let sql = r#"
CREATE TABLE json_data (
my_json JSON(format = "partial", fields = Struct<i Int, "o.a" String, "o.b" String, `x.y.z` Float64>),
ts TIMESTAMP TIME INDEX,
)"#;
let options = parse(sql).unwrap();
assert_eq!(options.len(), 2);
let option = options.value(JSON_OPT_FIELDS);
let expected = OptionValue::try_new(Expr::Struct {
values: vec![],
fields: vec![
StructField {
field_name: Some(Ident::new("i")),
field_type: DataType::Int(None),
options: None,
},
StructField {
field_name: Some(Ident::with_quote('"', "o.a")),
field_type: DataType::String(None),
options: None,
},
StructField {
field_name: Some(Ident::with_quote('"', "o.b")),
field_type: DataType::String(None),
options: None,
},
StructField {
field_name: Some(Ident::with_quote('`', "x.y.z")),
field_type: DataType::Float64,
options: None,
},
],
})
.ok();
assert_eq!(option, expected.as_ref());
let sql = r#"
CREATE TABLE json_data (
my_json JSON(format = "partial", unstructured_keys = ["k", "foo.bar", "a.b.c"]),
ts TIMESTAMP TIME INDEX,
)"#;
let options = parse(sql).unwrap();
assert_eq!(options.len(), 2);
assert_eq!(
options.value(JSON_OPT_FORMAT).and_then(|x| x.as_string()),
Some(JSON_FORMAT_PARTIAL)
);
let expected = vec!["k", "foo.bar", "a.b.c"];
assert_eq!(
options
.value(JSON_OPT_UNSTRUCTURED_KEYS)
.and_then(|x| x.as_list()),
Some(expected)
);
let sql = r#"
CREATE TABLE json_data (
my_json JSON(format = "structured"),
ts TIMESTAMP TIME INDEX,
)"#;
let options = parse(sql).unwrap();
assert_eq!(options.len(), 1);
assert_eq!(
options.value(JSON_OPT_FORMAT).and_then(|x| x.as_string()),
Some(JSON_FORMAT_FULL_STRUCTURED)
);
let sql = r#"
CREATE TABLE json_data (
my_json JSON(format = "raw"),
ts TIMESTAMP TIME INDEX,
)"#;
let options = parse(sql).unwrap();
assert_eq!(options.len(), 1);
assert_eq!(
options.value(JSON_OPT_FORMAT).and_then(|x| x.as_string()),
Some(JSON_FORMAT_RAW)
);
let sql = r#"
CREATE TABLE json_data (
my_json JSON(),
ts TIMESTAMP TIME INDEX,
)"#;
let options = parse(sql).unwrap();
assert!(options.is_empty());
let sql = r#"
CREATE TABLE json_data (
my_json JSON,
ts TIMESTAMP TIME INDEX,
)"#;
let options = parse(sql);
assert!(options.is_none());
}
}

47
src/sql/src/statements.rs
View File

@@ -40,7 +40,6 @@ use api::v1::SemanticType;
use common_sql::default_constraint::parse_column_default_constraint;
use common_time::timezone::Timezone;
use datatypes::extension::json::{JsonExtensionType, JsonMetadata};
use datatypes::json::JsonStructureSettings;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{COMMENT_KEY, ColumnDefaultConstraint, ColumnSchema};
use datatypes::types::json_type::JsonNativeType;
@@ -55,10 +54,10 @@ use crate::ast::{
};
use crate::error::{
self, ConvertToGrpcDataTypeSnafu, ConvertValueSnafu, Result,
SerializeColumnDefaultConstraintSnafu, SetFulltextOptionSnafu, SetJsonStructureSettingsSnafu,
SerializeColumnDefaultConstraintSnafu, SetFulltextOptionSnafu, SetJsonSettingsSnafu,
SetSkippingIndexOptionSnafu, SetVectorIndexOptionSnafu, SqlCommonSnafu,
};
use crate::statements::create::{Column, ColumnExtensions};
use crate::statements::create::Column;
pub use crate::statements::option_map::OptionMap;
pub(crate) use crate::statements::transform::transform_statements;
@@ -110,7 +109,7 @@ pub fn column_to_schema(
&& !is_time_index;
let name = column.name().value.clone();
let data_type = sql_data_type_to_concrete_data_type(column.data_type(), &column.extensions)?;
let data_type = sql_data_type_to_concrete_data_type(column.data_type())?;
let default_constraint =
parse_column_default_constraint(&name, &data_type, column.options(), timezone)
.context(SqlCommonSnafu)?;
@@ -164,16 +163,13 @@ pub fn column_to_schema(
false
};
if is_json2_column {
let settings = column
.extensions
.build_json_structure_settings()?
.unwrap_or_default();
let settings = column.extensions.build_json_settings()?.unwrap_or_default();
let extension = JsonExtensionType::new(Arc::new(JsonMetadata {
json_structure_settings: Some(settings.clone()),
json_settings: Some(settings.clone()),
}));
column_schema
.with_extension_type(&extension)
.with_context(|_| SetJsonStructureSettingsSnafu {
.with_context(|_| SetJsonSettingsSnafu {
value: format!("{settings:?}"),
})?;
}
@@ -187,7 +183,7 @@ pub fn sql_column_def_to_grpc_column_def(
timezone: Option<&Timezone>,
) -> Result<api::v1::ColumnDef> {
let name = col.name.value.clone();
let data_type = sql_data_type_to_concrete_data_type(&col.data_type, &Default::default())?;
let data_type = sql_data_type_to_concrete_data_type(&col.data_type)?;
let is_nullable = col
.options
@@ -228,10 +224,7 @@ pub fn sql_column_def_to_grpc_column_def(
})
}
pub fn sql_data_type_to_concrete_data_type(
data_type: &SqlDataType,
column_extensions: &ColumnExtensions,
) -> Result<ConcreteDataType> {
pub fn sql_data_type_to_concrete_data_type(data_type: &SqlDataType) -> Result<ConcreteDataType> {
match data_type {
SqlDataType::BigInt(_) | SqlDataType::Int64 => Ok(ConcreteDataType::int64_datatype()),
SqlDataType::BigIntUnsigned(_) => Ok(ConcreteDataType::uint64_datatype()),
@@ -299,19 +292,9 @@ pub fn sql_data_type_to_concrete_data_type(
Ok(ConcreteDataType::vector_datatype(dim))
}
JSON2_TYPE_NAME if args.is_empty() => {
let native_type = column_extensions
.build_json_structure_settings()?
.and_then(|x| match x {
JsonStructureSettings::Structured(Some(fields))
| JsonStructureSettings::PartialUnstructuredByKey {
fields: Some(fields),
..
} => Some(JsonNativeType::from(&ConcreteDataType::Struct(fields))),
JsonStructureSettings::UnstructuredRaw => Some(JsonNativeType::Variant),
_ => None,
})
.unwrap_or(JsonNativeType::Object(Default::default()));
let format = JsonFormat::Json2(Box::new(native_type));
// Currently, JSON2 is not inferred as any native type initially.
// TODO(fys): infer it later from type hints.
let format = JsonFormat::Json2(Box::new(JsonNativeType::Null));
Ok(ConcreteDataType::Json(JsonType::new(format)))
}
_ => error::SqlTypeNotSupportedSnafu {
@@ -390,7 +373,7 @@ mod tests {
fn check_type(sql_type: SqlDataType, data_type: ConcreteDataType) {
assert_eq!(
data_type,
sql_data_type_to_concrete_data_type(&sql_type, &Default::default()).unwrap()
sql_data_type_to_concrete_data_type(&sql_type).unwrap()
);
}
@@ -744,7 +727,7 @@ mod tests {
vector_options: None,
skipping_index_options: None,
inverted_index_options: None,
json_datatype_options: None,
json_type_hints: vec![],
vector_index_options: None,
},
};
@@ -776,7 +759,7 @@ mod tests {
vector_options: None,
skipping_index_options: None,
inverted_index_options: None,
json_datatype_options: None,
json_type_hints: vec![],
vector_index_options: Some(OptionMap::from([
("metric".to_string(), "cosine".to_string()),
("connectivity".to_string(), "32".to_string()),
@@ -817,7 +800,7 @@ mod tests {
vector_options: None,
skipping_index_options: None,
inverted_index_options: None,
json_datatype_options: None,
json_type_hints: vec![],
vector_index_options: Some(OptionMap::default()),
},
};

554
src/sql/src/statements/create.rs
View File

@@ -12,47 +12,40 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use std::fmt::{Display, Formatter};
use std::sync::Arc;
use common_catalog::consts::FILE_ENGINE;
use datatypes::data_type::ConcreteDataType;
use datatypes::json::JsonStructureSettings;
use common_sql::default_constraint::parse_column_default_constraint;
use datatypes::json::JsonSettings;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{
FulltextOptions, SkippingIndexOptions, VectorDistanceMetric, VectorIndexEngineType,
VectorIndexOptions,
ColumnDefaultConstraint, FulltextOptions, SkippingIndexOptions, VectorDistanceMetric,
VectorIndexEngineType, VectorIndexOptions,
};
use datatypes::types::StructType;
use itertools::Itertools;
use serde::Serialize;
use snafu::{OptionExt, ResultExt};
use sqlparser::ast::{ColumnOptionDef, DataType, Expr};
use snafu::ResultExt;
use sqlparser::ast::{ColumnOption, ColumnOptionDef, DataType, Expr};
use sqlparser_derive::{Visit, VisitMut};
use crate::ast::{ColumnDef, Ident, ObjectName, Value as SqlValue};
use crate::dialect::GreptimeDbDialect;
use crate::error::{
InvalidFlowQuerySnafu, InvalidJsonStructureSettingSnafu, InvalidSqlSnafu, Result,
SetFulltextOptionSnafu, SetSkippingIndexOptionSnafu,
InvalidFlowQuerySnafu, InvalidSqlSnafu, Result, SetFulltextOptionSnafu,
SetSkippingIndexOptionSnafu,
};
use crate::parser::ParserContext;
use crate::statements::query::Query as GtQuery;
use crate::statements::statement::Statement;
use crate::statements::tql::Tql;
use crate::statements::{OptionMap, sql_data_type_to_concrete_data_type};
use crate::util::OptionValue;
use crate::statements::{OptionMap, sql_data_type_to_concrete_data_type, value_to_sql_value};
const LINE_SEP: &str = ",\n";
const COMMA_SEP: &str = ", ";
const INDENT: usize = 2;
pub const VECTOR_OPT_DIM: &str = "dim";
pub const JSON_OPT_UNSTRUCTURED_KEYS: &str = "unstructured_keys";
pub const JSON_OPT_FORMAT: &str = "format";
pub(crate) const JSON_OPT_FIELDS: &str = "fields";
pub const JSON_FORMAT_FULL_STRUCTURED: &str = "structured";
pub const JSON_FORMAT_RAW: &str = "raw";
pub const JSON_FORMAT_PARTIAL: &str = "partial";
macro_rules! format_indent {
($fmt: expr, $arg: expr) => {
format!($fmt, format_args!("{: >1$}", "", INDENT), $arg)
@@ -143,7 +136,16 @@ pub struct ColumnExtensions {
pub inverted_index_options: Option<OptionMap>,
/// Vector index options for HNSW-based vector similarity search.
pub vector_index_options: Option<OptionMap>,
pub json_datatype_options: Option<OptionMap>,
pub json_type_hints: Vec<JsonTypeHint>,
}
#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]
pub struct JsonTypeHint {
pub path: Vec<String>,
pub data_type: DataType,
pub nullable: bool,
pub default: Option<Expr>,
pub inverted_index: bool,
}
impl Column {
@@ -178,14 +180,11 @@ impl Display for Column {
}
write!(f, "{} {}", self.column_def.name, self.column_def.data_type)?;
if let Some(options) = &self.extensions.json_datatype_options {
if !self.extensions.json_type_hints.is_empty() {
write!(
f,
"({})",
options
.entries()
.map(|(k, v)| format!("{k} = {v}"))
.join(COMMA_SEP)
"{}",
format_json_type_hints(&self.extensions.json_type_hints)
)?;
}
for option in &self.column_def.options {
@@ -335,106 +334,168 @@ impl ColumnExtensions {
Ok(Some(result))
}
pub fn build_json_structure_settings(&self) -> Result<Option<JsonStructureSettings>> {
let Some(options) = self.json_datatype_options.as_ref() else {
pub fn build_json_settings(&self) -> Result<Option<JsonSettings>> {
if self.json_type_hints.is_empty() {
return Ok(None);
};
let unstructured_keys = options
.value(JSON_OPT_UNSTRUCTURED_KEYS)
.and_then(|v| {
v.as_list().map(|x| {
x.into_iter()
.map(|x| x.to_string())
.collect::<HashSet<String>>()
})
})
.unwrap_or_default();
let fields = if let Some(value) = options.value(JSON_OPT_FIELDS) {
let fields = value
.as_struct_fields()
.context(InvalidJsonStructureSettingSnafu {
reason: format!(r#"expect "{JSON_OPT_FIELDS}" a struct, actual: "{value}""#,),
})?;
let fields = fields
.iter()
.map(|field| {
let name = field.field_name.as_ref().map(|x| x.value.clone()).context(
InvalidJsonStructureSettingSnafu {
reason: format!(r#"missing field name in "{field}""#),
},
)?;
let datatype = sql_data_type_to_concrete_data_type(
&field.field_type,
&Default::default(),
)?;
Ok(datatypes::types::StructField::new(name, datatype, true))
})
.collect::<Result<_>>()?;
Some(StructType::new(Arc::new(fields)))
} else {
None
};
let format = options
.get(JSON_OPT_FORMAT)
.unwrap_or(JSON_FORMAT_FULL_STRUCTURED);
let settings = match format {
JSON_FORMAT_FULL_STRUCTURED => JsonStructureSettings::Structured(fields),
JSON_FORMAT_PARTIAL => {
let fields = fields.map(|fields| {
let mut fields = Arc::unwrap_or_clone(fields.fields());
fields.push(datatypes::types::StructField::new(
JsonStructureSettings::RAW_FIELD.to_string(),
ConcreteDataType::string_datatype(),
true,
));
StructType::new(Arc::new(fields))
});
JsonStructureSettings::PartialUnstructuredByKey {
fields,
unstructured_keys,
}
}
JSON_FORMAT_RAW => JsonStructureSettings::UnstructuredRaw,
_ => {
return InvalidSqlSnafu {
msg: format!("unknown JSON datatype 'format': {format}"),
}
.fail();
}
};
Ok(Some(settings))
}
pub fn set_json_structure_settings(&mut self, settings: JsonStructureSettings) {
let mut map = OptionMap::default();
let format = match settings {
JsonStructureSettings::Structured(_) => JSON_FORMAT_FULL_STRUCTURED,
JsonStructureSettings::PartialUnstructuredByKey { .. } => JSON_FORMAT_PARTIAL,
JsonStructureSettings::UnstructuredRaw => JSON_FORMAT_RAW,
};
map.insert(JSON_OPT_FORMAT.to_string(), format.to_string());
if let JsonStructureSettings::PartialUnstructuredByKey {
fields: _,
unstructured_keys,
} = settings
{
let value = OptionValue::from(
unstructured_keys
.iter()
.map(|x| x.as_str())
.sorted()
.collect::<Vec<_>>(),
);
map.insert_options(JSON_OPT_UNSTRUCTURED_KEYS, value);
}
self.json_datatype_options = Some(map);
Ok(Some(JsonSettings::new(
self.json_type_hints
.iter()
.map(|hint| {
Ok(datatypes::json::JsonTypeHint {
path: hint.path.clone(),
data_type: json_type_hint_concrete_data_type(&hint.data_type)?,
nullable: hint.nullable,
default_constraint: build_json_type_hint_default_constraint(hint)?,
inverted_index: hint.inverted_index,
})
})
.collect::<Result<Vec<_>>>()?,
)))
}
pub fn set_json_settings(&mut self, settings: JsonSettings) -> Result<()> {
self.json_type_hints = settings
.type_hints
.into_iter()
.map(|hint| {
let data_type = json_type_hint_sql_data_type(&hint.data_type)?;
let default = hint
.default_constraint
.map(|constraint| column_default_constraint_to_expr(&constraint))
.transpose()?;
Ok(JsonTypeHint {
path: hint.path,
data_type,
nullable: hint.nullable,
default,
inverted_index: hint.inverted_index,
})
})
.collect::<Result<Vec<_>>>()?;
Ok(())
}
}
fn build_json_type_hint_default_constraint(
hint: &JsonTypeHint,
) -> Result<Option<ColumnDefaultConstraint>> {
let Some(default) = &hint.default else {
return Ok(None);
};
let data_type = json_type_hint_concrete_data_type(&hint.data_type)?;
let opts = [ColumnOptionDef {
name: None,
option: ColumnOption::Default(default.clone()),
}];
// Use the JSON path as the column name context for default value parsing errors.
let json_path = hint.path.join(".");
let default_constraint = parse_column_default_constraint(&json_path, &data_type, &opts, None)
.context(crate::error::SqlCommonSnafu)?;
if let Some(constraint) = &default_constraint {
constraint
.validate(&data_type, hint.nullable)
.map_err(|e| {
InvalidSqlSnafu {
msg: format!("invalid DEFAULT for JSON2 type hint '{}': {e}", json_path),
}
.build()
})?;
}
Ok(default_constraint)
}
fn json_type_hint_concrete_data_type(data_type: &DataType) -> Result<ConcreteDataType> {
let data_type = sql_data_type_to_concrete_data_type(data_type)?;
normalize_json_type_hint_concrete_data_type(&data_type)
}
fn normalize_json_type_hint_concrete_data_type(
data_type: &ConcreteDataType,
) -> Result<ConcreteDataType> {
let normalized = match data_type {
ConcreteDataType::String(_) => ConcreteDataType::string_datatype(),
ConcreteDataType::Int8(_)
| ConcreteDataType::Int16(_)
| ConcreteDataType::Int32(_)
| ConcreteDataType::Int64(_) => ConcreteDataType::int64_datatype(),
ConcreteDataType::UInt8(_)
| ConcreteDataType::UInt16(_)
| ConcreteDataType::UInt32(_)
| ConcreteDataType::UInt64(_) => ConcreteDataType::uint64_datatype(),
ConcreteDataType::Float32(_) | ConcreteDataType::Float64(_) => {
ConcreteDataType::float64_datatype()
}
ConcreteDataType::Boolean(_) => ConcreteDataType::boolean_datatype(),
_ => {
return InvalidSqlSnafu {
msg: format!("unsupported JSON2 type hint data type: {data_type}"),
}
.fail();
}
};
Ok(normalized)
}
fn json_type_hint_sql_data_type(data_type: &ConcreteDataType) -> Result<DataType> {
let data_type = normalize_json_type_hint_concrete_data_type(data_type)?;
let sql_type = match data_type {
ConcreteDataType::String(_) => DataType::String(None),
ConcreteDataType::Int64(_) => DataType::BigInt(None),
ConcreteDataType::UInt64(_) => DataType::BigIntUnsigned(None),
ConcreteDataType::Float64(_) => DataType::Double(sqlparser::ast::ExactNumberInfo::None),
ConcreteDataType::Boolean(_) => DataType::Boolean,
_ => unreachable!("JSON2 type hint data type should have been normalized"),
};
Ok(sql_type)
}
fn column_default_constraint_to_expr(constraint: &ColumnDefaultConstraint) -> Result<Expr> {
match constraint {
ColumnDefaultConstraint::Value(value) => Ok(Expr::Value(value_to_sql_value(value)?.into())),
ColumnDefaultConstraint::Function(function) => {
ParserContext::parse_function(function, &GreptimeDbDialect {})
}
}
}
fn format_json_type_hint(hint: &JsonTypeHint) -> String {
let path = hint
.path
.iter()
.map(|segment| format_json_path_segment(segment))
.join(".");
let nullability = if hint.nullable { " NULL" } else { " NOT NULL" };
let default = hint
.default
.as_ref()
.map(|expr| format!(" DEFAULT {expr}"))
.unwrap_or_default();
let inverted_index = if hint.inverted_index {
" INVERTED INDEX"
} else {
""
};
format!(
"{} {}{}{}{}",
path, hint.data_type, nullability, default, inverted_index
)
}
fn format_json_type_hints(hints: &[JsonTypeHint]) -> String {
format!(
"(\n {}\n )",
hints.iter().map(format_json_type_hint).join(",\n ")
)
}
fn format_json_path_segment(segment: &str) -> String {
format!("\"{}\"", segment.replace('"', "\"\""))
}
/// Partition on columns or values.
@@ -720,6 +781,11 @@ impl Display for CreateView {
mod tests {
use std::assert_matches;
use datatypes::json::{JsonSettings, JsonTypeHint as DatatypeJsonTypeHint};
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::ColumnDefaultConstraint;
use datatypes::value::Value;
use crate::dialect::GreptimeDbDialect;
use crate::error::Error;
use crate::parser::{ParseOptions, ParserContext};
@@ -889,6 +955,240 @@ ENGINE=mito
);
}
#[test]
fn test_display_json2_type_hints_quotes_path_segments() {
let sql = r#"CREATE TABLE traces (
log_json_data JSON2 (
"service.name" STRING,
"a.b"."c" INT64 NOT NULL,
a."b.c" STRING
),
ts TIMESTAMP TIME INDEX
)"#;
let result =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap();
match &result[0] {
Statement::CreateTable(c) => {
let new_sql = format!("\n{}", c);
assert_eq!(
r#"
CREATE TABLE traces (
log_json_data JSON2(
"service.name" STRING NULL,
"a.b"."c" BIGINT NOT NULL,
"a"."b.c" STRING NULL
),
ts TIMESTAMP NOT NULL,
TIME INDEX (ts)
)
ENGINE=mito
"#,
&new_sql
);
let new_result = ParserContext::create_with_dialect(
&new_sql,
&GreptimeDbDialect {},
ParseOptions::default(),
)
.unwrap();
assert_eq!(result, new_result);
}
_ => unreachable!(),
}
}
#[test]
fn test_display_json2_type_hints_quotes_numeric_segments() {
let sql = r#"CREATE TABLE traces (
log_json_data JSON2 (
"1abc" STRING,
a."2b" INT64 NOT NULL
),
ts TIMESTAMP TIME INDEX
)"#;
let result =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap();
match &result[0] {
Statement::CreateTable(c) => {
let new_sql = format!("\n{}", c);
assert_eq!(
r#"
CREATE TABLE traces (
log_json_data JSON2(
"1abc" STRING NULL,
"a"."2b" BIGINT NOT NULL
),
ts TIMESTAMP NOT NULL,
TIME INDEX (ts)
)
ENGINE=mito
"#,
&new_sql
);
let new_result = ParserContext::create_with_dialect(
&new_sql,
&GreptimeDbDialect {},
ParseOptions::default(),
)
.unwrap();
assert_eq!(result, new_result);
}
_ => unreachable!(),
}
}
#[test]
fn test_json2_type_hint_default_builds_default_constraint() {
let sql = r#"CREATE TABLE traces (
log_json_data JSON2 (
status_code INT64 DEFAULT -5,
duration FLOAT64 DEFAULT +1.5,
error BOOLEAN DEFAULT false,
message STRING DEFAULT 'unknown'
),
ts TIMESTAMP TIME INDEX
)"#;
let result =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap();
let Statement::CreateTable(create_table) = &result[0] else {
unreachable!()
};
let settings = create_table.columns[0]
.extensions
.build_json_settings()
.unwrap()
.unwrap();
let hints = settings.type_hints;
assert_eq!(hints[0].data_type, ConcreteDataType::int64_datatype());
assert_eq!(
hints[0].default_constraint,
Some(ColumnDefaultConstraint::Value(Value::Int64(-5)))
);
assert_eq!(hints[1].data_type, ConcreteDataType::float64_datatype());
assert_eq!(
hints[1].default_constraint,
Some(ColumnDefaultConstraint::Value(Value::Float64(1.5.into())))
);
assert_eq!(hints[2].data_type, ConcreteDataType::boolean_datatype());
assert_eq!(
hints[2].default_constraint,
Some(ColumnDefaultConstraint::Value(Value::Boolean(false)))
);
assert_eq!(hints[3].data_type, ConcreteDataType::string_datatype());
assert_eq!(
hints[3].default_constraint,
Some(ColumnDefaultConstraint::Value(Value::String(
"unknown".into()
)))
);
}
#[test]
fn test_json2_type_hint_not_null_default_null_is_rejected() {
let sql = r#"CREATE TABLE traces (
log_json_data JSON2 (
status_code INT64 NOT NULL DEFAULT NULL
),
ts TIMESTAMP TIME INDEX
)"#;
let result =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap();
let Statement::CreateTable(create_table) = &result[0] else {
unreachable!()
};
let err = create_table.columns[0]
.extensions
.build_json_settings()
.unwrap_err();
assert!(
err.to_string()
.contains("Default value should not be null for non null column")
);
}
#[test]
fn test_set_json_settings_normalizes_type_hint_sql_types() {
let mut extensions = super::ColumnExtensions::default();
extensions
.set_json_settings(JsonSettings::new(vec![
DatatypeJsonTypeHint {
path: vec!["i".to_string()],
data_type: ConcreteDataType::int32_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
DatatypeJsonTypeHint {
path: vec!["f".to_string()],
data_type: ConcreteDataType::float32_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
DatatypeJsonTypeHint {
path: vec!["u".to_string()],
data_type: ConcreteDataType::uint32_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
DatatypeJsonTypeHint {
path: vec!["s".to_string()],
data_type: ConcreteDataType::string_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
DatatypeJsonTypeHint {
path: vec!["b".to_string()],
data_type: ConcreteDataType::boolean_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
]))
.unwrap();
assert_eq!(
extensions
.json_type_hints
.iter()
.map(|hint| hint.data_type.to_string())
.collect::<Vec<_>>(),
vec!["BIGINT", "DOUBLE", "BIGINT UNSIGNED", "STRING", "BOOLEAN"]
);
}
#[test]
fn test_set_json_settings_rejects_unsupported_type_hint_type() {
let mut extensions = super::ColumnExtensions::default();
let err = extensions
.set_json_settings(JsonSettings::new(vec![DatatypeJsonTypeHint {
path: vec!["u".to_string()],
data_type: ConcreteDataType::date_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
}]))
.unwrap_err();
assert!(
err.to_string()
.contains("unsupported JSON2 type hint data type")
);
}
#[test]
fn test_display_create_database() {
let sql = r"create database test;";
@@ -1068,7 +1368,7 @@ AS SELECT number FROM numbers_input where number > 10"#,
vector_options: None,
skipping_index_options: None,
inverted_index_options: None,
json_datatype_options: None,
json_type_hints: vec![],
vector_index_options: Some(OptionMap::from([(
"connectivity".to_string(),
"0".to_string(),
@@ -1089,7 +1389,7 @@ AS SELECT number FROM numbers_input where number > 10"#,
vector_options: None,
skipping_index_options: None,
inverted_index_options: None,
json_datatype_options: None,
json_type_hints: vec![],
vector_index_options: Some(OptionMap::from([(
"expansion_add".to_string(),
"0".to_string(),
@@ -1110,7 +1410,7 @@ AS SELECT number FROM numbers_input where number > 10"#,
vector_options: None,
skipping_index_options: None,
inverted_index_options: None,
json_datatype_options: None,
json_type_hints: vec![],
vector_index_options: Some(OptionMap::from([(
"expansion_search".to_string(),
"0".to_string(),
@@ -1131,7 +1431,7 @@ AS SELECT number FROM numbers_input where number > 10"#,
vector_options: None,
skipping_index_options: None,
inverted_index_options: None,
json_datatype_options: None,
json_type_hints: vec![],
vector_index_options: Some(OptionMap::from([
("connectivity".to_string(), "32".to_string()),
("expansion_add".to_string(), "200".to_string()),

11
src/sql/src/statements/transform/type_alias.rs
View File

@@ -117,9 +117,7 @@ impl TransformRule for TypeAliasTransformRule {
} if get_type_by_alias(data_type).is_some() => {
// Safety: checked in the match arm.
let new_type = get_type_by_alias(data_type).unwrap();
if let Ok(new_type) =
sql_data_type_to_concrete_data_type(&new_type, &Default::default())
{
if let Ok(new_type) = sql_data_type_to_concrete_data_type(&new_type) {
*expr = Expr::Function(cast_expr_to_arrow_cast_func(
(**cast_expr).clone(),
new_type.as_arrow_type().to_string(),
@@ -134,10 +132,9 @@ impl TransformRule for TypeAliasTransformRule {
expr: cast_expr,
..
} => {
if let Ok(concrete_type) = sql_data_type_to_concrete_data_type(
&DataType::Timestamp(*precision, *zone),
&Default::default(),
) {
if let Ok(concrete_type) =
sql_data_type_to_concrete_data_type(&DataType::Timestamp(*precision, *zone))
{
let new_type = concrete_type.as_arrow_type();
*expr = Expr::Function(cast_expr_to_arrow_cast_func(
(**cast_expr).clone(),

9
src/sql/src/util.rs
View File

@@ -26,7 +26,7 @@ use promql_parser::parser::{
use serde::Serialize;
use snafu::ensure;
use sqlparser::ast::{
Array, Expr, Ident, ObjectName, ObjectNamePart, SetExpr, SqlOption, StructField, TableFactor,
Array, Expr, Ident, ObjectName, ObjectNamePart, SetExpr, SqlOption, TableFactor,
TableWithJoins, Value, ValueWithSpan,
};
use sqlparser_derive::{Visit, VisitMut};
@@ -128,13 +128,6 @@ impl OptionValue {
_ => None,
}
}
pub(crate) fn as_struct_fields(&self) -> Option<&[StructField]> {
match &self.0 {
Expr::Struct { fields, .. } => Some(fields),
_ => None,
}
}
}
impl From<String> for OptionValue {

24
tests-integration/src/tests/instance_test.rs
View File

@@ -3009,17 +3009,17 @@ CREATE TABLE b (
let output = execute_sql(&instance, "SHOW CREATE TABLE b").await.data;
let expected = r#"
+-------+-----------------------------------------+
| Table | Create Table |
+-------+-----------------------------------------+
| b | CREATE TABLE IF NOT EXISTS "b" ( |
| | "j" JSON(format = 'structured') NULL, |
| | "ts" TIMESTAMP(3) NOT NULL, |
| | TIME INDEX ("ts") |
| | ) |
| | |
| | ENGINE=mito |
| | |
+-------+-----------------------------------------+"#;
+-------+----------------------------------+
| Table | Create Table |
+-------+----------------------------------+
| b | CREATE TABLE IF NOT EXISTS "b" ( |
| | "j" JSON2 NULL, |
| | "ts" TIMESTAMP(3) NOT NULL, |
| | TIME INDEX ("ts") |
| | ) |
| | |
| | ENGINE=mito |
| | |
+-------+----------------------------------+"#;
check_output_stream(output, expected).await;
}

2
tests-integration/tests/jsonbench.rs
View File

@@ -295,7 +295,7 @@ async fn desc_table(frontend: &Arc<Instance>) {
+---------+----------------------+-----+------+---------+---------------+
| Column | Type | Key | Null | Default | Semantic Type |
+---------+----------------------+-----+------+---------+---------------+
| data | Json2{} | | YES | | FIELD |
| data | Json2 | | YES | | FIELD |
| time_us | TimestampMicrosecond | PRI | NO | | TIMESTAMP |
+---------+----------------------+-----+------+---------+---------------+"#;
execute_sql_and_expect(frontend, sql, expected).await;

52
tests/cases/standalone/common/types/json/json2.result
View File

@@ -153,11 +153,37 @@ select j.c, j.y from json2_table order by ts;
select j from json2_table order by ts;
Error: 3001(EngineExecuteQuery), Failed to align JSON array, reason: Invalid argument error: use StructArray::try_new_with_length or StructArray::new_empty_fields to create a struct array with no fields so that the length can be set correctly
+--------------------+
| j |
+--------------------+
| {__json_plain__: } |
| {__json_plain__: } |
| {__json_plain__: } |
| {__json_plain__: } |
| {__json_plain__: } |
| {__json_plain__: } |
| {__json_plain__: } |
| {__json_plain__: } |
| {__json_plain__: } |
| {__json_plain__: } |
+--------------------+
select * from json2_table order by ts;
Error: 3001(EngineExecuteQuery), Failed to align JSON array, reason: Invalid argument error: use StructArray::try_new_with_length or StructArray::new_empty_fields to create a struct array with no fields so that the length can be set correctly
+-------------------------+--------------------+
| ts | j |
+-------------------------+--------------------+
| 1970-01-01T00:00:00.001 | {__json_plain__: } |
| 1970-01-01T00:00:00.002 | {__json_plain__: } |
| 1970-01-01T00:00:00.003 | {__json_plain__: } |
| 1970-01-01T00:00:00.004 | {__json_plain__: } |
| 1970-01-01T00:00:00.005 | {__json_plain__: } |
| 1970-01-01T00:00:00.006 | {__json_plain__: } |
| 1970-01-01T00:00:00.007 | {__json_plain__: } |
| 1970-01-01T00:00:00.008 | {__json_plain__: } |
| 1970-01-01T00:00:00.009 | {__json_plain__: } |
| 1970-01-01T00:00:00.010 | {__json_plain__: } |
+-------------------------+--------------------+
select j.a.b + 1 from json2_table order by ts;
@@ -210,3 +236,25 @@ drop table json2_table;
Affected Rows: 0
create table json2_default_null_ok (
ts timestamp time index,
j json2(
a int64 null default null
)
);
Affected Rows: 0
drop table json2_default_null_ok;
Affected Rows: 0
create table json2_default_null_check (
ts timestamp time index,
j json2(
a int64 not null default null
)
);
Error: 2000(InvalidSyntax), Invalid SQL, error: invalid DEFAULT for JSON2 type hint 'a': Default value should not be null for non null column

16
tests/cases/standalone/common/types/json/json2.sql
View File

@@ -60,3 +60,19 @@ select abs(j.c) from json2_table order by ts;
select j.d from json2_table order by ts;
drop table json2_table;
create table json2_default_null_ok (
ts timestamp time index,
j json2(
a int64 null default null
)
);
drop table json2_default_null_ok;
create table json2_default_null_check (
ts timestamp time index,
j json2(
a int64 not null default null
)
);

92
tests/cases/standalone/common/types/json/json2_type_hints.result Normal file
View File

@@ -0,0 +1,92 @@
CREATE TABLE json2_type_hints (
ts TIMESTAMP TIME INDEX,
j JSON2 (
user.age BIGINT NOT NULL DEFAULT 18,
user.name STRING DEFAULT 'unknown',
user.active BOOLEAN NULL,
score DOUBLE NULL DEFAULT 1.5
)
);
Affected Rows: 0
SHOW CREATE TABLE json2_type_hints;
+------------------+--------------------------------------------------+
| Table | Create Table |
+------------------+--------------------------------------------------+
| json2_type_hints | CREATE TABLE IF NOT EXISTS "json2_type_hints" ( |
| | "ts" TIMESTAMP(3) NOT NULL, |
| | "j" JSON2( |
| | "user"."age" BIGINT NOT NULL DEFAULT 18, |
| | "user"."name" STRING NULL DEFAULT 'unknown', |
| | "user"."active" BOOLEAN NULL, |
| | "score" DOUBLE NULL DEFAULT 1.5 |
| | ) NULL, |
| | TIME INDEX ("ts") |
| | ) |
| | |
| | ENGINE=mito |
| | |
+------------------+--------------------------------------------------+
INSERT INTO json2_type_hints
VALUES
(1, '{"user":{"age":42,"name":"Alice","active":true},"score":3.25}'),
(2, '{"user":{"name":"Bob"}}'),
(3, '{}');
Affected Rows: 3
SELECT
j.user.age,
j.user.name,
j.user.active,
j.score
FROM json2_type_hints
ORDER BY ts;
+-----------------------------------------------+------------------------------------------------+--------------------------------------------------+--------------------------------------------+
| json_get(json2_type_hints.j,Utf8("user.age")) | json_get(json2_type_hints.j,Utf8("user.name")) | json_get(json2_type_hints.j,Utf8("user.active")) | json_get(json2_type_hints.j,Utf8("score")) |
+-----------------------------------------------+------------------------------------------------+--------------------------------------------------+--------------------------------------------+
| 42 | Alice | true | 3.25 |
| 18 | Bob | | 1.5 |
| 18 | unknown | | 1.5 |
+-----------------------------------------------+------------------------------------------------+--------------------------------------------------+--------------------------------------------+
INSERT INTO json2_type_hints
VALUES (4, '{"user":{"age":"bad"}}');
Error: 1004(InvalidArguments), Invalid JSON: JSON value at user.age does not match JSON2 type hint Int64
CREATE TABLE json2_type_hints_required (
ts TIMESTAMP TIME INDEX,
j JSON2 (
user.age BIGINT NOT NULL
)
);
Affected Rows: 0
INSERT INTO json2_type_hints_required
VALUES (1, '{}');
Error: 1004(InvalidArguments), Invalid JSON: missing non-null JSON2 type hint path user.age
CREATE TABLE json2_type_hints_timestamp (
ts TIMESTAMP TIME INDEX,
j JSON2 (
event_time TIMESTAMP
)
);
Error: 2000(InvalidSyntax), Invalid SQL, error: unsupported JSON2 type hint data type: TIMESTAMP
DROP TABLE json2_type_hints;
Affected Rows: 0
DROP TABLE json2_type_hints_required;
Affected Rows: 0

49
tests/cases/standalone/common/types/json/json2_type_hints.sql Normal file
View File

@@ -0,0 +1,49 @@
CREATE TABLE json2_type_hints (
ts TIMESTAMP TIME INDEX,
j JSON2 (
user.age BIGINT NOT NULL DEFAULT 18,
user.name STRING DEFAULT 'unknown',
user.active BOOLEAN NULL,
score DOUBLE NULL DEFAULT 1.5
)
);
SHOW CREATE TABLE json2_type_hints;
INSERT INTO json2_type_hints
VALUES
(1, '{"user":{"age":42,"name":"Alice","active":true},"score":3.25}'),
(2, '{"user":{"name":"Bob"}}'),
(3, '{}');
SELECT
j.user.age,
j.user.name,
j.user.active,
j.score
FROM json2_type_hints
ORDER BY ts;
INSERT INTO json2_type_hints
VALUES (4, '{"user":{"age":"bad"}}');
CREATE TABLE json2_type_hints_required (
ts TIMESTAMP TIME INDEX,
j JSON2 (
user.age BIGINT NOT NULL
)
);
INSERT INTO json2_type_hints_required
VALUES (1, '{}');
CREATE TABLE json2_type_hints_timestamp (
ts TIMESTAMP TIME INDEX,
j JSON2 (
event_time TIMESTAMP
)
);
DROP TABLE json2_type_hints;
DROP TABLE json2_type_hints_required;