rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-16 21:10:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: simple read write new json type values (#7175)

Browse Source 
feat: basic json read and write

Signed-off-by: luofucong <luofc@foxmail.com>
This commit is contained in:
LFC
2025-11-27 20:40:35 +08:00
committed by GitHub
parent 4c07d2d5de
commit fdab75ce27
35 changed files with 1049 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/sql/src/parser.rs
View File

@@ -353,7 +353,8 @@ mod tests {
let ts_col = columns.first().unwrap();
assert_eq!(
expected_type,
sql_data_type_to_concrete_data_type(ts_col.data_type()).unwrap()
sql_data_type_to_concrete_data_type(ts_col.data_type(), &Default::default())
.unwrap()
);
}
_ => unreachable!(),

5
src/sql/src/parsers/create_parser.rs
View File

@@ -669,8 +669,7 @@ impl<'a> ParserContext<'a> {
// Must immediately parse the JSON datatype format because it is closely after the "JSON"
// datatype, like this: "JSON(format = ...)".
if matches!(data_type, DataType::JSON) {
let options = json::parse_json_datatype_options(parser)?;
extensions.json_datatype_options = Some(options);
extensions.json_datatype_options = json::parse_json_datatype_options(parser)?;
}
let mut options = vec![];
@@ -856,7 +855,7 @@ impl<'a> ParserContext<'a> {
);
let column_type = get_unalias_type(column_type);
let data_type = sql_data_type_to_concrete_data_type(&column_type)?;
let data_type = sql_data_type_to_concrete_data_type(&column_type, column_extensions)?;
ensure!(
data_type == ConcreteDataType::string_datatype(),
InvalidColumnOptionSnafu {

21
src/sql/src/parsers/create_parser/json.rs
View File

@@ -20,7 +20,7 @@ use crate::error::{Result, SyntaxSnafu};
use crate::statements::OptionMap;
use crate::util;
pub(super) fn parse_json_datatype_options(parser: &mut Parser<'_>) -> Result<OptionMap> {
pub(super) fn parse_json_datatype_options(parser: &mut Parser<'_>) -> Result<Option<OptionMap>> {
if parser.consume_token(&Token::LParen) {
let result = parser
.parse_comma_separated0(Parser::parse_sql_option, Token::RParen)
@@ -32,9 +32,9 @@ pub(super) fn parse_json_datatype_options(parser: &mut Parser<'_>) -> Result<Opt
.collect::<Result<Vec<_>>>()
})?;
parser.expect_token(&Token::RParen).context(SyntaxSnafu)?;
Ok(OptionMap::new(result))
Ok(Some(OptionMap::new(result)))
} else {
Ok(OptionMap::default())
Ok(None)
}
}
@@ -53,7 +53,7 @@ mod tests {
#[test]
fn test_parse_json_datatype_options() {
fn parse(sql: &str) -> OptionMap {
fn parse(sql: &str) -> Option<OptionMap> {
let Statement::CreateTable(mut create_table) = ParserContext::create_with_dialect(
sql,
&GreptimeDbDialect {},
@@ -72,8 +72,7 @@ mod tests {
assert_eq!(column_def.data_type, DataType::JSON);
assert!(column_def.options.is_empty());
assert!(extensions.json_datatype_options.is_some());
extensions.json_datatype_options.unwrap()
extensions.json_datatype_options
}
let sql = r#"
@@ -81,7 +80,7 @@ CREATE TABLE json_data (
my_json JSON(format = "partial", unstructured_keys = ["k", "foo.bar", "a.b.c"]),
ts TIMESTAMP TIME INDEX,
)"#;
let options = parse(sql);
let options = parse(sql).unwrap();
assert_eq!(options.len(), 2);
assert_eq!(
options.value(JSON_OPT_FORMAT).and_then(|x| x.as_string()),
@@ -100,7 +99,7 @@ CREATE TABLE json_data (
my_json JSON(format = "structured"),
ts TIMESTAMP TIME INDEX,
)"#;
let options = parse(sql);
let options = parse(sql).unwrap();
assert_eq!(options.len(), 1);
assert_eq!(
options.value(JSON_OPT_FORMAT).and_then(|x| x.as_string()),
@@ -112,7 +111,7 @@ CREATE TABLE json_data (
my_json JSON(format = "raw"),
ts TIMESTAMP TIME INDEX,
)"#;
let options = parse(sql);
let options = parse(sql).unwrap();
assert_eq!(options.len(), 1);
assert_eq!(
options.value(JSON_OPT_FORMAT).and_then(|x| x.as_string()),
@@ -124,7 +123,7 @@ CREATE TABLE json_data (
my_json JSON(),
ts TIMESTAMP TIME INDEX,
)"#;
let options = parse(sql);
let options = parse(sql).unwrap();
assert!(options.is_empty());
let sql = r#"
@@ -133,6 +132,6 @@ CREATE TABLE json_data (
ts TIMESTAMP TIME INDEX,
)"#;
let options = parse(sql);
assert!(options.is_empty());
assert!(options.is_none());
}
}

25
src/sql/src/statements.rs
View File

@@ -41,7 +41,8 @@ use common_time::timezone::Timezone;
use datatypes::extension::json::{JsonExtensionType, JsonMetadata};
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{COMMENT_KEY, ColumnDefaultConstraint, ColumnSchema};
use datatypes::types::TimestampType;
use datatypes::types::json_type::JsonNativeType;
use datatypes::types::{JsonFormat, JsonType, TimestampType};
use datatypes::value::Value;
use snafu::ResultExt;
use sqlparser::ast::{ExactNumberInfo, Ident};
@@ -55,7 +56,7 @@ use crate::error::{
SerializeColumnDefaultConstraintSnafu, SetFulltextOptionSnafu, SetJsonStructureSettingsSnafu,
SetSkippingIndexOptionSnafu, SqlCommonSnafu,
};
use crate::statements::create::Column;
use crate::statements::create::{Column, ColumnExtensions};
pub use crate::statements::option_map::OptionMap;
pub(crate) use crate::statements::transform::transform_statements;
@@ -109,7 +110,7 @@ pub fn column_to_schema(
&& !is_time_index;
let name = column.name().value.clone();
let data_type = sql_data_type_to_concrete_data_type(column.data_type())?;
let data_type = sql_data_type_to_concrete_data_type(column.data_type(), &column.extensions)?;
let default_constraint =
parse_column_default_constraint(&name, &data_type, column.options(), timezone)
.context(SqlCommonSnafu)?;
@@ -171,7 +172,7 @@ pub fn sql_column_def_to_grpc_column_def(
timezone: Option<&Timezone>,
) -> Result<api::v1::ColumnDef> {
let name = col.name.value.clone();
let data_type = sql_data_type_to_concrete_data_type(&col.data_type)?;
let data_type = sql_data_type_to_concrete_data_type(&col.data_type, &Default::default())?;
let is_nullable = col
.options
@@ -217,7 +218,10 @@ pub fn sql_column_def_to_grpc_column_def(
})
}
pub fn sql_data_type_to_concrete_data_type(data_type: &SqlDataType) -> Result<ConcreteDataType> {
pub fn sql_data_type_to_concrete_data_type(
data_type: &SqlDataType,
column_extensions: &ColumnExtensions,
) -> Result<ConcreteDataType> {
match data_type {
SqlDataType::BigInt(_) | SqlDataType::Int64 => Ok(ConcreteDataType::int64_datatype()),
SqlDataType::BigIntUnsigned(_) => Ok(ConcreteDataType::uint64_datatype()),
@@ -269,7 +273,14 @@ pub fn sql_data_type_to_concrete_data_type(data_type: &SqlDataType) -> Result<Co
Ok(ConcreteDataType::decimal128_datatype(*p as u8, *s as i8))
}
},
SqlDataType::JSON => Ok(ConcreteDataType::json_datatype()),
SqlDataType::JSON => {
let format = if column_extensions.json_datatype_options.is_some() {
JsonFormat::Native(Box::new(JsonNativeType::Null))
} else {
JsonFormat::Jsonb
};
Ok(ConcreteDataType::Json(JsonType::new(format)))
}
// Vector type
SqlDataType::Custom(name, d)
if name.0.as_slice().len() == 1
@@ -354,7 +365,7 @@ mod tests {
fn check_type(sql_type: SqlDataType, data_type: ConcreteDataType) {
assert_eq!(
data_type,
sql_data_type_to_concrete_data_type(&sql_type).unwrap()
sql_data_type_to_concrete_data_type(&sql_type, &Default::default()).unwrap()
);
}

11
src/sql/src/statements/transform/type_alias.rs
View File

@@ -117,7 +117,9 @@ impl TransformRule for TypeAliasTransformRule {
} if get_type_by_alias(data_type).is_some() => {
// Safety: checked in the match arm.
let new_type = get_type_by_alias(data_type).unwrap();
if let Ok(new_type) = sql_data_type_to_concrete_data_type(&new_type) {
if let Ok(new_type) =
sql_data_type_to_concrete_data_type(&new_type, &Default::default())
{
*expr = Expr::Function(cast_expr_to_arrow_cast_func(
(**cast_expr).clone(),
new_type.as_arrow_type().to_string(),
@@ -132,9 +134,10 @@ impl TransformRule for TypeAliasTransformRule {
expr: cast_expr,
..
} => {
if let Ok(concrete_type) =
sql_data_type_to_concrete_data_type(&DataType::Timestamp(*precision, *zone))
{
if let Ok(concrete_type) = sql_data_type_to_concrete_data_type(
&DataType::Timestamp(*precision, *zone),
&Default::default(),
) {
let new_type = concrete_type.as_arrow_type();
*expr = Expr::Function(cast_expr_to_arrow_cast_func(
(**cast_expr).clone(),