rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-05 03:25:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(json2)!: remove nullable and default type hint options (#9213)

Browse Source 
feat(json2): remove nullable and default type hint options
This commit is contained in:
fys
2026-09-18 07:08:35 +00:00
committed by GitHub
parent 5ef8a46e3f
commit 5b631fd8eb
18 changed files with 112 additions and 596 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/sql/src/parsers/alter_parser.rs
+2 -4
View File
@@ -1565,8 +1565,8 @@ ALTER TABLE metrics REPARTITION
MODIFY COLUMN attrs JSON2 (
max_auto_expanded_paths = 2000,
trace_id STRING,
user.id STRING NOT NULL,
user.name STRING DEFAULT 'anonymous',
user.id STRING,
user.name STRING,
request_id STRING INVERTED INDEX
)"#;
let mut statements =
@@ -1590,8 +1590,6 @@ MODIFY COLUMN attrs JSON2 (
assert_eq!(Some(2000), options.max_auto_expanded_paths);
assert_eq!(4, options.type_hints.len());
assert_eq!(vec!["user", "id"], options.type_hints[1].path);
assert!(!options.type_hints[1].nullable);
assert!(options.type_hints[2].default.is_some());
assert!(options.type_hints[3].inverted_index);
let formatted = alter_table.to_string();
src/sql/src/parsers/create_parser/json.rs
+34 -131
View File
@@ -15,7 +15,7 @@
use datatypes::extension::json::JSON2_REMAINDER_FIELD_NAME;
use datatypes::json::JSON2_MAX_STRUCTURED_DEPTH;
use snafu::{ResultExt, ensure};
use sqlparser::ast::{DataType, ExactNumberInfo, Expr, ObjectName, UnaryOperator};
use sqlparser::ast::{DataType, ExactNumberInfo, ObjectName};
use sqlparser::dialect::keywords::Keyword;
use sqlparser::parser::Parser;
use sqlparser::tokenizer::Token;
@@ -157,49 +157,22 @@ fn parse_json2_type_hint(parser: &mut Parser<'_>) -> Result<JsonTypeHint> {
let data_type = parser.parse_data_type().context(SyntaxSnafu)?;
let data_type = normalize_json2_type_hint_type(data_type)?;
let mut nullable = true;
let mut nullable_set = false;
let mut default = None;
let mut inverted_index = false;
loop {
if parser.parse_keywords(&[Keyword::NOT, Keyword::NULL]) {
ensure!(
!nullable_set,
InvalidSqlSnafu {
msg: format!(
"NULL/NOT NULL option already specified for JSON2 type hint '{}'",
path.join(".")
)
}
);
nullable = false;
nullable_set = true;
} else if parser.parse_keyword(Keyword::NULL) {
ensure!(
!nullable_set,
InvalidSqlSnafu {
msg: format!(
"NULL/NOT NULL option already specified for JSON2 type hint '{}'",
path.join(".")
)
}
);
nullable = true;
nullable_set = true;
if parser.parse_keywords(&[Keyword::NOT, Keyword::NULL])
|| parser.parse_keyword(Keyword::NULL)
{
return InvalidSqlSnafu {
msg: "JSON2 type hint NULL/NOT NULL is not supported; hinted fields are always nullable"
.to_string(),
}
.fail();
} else if parser.parse_keyword(Keyword::DEFAULT) {
ensure!(
default.is_none(),
InvalidSqlSnafu {
msg: format!(
"duplicated DEFAULT option for JSON2 type hint '{}'",
path.join(".")
)
}
);
let expr = parser.parse_expr().context(SyntaxSnafu)?;
ensure_json2_default_expr_is_literal(&expr)?;
default = Some(expr);
return InvalidSqlSnafu {
msg: "JSON2 type hint DEFAULT is not supported".to_string(),
}
.fail();
} else if let Token::Word(word) = parser.peek_token().token
&& word.value.eq_ignore_ascii_case(INVERTED)
{
@@ -242,8 +215,6 @@ fn parse_json2_type_hint(parser: &mut Parser<'_>) -> Result<JsonTypeHint> {
Ok(JsonTypeHint {
path,
data_type,
nullable,
default,
inverted_index,
})
}
@@ -298,24 +269,6 @@ fn normalize_json2_type_hint_type(data_type: DataType) -> Result<DataType> {
Ok(normalized)
}
fn ensure_json2_default_expr_is_literal(expr: &Expr) -> Result<()> {
let is_literal = match expr {
Expr::Value(_) => true,
Expr::UnaryOp { op, expr } => {
matches!(op, UnaryOperator::Plus | UnaryOperator::Minus)
&& matches!(expr.as_ref(), Expr::Value(_))
}
_ => false,
};
ensure!(
is_literal,
InvalidSqlSnafu {
msg: "JSON2 type hint DEFAULT only supports literal values",
}
);
Ok(())
}
fn ensure_no_path_conflict(hints: &[JsonTypeHint], path: &[String]) -> Result<()> {
for hint in hints {
ensure!(
@@ -375,10 +328,10 @@ mod tests {
r#"
CREATE TABLE traces (
log_json_data JSON2 (
"service.name" STRING NOT NULL DEFAULT 'null' INVERTED INDEX,
http.method STRING NOT NULL,
status_code INT64 NOT NULL,
comment STRING NULL,
"service.name" STRING INVERTED INDEX,
http.method STRING,
status_code INT64,
comment STRING,
),
ts TIMESTAMP TIME INDEX,
)"#,
@@ -393,29 +346,17 @@ CREATE TABLE traces (
assert_eq!(hints[0].path, vec!["service.name"]);
assert_eq!(hints[0].data_type, DataType::String(None));
assert!(!hints[0].nullable);
assert_eq!(
hints[0]
.default
.as_ref()
.map(|expr| expr.to_string())
.as_deref(),
Some("'null'")
);
assert!(hints[0].inverted_index);
assert_eq!(hints[1].path, vec!["http", "method"]);
assert_eq!(hints[1].data_type, DataType::String(None));
assert!(!hints[1].nullable);
assert!(!hints[1].inverted_index);
assert_eq!(hints[2].path, vec!["status_code"]);
assert_eq!(hints[2].data_type, DataType::BigInt(None));
assert!(!hints[2].nullable);
assert_eq!(hints[3].path, vec!["comment"]);
assert_eq!(hints[3].data_type, DataType::String(None));
assert!(hints[3].nullable);
}
#[test]
@@ -484,7 +425,7 @@ CREATE TABLE traces (
}
#[test]
fn test_parse_json2_type_hint_default_nullable() {
fn test_parse_json2_type_hint_defaults_to_nullable() {
let column = parse_json2_column(
r#"
CREATE TABLE traces (
@@ -495,7 +436,7 @@ CREATE TABLE traces (
let hints = column.extensions.json2_options.unwrap().type_hints;
assert_eq!(hints.len(), 1);
assert!(hints[0].nullable);
assert_eq!(hints[0].data_type, DataType::String(None));
}
#[test]
@@ -560,57 +501,19 @@ CREATE TABLE traces (
}
#[test]
fn test_parse_json2_type_hint_default_accepts_signed_literals() {
let column = parse_json2_column(
r#"
CREATE TABLE traces (
log_json_data JSON2 (
negative_int INT64 DEFAULT -5,
positive_float FLOAT64 DEFAULT +1.5
),
ts TIMESTAMP TIME INDEX,
)"#,
);
let hints = column.extensions.json2_options.unwrap().type_hints;
assert_eq!(hints.len(), 2);
assert_eq!(
hints[0]
.default
.as_ref()
.map(|expr| expr.to_string())
.as_deref(),
Some("-5")
);
assert_eq!(
hints[1]
.default
.as_ref()
.map(|expr| expr.to_string())
.as_deref(),
Some("+1.5")
);
}
#[test]
fn test_parse_json2_type_hint_default_rejects_function() {
let result = ParserContext::create_with_dialect(
r#"
CREATE TABLE traces (
log_json_data JSON2 (status_code INT64 DEFAULT abs(-1)),
ts TIMESTAMP TIME INDEX,
)"#,
&GreptimeDbDialect {},
ParseOptions::default(),
);
assert!(result.is_err());
assert!(
result
.unwrap_err()
.to_string()
.contains("DEFAULT only supports literal values")
);
fn test_parse_json2_type_hint_rejects_default() {
for default in ["-5", "abs(-1)"] {
let sql = format!(
"CREATE TABLE traces (log_json_data JSON2 (status_code INT64 DEFAULT {default}), ts TIMESTAMP TIME INDEX)"
);
let err = ParserContext::create_with_dialect(
&sql,
&GreptimeDbDialect {},
ParseOptions::default(),
)
.unwrap_err();
assert!(err.to_string().contains("DEFAULT is not supported"));
}
}
#[test]
@@ -646,7 +549,7 @@ CREATE TABLE traces (
}
#[test]
fn test_parse_json2_type_hint_rejects_duplicated_nullability() {
fn test_parse_json2_type_hint_rejects_nullability() {
for sql in [
r#"
CREATE TABLE traces (
@@ -680,7 +583,7 @@ CREATE TABLE traces (
result
.unwrap_err()
.to_string()
.contains("NULL/NOT NULL option already specified")
.contains("NULL/NOT NULL is not supported")
);
}
}
src/sql/src/statements.rs
-2
View File
@@ -678,8 +678,6 @@ mod tests {
vec![datatypes::json::JsonTypeHint {
path: vec!["kind".to_string()],
data_type: ConcreteDataType::string_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
}],
None,
src/sql/src/statements/create.rs
+21 -141
View File
@@ -16,30 +16,27 @@ use std::collections::HashMap;
use std::fmt::{Display, Formatter};
use common_catalog::consts::FILE_ENGINE;
use common_sql::default_constraint::parse_column_default_constraint;
use datatypes::json::{JSON2_DEFAULT_MAX_AUTO_EXPANDED_PATHS, JsonSettings};
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{
ColumnDefaultConstraint, FulltextOptions, SkippingIndexOptions, VectorDistanceMetric,
VectorIndexEngineType, VectorIndexOptions,
FulltextOptions, SkippingIndexOptions, VectorDistanceMetric, VectorIndexEngineType,
VectorIndexOptions,
};
use itertools::Itertools;
use serde::Serialize;
use snafu::ResultExt;
use sqlparser::ast::{ColumnOption, ColumnOptionDef, DataType, Expr};
use sqlparser::ast::{ColumnOptionDef, DataType, Expr};
use sqlparser_derive::{Visit, VisitMut};
use crate::ast::{ColumnDef, Ident, ObjectName, Value as SqlValue};
use crate::dialect::GreptimeDbDialect;
use crate::error::{
InvalidFlowQuerySnafu, InvalidSqlSnafu, Result, SetFulltextOptionSnafu,
SetSkippingIndexOptionSnafu,
};
use crate::parser::ParserContext;
use crate::statements::query::Query as GtQuery;
use crate::statements::statement::Statement;
use crate::statements::tql::Tql;
use crate::statements::{OptionMap, sql_data_type_to_concrete_data_type, value_to_sql_value};
use crate::statements::{OptionMap, sql_data_type_to_concrete_data_type};
const LINE_SEP: &str = ",\n";
const COMMA_SEP: &str = ", ";
@@ -164,8 +161,6 @@ impl Display for Json2Options {
pub struct JsonTypeHint {
pub path: Vec<String>,
pub data_type: DataType,
pub nullable: bool,
pub default: Option<Expr>,
pub inverted_index: bool,
}
@@ -363,8 +358,6 @@ impl ColumnExtensions {
Ok(datatypes::json::JsonTypeHint {
path: hint.path.clone(),
data_type: json_type_hint_concrete_data_type(&hint.data_type)?,
nullable: hint.nullable,
default_constraint: build_json_type_hint_default_constraint(hint)?,
inverted_index: hint.inverted_index,
})
})
@@ -384,15 +377,9 @@ impl ColumnExtensions {
.into_iter()
.map(|hint| {
let data_type = json_type_hint_sql_data_type(&hint.data_type)?;
let default = hint
.default_constraint
.map(|constraint| column_default_constraint_to_expr(&constraint))
.transpose()?;
Ok(JsonTypeHint {
path: hint.path,
data_type,
nullable: hint.nullable,
default,
inverted_index: hint.inverted_index,
})
})
@@ -406,38 +393,6 @@ impl ColumnExtensions {
}
}
fn build_json_type_hint_default_constraint(
hint: &JsonTypeHint,
) -> Result<Option<ColumnDefaultConstraint>> {
let Some(default) = &hint.default else {
return Ok(None);
};
let data_type = json_type_hint_concrete_data_type(&hint.data_type)?;
let opts = [ColumnOptionDef {
name: None,
option: ColumnOption::Default(default.clone()),
}];
// Use the JSON path as the column name context for default value parsing errors.
let json_path = hint.path.join(".");
let default_constraint = parse_column_default_constraint(&json_path, &data_type, &opts, None)
.context(crate::error::SqlCommonSnafu)?;
if let Some(constraint) = &default_constraint {
constraint
.validate(&data_type, hint.nullable)
.map_err(|e| {
InvalidSqlSnafu {
msg: format!("invalid DEFAULT for JSON2 type hint '{}': {e}", json_path),
}
.build()
})?;
}
Ok(default_constraint)
}
fn json_type_hint_concrete_data_type(data_type: &DataType) -> Result<ConcreteDataType> {
let data_type = sql_data_type_to_concrete_data_type(data_type)?;
normalize_json_type_hint_concrete_data_type(&data_type)
@@ -483,36 +438,18 @@ fn json_type_hint_sql_data_type(data_type: &ConcreteDataType) -> Result<DataType
Ok(sql_type)
}
fn column_default_constraint_to_expr(constraint: &ColumnDefaultConstraint) -> Result<Expr> {
match constraint {
ColumnDefaultConstraint::Value(value) => Ok(Expr::Value(value_to_sql_value(value)?.into())),
ColumnDefaultConstraint::Function(function) => {
ParserContext::parse_function(function, &GreptimeDbDialect {})
}
}
}
fn format_json_type_hint(hint: &JsonTypeHint) -> String {
let path = hint
.path
.iter()
.map(|segment| format_json_path_segment(segment))
.join(".");
let nullability = if hint.nullable { " NULL" } else { " NOT NULL" };
let default = hint
.default
.as_ref()
.map(|expr| format!(" DEFAULT {expr}"))
.unwrap_or_default();
let inverted_index = if hint.inverted_index {
" INVERTED INDEX"
} else {
""
};
format!(
"{} {}{}{}{}",
path, hint.data_type, nullability, default, inverted_index
)
format!("{} {}{}", path, hint.data_type, inverted_index)
}
fn format_json_path_segment(segment: &str) -> String {
@@ -817,8 +754,6 @@ mod tests {
use datatypes::json::{JsonSettings, JsonTypeHint as DatatypeJsonTypeHint};
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::ColumnDefaultConstraint;
use datatypes::value::Value;
use super::*;
use crate::dialect::GreptimeDbDialect;
@@ -995,7 +930,7 @@ ENGINE=mito
let sql = r#"CREATE TABLE traces (
log_json_data JSON2 (
"service.name" STRING,
"a.b"."c" INT64 NOT NULL,
"a.b"."c" INT64,
a."b.c" STRING
),
ts TIMESTAMP TIME INDEX
@@ -1011,9 +946,9 @@ ENGINE=mito
r#"
CREATE TABLE traces (
log_json_data JSON2(
"service.name" STRING NULL,
"a.b"."c" BIGINT NOT NULL,
"a"."b.c" STRING NULL
"service.name" STRING,
"a.b"."c" BIGINT,
"a"."b.c" STRING
),
ts TIMESTAMP NOT NULL,
TIME INDEX (ts)
@@ -1039,7 +974,7 @@ ENGINE=mito
fn test_parse_json2_max_auto_expanded_paths_option() -> Result<()> {
let sql = r#"CREATE TABLE traces (
log_json_data JSON2 (
status_code INT64 NOT NULL,
status_code INT64,
max_auto_expanded_paths = 1
),
ts TIMESTAMP TIME INDEX
@@ -1065,7 +1000,7 @@ ENGINE=mito
let sql = r#"CREATE TABLE traces (
log_json_data JSON2 (
"1abc" STRING,
a."2b" INT64 NOT NULL
a."2b" INT64
),
ts TIMESTAMP TIME INDEX
)"#;
@@ -1080,8 +1015,8 @@ ENGINE=mito
r#"
CREATE TABLE traces (
log_json_data JSON2(
"1abc" STRING NULL,
"a"."2b" BIGINT NOT NULL
"1abc" STRING,
"a"."2b" BIGINT
),
ts TIMESTAMP NOT NULL,
TIME INDEX (ts)
@@ -1104,7 +1039,7 @@ ENGINE=mito
}
#[test]
fn test_json2_type_hint_default_builds_default_constraint() {
fn test_json2_type_hint_rejects_default() {
let sql = r#"CREATE TABLE traces (
log_json_data JSON2 (
status_code INT64 DEFAULT -5,
@@ -1114,67 +1049,24 @@ ENGINE=mito
),
ts TIMESTAMP TIME INDEX
)"#;
let result =
let err =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap();
let Statement::CreateTable(create_table) = &result[0] else {
unreachable!()
};
let settings = create_table.columns[0]
.extensions
.build_json_settings()
.unwrap()
.unwrap();
let hints = settings.type_hints();
assert_eq!(hints[0].data_type, ConcreteDataType::int64_datatype());
assert_eq!(
hints[0].default_constraint,
Some(ColumnDefaultConstraint::Value(Value::Int64(-5)))
);
assert_eq!(hints[1].data_type, ConcreteDataType::float64_datatype());
assert_eq!(
hints[1].default_constraint,
Some(ColumnDefaultConstraint::Value(Value::Float64(1.5.into())))
);
assert_eq!(hints[2].data_type, ConcreteDataType::boolean_datatype());
assert_eq!(
hints[2].default_constraint,
Some(ColumnDefaultConstraint::Value(Value::Boolean(false)))
);
assert_eq!(hints[3].data_type, ConcreteDataType::string_datatype());
assert_eq!(
hints[3].default_constraint,
Some(ColumnDefaultConstraint::Value(Value::String(
"unknown".into()
)))
);
.unwrap_err();
assert!(err.to_string().contains("DEFAULT is not supported"));
}
#[test]
fn test_json2_type_hint_not_null_default_null_is_rejected() {
fn test_json2_type_hint_rejects_not_null() {
let sql = r#"CREATE TABLE traces (
log_json_data JSON2 (
status_code INT64 NOT NULL DEFAULT NULL
),
ts TIMESTAMP TIME INDEX
)"#;
let result =
let err =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap();
let Statement::CreateTable(create_table) = &result[0] else {
unreachable!()
};
let err = create_table.columns[0]
.extensions
.build_json_settings()
.unwrap_err();
assert!(
err.to_string()
.contains("Default value should not be null for non null column")
);
.unwrap_err();
assert!(err.to_string().contains("NULL/NOT NULL is not supported"));
}
#[test]
@@ -1185,36 +1077,26 @@ ENGINE=mito
DatatypeJsonTypeHint {
path: vec!["i".to_string()],
data_type: ConcreteDataType::int32_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
DatatypeJsonTypeHint {
path: vec!["f".to_string()],
data_type: ConcreteDataType::float32_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
DatatypeJsonTypeHint {
path: vec!["u".to_string()],
data_type: ConcreteDataType::uint32_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
DatatypeJsonTypeHint {
path: vec!["s".to_string()],
data_type: ConcreteDataType::string_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
DatatypeJsonTypeHint {
path: vec!["b".to_string()],
data_type: ConcreteDataType::boolean_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
],
@@ -1241,8 +1123,6 @@ ENGINE=mito
vec![DatatypeJsonTypeHint {
path: vec!["u".to_string()],
data_type: ConcreteDataType::date_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
}],
None,