rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-17 01:12:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add JSON2 storage layout settings

Browse Source 
Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
discord9
2026-08-21 11:51:21 +08:00
parent d18430b874
commit fed5c80d7e
8 changed files with 482 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/sql/src/error.rs
+10 -1
View File
@@ -339,6 +339,13 @@ pub enum Error {
#[snafu(implicit)]
location: Location,
},
#[snafu(transparent)]
Datatypes {
#[snafu(implicit)]
location: Location,
source: datatypes::error::Error,
},
}
impl ErrorExt for Error {
@@ -384,7 +391,9 @@ impl ErrorExt for Error {
#[cfg(feature = "enterprise")]
InvalidTriggerWebhookOption { .. } => StatusCode::InvalidArguments,
SerializeColumnDefaultConstraint { source, .. } => source.status_code(),
SerializeColumnDefaultConstraint { source, .. } | Datatypes { source, .. } => {
source.status_code()
}
ConvertToGrpcDataType { source, .. } => source.status_code(),
SqlCommon { source, .. } => source.status_code(),
src/sql/src/parsers/create_parser.rs
+2 -2
View File
@@ -711,8 +711,8 @@ impl<'a> ParserContext<'a> {
let mut extensions = ColumnExtensions::default();
let data_type =
if let Some((data_type, type_hints)) = json::parse_json2_type_and_hints(parser)? {
extensions.json_type_hints = type_hints;
if let Some((data_type, options)) = json::parse_json2_type_and_options(parser)? {
extensions.json2_options = options;
data_type
} else {
parser.parse_data_type().context(SyntaxSnafu)?
src/sql/src/parsers/create_parser/json.rs
+134 -20
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use datatypes::extension::json::JSON2_REMAINDER_FIELD_NAME;
use datatypes::json::JSON2_MAX_STRUCTURED_DEPTH;
use snafu::{ResultExt, ensure};
use sqlparser::ast::{DataType, ExactNumberInfo, Expr, ObjectName, UnaryOperator};
@@ -22,14 +23,15 @@ use sqlparser::tokenizer::Token;
use crate::ast::Ident;
use crate::error::{InvalidSqlSnafu, Result, SyntaxSnafu};
use crate::parsers::create_parser::{INVERTED, SKIPPING};
use crate::statements::create::JsonTypeHint;
use crate::statements::create::{Json2Options, JsonTypeHint};
use crate::statements::transform::type_alias::get_type_by_alias;
const JSON2_TYPE_NAME: &str = "JSON2";
const MAX_AUTO_EXPANDED_PATHS: &str = "max_auto_expanded_paths";
pub(super) fn parse_json2_type_and_hints(
pub(super) fn parse_json2_type_and_options(
parser: &mut Parser<'_>,
) -> Result<Option<(DataType, Vec<JsonTypeHint>)>> {
) -> Result<Option<(DataType, Option<Json2Options>)>> {
let token = parser.peek_token();
let Token::Word(word) = &token.token else {
return Ok(None);
@@ -41,26 +43,62 @@ pub(super) fn parse_json2_type_and_hints(
parser.next_token();
let data_type = DataType::Custom(ObjectName::from(vec![Ident::new(JSON2_TYPE_NAME)]), vec![]);
let type_hints = if parser.consume_token(&Token::LParen) {
parse_json2_type_hints(parser)?
let options = if parser.consume_token(&Token::LParen) {
parse_json2_options(parser)?
} else {
vec![]
None
};
Ok(Some((data_type, type_hints)))
Ok(Some((data_type, options)))
}
fn parse_json2_type_hints(parser: &mut Parser<'_>) -> Result<Vec<JsonTypeHint>> {
let mut hints = Vec::new();
fn parse_json2_options(parser: &mut Parser<'_>) -> Result<Option<Json2Options>> {
if parser.consume_token(&Token::RParen) {
return Ok(hints);
return Ok(None);
}
let mut max_auto_expanded_paths = None;
let mut type_hints = Vec::new();
loop {
let hint = parse_json2_type_hint(parser)?;
ensure_no_path_conflict(&hints, &hint.path)?;
hints.push(hint);
let token = parser.peek_token();
let is_max_auto_expanded_paths = matches!(
&token.token,
Token::Word(word)
if word.quote_style.is_none()
&& word.value.eq_ignore_ascii_case(MAX_AUTO_EXPANDED_PATHS)
);
if is_max_auto_expanded_paths {
parser.next_token();
ensure!(
max_auto_expanded_paths.is_none(),
InvalidSqlSnafu {
msg: format!("duplicated JSON2 option '{MAX_AUTO_EXPANDED_PATHS}'")
}
);
parser.expect_token(&Token::Eq).context(SyntaxSnafu)?;
let token = parser.next_token();
let Token::Number(value, _) = token.token else {
return InvalidSqlSnafu {
msg: format!(
"JSON2 option '{MAX_AUTO_EXPANDED_PATHS}' expects a non-negative integer"
),
}
.fail();
};
max_auto_expanded_paths = Some(value.parse::<u32>().map_err(|_| {
InvalidSqlSnafu {
msg: format!(
"JSON2 option '{MAX_AUTO_EXPANDED_PATHS}' expects a non-negative integer"
),
}
.build()
})?);
} else {
let hint = parse_json2_type_hint(parser)?;
ensure_no_path_conflict(&type_hints, &hint.path)?;
type_hints.push(hint);
}
if parser.consume_token(&Token::Comma) {
if parser.consume_token(&Token::RParen) {
@@ -72,11 +110,22 @@ fn parse_json2_type_hints(parser: &mut Parser<'_>) -> Result<Vec<JsonTypeHint>>
}
}
Ok(hints)
Ok(Some(Json2Options {
max_auto_expanded_paths,
type_hints,
}))
}
fn parse_json2_type_hint(parser: &mut Parser<'_>) -> Result<JsonTypeHint> {
let path = parse_json2_path(parser)?;
ensure!(
path.first().is_none_or(|x| x != JSON2_REMAINDER_FIELD_NAME),
InvalidSqlSnafu {
msg: format!(
"JSON2 type hint path cannot be rooted at reserved field '{JSON2_REMAINDER_FIELD_NAME}'"
)
}
);
ensure!(
path.len() <= JSON2_MAX_STRUCTURED_DEPTH,
InvalidSqlSnafu {
@@ -309,7 +358,7 @@ CREATE TABLE traces (
column.column_def.data_type,
DataType::Custom(_, _)
));
let hints = column.extensions.json_type_hints;
let hints = column.extensions.json2_options.unwrap().type_hints;
assert_eq!(hints.len(), 4);
assert_eq!(hints[0].path, vec!["service.name"]);
@@ -339,6 +388,71 @@ CREATE TABLE traces (
assert!(hints[3].nullable);
}
#[test]
fn test_parse_json2_max_auto_expanded_paths() {
let column = parse_json2_column(
r#"
CREATE TABLE traces (
log_json_data JSON2 (
http.method STRING,
max_auto_expanded_paths = 0
),
ts TIMESTAMP TIME INDEX,
)"#,
);
let options = column.extensions.json2_options.unwrap();
assert_eq!(options.max_auto_expanded_paths, Some(0));
assert_eq!(options.type_hints.len(), 1);
let empty = parse_json2_column(
r#"
CREATE TABLE traces (
log_json_data JSON2 (),
ts TIMESTAMP TIME INDEX,
)"#,
);
assert!(empty.extensions.json2_options.is_none());
let quoted = parse_json2_column(
r#"
CREATE TABLE traces (
log_json_data JSON2 (
"max_auto_expanded_paths" STRING,
nested."!__remainder__!" STRING
),
ts TIMESTAMP TIME INDEX,
)"#,
);
let options = quoted.extensions.json2_options.unwrap();
assert_eq!(options.max_auto_expanded_paths, None);
assert_eq!(options.type_hints.len(), 2);
}
#[test]
fn test_parse_json2_max_auto_expanded_paths_rejects_invalid_options() {
for options in [
"max_auto_expanded_paths = 0, max_auto_expanded_paths = 1",
"max_auto_expanded_paths = -1",
"max_auto_expanded_paths = 1.5",
"max_auto_expanded_paths = 4294967296",
r#""!__remainder__!".value STRING"#,
] {
let sql = format!(
"CREATE TABLE traces (log_json_data JSON2 ({options}), ts TIMESTAMP TIME INDEX)"
);
assert!(
ParserContext::create_with_dialect(
&sql,
&GreptimeDbDialect {},
ParseOptions::default()
)
.is_err(),
"{options}"
);
}
}
#[test]
fn test_parse_json2_type_hint_default_nullable() {
let column = parse_json2_column(
@@ -349,7 +463,7 @@ CREATE TABLE traces (
)"#,
);
let hints = column.extensions.json_type_hints;
let hints = column.extensions.json2_options.unwrap().type_hints;
assert_eq!(hints.len(), 1);
assert!(hints[0].nullable);
}
@@ -369,7 +483,7 @@ CREATE TABLE traces (
)"#,
);
let hints = column.extensions.json_type_hints;
let hints = column.extensions.json2_options.unwrap().type_hints;
assert_eq!(hints.len(), 4);
assert_eq!(hints[0].path, vec!["a", "b"]);
assert_eq!(hints[1].path, vec!["x", "y"]);
@@ -402,7 +516,7 @@ CREATE TABLE traces (
)"#,
);
let hints = column.extensions.json_type_hints;
let hints = column.extensions.json2_options.unwrap().type_hints;
assert_eq!(hints.len(), 14);
for hint in hints.iter().take(6) {
assert_eq!(hint.data_type, DataType::BigInt(None));
@@ -428,7 +542,7 @@ CREATE TABLE traces (
)"#,
);
let hints = column.extensions.json_type_hints;
let hints = column.extensions.json2_options.unwrap().type_hints;
assert_eq!(hints.len(), 2);
assert_eq!(
hints[0]
src/sql/src/statements.rs
+3 -15
View File
@@ -718,11 +718,7 @@ mod tests {
"true".to_string(),
),
])),
vector_options: None,
skipping_index_options: None,
inverted_index_options: None,
json_type_hints: vec![],
vector_index_options: None,
..Default::default()
},
};
@@ -749,17 +745,13 @@ mod tests {
options: vec![],
},
extensions: ColumnExtensions {
fulltext_index_options: None,
vector_options: None,
skipping_index_options: None,
inverted_index_options: None,
json_type_hints: vec![],
vector_index_options: Some(OptionMap::from([
("metric".to_string(), "cosine".to_string()),
("connectivity".to_string(), "32".to_string()),
("expansion_add".to_string(), "200".to_string()),
("expansion_search".to_string(), "100".to_string()),
])),
..Default::default()
},
};
@@ -790,12 +782,8 @@ mod tests {
options: vec![],
},
extensions: ColumnExtensions {
fulltext_index_options: None,
vector_options: None,
skipping_index_options: None,
inverted_index_options: None,
json_type_hints: vec![],
vector_index_options: Some(OptionMap::default()),
..Default::default()
},
};
src/sql/src/statements/create.rs
+80 -64
View File
@@ -136,7 +136,28 @@ pub struct ColumnExtensions {
pub inverted_index_options: Option<OptionMap>,
/// Vector index options for HNSW-based vector similarity search.
pub vector_index_options: Option<OptionMap>,
pub json_type_hints: Vec<JsonTypeHint>,
/// JSON2-specific column options.
pub json2_options: Option<Json2Options>,
}
/// JSON2-specific options represented in the SQL AST.
#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]
pub struct Json2Options {
/// Maximum number of unhinted JSON2 paths expanded into Arrow fields.
pub(crate) max_auto_expanded_paths: Option<u32>,
/// Paths stored as explicitly typed JSON2 fields.
pub(crate) type_hints: Vec<JsonTypeHint>,
}
impl Display for Json2Options {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut options = Vec::with_capacity(self.type_hints.len() + 1);
if let Some(max) = self.max_auto_expanded_paths {
options.push(format!("max_auto_expanded_paths = {max}"));
}
options.extend(self.type_hints.iter().map(format_json_type_hint));
write!(f, "(\n {}\n )", options.iter().join(",\n "))
}
}
#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]
@@ -180,12 +201,8 @@ impl Display for Column {
}
write!(f, "{} {}", self.column_def.name, self.column_def.data_type)?;
if !self.extensions.json_type_hints.is_empty() {
write!(
f,
"{}",
format_json_type_hints(&self.extensions.json_type_hints)
)?;
if let Some(options) = &self.extensions.json2_options {
write!(f, "{options}")?;
}
for option in &self.column_def.options {
write!(f, " {option}")?;
@@ -335,29 +352,30 @@ impl ColumnExtensions {
}
pub fn build_json_settings(&self) -> Result<Option<JsonSettings>> {
if self.json_type_hints.is_empty() {
let Some(options) = &self.json2_options else {
return Ok(None);
}
};
Ok(Some(JsonSettings::new(
self.json_type_hints
.iter()
.map(|hint| {
Ok(datatypes::json::JsonTypeHint {
path: hint.path.clone(),
data_type: json_type_hint_concrete_data_type(&hint.data_type)?,
nullable: hint.nullable,
default_constraint: build_json_type_hint_default_constraint(hint)?,
inverted_index: hint.inverted_index,
})
let type_hints = options
.type_hints
.iter()
.map(|hint| {
Ok(datatypes::json::JsonTypeHint {
path: hint.path.clone(),
data_type: json_type_hint_concrete_data_type(&hint.data_type)?,
nullable: hint.nullable,
default_constraint: build_json_type_hint_default_constraint(hint)?,
inverted_index: hint.inverted_index,
})
.collect::<Result<Vec<_>>>()?,
)))
})
.collect::<Result<Vec<_>>>()?;
let settings = JsonSettings::try_new(type_hints, options.max_auto_expanded_paths)?;
Ok(Some(settings))
}
pub fn set_json_settings(&mut self, settings: JsonSettings) -> Result<()> {
self.json_type_hints = settings
.type_hints
let (type_hints, max_auto_expanded_paths) = settings.into_parts();
let type_hints = type_hints
.into_iter()
.map(|hint| {
let data_type = json_type_hint_sql_data_type(&hint.data_type)?;
@@ -374,6 +392,11 @@ impl ColumnExtensions {
})
})
.collect::<Result<Vec<_>>>()?;
self.json2_options = (max_auto_expanded_paths.is_some() || !type_hints.is_empty())
.then_some(Json2Options {
max_auto_expanded_paths,
type_hints,
});
Ok(())
}
}
@@ -487,13 +510,6 @@ fn format_json_type_hint(hint: &JsonTypeHint) -> String {
)
}
fn format_json_type_hints(hints: &[JsonTypeHint]) -> String {
format!(
"(\n {}\n )",
hints.iter().map(format_json_type_hint).join(",\n ")
)
}
fn format_json_path_segment(segment: &str) -> String {
format!("\"{}\"", segment.replace('"', "\"\""))
}
@@ -786,6 +802,7 @@ mod tests {
use datatypes::schema::ColumnDefaultConstraint;
use datatypes::value::Value;
use super::*;
use crate::dialect::GreptimeDbDialect;
use crate::error::Error;
use crate::parser::{ParseOptions, ParserContext};
@@ -1066,7 +1083,7 @@ ENGINE=mito
.build_json_settings()
.unwrap()
.unwrap();
let hints = settings.type_hints;
let hints = settings.type_hints();
assert_eq!(hints[0].data_type, ConcreteDataType::int64_datatype());
assert_eq!(
@@ -1118,10 +1135,10 @@ ENGINE=mito
}
#[test]
fn test_set_json_settings_normalizes_type_hint_sql_types() {
fn test_set_json_settings_normalizes_type_hint_sql_types() -> Result<()> {
let mut extensions = super::ColumnExtensions::default();
extensions
.set_json_settings(JsonSettings::new(vec![
let settings = JsonSettings::try_new(
vec![
DatatypeJsonTypeHint {
path: vec!["i".to_string()],
data_type: ConcreteDataType::int32_datatype(),
@@ -1157,36 +1174,51 @@ ENGINE=mito
default_constraint: None,
inverted_index: false,
},
]))
.unwrap();
],
None,
)?;
extensions.set_json_settings(settings)?;
assert_eq!(
extensions
.json_type_hints
.json2_options
.unwrap()
.type_hints
.iter()
.map(|hint| hint.data_type.to_string())
.collect::<Vec<_>>(),
vec!["BIGINT", "DOUBLE", "BIGINT UNSIGNED", "STRING", "BOOLEAN"]
);
Ok(())
}
#[test]
fn test_set_json_settings_rejects_unsupported_type_hint_type() {
let mut extensions = super::ColumnExtensions::default();
let err = extensions
.set_json_settings(JsonSettings::new(vec![DatatypeJsonTypeHint {
fn test_set_json_settings_rejects_unsupported_type_hint_type() -> Result<()> {
let err = JsonSettings::try_new(
vec![DatatypeJsonTypeHint {
path: vec!["u".to_string()],
data_type: ConcreteDataType::date_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
}]))
.unwrap_err();
}],
None,
)
.unwrap_err();
assert!(
err.to_string()
.contains("unsupported JSON2 type hint data type")
);
Ok(())
}
#[test]
fn test_set_empty_json_settings_omits_json2_options() -> Result<()> {
let mut extensions = ColumnExtensions::default();
extensions.set_json_settings(JsonSettings::default())?;
assert!(extensions.json2_options.is_none());
Ok(())
}
#[test]
@@ -1364,15 +1396,11 @@ AS SELECT number FROM numbers_input where number > 10"#,
// Test zero connectivity should fail
let extensions = ColumnExtensions {
fulltext_index_options: None,
vector_options: None,
skipping_index_options: None,
inverted_index_options: None,
json_type_hints: vec![],
vector_index_options: Some(OptionMap::from([(
"connectivity".to_string(),
"0".to_string(),
)])),
..Default::default()
};
let result = extensions.build_vector_index_options();
assert!(result.is_err());
@@ -1385,15 +1413,11 @@ AS SELECT number FROM numbers_input where number > 10"#,
// Test zero expansion_add should fail
let extensions = ColumnExtensions {
fulltext_index_options: None,
vector_options: None,
skipping_index_options: None,
inverted_index_options: None,
json_type_hints: vec![],
vector_index_options: Some(OptionMap::from([(
"expansion_add".to_string(),
"0".to_string(),
)])),
..Default::default()
};
let result = extensions.build_vector_index_options();
assert!(result.is_err());
@@ -1406,15 +1430,11 @@ AS SELECT number FROM numbers_input where number > 10"#,
// Test zero expansion_search should fail
let extensions = ColumnExtensions {
fulltext_index_options: None,
vector_options: None,
skipping_index_options: None,
inverted_index_options: None,
json_type_hints: vec![],
vector_index_options: Some(OptionMap::from([(
"expansion_search".to_string(),
"0".to_string(),
)])),
..Default::default()
};
let result = extensions.build_vector_index_options();
assert!(result.is_err());
@@ -1427,16 +1447,12 @@ AS SELECT number FROM numbers_input where number > 10"#,
// Test valid values should succeed
let extensions = ColumnExtensions {
fulltext_index_options: None,
vector_options: None,
skipping_index_options: None,
inverted_index_options: None,
json_type_hints: vec![],
vector_index_options: Some(OptionMap::from([
("connectivity".to_string(), "32".to_string()),
("expansion_add".to_string(), "200".to_string()),
("expansion_search".to_string(), "100".to_string()),
])),
..Default::default()
};
let result = extensions.build_vector_index_options();
assert!(result.is_ok());