rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2025-12-26 08:00:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor!: unify sql options into OptionMap (#3792)

Browse Source 
* unify sql options into OptionMap

Signed-off-by: tison <wander4096@gmail.com>

* fixup

Signed-off-by: tison <wander4096@gmail.com>

* Update src/sql/src/util.rs

* drop legacy regions option

Signed-off-by: tison <wander4096@gmail.com>

* fixup

Signed-off-by: tison <wander4096@gmail.com>

* fixup

Signed-off-by: tison <wander4096@gmail.com>

---------

Signed-off-by: tison <wander4096@gmail.com>
This commit is contained in:
tison
2024-04-25 12:06:05 +08:00
committed by GitHub
parent 9524ec83bc
commit bba3108e0d
23 changed files with 110 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/schema-structs.md
View File

@@ -73,7 +73,7 @@ CREATE TABLE cpu (
usage_system DOUBLE, usage_system DOUBLE,
datacenter STRING, datacenter STRING,
TIME INDEX (ts), TIME INDEX (ts),
PRIMARY KEY(datacenter, host)) ENGINE=mito WITH(regions=1); PRIMARY KEY(datacenter, host)) ENGINE=mito;
``` ```
Then the table's `TableMeta` may look like this: Then the table's `TableMeta` may look like this:
@@ -249,7 +249,7 @@ CREATE TABLE cpu (
usage_system DOUBLE, usage_system DOUBLE,
datacenter STRING, datacenter STRING,
TIME INDEX (ts), TIME INDEX (ts),
PRIMARY KEY(datacenter, host)) ENGINE=mito WITH(regions=1); PRIMARY KEY(datacenter, host)) ENGINE=mito;
select ts, usage_system from cpu; select ts, usage_system from cpu;
``` ```

4
src/cmd/src/cli/export.rs
View File

@@ -492,9 +492,7 @@ mod tests {
) )
ENGINE=mito ENGINE=mito
WITH( ;
regions = 1
);
"#; "#;
assert_eq!(res.trim(), expect.trim()); assert_eq!(res.trim(), expect.trim());

2
src/frontend/src/instance.rs
View File

@@ -650,7 +650,7 @@ mod tests {
ts TIMESTAMP, ts TIMESTAMP,
TIME INDEX (ts), TIME INDEX (ts),
PRIMARY KEY(host) PRIMARY KEY(host)
) engine=mito with(regions=1);"#; ) engine=mito;"#;
replace_test(sql, plugins.clone(), &query_ctx); replace_test(sql, plugins.clone(), &query_ctx);
// test drop table // test drop table

6
src/operator/src/expr_factory.rs
View File

@@ -36,7 +36,6 @@ use sql::ast::{ColumnDef, ColumnOption, TableConstraint};
use sql::statements::alter::{AlterTable, AlterTableOperation}; use sql::statements::alter::{AlterTable, AlterTableOperation};
use sql::statements::create::{CreateExternalTable, CreateTable, TIME_INDEX}; use sql::statements::create::{CreateExternalTable, CreateTable, TIME_INDEX};
use sql::statements::{column_def_to_schema, sql_column_def_to_grpc_column_def}; use sql::statements::{column_def_to_schema, sql_column_def_to_grpc_column_def};
use sql::util::to_lowercase_options_map;
use table::requests::{TableOptions, FILE_TABLE_META_KEY}; use table::requests::{TableOptions, FILE_TABLE_META_KEY};
use table::table_reference::TableReference; use table::table_reference::TableReference;
@@ -190,8 +189,7 @@ pub fn create_to_expr(create: &CreateTable, query_ctx: QueryContextRef) -> Resul
let time_index = find_time_index(&create.constraints)?; let time_index = find_time_index(&create.constraints)?;
let table_options = HashMap::from( let table_options = HashMap::from(
&TableOptions::try_from(&to_lowercase_options_map(&create.options)) &TableOptions::try_from(create.options.as_ref()).context(UnrecognizedTableOptionSnafu)?,
.context(UnrecognizedTableOptionSnafu)?,
); );
let primary_keys = find_primary_keys(&create.columns, &create.constraints)?; let primary_keys = find_primary_keys(&create.columns, &create.constraints)?;
@@ -502,7 +500,7 @@ mod tests {
#[test] #[test]
fn test_create_to_expr() { fn test_create_to_expr() {
let sql = "CREATE TABLE monitor (host STRING,ts TIMESTAMP,TIME INDEX (ts),PRIMARY KEY(host)) ENGINE=mito WITH(regions=1, ttl='3days', write_buffer_size='1024KB');"; let sql = "CREATE TABLE monitor (host STRING,ts TIMESTAMP,TIME INDEX (ts),PRIMARY KEY(host)) ENGINE=mito WITH(ttl='3days', write_buffer_size='1024KB');";
let stmt = let stmt =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap() .unwrap()

59
src/query/src/sql/show_create_table.rs
View File

@@ -14,64 +14,35 @@
//! Implementation of `SHOW CREATE TABLE` statement. //! Implementation of `SHOW CREATE TABLE` statement.
use std::fmt::Display; use std::collections::HashMap;
use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema, SchemaRef, COMMENT_KEY}; use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema, SchemaRef, COMMENT_KEY};
use humantime::format_duration; use humantime::format_duration;
use snafu::ResultExt; use snafu::ResultExt;
use sql::ast::{ use sql::ast::{
ColumnDef, ColumnOption, ColumnOptionDef, Expr, Ident, ObjectName, SqlOption, TableConstraint, ColumnDef, ColumnOption, ColumnOptionDef, Expr, Ident, ObjectName, TableConstraint,
Value as SqlValue,
}; };
use sql::dialect::GreptimeDbDialect; use sql::dialect::GreptimeDbDialect;
use sql::parser::ParserContext; use sql::parser::ParserContext;
use sql::statements::create::{CreateTable, TIME_INDEX}; use sql::statements::create::{CreateTable, TIME_INDEX};
use sql::statements::{self}; use sql::statements::{self, OptionMap};
use table::metadata::{TableInfoRef, TableMeta}; use table::metadata::{TableInfoRef, TableMeta};
use table::requests::FILE_TABLE_META_KEY; use table::requests::{FILE_TABLE_META_KEY, TTL_KEY, WRITE_BUFFER_SIZE_KEY};
use crate::error::{ConvertSqlTypeSnafu, ConvertSqlValueSnafu, Result, SqlSnafu}; use crate::error::{ConvertSqlTypeSnafu, ConvertSqlValueSnafu, Result, SqlSnafu};
#[inline] fn create_sql_options(table_meta: &TableMeta) -> OptionMap {
fn number_value<T: Display>(n: T) -> SqlValue {
SqlValue::Number(format!("{}", n), false)
}
#[inline]
fn string_value(s: impl Into<String>) -> SqlValue {
SqlValue::SingleQuotedString(s.into())
}
#[inline]
fn sql_option(name: &str, value: SqlValue) -> SqlOption {
SqlOption {
name: name.into(),
value: Expr::Value(value),
}
}
fn create_sql_options(table_meta: &TableMeta) -> Vec<SqlOption> {
let table_opts = &table_meta.options; let table_opts = &table_meta.options;
let mut options = Vec::with_capacity(4 + table_opts.extra_options.len()); let mut options = HashMap::with_capacity(4 + table_opts.extra_options.len());
if !table_meta.region_numbers.is_empty() {
options.push(sql_option(
"regions",
number_value(table_meta.region_numbers.len()),
));
}
if let Some(write_buffer_size) = table_opts.write_buffer_size { if let Some(write_buffer_size) = table_opts.write_buffer_size {
options.push(sql_option( options.insert(
"write_buffer_size", WRITE_BUFFER_SIZE_KEY.to_string(),
string_value(write_buffer_size.to_string()), write_buffer_size.to_string(),
)); );
} }
if let Some(ttl) = table_opts.ttl { if let Some(ttl) = table_opts.ttl {
options.push(sql_option( options.insert(TTL_KEY.to_string(), format_duration(ttl).to_string());
"ttl",
string_value(format_duration(ttl).to_string()),
));
} }
for (k, v) in table_opts for (k, v) in table_opts
@@ -79,10 +50,10 @@ fn create_sql_options(table_meta: &TableMeta) -> Vec<SqlOption> {
.iter() .iter()
.filter(|(k, _)| k != &FILE_TABLE_META_KEY) .filter(|(k, _)| k != &FILE_TABLE_META_KEY)
{ {
options.push(sql_option(k, string_value(v))); options.insert(k.to_string(), v.to_string());
} }
options OptionMap { map: options }
} }
#[inline] #[inline]
@@ -265,9 +236,7 @@ CREATE TABLE IF NOT EXISTS "system_metrics" (
PRIMARY KEY ("id", "host") PRIMARY KEY ("id", "host")
) )
ENGINE=mito ENGINE=mito
WITH( "#,
regions = 3
)"#,
sql sql
); );
} }

9
src/sql/src/error.rs
View File

@@ -21,6 +21,7 @@ use common_time::timestamp::TimeUnit;
use common_time::Timestamp; use common_time::Timestamp;
use datatypes::prelude::{ConcreteDataType, Value}; use datatypes::prelude::{ConcreteDataType, Value};
use snafu::{Location, Snafu}; use snafu::{Location, Snafu};
use sqlparser::ast::Ident;
use sqlparser::parser::ParserError; use sqlparser::parser::ParserError;
use crate::ast::{Expr, Value as SqlValue}; use crate::ast::{Expr, Value as SqlValue};
@@ -141,6 +142,13 @@ pub enum Error {
#[snafu(display("Unrecognized table option key: {}", key))] #[snafu(display("Unrecognized table option key: {}", key))]
InvalidTableOption { key: String, location: Location }, InvalidTableOption { key: String, location: Location },
#[snafu(display("Unrecognized table option key: {}, value: {}", key, value))]
InvalidTableOptionValue {
key: Ident,
value: Expr,
location: Location,
},
#[snafu(display("Failed to serialize column default constraint"))] #[snafu(display("Failed to serialize column default constraint"))]
SerializeColumnDefaultConstraint { SerializeColumnDefaultConstraint {
location: Location, location: Location,
@@ -201,6 +209,7 @@ impl ErrorExt for Error {
| InvalidDefault { .. } => StatusCode::InvalidSyntax, | InvalidDefault { .. } => StatusCode::InvalidSyntax,
InvalidColumnOption { .. } InvalidColumnOption { .. }
| InvalidTableOptionValue { .. }
| InvalidDatabaseName { .. } | InvalidDatabaseName { .. }
| ColumnTypeMismatch { .. } | ColumnTypeMismatch { .. }
| InvalidTableName { .. } | InvalidTableName { .. }

12
src/sql/src/parsers/copy_parser.rs
View File

@@ -129,10 +129,8 @@ impl<'a> ParserContext<'a> {
let with = options let with = options
.into_iter() .into_iter()
.filter_map(|option| { .map(parse_option_string)
parse_option_string(option.value).map(|v| (option.name.value.to_lowercase(), v)) .collect::<Result<With>>()?;
})
.collect();
let connection_options = self let connection_options = self
.parser .parser
@@ -141,10 +139,8 @@ impl<'a> ParserContext<'a> {
let connection = connection_options let connection = connection_options
.into_iter() .into_iter()
.filter_map(|option| { .map(parse_option_string)
parse_option_string(option.value).map(|v| (option.name.value.to_lowercase(), v)) .collect::<Result<Connection>>()?;
})
.collect();
Ok((with, connection, location)) Ok((with, connection, location))
} }

77
src/sql/src/parsers/create_parser.rs
View File

@@ -34,8 +34,8 @@ use crate::parser::ParserContext;
use crate::statements::create::{ use crate::statements::create::{
CreateDatabase, CreateExternalTable, CreateTable, CreateTableLike, Partitions, TIME_INDEX, CreateDatabase, CreateExternalTable, CreateTable, CreateTableLike, Partitions, TIME_INDEX,
}; };
use crate::statements::get_data_type_by_alias_name;
use crate::statements::statement::Statement; use crate::statements::statement::Statement;
use crate::statements::{get_data_type_by_alias_name, OptionMap};
use crate::util::parse_option_string; use crate::util::parse_option_string;
pub const ENGINE: &str = "ENGINE"; pub const ENGINE: &str = "ENGINE";
@@ -73,32 +73,12 @@ impl<'a> ParserContext<'a> {
} }
let engine = self.parse_table_engine(common_catalog::consts::FILE_ENGINE)?; let engine = self.parse_table_engine(common_catalog::consts::FILE_ENGINE)?;
let options = self let options = self.parse_create_table_options()?;
.parser
.parse_options(Keyword::WITH)
.context(SyntaxSnafu)?
.into_iter()
.filter_map(|option| {
if let Some(v) = parse_option_string(option.value) {
Some((option.name.value.to_lowercase(), v))
} else {
None
}
})
.collect::<HashMap<String, String>>();
for key in options.keys() {
ensure!(
validate_table_option(key),
InvalidTableOptionSnafu {
key: key.to_string()
}
);
}
Ok(Statement::CreateExternalTable(CreateExternalTable { Ok(Statement::CreateExternalTable(CreateExternalTable {
name: table_name, name: table_name,
columns, columns,
constraints, constraints,
options: options.into(), options,
if_not_exists, if_not_exists,
engine, engine,
})) }))
@@ -149,20 +129,7 @@ impl<'a> ParserContext<'a> {
} }
let engine = self.parse_table_engine(default_engine())?; let engine = self.parse_table_engine(default_engine())?;
let options = self let options = self.parse_create_table_options()?;
.parser
.parse_options(Keyword::WITH)
.context(error::SyntaxSnafu)?;
for option in options.iter() {
ensure!(
validate_table_option(&option.name.value),
InvalidTableOptionSnafu {
key: option.name.value.to_string()
}
);
}
// Sorts options so that `test_display_create_table` can always pass.
let options = options.into_iter().sorted().collect();
let create_table = CreateTable { let create_table = CreateTable {
if_not_exists, if_not_exists,
name: table_name, name: table_name,
@@ -177,6 +144,25 @@ impl<'a> ParserContext<'a> {
Ok(Statement::CreateTable(create_table)) Ok(Statement::CreateTable(create_table))
} }
fn parse_create_table_options(&mut self) -> Result<OptionMap> {
let options = self
.parser
.parse_options(Keyword::WITH)
.context(SyntaxSnafu)?
.into_iter()
.map(parse_option_string)
.collect::<Result<HashMap<String, String>>>()?;
for key in options.keys() {
ensure!(
validate_table_option(key),
InvalidTableOptionSnafu {
key: key.to_string()
}
);
}
Ok(options.into())
}
/// "PARTITION BY ..." syntax: /// "PARTITION BY ..." syntax:
// TODO(ruihang): docs // TODO(ruihang): docs
fn parse_partitions(&mut self) -> Result<Option<Partitions>> { fn parse_partitions(&mut self) -> Result<Option<Partitions>> {
@@ -1415,7 +1401,7 @@ ENGINE=mito";
memory float64, memory float64,
TIME INDEX (ts), TIME INDEX (ts),
PRIMARY KEY(ts, host)) engine=mito PRIMARY KEY(ts, host)) engine=mito
with(regions=1); with(ttl='10s');
"; ";
let result = let result =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
@@ -1449,9 +1435,9 @@ ENGINE=mito";
} }
); );
let options = &c.options; let options = &c.options;
assert_eq!(1, options.len()); assert_eq!(1, options.map.len());
assert_eq!("regions", &options[0].name.to_string()); let (k, v) = options.map.iter().next().unwrap();
assert_eq!("1", &options[0].value.to_string()); assert_eq!(("ttl", "10s"), (k.as_str(), v.as_str()));
} }
_ => unreachable!(), _ => unreachable!(),
} }
@@ -1465,8 +1451,7 @@ ENGINE=mito";
cpu float64 default 0, cpu float64 default 0,
memory float64, memory float64,
TIME INDEX (ts, host), TIME INDEX (ts, host),
PRIMARY KEY(ts, host)) engine=mito PRIMARY KEY(ts, host)) engine=mito;
with(regions=1);
"; ";
let result = let result =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default());
@@ -1483,8 +1468,7 @@ ENGINE=mito";
cpu float64 default 0, cpu float64 default 0,
memory float64, memory float64,
TIME INDEX (ts, host), TIME INDEX (ts, host),
PRIMARY KEY(ts, host)) engine=mito PRIMARY KEY(ts, host)) engine=mito;
with(regions=1);
"; ";
let result = let result =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default());
@@ -1498,8 +1482,7 @@ ENGINE=mito";
t timestamp, t timestamp,
memory float64, memory float64,
TIME INDEX (t), TIME INDEX (t),
PRIMARY KEY(ts, host)) engine=mito PRIMARY KEY(ts, host)) engine=mito;
with(regions=1);
"; ";
let result = let result =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default());

30
src/sql/src/statements/create.rs
View File

@@ -19,7 +19,7 @@ use itertools::Itertools;
use sqlparser::ast::Expr; use sqlparser::ast::Expr;
use sqlparser_derive::{Visit, VisitMut}; use sqlparser_derive::{Visit, VisitMut};
use crate::ast::{ColumnDef, Ident, ObjectName, SqlOption, TableConstraint, Value as SqlValue}; use crate::ast::{ColumnDef, Ident, ObjectName, TableConstraint, Value as SqlValue};
use crate::statements::{redact_and_sort_options, OptionMap}; use crate::statements::{redact_and_sort_options, OptionMap};
const LINE_SEP: &str = ",\n"; const LINE_SEP: &str = ",\n";
@@ -86,8 +86,8 @@ pub struct CreateTable {
pub columns: Vec<ColumnDef>, pub columns: Vec<ColumnDef>,
pub engine: String, pub engine: String,
pub constraints: Vec<TableConstraint>, pub constraints: Vec<TableConstraint>,
/// Table options in `WITH`. /// Table options in `WITH`. All keys are lowercase.
pub options: Vec<SqlOption>, pub options: OptionMap,
pub partitions: Option<Partitions>, pub partitions: Option<Partitions>,
} }
@@ -155,11 +155,9 @@ impl Display for CreateTable {
writeln!(f, "{partitions}")?; writeln!(f, "{partitions}")?;
} }
writeln!(f, "ENGINE={}", &self.engine)?; writeln!(f, "ENGINE={}", &self.engine)?;
if !self.options.is_empty() { if !self.options.map.is_empty() {
writeln!(f, "WITH(")?; let options = redact_and_sort_options(&self.options);
let options: Vec<&SqlOption> = self.options.iter().sorted().collect(); write!(f, "WITH(\n{}\n)", format_list_indent!(options))?;
writeln!(f, "{}", format_list_indent!(options))?;
write!(f, ")")?;
} }
Ok(()) Ok(())
} }
@@ -198,8 +196,7 @@ pub struct CreateExternalTable {
pub name: ObjectName, pub name: ObjectName,
pub columns: Vec<ColumnDef>, pub columns: Vec<ColumnDef>,
pub constraints: Vec<TableConstraint>, pub constraints: Vec<TableConstraint>,
/// Table options in `WITH`. /// Table options in `WITH`. All keys are lowercase.
/// All keys are lowercase.
pub options: OptionMap, pub options: OptionMap,
pub if_not_exists: bool, pub if_not_exists: bool,
pub engine: String, pub engine: String,
@@ -218,9 +215,7 @@ impl Display for CreateExternalTable {
writeln!(f, "ENGINE={}", &self.engine)?; writeln!(f, "ENGINE={}", &self.engine)?;
if !self.options.map.is_empty() { if !self.options.map.is_empty() {
let options = redact_and_sort_options(&self.options); let options = redact_and_sort_options(&self.options);
writeln!(f, "WITH(")?; write!(f, "WITH(\n{}\n)", format_list_indent!(options))?;
writeln!(f, "{}", format_list_indent!(options))?;
write!(f, ")")?;
} }
Ok(()) Ok(())
} }
@@ -266,7 +261,7 @@ mod tests {
host > 'a', host > 'a',
) )
engine=mito engine=mito
with(regions=1, ttl='7d', storage='File'); with(ttl='7d', storage='File');
"; ";
let result = let result =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
@@ -292,7 +287,6 @@ PARTITION ON COLUMNS (host) (
) )
ENGINE=mito ENGINE=mito
WITH( WITH(
regions = 1,
storage = 'File', storage = 'File',
ttl = '7d' ttl = '7d'
)"#, )"#,
@@ -369,14 +363,14 @@ ENGINE=mito
) )
PARTITION ON COLUMNS (host) () PARTITION ON COLUMNS (host) ()
engine=mito engine=mito
with(regions=1, ttl='7d', 'compaction.type'='world'); with(ttl='7d', 'compaction.type'='world');
"; ";
let result = let result =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap(); .unwrap();
match &result[0] { match &result[0] {
Statement::CreateTable(c) => { Statement::CreateTable(c) => {
assert_eq!(3, c.options.len()); assert_eq!(2, c.options.map.len());
} }
_ => unreachable!(), _ => unreachable!(),
} }
@@ -391,7 +385,7 @@ ENGINE=mito
) )
PARTITION ON COLUMNS (host) () PARTITION ON COLUMNS (host) ()
engine=mito engine=mito
with(regions=1, ttl='7d', hello='world'); with(ttl='7d', hello='world');
"; ";
let result = let result =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default());

3
src/sql/src/statements/tql.rs
View File

@@ -33,7 +33,6 @@ impl Display for Tql {
} }
} }
// TODO: encapsulate shard TQL args into a struct and implement Display for it.
fn format_tql( fn format_tql(
f: &mut std::fmt::Formatter<'_>, f: &mut std::fmt::Formatter<'_>,
start: &str, start: &str,
@@ -61,7 +60,7 @@ pub struct TqlEval {
impl Display for TqlEval { impl Display for TqlEval {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "TQL EVAL")?; write!(f, "TQL EVAL ")?;
format_tql( format_tql(
f, f,
&self.start, &self.start,

36
src/sql/src/util.rs
View File

@@ -12,13 +12,14 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use std::collections::HashMap;
use std::fmt::{Display, Formatter}; use std::fmt::{Display, Formatter};
use std::sync::LazyLock; use std::sync::LazyLock;
use regex::Regex; use regex::Regex;
use sqlparser::ast::{Expr, ObjectName, SqlOption, Value}; use sqlparser::ast::{Expr, ObjectName, SqlOption, Value};
use crate::error::{InvalidTableOptionValueSnafu, Result};
static SQL_SECRET_PATTERNS: LazyLock<Vec<Regex>> = LazyLock::new(|| { static SQL_SECRET_PATTERNS: LazyLock<Vec<Regex>> = LazyLock::new(|| {
vec![ vec![
Regex::new(r#"(?i)access_key_id=["']([^"']*)["'].*"#).unwrap(), Regex::new(r#"(?i)access_key_id=["']([^"']*)["'].*"#).unwrap(),
@@ -47,29 +48,16 @@ pub fn format_raw_object_name(name: &ObjectName) -> String {
format!("{}", Inner { name }) format!("{}", Inner { name })
} }
pub fn parse_option_string(value: Expr) -> Option<String> { pub fn parse_option_string(option: SqlOption) -> Result<(String, String)> {
match value { let (key, value) = (option.name, option.value);
Expr::Value(Value::SingleQuotedString(v)) | Expr::Value(Value::DoubleQuotedString(v)) => { let v = match value {
Some(v) Expr::Value(Value::SingleQuotedString(v)) | Expr::Value(Value::DoubleQuotedString(v)) => v,
} Expr::Identifier(v) => v.value,
_ => None, Expr::Value(Value::Number(v, _)) => v.to_string(),
} value => return InvalidTableOptionValueSnafu { key, value }.fail(),
} };
let k = key.value.to_lowercase();
/// Converts options to HashMap<String, String>. Ok((k, v))
/// All keys are lowercase.
pub fn to_lowercase_options_map(opts: &[SqlOption]) -> HashMap<String, String> {
let mut map = HashMap::with_capacity(opts.len());
for SqlOption { name, value } in opts {
let value_str = match value {
Expr::Value(Value::SingleQuotedString(s))
| Expr::Value(Value::DoubleQuotedString(s)) => s.clone(),
Expr::Identifier(i) => i.value.clone(),
_ => value.to_string(),
};
let _ = map.insert(name.value.to_lowercase().clone(), value_str);
}
map
} }
/// Use regex to match and replace common seen secret values in SQL. /// Use regex to match and replace common seen secret values in SQL.

5
src/table/src/requests.rs
View File

@@ -53,7 +53,6 @@ pub fn validate_table_option(key: &str) -> bool {
// common keys: // common keys:
WRITE_BUFFER_SIZE_KEY, WRITE_BUFFER_SIZE_KEY,
TTL_KEY, TTL_KEY,
REGIONS_KEY,
STORAGE_KEY, STORAGE_KEY,
// file engine keys: // file engine keys:
FILE_TABLE_LOCATION_KEY, FILE_TABLE_LOCATION_KEY,
@@ -80,7 +79,6 @@ pub struct TableOptions {
pub const WRITE_BUFFER_SIZE_KEY: &str = "write_buffer_size"; pub const WRITE_BUFFER_SIZE_KEY: &str = "write_buffer_size";
pub const TTL_KEY: &str = "ttl"; pub const TTL_KEY: &str = "ttl";
pub const REGIONS_KEY: &str = "regions";
pub const STORAGE_KEY: &str = "storage"; pub const STORAGE_KEY: &str = "storage";
impl TryFrom<&HashMap<String, String>> for TableOptions { impl TryFrom<&HashMap<String, String>> for TableOptions {
@@ -113,7 +111,7 @@ impl TryFrom<&HashMap<String, String>> for TableOptions {
options.ttl = Some(ttl_value); options.ttl = Some(ttl_value);
} }
options.extra_options = HashMap::from_iter(value.iter().filter_map(|(k, v)| { options.extra_options = HashMap::from_iter(value.iter().filter_map(|(k, v)| {
if k != WRITE_BUFFER_SIZE_KEY && k != REGIONS_KEY && k != TTL_KEY { if k != WRITE_BUFFER_SIZE_KEY && k != TTL_KEY {
Some((k.clone(), v.clone())) Some((k.clone(), v.clone()))
} else { } else {
None None
@@ -281,7 +279,6 @@ mod tests {
assert!(validate_table_option(FILE_TABLE_FORMAT_KEY)); assert!(validate_table_option(FILE_TABLE_FORMAT_KEY));
assert!(validate_table_option(FILE_TABLE_PATTERN_KEY)); assert!(validate_table_option(FILE_TABLE_PATTERN_KEY));
assert!(validate_table_option(TTL_KEY)); assert!(validate_table_option(TTL_KEY));
assert!(validate_table_option(REGIONS_KEY));
assert!(validate_table_option(WRITE_BUFFER_SIZE_KEY)); assert!(validate_table_option(WRITE_BUFFER_SIZE_KEY));
assert!(validate_table_option(STORAGE_KEY)); assert!(validate_table_option(STORAGE_KEY));
assert!(!validate_table_option("foo")); assert!(!validate_table_option("foo"));

4
tests-integration/src/instance.rs
View File

@@ -357,7 +357,7 @@ mod tests {
disk_util DOUBLE DEFAULT 9.9, disk_util DOUBLE DEFAULT 9.9,
TIME INDEX (ts), TIME INDEX (ts),
PRIMARY KEY(host) PRIMARY KEY(host)
) engine=mito with(regions=1);"#; ) engine=mito;"#;
let output = SqlQueryHandler::do_query(&*instance, sql, QueryContext::arc()) let output = SqlQueryHandler::do_query(&*instance, sql, QueryContext::arc())
.await .await
.remove(0) .remove(0)
@@ -419,7 +419,7 @@ mod tests {
disk_util DOUBLE DEFAULT 9.9, disk_util DOUBLE DEFAULT 9.9,
TIME INDEX (ts), TIME INDEX (ts),
PRIMARY KEY(host) PRIMARY KEY(host)
) engine=mito with(regions=1);"#; ) engine=mito;"#;
let output = SqlQueryHandler::do_query(&*instance, sql, query_ctx.clone()) let output = SqlQueryHandler::do_query(&*instance, sql, query_ctx.clone())
.await .await
.remove(0) .remove(0)

19
tests-integration/src/tests/instance_test.rs
View File

@@ -134,9 +134,7 @@ PARTITION ON COLUMNS (n) (
| | n >= 10 AND n < 100 | | | n >= 10 AND n < 100 |
| | ) | | | ) |
| | ENGINE=mito | | | ENGINE=mito |
| | WITH( | | | |
| | regions = 4 |
| | ) |
+-------+-------------------------------------+"# +-------+-------------------------------------+"#
} else { } else {
r#"+-------+-------------------------------------+ r#"+-------+-------------------------------------+
@@ -151,9 +149,7 @@ PARTITION ON COLUMNS (n) (
| | ) | | | ) |
| | | | | |
| | ENGINE=mito | | | ENGINE=mito |
| | WITH( | | | |
| | regions = 1 |
| | ) |
+-------+-------------------------------------+"# +-------+-------------------------------------+"#
}; };
@@ -228,8 +224,7 @@ async fn test_show_create_external_table(instance: Arc<dyn MockInstance>) {
ENGINE=file ENGINE=file
WITH( WITH(
format = 'csv', format = 'csv',
location = '{location}', location = '{location}'
regions = 1
)"# )"#
); );
assert_eq!(actual.to_string(), expect); assert_eq!(actual.to_string(), expect);
@@ -560,7 +555,7 @@ async fn test_execute_create(instance: Arc<dyn MockInstance>) {
memory double, memory double,
TIME INDEX (ts), TIME INDEX (ts),
PRIMARY KEY(host) PRIMARY KEY(host)
) engine=mito with(regions=1);"#, ) engine=mito;"#,
) )
.await .await
.data; .data;
@@ -1438,7 +1433,7 @@ async fn test_insert_with_default_value_for_type(instance: Arc<Instance>, type_n
cpu double default 0, cpu double default 0,
TIME INDEX (ts), TIME INDEX (ts),
PRIMARY KEY(host) PRIMARY KEY(host)
) engine=mito with(regions=1);"#, ) engine=mito;"#,
); );
let output = execute_sql(&instance, &create_sql).await.data; let output = execute_sql(&instance, &create_sql).await.data;
assert!(matches!(output, OutputData::AffectedRows(0))); assert!(matches!(output, OutputData::AffectedRows(0)));
@@ -1555,7 +1550,7 @@ async fn test_delete(instance: Arc<dyn MockInstance>) {
memory double, memory double,
TIME INDEX (ts), TIME INDEX (ts),
PRIMARY KEY(host) PRIMARY KEY(host)
) engine=mito with(regions=1);"#, ) engine=mito;"#,
) )
.await .await
.data; .data;
@@ -2085,7 +2080,6 @@ PARTITION ON COLUMNS ("a") (
) )
ENGINE=mito ENGINE=mito
WITH( WITH(
regions = 1,
storage = '{storage_name}' storage = '{storage_name}'
)"# )"#
) )
@@ -2100,7 +2094,6 @@ WITH(
ENGINE=mito ENGINE=mito
WITH( WITH(
regions = 1,
storage = '{storage_name}' storage = '{storage_name}'
)"# )"#
) )

2
tests/cases/standalone/common/basic.result
View File

@@ -64,7 +64,7 @@ create table foo (
cpu double default 0, cpu double default 0,
TIME INDEX (ts), TIME INDEX (ts),
PRIMARY KEY(host) PRIMARY KEY(host)
) engine=mito with(regions=1); ) engine=mito;
Affected Rows: 0 Affected Rows: 0

2
tests/cases/standalone/common/basic.sql
View File

@@ -31,7 +31,7 @@ create table foo (
cpu double default 0, cpu double default 0,
TIME INDEX (ts), TIME INDEX (ts),
PRIMARY KEY(host) PRIMARY KEY(host)
) engine=mito with(regions=1); ) engine=mito;
insert into foo (host, cpu, ts) values ('host1', 1.1, '2000-01-01 00:00:00+00:00'); insert into foo (host, cpu, ts) values ('host1', 1.1, '2000-01-01 00:00:00+00:00');

4
tests/cases/standalone/common/create/create_type_alias.result
View File

@@ -46,9 +46,7 @@ SHOW CREATE TABLE data_types;
| | ) | | | ) |
| | | | | |
| | ENGINE=mito | | | ENGINE=mito |
| | WITH( | | | |
| | regions = 1 |
| | ) |
+------------+------------------------------------------------------------+ +------------+------------------------------------------------------------+
DESC TABLE data_types; DESC TABLE data_types;

5
tests/cases/standalone/common/create/create_with_options.result
View File

@@ -30,7 +30,7 @@ create table if not exists test_opts(
PRIMARY KEY(host) PRIMARY KEY(host)
) )
engine=mito engine=mito
with(regions=1, ttl='7d', 'compaction.type'='twcs', 'compaction.twcs.time_window'='1d'); with(ttl='7d', 'compaction.type'='twcs', 'compaction.twcs.time_window'='1d');
Affected Rows: 0 Affected Rows: 0
@@ -47,7 +47,7 @@ create table if not exists test_opts(
PRIMARY KEY(host) PRIMARY KEY(host)
) )
engine=mito engine=mito
with('regions'=1, 'ttl'='7d', 'compaction.type'='twcs', 'compaction.twcs.time_window'='1d'); with('ttl'='7d', 'compaction.type'='twcs', 'compaction.twcs.time_window'='1d');
Affected Rows: 0 Affected Rows: 0
@@ -65,7 +65,6 @@ create table if not exists test_mito_options(
) )
engine=mito engine=mito
with( with(
'regions'=1,
'ttl'='7d', 'ttl'='7d',
'compaction.type'='twcs', 'compaction.type'='twcs',
'compaction.twcs.max_active_window_files'='8', 'compaction.twcs.max_active_window_files'='8',

5
tests/cases/standalone/common/create/create_with_options.sql
View File

@@ -28,7 +28,7 @@ create table if not exists test_opts(
PRIMARY KEY(host) PRIMARY KEY(host)
) )
engine=mito engine=mito
with(regions=1, ttl='7d', 'compaction.type'='twcs', 'compaction.twcs.time_window'='1d'); with(ttl='7d', 'compaction.type'='twcs', 'compaction.twcs.time_window'='1d');
drop table test_opts; drop table test_opts;
@@ -41,7 +41,7 @@ create table if not exists test_opts(
PRIMARY KEY(host) PRIMARY KEY(host)
) )
engine=mito engine=mito
with('regions'=1, 'ttl'='7d', 'compaction.type'='twcs', 'compaction.twcs.time_window'='1d'); with('ttl'='7d', 'compaction.type'='twcs', 'compaction.twcs.time_window'='1d');
drop table test_opts; drop table test_opts;
@@ -55,7 +55,6 @@ create table if not exists test_mito_options(
) )
engine=mito engine=mito
with( with(
'regions'=1,
'ttl'='7d', 'ttl'='7d',
'compaction.type'='twcs', 'compaction.type'='twcs',
'compaction.twcs.max_active_window_files'='8', 'compaction.twcs.max_active_window_files'='8',

12
tests/cases/standalone/common/create/current_timestamp.result
View File

@@ -13,9 +13,7 @@ show create table t1;
| | ) | | | ) |
| | | | | |
| | ENGINE=mito | | | ENGINE=mito |
| | WITH( | | | |
| | regions = 1 |
| | ) |
+-------+-----------------------------------------------------------+ +-------+-----------------------------------------------------------+
create table t2 (ts timestamp time index default currEnt_tImEsTamp()); create table t2 (ts timestamp time index default currEnt_tImEsTamp());
@@ -33,9 +31,7 @@ show create table t2;
| | ) | | | ) |
| | | | | |
| | ENGINE=mito | | | ENGINE=mito |
| | WITH( | | | |
| | regions = 1 |
| | ) |
+-------+-----------------------------------------------------------+ +-------+-----------------------------------------------------------+
create table t3 (ts timestamp time index default now()); create table t3 (ts timestamp time index default now());
@@ -53,9 +49,7 @@ show create table t3;
| | ) | | | ) |
| | | | | |
| | ENGINE=mito | | | ENGINE=mito |
| | WITH( | | | |
| | regions = 1 |
| | ) |
+-------+---------------------------------------------+ +-------+---------------------------------------------+
create table t4 (ts timestamp time index default now); create table t4 (ts timestamp time index default now);

2
tests/cases/standalone/common/drop/drop_table.result
View File

@@ -8,7 +8,7 @@ create table foo (
cpu double default 0, cpu double default 0,
TIME INDEX (ts), TIME INDEX (ts),
PRIMARY KEY(host) PRIMARY KEY(host)
) engine=mito with(regions=1); ) engine=mito;
Affected Rows: 0 Affected Rows: 0

2
tests/cases/standalone/common/drop/drop_table.sql
View File

@@ -6,7 +6,7 @@ create table foo (
cpu double default 0, cpu double default 0,
TIME INDEX (ts), TIME INDEX (ts),
PRIMARY KEY(host) PRIMARY KEY(host)
) engine=mito with(regions=1); ) engine=mito;
DROP TABLE IF EXISTS foo; DROP TABLE IF EXISTS foo;

5
tests/cases/standalone/common/show/show_create.result
View File

@@ -41,7 +41,6 @@ SHOW CREATE TABLE system_metrics;
| | ) | | | ) |
| | ENGINE=mito | | | ENGINE=mito |
| | WITH( | | | WITH( |
| | regions = 3, |
| | ttl = '7days', | | | ttl = '7days', |
| | write_buffer_size = '1.0KiB' | | | write_buffer_size = '1.0KiB' |
| | ) | | | ) |
@@ -68,9 +67,7 @@ show create table table_without_partition;
| | ) | | | ) |
| | | | | |
| | ENGINE=mito | | | ENGINE=mito |
| | WITH( | | | |
| | regions = 1 |
| | ) |
+-------------------------+-----------------------------------------------------------+ +-------------------------+-----------------------------------------------------------+
drop table table_without_partition; drop table table_without_partition;