refactor!: unify sql options into OptionMap (#3792)

* unify sql options into OptionMap Signed-off-by: tison <wander4096@gmail.com> * fixup Signed-off-by: tison <wander4096@gmail.com> * Update src/sql/src/util.rs * drop legacy regions option Signed-off-by: tison <wander4096@gmail.com> * fixup Signed-off-by: tison <wander4096@gmail.com> * fixup Signed-off-by: tison <wander4096@gmail.com> --------- Signed-off-by: tison <wander4096@gmail.com>
2026-01-10 15:22:56 +00:00 · 2024-04-25 12:06:05 +08:00
parent 9524ec83bc
commit bba3108e0d
23 changed files with 110 additions and 199 deletions
--- a/src/sql/src/error.rs
+++ b/src/sql/src/error.rs
@@ -21,6 +21,7 @@ use common_time::timestamp::TimeUnit;
 use common_time::Timestamp;
 use datatypes::prelude::{ConcreteDataType, Value};
 use snafu::{Location, Snafu};
+use sqlparser::ast::Ident;
 use sqlparser::parser::ParserError;

 use crate::ast::{Expr, Value as SqlValue};
@@ -141,6 +142,13 @@ pub enum Error {
    #[snafu(display("Unrecognized table option key: {}", key))]
    InvalidTableOption { key: String, location: Location },

+    #[snafu(display("Unrecognized table option key: {}, value: {}", key, value))]
+    InvalidTableOptionValue {
+        key: Ident,
+        value: Expr,
+        location: Location,
+    },
+
    #[snafu(display("Failed to serialize column default constraint"))]
    SerializeColumnDefaultConstraint {
        location: Location,
@@ -201,6 +209,7 @@ impl ErrorExt for Error {
            | InvalidDefault { .. } => StatusCode::InvalidSyntax,

            InvalidColumnOption { .. }
+            | InvalidTableOptionValue { .. }
            | InvalidDatabaseName { .. }
            | ColumnTypeMismatch { .. }
            | InvalidTableName { .. }
--- a/src/sql/src/parsers/copy_parser.rs
+++ b/src/sql/src/parsers/copy_parser.rs
@@ -129,10 +129,8 @@ impl<'a> ParserContext<'a> {

        let with = options
            .into_iter()
-            .filter_map(|option| {
-                parse_option_string(option.value).map(|v| (option.name.value.to_lowercase(), v))
-            })
-            .collect();
+            .map(parse_option_string)
+            .collect::<Result<With>>()?;

        let connection_options = self
            .parser
@@ -141,10 +139,8 @@ impl<'a> ParserContext<'a> {

        let connection = connection_options
            .into_iter()
-            .filter_map(|option| {
-                parse_option_string(option.value).map(|v| (option.name.value.to_lowercase(), v))
-            })
-            .collect();
+            .map(parse_option_string)
+            .collect::<Result<Connection>>()?;

        Ok((with, connection, location))
    }
--- a/src/sql/src/parsers/create_parser.rs
+++ b/src/sql/src/parsers/create_parser.rs
@@ -34,8 +34,8 @@ use crate::parser::ParserContext;
 use crate::statements::create::{
    CreateDatabase, CreateExternalTable, CreateTable, CreateTableLike, Partitions, TIME_INDEX,
 };
-use crate::statements::get_data_type_by_alias_name;
 use crate::statements::statement::Statement;
+use crate::statements::{get_data_type_by_alias_name, OptionMap};
 use crate::util::parse_option_string;

 pub const ENGINE: &str = "ENGINE";
@@ -73,32 +73,12 @@ impl<'a> ParserContext<'a> {
        }

        let engine = self.parse_table_engine(common_catalog::consts::FILE_ENGINE)?;
-        let options = self
-            .parser
-            .parse_options(Keyword::WITH)
-            .context(SyntaxSnafu)?
-            .into_iter()
-            .filter_map(|option| {
-                if let Some(v) = parse_option_string(option.value) {
-                    Some((option.name.value.to_lowercase(), v))
-                } else {
-                    None
-                }
-            })
-            .collect::<HashMap<String, String>>();
-        for key in options.keys() {
-            ensure!(
-                validate_table_option(key),
-                InvalidTableOptionSnafu {
-                    key: key.to_string()
-                }
-            );
-        }
+        let options = self.parse_create_table_options()?;
        Ok(Statement::CreateExternalTable(CreateExternalTable {
            name: table_name,
            columns,
            constraints,
-            options: options.into(),
+            options,
            if_not_exists,
            engine,
        }))
@@ -149,20 +129,7 @@ impl<'a> ParserContext<'a> {
        }

        let engine = self.parse_table_engine(default_engine())?;
-        let options = self
-            .parser
-            .parse_options(Keyword::WITH)
-            .context(error::SyntaxSnafu)?;
-        for option in options.iter() {
-            ensure!(
-                validate_table_option(&option.name.value),
-                InvalidTableOptionSnafu {
-                    key: option.name.value.to_string()
-                }
-            );
-        }
-        // Sorts options so that `test_display_create_table` can always pass.
-        let options = options.into_iter().sorted().collect();
+        let options = self.parse_create_table_options()?;
        let create_table = CreateTable {
            if_not_exists,
            name: table_name,
@@ -177,6 +144,25 @@ impl<'a> ParserContext<'a> {
        Ok(Statement::CreateTable(create_table))
    }

+    fn parse_create_table_options(&mut self) -> Result<OptionMap> {
+        let options = self
+            .parser
+            .parse_options(Keyword::WITH)
+            .context(SyntaxSnafu)?
+            .into_iter()
+            .map(parse_option_string)
+            .collect::<Result<HashMap<String, String>>>()?;
+        for key in options.keys() {
+            ensure!(
+                validate_table_option(key),
+                InvalidTableOptionSnafu {
+                    key: key.to_string()
+                }
+            );
+        }
+        Ok(options.into())
+    }
+
    /// "PARTITION BY ..." syntax:
    // TODO(ruihang): docs
    fn parse_partitions(&mut self) -> Result<Option<Partitions>> {
@@ -1415,7 +1401,7 @@ ENGINE=mito";
                             memory float64,
                             TIME INDEX (ts),
                             PRIMARY KEY(ts, host)) engine=mito
-                             with(regions=1);
+                             with(ttl='10s');
         ";
        let result =
            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
@@ -1449,9 +1435,9 @@ ENGINE=mito";
                    }
                );
                let options = &c.options;
-                assert_eq!(1, options.len());
-                assert_eq!("regions", &options[0].name.to_string());
-                assert_eq!("1", &options[0].value.to_string());
+                assert_eq!(1, options.map.len());
+                let (k, v) = options.map.iter().next().unwrap();
+                assert_eq!(("ttl", "10s"), (k.as_str(), v.as_str()));
            }
            _ => unreachable!(),
        }
@@ -1465,8 +1451,7 @@ ENGINE=mito";
                             cpu float64 default 0,
                             memory float64,
                             TIME INDEX (ts, host),
-                             PRIMARY KEY(ts, host)) engine=mito
-                             with(regions=1);
+                             PRIMARY KEY(ts, host)) engine=mito;
         ";
        let result =
            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default());
@@ -1483,8 +1468,7 @@ ENGINE=mito";
                             cpu float64 default 0,
                             memory float64,
                             TIME INDEX (ts, host),
-                             PRIMARY KEY(ts, host)) engine=mito
-                             with(regions=1);
+                             PRIMARY KEY(ts, host)) engine=mito;
         ";
        let result =
            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default());
@@ -1498,8 +1482,7 @@ ENGINE=mito";
                             t timestamp,
                             memory float64,
                             TIME INDEX (t),
-                             PRIMARY KEY(ts, host)) engine=mito
-                             with(regions=1);
+                             PRIMARY KEY(ts, host)) engine=mito;
         ";
        let result =
            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default());
--- a/src/sql/src/statements/create.rs
+++ b/src/sql/src/statements/create.rs
@@ -19,7 +19,7 @@ use itertools::Itertools;
 use sqlparser::ast::Expr;
 use sqlparser_derive::{Visit, VisitMut};

-use crate::ast::{ColumnDef, Ident, ObjectName, SqlOption, TableConstraint, Value as SqlValue};
+use crate::ast::{ColumnDef, Ident, ObjectName, TableConstraint, Value as SqlValue};
 use crate::statements::{redact_and_sort_options, OptionMap};

 const LINE_SEP: &str = ",\n";
@@ -86,8 +86,8 @@ pub struct CreateTable {
    pub columns: Vec<ColumnDef>,
    pub engine: String,
    pub constraints: Vec<TableConstraint>,
-    /// Table options in `WITH`.
-    pub options: Vec<SqlOption>,
+    /// Table options in `WITH`. All keys are lowercase.
+    pub options: OptionMap,
    pub partitions: Option<Partitions>,
 }

@@ -155,11 +155,9 @@ impl Display for CreateTable {
            writeln!(f, "{partitions}")?;
        }
        writeln!(f, "ENGINE={}", &self.engine)?;
-        if !self.options.is_empty() {
-            writeln!(f, "WITH(")?;
-            let options: Vec<&SqlOption> = self.options.iter().sorted().collect();
-            writeln!(f, "{}", format_list_indent!(options))?;
-            write!(f, ")")?;
+        if !self.options.map.is_empty() {
+            let options = redact_and_sort_options(&self.options);
+            write!(f, "WITH(\n{}\n)", format_list_indent!(options))?;
        }
        Ok(())
    }
@@ -198,8 +196,7 @@ pub struct CreateExternalTable {
    pub name: ObjectName,
    pub columns: Vec<ColumnDef>,
    pub constraints: Vec<TableConstraint>,
-    /// Table options in `WITH`.
-    /// All keys are lowercase.
+    /// Table options in `WITH`. All keys are lowercase.
    pub options: OptionMap,
    pub if_not_exists: bool,
    pub engine: String,
@@ -218,9 +215,7 @@ impl Display for CreateExternalTable {
        writeln!(f, "ENGINE={}", &self.engine)?;
        if !self.options.map.is_empty() {
            let options = redact_and_sort_options(&self.options);
-            writeln!(f, "WITH(")?;
-            writeln!(f, "{}", format_list_indent!(options))?;
-            write!(f, ")")?;
+            write!(f, "WITH(\n{}\n)", format_list_indent!(options))?;
        }
        Ok(())
    }
@@ -266,7 +261,7 @@ mod tests {
                            host > 'a',
                       )
                       engine=mito
-                       with(regions=1, ttl='7d', storage='File');
+                       with(ttl='7d', storage='File');
         ";
        let result =
            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
@@ -292,7 +287,6 @@ PARTITION ON COLUMNS (host) (
 )
 ENGINE=mito
 WITH(
-  regions = 1,
  storage = 'File',
  ttl = '7d'
 )"#,
@@ -369,14 +363,14 @@ ENGINE=mito
      )
      PARTITION ON COLUMNS (host) ()
      engine=mito
-      with(regions=1, ttl='7d', 'compaction.type'='world');
+      with(ttl='7d', 'compaction.type'='world');
 ";
        let result =
            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
                .unwrap();
        match &result[0] {
            Statement::CreateTable(c) => {
-                assert_eq!(3, c.options.len());
+                assert_eq!(2, c.options.map.len());
            }
            _ => unreachable!(),
        }
@@ -391,7 +385,7 @@ ENGINE=mito
      )
      PARTITION ON COLUMNS (host) ()
      engine=mito
-      with(regions=1, ttl='7d', hello='world');
+      with(ttl='7d', hello='world');
 ";
        let result =
            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default());
--- a/src/sql/src/statements/tql.rs
+++ b/src/sql/src/statements/tql.rs
@@ -33,7 +33,6 @@ impl Display for Tql {
    }
 }

-// TODO: encapsulate shard TQL args into a struct and implement Display for it.
 fn format_tql(
    f: &mut std::fmt::Formatter<'_>,
    start: &str,
@@ -61,7 +60,7 @@ pub struct TqlEval {

 impl Display for TqlEval {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "TQL EVAL")?;
+        write!(f, "TQL EVAL ")?;
        format_tql(
            f,
            &self.start,
--- a/src/sql/src/util.rs
+++ b/src/sql/src/util.rs
@@ -12,13 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use std::collections::HashMap;
 use std::fmt::{Display, Formatter};
 use std::sync::LazyLock;

 use regex::Regex;
 use sqlparser::ast::{Expr, ObjectName, SqlOption, Value};

+use crate::error::{InvalidTableOptionValueSnafu, Result};
+
 static SQL_SECRET_PATTERNS: LazyLock<Vec<Regex>> = LazyLock::new(|| {
    vec![
        Regex::new(r#"(?i)access_key_id=["']([^"']*)["'].*"#).unwrap(),
@@ -47,29 +48,16 @@ pub fn format_raw_object_name(name: &ObjectName) -> String {
    format!("{}", Inner { name })
 }

-pub fn parse_option_string(value: Expr) -> Option<String> {
-    match value {
-        Expr::Value(Value::SingleQuotedString(v)) | Expr::Value(Value::DoubleQuotedString(v)) => {
-            Some(v)
-        }
-        _ => None,
-    }
-}
-
-/// Converts options to HashMap<String, String>.
-/// All keys are lowercase.
-pub fn to_lowercase_options_map(opts: &[SqlOption]) -> HashMap<String, String> {
-    let mut map = HashMap::with_capacity(opts.len());
-    for SqlOption { name, value } in opts {
-        let value_str = match value {
-            Expr::Value(Value::SingleQuotedString(s))
-            | Expr::Value(Value::DoubleQuotedString(s)) => s.clone(),
-            Expr::Identifier(i) => i.value.clone(),
-            _ => value.to_string(),
-        };
-        let _ = map.insert(name.value.to_lowercase().clone(), value_str);
-    }
-    map
+pub fn parse_option_string(option: SqlOption) -> Result<(String, String)> {
+    let (key, value) = (option.name, option.value);
+    let v = match value {
+        Expr::Value(Value::SingleQuotedString(v)) | Expr::Value(Value::DoubleQuotedString(v)) => v,
+        Expr::Identifier(v) => v.value,
+        Expr::Value(Value::Number(v, _)) => v.to_string(),
+        value => return InvalidTableOptionValueSnafu { key, value }.fail(),
+    };
+    let k = key.value.to_lowercase();
+    Ok((k, v))
 }

 /// Use regex to match and replace common seen secret values in SQL.