rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-20 06:50:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: alter fulltext options (#4952)

Browse Source 
* feat(WIP): alter fulltext index

Co-Authored-By: irenjj <renj.jiang@gmail.com>

* feat: alter column fulltext option

Co-Authored-By: irenjj <renj.jiang@gmail.com>

* chore: fmt

* test: add unit and integration tests

Co-Authored-By: irenjj <renj.jiang@gmail.com>

* test: update sqlness test

* chore: new line

* chore: lock file update

* chore: apply review comments

* test: update sqlness test

* test: update sqlness test

* fix: convert

* chore: apply review comments

* fix: toml fmt

* fix: tests

* test: add test for mito

* chore: error message

* fix: test

* fix: test

* fix: wrong comment

* chore: change proto rev

* chore: apply review comments

* chore: apply review comments

* chore: fmt

---------

Co-authored-by: irenjj <renj.jiang@gmail.com>
This commit is contained in:
Yohan Wal
2024-11-12 11:04:04 +08:00
committed by GitHub
parent cbf21e53a9
commit 84aa5b7b22
30 changed files with 1043 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/sql/src/error.rs
View File

@@ -319,13 +319,6 @@ pub enum Error {
location: Location,
},
#[snafu(display("Invalid fulltext option: {}", msg))]
FulltextInvalidOption {
msg: String,
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Failed to set fulltext option"))]
SetFulltextOption {
source: datatypes::error::Error,
@@ -366,8 +359,7 @@ impl ErrorExt for Error {
| Simplification { .. }
| InvalidInterval { .. }
| InvalidUnaryOp { .. }
| UnsupportedUnaryOp { .. }
| FulltextInvalidOption { .. } => StatusCode::InvalidArguments,
| UnsupportedUnaryOp { .. } => StatusCode::InvalidArguments,
SerializeColumnDefaultConstraint { source, .. } => source.status_code(),
ConvertToGrpcDataType { source, .. } => source.status_code(),

4
src/sql/src/lib.rs
View File

@@ -24,7 +24,5 @@ pub mod parsers;
pub mod statements;
pub mod util;
pub use parsers::create_parser::{
COLUMN_FULLTEXT_OPT_KEY_ANALYZER, COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE, ENGINE, MAXVALUE,
};
pub use parsers::create_parser::{ENGINE, MAXVALUE};
pub use parsers::tql_parser::TQL;

107
src/sql/src/parsers/alter_parser.rs
View File

@@ -12,16 +12,25 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use common_query::AddColumnLocation;
use snafu::ResultExt;
use datatypes::schema::COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE;
use snafu::{ensure, ResultExt};
use sqlparser::keywords::Keyword;
use sqlparser::parser::{Parser, ParserError};
use sqlparser::tokenizer::Token;
use crate::error::{self, Result};
use crate::error::{self, InvalidColumnOptionSnafu, Result, SetFulltextOptionSnafu};
use crate::parser::ParserContext;
use crate::parsers::utils::validate_column_fulltext_create_option;
use crate::statements::alter::{AlterTable, AlterTableOperation, ChangeTableOption};
use crate::statements::statement::Statement;
use crate::util::parse_option_string;
fn validate_column_fulltext_alter_option(key: &str) -> bool {
key == COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE || validate_column_fulltext_create_option(key)
}
impl ParserContext<'_> {
pub(crate) fn parse_alter(&mut self) -> Result<Statement> {
@@ -143,12 +152,41 @@ impl ParserContext<'_> {
.parse_identifier(false)
.context(error::SyntaxSnafu)?,
);
let target_type = self.parser.parse_data_type().context(error::SyntaxSnafu)?;
Ok(AlterTableOperation::ChangeColumnType {
column_name,
target_type,
})
if self.parser.parse_keyword(Keyword::SET) {
self.parser
.expect_keyword(Keyword::FULLTEXT)
.context(error::SyntaxSnafu)?;
let options = self
.parser
.parse_options(Keyword::WITH)
.context(error::SyntaxSnafu)?
.into_iter()
.map(parse_option_string)
.collect::<Result<HashMap<String, String>>>()?;
for key in options.keys() {
ensure!(
validate_column_fulltext_alter_option(key),
InvalidColumnOptionSnafu {
name: column_name.to_string(),
msg: format!("invalid FULLTEXT option: {key}"),
}
);
}
Ok(AlterTableOperation::ChangeColumnFulltext {
column_name,
options: options.try_into().context(SetFulltextOptionSnafu)?,
})
} else {
let target_type = self.parser.parse_data_type().context(error::SyntaxSnafu)?;
Ok(AlterTableOperation::ChangeColumnType {
column_name,
target_type,
})
}
}
}
@@ -173,6 +211,7 @@ mod tests {
use std::assert_matches::assert_matches;
use common_error::ext::ErrorExt;
use datatypes::schema::{FulltextAnalyzer, FulltextOptions};
use sqlparser::ast::{ColumnOption, DataType};
use super::*;
@@ -515,4 +554,58 @@ mod tests {
)
.unwrap_err();
}
#[test]
fn test_parse_alter_column_fulltext() {
let sql = "ALTER TABLE test_table MODIFY COLUMN a SET FULLTEXT WITH(enable='true',analyzer='English',case_sensitive='false')";
let mut result =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap();
assert_eq!(1, result.len());
let statement = result.remove(0);
assert_matches!(statement, Statement::Alter { .. });
match statement {
Statement::Alter(alter_table) => {
assert_eq!("test_table", alter_table.table_name().0[0].value);
let alter_operation = alter_table.alter_operation();
assert_matches!(
alter_operation,
AlterTableOperation::ChangeColumnFulltext { .. }
);
match alter_operation {
AlterTableOperation::ChangeColumnFulltext {
column_name,
options,
} => {
assert_eq!("a", column_name.value);
assert_eq!(
FulltextOptions {
enable: true,
analyzer: FulltextAnalyzer::English,
case_sensitive: false
},
*options
);
}
_ => unreachable!(),
}
}
_ => unreachable!(),
}
let invalid_sql = "ALTER TABLE test_table MODIFY COLUMN a SET FULLTEXT WITH('abcd'='true')";
let result = ParserContext::create_with_dialect(
invalid_sql,
&GreptimeDbDialect {},
ParseOptions::default(),
)
.unwrap_err();
let err = result.to_string();
assert_eq!(
err,
"Invalid column option, column name: a, error: invalid FULLTEXT option: abcd"
);
}
}

14
src/sql/src/parsers/create_parser.rs
View File

@@ -36,6 +36,7 @@ use crate::error::{
SyntaxSnafu, UnexpectedSnafu, UnsupportedSnafu,
};
use crate::parser::{ParserContext, FLOW};
use crate::parsers::utils::validate_column_fulltext_create_option;
use crate::statements::create::{
Column, ColumnExtensions, CreateDatabase, CreateExternalTable, CreateFlow, CreateTable,
CreateTableLike, CreateView, Partitions, TableConstraint,
@@ -59,17 +60,6 @@ fn validate_database_option(key: &str) -> bool {
[DB_OPT_KEY_TTL].contains(&key)
}
pub const COLUMN_FULLTEXT_OPT_KEY_ANALYZER: &str = "analyzer";
pub const COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE: &str = "case_sensitive";
fn validate_column_fulltext_option(key: &str) -> bool {
[
COLUMN_FULLTEXT_OPT_KEY_ANALYZER,
COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE,
]
.contains(&key)
}
/// Parses create [table] statement
impl<'a> ParserContext<'a> {
pub(crate) fn parse_create(&mut self) -> Result<Statement> {
@@ -706,7 +696,7 @@ impl<'a> ParserContext<'a> {
for key in options.keys() {
ensure!(
validate_column_fulltext_option(key),
validate_column_fulltext_create_option(key),
InvalidColumnOptionSnafu {
name: column_name.to_string(),
msg: format!("invalid FULLTEXT option: {key}"),

9
src/sql/src/parsers/utils.rs
View File

@@ -26,6 +26,7 @@ use datafusion_expr::{AggregateUDF, ScalarUDF, TableSource, WindowUDF};
use datafusion_sql::planner::{ContextProvider, SqlToRel};
use datafusion_sql::TableReference;
use datatypes::arrow::datatypes::DataType;
use datatypes::schema::{COLUMN_FULLTEXT_OPT_KEY_ANALYZER, COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE};
use snafu::ResultExt;
use crate::error::{
@@ -110,3 +111,11 @@ impl ContextProvider for StubContextProvider {
self.state.window_functions().keys().cloned().collect()
}
}
pub fn validate_column_fulltext_create_option(key: &str) -> bool {
[
COLUMN_FULLTEXT_OPT_KEY_ANALYZER,
COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE,
]
.contains(&key)
}

5
src/sql/src/statements.rs
View File

@@ -684,7 +684,9 @@ mod tests {
use api::v1::ColumnDataType;
use common_time::timestamp::TimeUnit;
use common_time::timezone::set_default_timezone;
use datatypes::schema::FulltextAnalyzer;
use datatypes::schema::{
FulltextAnalyzer, COLUMN_FULLTEXT_OPT_KEY_ANALYZER, COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE,
};
use datatypes::types::BooleanType;
use datatypes::value::OrderedFloat;
@@ -692,7 +694,6 @@ mod tests {
use crate::ast::TimezoneInfo;
use crate::statements::create::ColumnExtensions;
use crate::statements::ColumnOption;
use crate::{COLUMN_FULLTEXT_OPT_KEY_ANALYZER, COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE};
fn check_type(sql_type: SqlDataType, data_type: ConcreteDataType) {
assert_eq!(

34
src/sql/src/statements/alter.rs
View File

@@ -16,6 +16,7 @@ use std::fmt::{Debug, Display};
use api::v1;
use common_query::AddColumnLocation;
use datatypes::schema::FulltextOptions;
use itertools::Itertools;
use sqlparser::ast::{ColumnDef, DataType, Ident, ObjectName, TableConstraint};
use sqlparser_derive::{Visit, VisitMut};
@@ -75,6 +76,11 @@ pub enum AlterTableOperation {
DropColumn { name: Ident },
/// `RENAME <new_table_name>`
RenameTable { new_table_name: String },
/// `MODIFY COLUMN <column_name> SET FULLTEXT [WITH <options>]`
ChangeColumnFulltext {
column_name: Ident,
options: FulltextOptions,
},
}
impl Display for AlterTableOperation {
@@ -117,6 +123,13 @@ impl Display for AlterTableOperation {
Ok(())
}
AlterTableOperation::ChangeColumnFulltext {
column_name,
options,
} => write!(
f,
r#"MODIFY COLUMN {column_name} SET FULLTEXT WITH({options})"#,
),
}
}
}
@@ -229,5 +242,26 @@ ALTER TABLE monitor RENAME monitor_new"#,
unreachable!();
}
}
let sql = "ALTER TABLE monitor MODIFY COLUMN a SET FULLTEXT WITH(enable='true',analyzer='English',case_sensitive='false')";
let stmts =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap();
assert_eq!(1, stmts.len());
assert_matches!(&stmts[0], Statement::Alter { .. });
match &stmts[0] {
Statement::Alter(set) => {
let new_sql = format!("\n{}", set);
assert_eq!(
r#"
ALTER TABLE monitor MODIFY COLUMN a SET FULLTEXT WITH(enable=true, analyzer=English, case_sensitive=false)"#,
&new_sql
);
}
_ => {
unreachable!();
}
}
}
}

39
src/sql/src/statements/create.rs
View File

@@ -12,19 +12,20 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use std::fmt::{Display, Formatter};
use common_catalog::consts::FILE_ENGINE;
use datatypes::schema::{FulltextAnalyzer, FulltextOptions};
use datatypes::schema::FulltextOptions;
use itertools::Itertools;
use snafu::ResultExt;
use sqlparser::ast::{ColumnOptionDef, DataType, Expr, Query};
use sqlparser_derive::{Visit, VisitMut};
use crate::ast::{ColumnDef, Ident, ObjectName, Value as SqlValue};
use crate::error::{FulltextInvalidOptionSnafu, Result};
use crate::error::{Result, SetFulltextOptionSnafu};
use crate::statements::statement::Statement;
use crate::statements::OptionMap;
use crate::{COLUMN_FULLTEXT_OPT_KEY_ANALYZER, COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE};
const LINE_SEP: &str = ",\n";
const COMMA_SEP: &str = ", ";
@@ -156,36 +157,8 @@ impl ColumnExtensions {
return Ok(None);
};
let mut fulltext = FulltextOptions {
enable: true,
..Default::default()
};
if let Some(analyzer) = options.get(COLUMN_FULLTEXT_OPT_KEY_ANALYZER) {
match analyzer.to_ascii_lowercase().as_str() {
"english" => fulltext.analyzer = FulltextAnalyzer::English,
"chinese" => fulltext.analyzer = FulltextAnalyzer::Chinese,
_ => {
return FulltextInvalidOptionSnafu {
msg: format!("{analyzer}, expected: 'English' | 'Chinese'"),
}
.fail();
}
}
}
if let Some(case_sensitive) = options.get(COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE) {
match case_sensitive.to_ascii_lowercase().as_str() {
"true" => fulltext.case_sensitive = true,
"false" => fulltext.case_sensitive = false,
_ => {
return FulltextInvalidOptionSnafu {
msg: format!("{case_sensitive}, expected: 'true' | 'false'"),
}
.fail();
}
}
}
Ok(Some(fulltext))
let options: HashMap<String, String> = options.clone().into_map();
Ok(Some(options.try_into().context(SetFulltextOptionSnafu)?))
}
}