rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-27 18:30:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: introduce SKIPPING index (part 1) (#5155)

Browse Source 
* skip index parser

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* wip: sqlness

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* impl show create part

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* add empty line

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* change keyword to SKIPPING INDEX

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* rename local variables

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
This commit is contained in:
Ruihang Xia
2024-12-16 17:21:00 +08:00
committed by Yingwen
parent 7f71693b8e
commit b71d842615
14 changed files with 371 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/sql/src/error.rs
View File

@@ -326,6 +326,13 @@ pub enum Error {
location: Location,
},
#[snafu(display("Failed to set SKIPPING index option"))]
SetSkippingIndexOption {
source: datatypes::error::Error,
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Datatype error: {}", source))]
Datatype {
source: datatypes::error::Error,
@@ -375,7 +382,7 @@ impl ErrorExt for Error {
ConvertSqlValue { .. } | ConvertValue { .. } => StatusCode::Unsupported,
PermissionDenied { .. } => StatusCode::PermissionDenied,
SetFulltextOption { .. } => StatusCode::Unexpected,
SetFulltextOption { .. } | SetSkippingIndexOption { .. } => StatusCode::Unexpected,
}
}

105
src/sql/src/parsers/create_parser.rs
View File

@@ -36,7 +36,9 @@ use crate::error::{
SyntaxSnafu, UnexpectedSnafu, UnsupportedSnafu,
};
use crate::parser::{ParserContext, FLOW};
use crate::parsers::utils::validate_column_fulltext_create_option;
use crate::parsers::utils::{
validate_column_fulltext_create_option, validate_column_skipping_index_create_option,
};
use crate::statements::create::{
Column, ColumnExtensions, CreateDatabase, CreateExternalTable, CreateFlow, CreateTable,
CreateTableLike, CreateView, Partitions, TableConstraint, VECTOR_OPT_DIM,
@@ -53,6 +55,7 @@ pub const SINK: &str = "SINK";
pub const EXPIRE: &str = "EXPIRE";
pub const AFTER: &str = "AFTER";
pub const INVERTED: &str = "INVERTED";
pub const SKIPPING: &str = "SKIPPING";
const DB_OPT_KEY_TTL: &str = "ttl";
@@ -701,6 +704,49 @@ impl<'a> ParserContext<'a> {
column_extensions.vector_options = Some(options.into());
}
let mut is_index_declared = false;
if let Token::Word(word) = parser.peek_token().token
&& word.value.eq_ignore_ascii_case(SKIPPING)
{
parser.next_token();
// Consume `INDEX` keyword
ensure!(
parser.parse_keyword(Keyword::INDEX),
InvalidColumnOptionSnafu {
name: column_name.to_string(),
msg: "expect INDEX after SKIPPING keyword",
}
);
ensure!(
column_extensions.skipping_index_options.is_none(),
InvalidColumnOptionSnafu {
name: column_name.to_string(),
msg: "duplicated SKIPPING index option",
}
);
let options = parser
.parse_options(Keyword::WITH)
.context(error::SyntaxSnafu)?
.into_iter()
.map(parse_option_string)
.collect::<Result<HashMap<String, String>>>()?;
for key in options.keys() {
ensure!(
validate_column_skipping_index_create_option(key),
InvalidColumnOptionSnafu {
name: column_name.to_string(),
msg: format!("invalid SKIP option: {key}"),
}
);
}
column_extensions.skipping_index_options = Some(options.into());
is_index_declared |= true;
}
if parser.parse_keyword(Keyword::FULLTEXT) {
ensure!(
column_extensions.fulltext_options.is_none(),
@@ -738,10 +784,10 @@ impl<'a> ParserContext<'a> {
}
column_extensions.fulltext_options = Some(options.into());
Ok(true)
} else {
Ok(false)
is_index_declared |= true;
}
Ok(is_index_declared)
}
fn parse_optional_table_constraint(&mut self) -> Result<Option<TableConstraint>> {
@@ -2103,6 +2149,57 @@ CREATE TABLE log (
.contains("invalid FULLTEXT option"));
}
#[test]
fn test_parse_create_table_skip_options() {
let sql = r"
CREATE TABLE log (
ts TIMESTAMP TIME INDEX,
msg INT SKIPPING INDEX WITH (granularity='8192', type='bloom'),
)";
let result =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap();
if let Statement::CreateTable(c) = &result[0] {
c.columns.iter().for_each(|col| {
if col.name().value == "msg" {
assert!(!col
.extensions
.skipping_index_options
.as_ref()
.unwrap()
.is_empty());
}
});
} else {
panic!("should be create_table statement");
}
let sql = r"
CREATE TABLE log (
ts TIMESTAMP TIME INDEX,
msg INT SKIPPING INDEX,
)";
let result =
ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
.unwrap();
if let Statement::CreateTable(c) = &result[0] {
c.columns.iter().for_each(|col| {
if col.name().value == "msg" {
assert!(col
.extensions
.skipping_index_options
.as_ref()
.unwrap()
.is_empty());
}
});
} else {
panic!("should be create_table statement");
}
}
#[test]
fn test_parse_create_view_with_columns() {
let sql = "CREATE VIEW test () AS SELECT * FROM NUMBERS";

13
src/sql/src/parsers/utils.rs
View File

@@ -26,7 +26,10 @@ use datafusion_expr::{AggregateUDF, ScalarUDF, TableSource, WindowUDF};
use datafusion_sql::planner::{ContextProvider, SqlToRel};
use datafusion_sql::TableReference;
use datatypes::arrow::datatypes::DataType;
use datatypes::schema::{COLUMN_FULLTEXT_OPT_KEY_ANALYZER, COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE};
use datatypes::schema::{
COLUMN_FULLTEXT_OPT_KEY_ANALYZER, COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE,
COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY, COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE,
};
use snafu::ResultExt;
use crate::error::{
@@ -119,3 +122,11 @@ pub fn validate_column_fulltext_create_option(key: &str) -> bool {
]
.contains(&key)
}
pub fn validate_column_skipping_index_create_option(key: &str) -> bool {
[
COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY,
COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE,
]
.contains(&key)
}

10
src/sql/src/statements.rs
View File

@@ -58,7 +58,8 @@ use crate::error::{
self, ColumnTypeMismatchSnafu, ConvertSqlValueSnafu, ConvertToGrpcDataTypeSnafu,
ConvertValueSnafu, DatatypeSnafu, InvalidCastSnafu, InvalidSqlValueSnafu, InvalidUnaryOpSnafu,
ParseSqlValueSnafu, Result, SerializeColumnDefaultConstraintSnafu, SetFulltextOptionSnafu,
TimestampOverflowSnafu, UnsupportedDefaultValueSnafu, UnsupportedUnaryOpSnafu,
SetSkippingIndexOptionSnafu, TimestampOverflowSnafu, UnsupportedDefaultValueSnafu,
UnsupportedUnaryOpSnafu,
};
use crate::statements::create::Column;
pub use crate::statements::option_map::OptionMap;
@@ -513,6 +514,12 @@ pub fn column_to_schema(
.context(SetFulltextOptionSnafu)?;
}
if let Some(options) = column.extensions.build_skipping_index_options()? {
column_schema = column_schema
.with_skipping_options(options)
.context(SetSkippingIndexOptionSnafu)?;
}
Ok(column_schema)
}
@@ -1519,6 +1526,7 @@ mod tests {
.into(),
),
vector_options: None,
skipping_index_options: None,
},
};

26
src/sql/src/statements/create.rs
View File

@@ -16,7 +16,7 @@ use std::collections::HashMap;
use std::fmt::{Display, Formatter};
use common_catalog::consts::FILE_ENGINE;
use datatypes::schema::FulltextOptions;
use datatypes::schema::{FulltextOptions, SkippingIndexOptions};
use itertools::Itertools;
use serde::Serialize;
use snafu::ResultExt;
@@ -24,7 +24,7 @@ use sqlparser::ast::{ColumnOptionDef, DataType, Expr, Query};
use sqlparser_derive::{Visit, VisitMut};
use crate::ast::{ColumnDef, Ident, ObjectName, Value as SqlValue};
use crate::error::{Result, SetFulltextOptionSnafu};
use crate::error::{Result, SetFulltextOptionSnafu, SetSkippingIndexOptionSnafu};
use crate::statements::statement::Statement;
use crate::statements::OptionMap;
@@ -116,6 +116,8 @@ pub struct ColumnExtensions {
pub fulltext_options: Option<OptionMap>,
/// Vector options.
pub vector_options: Option<OptionMap>,
/// Skipping index options.
pub skipping_index_options: Option<OptionMap>,
}
impl Column {
@@ -158,6 +160,15 @@ impl Display for Column {
write!(f, " FULLTEXT")?;
}
}
if let Some(skipping_index_options) = &self.extensions.skipping_index_options {
if !skipping_index_options.is_empty() {
let options = skipping_index_options.kv_pairs();
write!(f, " SKIPPING INDEX WITH({})", format_list_comma!(options))?;
} else {
write!(f, " SKIPPING INDEX")?;
}
}
Ok(())
}
}
@@ -171,6 +182,17 @@ impl ColumnExtensions {
let options: HashMap<String, String> = options.clone().into_map();
Ok(Some(options.try_into().context(SetFulltextOptionSnafu)?))
}
pub fn build_skipping_index_options(&self) -> Result<Option<SkippingIndexOptions>> {
let Some(options) = self.skipping_index_options.as_ref() else {
return Ok(None);
};
let options: HashMap<String, String> = options.clone().into_map();
Ok(Some(
options.try_into().context(SetSkippingIndexOptionSnafu)?,
))
}
}
#[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]