diff --git a/src/catalog/src/system_schema/information_schema/key_column_usage.rs b/src/catalog/src/system_schema/information_schema/key_column_usage.rs index bd7951777d..937bd1dbce 100644 --- a/src/catalog/src/system_schema/information_schema/key_column_usage.rs +++ b/src/catalog/src/system_schema/information_schema/key_column_usage.rs @@ -228,12 +228,6 @@ impl InformationSchemaKeyColumnUsageBuilder { let keys = &table_info.meta.primary_key_indices; let schema = table.schema(); - // For compatibility, use primary key columns as inverted index columns. - let pk_as_inverted_index = !schema - .column_schemas() - .iter() - .any(|c| c.has_inverted_index_key()); - for (idx, column) in schema.column_schemas().iter().enumerate() { let mut constraints = vec![]; if column.is_time_index() { @@ -251,10 +245,6 @@ impl InformationSchemaKeyColumnUsageBuilder { // TODO(dimbtp): foreign key constraint not supported yet if keys.contains(&idx) { constraints.push(PRI_CONSTRAINT_NAME); - - if pk_as_inverted_index { - constraints.push(INVERTED_INDEX_CONSTRAINT_NAME); - } } if column.is_inverted_indexed() { constraints.push(INVERTED_INDEX_CONSTRAINT_NAME); diff --git a/src/datatypes/src/schema/column_schema.rs b/src/datatypes/src/schema/column_schema.rs index 5146b640fc..c403600ad1 100644 --- a/src/datatypes/src/schema/column_schema.rs +++ b/src/datatypes/src/schema/column_schema.rs @@ -183,12 +183,6 @@ impl ColumnSchema { self } - // Put a placeholder to invalidate schemas.all(!has_inverted_index_key). - pub fn insert_inverted_index_placeholder(&mut self) { - self.metadata - .insert(INVERTED_INDEX_KEY.to_string(), "".to_string()); - } - pub fn is_inverted_indexed(&self) -> bool { self.metadata .get(INVERTED_INDEX_KEY) diff --git a/src/operator/src/expr_factory.rs b/src/operator/src/expr_factory.rs index 4fa239a503..589aba9b1b 100644 --- a/src/operator/src/expr_factory.rs +++ b/src/operator/src/expr_factory.rs @@ -137,14 +137,8 @@ pub(crate) async fn create_external_expr( // expanded form let time_index = find_time_index(&create.constraints)?; let primary_keys = find_primary_keys(&create.columns, &create.constraints)?; - let inverted_index_cols = find_inverted_index_cols(&create.columns, &create.constraints)?; - let column_schemas = columns_to_column_schemas( - &create.columns, - &time_index, - &inverted_index_cols, - &primary_keys, - Some(&query_ctx.timezone()), - )?; + let column_schemas = + columns_to_column_schemas(&create.columns, &time_index, Some(&query_ctx.timezone()))?; (time_index, primary_keys, column_schemas) } else { // inferred form @@ -199,7 +193,6 @@ pub fn create_to_expr( ); let primary_keys = find_primary_keys(&create.columns, &create.constraints)?; - let inverted_index_cols = find_inverted_index_cols(&create.columns, &create.constraints)?; let expr = CreateTableExpr { catalog_name, @@ -210,7 +203,6 @@ pub fn create_to_expr( &create.columns, &time_index, &primary_keys, - &inverted_index_cols, Some(&query_ctx.timezone()), )?, time_index, @@ -378,71 +370,24 @@ pub fn find_time_index(constraints: &[TableConstraint]) -> Result { Ok(time_index.first().unwrap().to_string()) } -/// Finds the inverted index columns from the constraints. If no inverted index -/// columns are provided in the constraints, return `None`. -fn find_inverted_index_cols( - columns: &[SqlColumn], - constraints: &[TableConstraint], -) -> Result>> { - let inverted_index_cols = constraints.iter().find_map(|constraint| { - if let TableConstraint::InvertedIndex { columns } = constraint { - Some( - columns - .iter() - .map(|ident| ident.value.clone()) - .collect::>(), - ) - } else { - None - } - }); - - let Some(inverted_index_cols) = inverted_index_cols else { - return Ok(None); - }; - - for col in &inverted_index_cols { - if !columns.iter().any(|c| c.name().value == *col) { - return InvalidSqlSnafu { - err_msg: format!("inverted index column `{}` not found in column list", col), - } - .fail(); - } - } - - Ok(Some(inverted_index_cols)) -} - fn columns_to_expr( column_defs: &[SqlColumn], time_index: &str, primary_keys: &[String], - invereted_index_cols: &Option>, timezone: Option<&Timezone>, ) -> Result> { - let column_schemas = columns_to_column_schemas( - column_defs, - time_index, - invereted_index_cols, - primary_keys, - timezone, - )?; + let column_schemas = columns_to_column_schemas(column_defs, time_index, timezone)?; column_schemas_to_defs(column_schemas, primary_keys) } fn columns_to_column_schemas( columns: &[SqlColumn], time_index: &str, - invereted_index_cols: &Option>, - primary_keys: &[String], timezone: Option<&Timezone>, ) -> Result> { columns .iter() - .map(|c| { - column_to_schema(c, time_index, invereted_index_cols, primary_keys, timezone) - .context(ParseSqlSnafu) - }) + .map(|c| column_to_schema(c, time_index, timezone).context(ParseSqlSnafu)) .collect::>>() } diff --git a/src/query/src/sql/show_create_table.rs b/src/query/src/sql/show_create_table.rs index 1149a2fce0..5faa7bc43b 100644 --- a/src/query/src/sql/show_create_table.rs +++ b/src/query/src/sql/show_create_table.rs @@ -114,7 +114,7 @@ fn create_column(column_schema: &ColumnSchema, quote_style: char) -> Result Result Result( - schema: &'a SchemaRef, - engine: &str, -) -> Vec<&'a ColumnSchema> { - let is_metric_engine = is_metric_engine(engine); - if is_metric_engine { - schema - .column_schemas() - .iter() - .filter(|c| !is_metric_engine_internal_column(&c.name)) - .collect() - } else { - schema.column_schemas().iter().collect() - } -} - /// Returns the primary key columns for `SHOW CREATE TABLE` statement. /// /// For metric engine, it will only return the primary key columns that are not internal columns. @@ -206,20 +191,6 @@ fn create_table_constraints( constraints.push(TableConstraint::PrimaryKey { columns }); } - let column_schemas = column_schemas_for_show_create(schema, engine); - let inverted_index_set = column_schemas.iter().any(|c| c.has_inverted_index_key()); - if inverted_index_set { - let inverted_index_cols = column_schemas - .iter() - .filter(|c| c.is_inverted_indexed()) - .map(|c| Ident::with_quote(quote_style, &c.name)) - .collect::>(); - - constraints.push(TableConstraint::InvertedIndex { - columns: inverted_index_cols, - }); - } - constraints } @@ -353,14 +324,13 @@ mod tests { r#" CREATE TABLE IF NOT EXISTS "system_metrics" ( "id" INT UNSIGNED NULL SKIPPING INDEX WITH(granularity = '4096', type = 'BLOOM'), - "host" STRING NULL, + "host" STRING NULL INVERTED INDEX, "cpu" DOUBLE NULL, "disk" FLOAT NULL, "msg" STRING NULL FULLTEXT WITH(analyzer = 'English', case_sensitive = 'false'), "ts" TIMESTAMP(3) NOT NULL DEFAULT current_timestamp(), TIME INDEX ("ts"), - PRIMARY KEY ("id", "host"), - INVERTED INDEX ("host") + PRIMARY KEY ("id", "host") ) ENGINE=mito WITH( diff --git a/src/sql/src/parsers/create_parser.rs b/src/sql/src/parsers/create_parser.rs index 84f491d8e3..60d9424720 100644 --- a/src/sql/src/parsers/create_parser.rs +++ b/src/sql/src/parsers/create_parser.rs @@ -430,6 +430,7 @@ impl<'a> ParserContext<'a> { Ok(values) } + /// Parse the columns and constraints. fn parse_columns(&mut self) -> Result<(Vec, Vec)> { let mut columns = vec![]; let mut constraints = vec![]; @@ -666,6 +667,11 @@ impl<'a> ParserContext<'a> { } } + /// Parse a column option extensions. + /// + /// This function will handle: + /// - Vector type + /// - Indexes fn parse_column_extensions( parser: &mut Parser<'_>, column_name: &Ident, @@ -697,8 +703,10 @@ impl<'a> ParserContext<'a> { column_extensions.vector_options = Some(options.into()); } + // parse index options in column definition let mut is_index_declared = false; + // skipping index if let Token::Word(word) = parser.peek_token().token && word.value.eq_ignore_ascii_case(SKIPPING) { @@ -731,7 +739,7 @@ impl<'a> ParserContext<'a> { validate_column_skipping_index_create_option(key), InvalidColumnOptionSnafu { name: column_name.to_string(), - msg: format!("invalid SKIP option: {key}"), + msg: format!("invalid SKIPPING INDEX option: {key}"), } ); } @@ -740,12 +748,22 @@ impl<'a> ParserContext<'a> { is_index_declared |= true; } + // fulltext index if parser.parse_keyword(Keyword::FULLTEXT) { + // Consume `INDEX` keyword ensure!( - column_extensions.fulltext_options.is_none(), + parser.parse_keyword(Keyword::INDEX), InvalidColumnOptionSnafu { name: column_name.to_string(), - msg: "duplicated FULLTEXT option", + msg: "expect INDEX after FULLTEXT keyword", + } + ); + + ensure!( + column_extensions.fulltext_index_options.is_none(), + InvalidColumnOptionSnafu { + name: column_name.to_string(), + msg: "duplicated FULLTEXT INDEX option", } ); @@ -771,12 +789,54 @@ impl<'a> ParserContext<'a> { validate_column_fulltext_create_option(key), InvalidColumnOptionSnafu { name: column_name.to_string(), - msg: format!("invalid FULLTEXT option: {key}"), + msg: format!("invalid FULLTEXT INDEX option: {key}"), } ); } - column_extensions.fulltext_options = Some(options.into()); + column_extensions.fulltext_index_options = Some(options.into()); + is_index_declared |= true; + } + + // inverted index + if let Token::Word(word) = parser.peek_token().token + && word.value.eq_ignore_ascii_case(INVERTED) + { + parser.next_token(); + // Consume `INDEX` keyword + ensure!( + parser.parse_keyword(Keyword::INDEX), + InvalidColumnOptionSnafu { + name: column_name.to_string(), + msg: "expect INDEX after INVERTED keyword", + } + ); + + ensure!( + column_extensions.inverted_index_options.is_none(), + InvalidColumnOptionSnafu { + name: column_name.to_string(), + msg: "duplicated INVERTED index option", + } + ); + + // inverted index doesn't have options, skipping `WITH` + // try cache `WITH` and throw error + let with_token = parser.peek_token(); + ensure!( + with_token.token + != Token::Word(Word { + value: "WITH".to_string(), + keyword: Keyword::WITH, + quote_style: None, + }), + InvalidColumnOptionSnafu { + name: column_name.to_string(), + msg: "INVERTED index doesn't support options", + } + ); + + column_extensions.inverted_index_options = Some(OptionMap::default()); is_index_declared |= true; } @@ -836,28 +896,6 @@ impl<'a> ParserContext<'a> { column: columns.pop().unwrap(), })) } - TokenWithLocation { - token: Token::Word(w), - .. - } if w.value.eq_ignore_ascii_case(INVERTED) => { - self.parser - .expect_keyword(Keyword::INDEX) - .context(error::UnexpectedSnafu { - expected: "INDEX", - actual: self.peek_token_as_string(), - })?; - - let raw_columns = self - .parser - // allow empty list to unset inverted index - .parse_parenthesized_column_list(Mandatory, true) - .context(error::SyntaxSnafu)?; - let columns = raw_columns - .into_iter() - .map(Self::canonicalize_identifier) - .collect::>(); - Ok(Some(TableConstraint::InvertedIndex { columns })) - } _ => { self.parser.prev_token(); Ok(None) @@ -1037,6 +1075,8 @@ mod tests { use common_error::ext::ErrorExt; use sqlparser::ast::ColumnOption::NotNull; use sqlparser::ast::{BinaryOperator, Expr, ObjectName, Value}; + use sqlparser::dialect::GenericDialect; + use sqlparser::tokenizer::Tokenizer; use super::*; use crate::dialect::GreptimeDbDialect; @@ -1151,7 +1191,6 @@ mod tests { memory float64, TIME INDEX (ts), PRIMARY KEY(ts, host), - INVERTED INDEX(host) ) with(location='/var/data/city.csv',format='csv');"; let options = HashMap::from([ @@ -1187,12 +1226,6 @@ mod tests { columns: vec![Ident::new("ts"), Ident::new("host")] } ); - assert_eq!( - &constraints[2], - &TableConstraint::InvertedIndex { - columns: vec![Ident::new("host")] - } - ); } _ => unreachable!(), } @@ -1811,7 +1844,6 @@ ENGINE=mito"; memory float64, TIME INDEX (ts), PRIMARY KEY(ts, host), - INVERTED INDEX(host) ) engine=mito with(ttl='10s'); "; @@ -1844,12 +1876,7 @@ ENGINE=mito"; columns: vec![Ident::new("ts"), Ident::new("host")] } ); - assert_eq!( - &constraints[2], - &TableConstraint::InvertedIndex { - columns: vec![Ident::new("host")] - } - ); + // inverted index is merged into column options assert_eq!(1, c.options.len()); assert_eq!( [("ttl", "10s")].into_iter().collect::>(), @@ -1907,33 +1934,6 @@ ENGINE=mito"; assert_matches!(result, Err(crate::error::Error::InvalidTimeIndex { .. })); } - #[test] - fn test_inverted_index_empty_list() { - let sql = r"create table demo( - host string, - ts timestamp time index, - cpu float64 default 0, - memory float64, - TIME INDEX (ts), - inverted index() - ) engine=mito; - "; - let result = - ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) - .unwrap(); - - if let Statement::CreateTable(c) = &result[0] { - let tc = &c - .constraints - .iter() - .find(|c| matches!(c, TableConstraint::InvertedIndex { .. })) - .unwrap(); - assert_eq!(*tc, &TableConstraint::InvertedIndex { columns: vec![] }); - } else { - unreachable!("should be create table statement"); - } - } - #[test] fn test_invalid_column_name() { let sql = "create table foo(user string, i timestamp time index)"; @@ -2019,7 +2019,7 @@ non TIMESTAMP(6) TIME INDEX, let sql1 = r" CREATE TABLE log ( ts TIMESTAMP TIME INDEX, - msg TEXT FULLTEXT, + msg TEXT FULLTEXT INDEX, )"; let result1 = ParserContext::create_with_dialect( sql1, @@ -2031,7 +2031,12 @@ CREATE TABLE log ( if let Statement::CreateTable(c) = &result1[0] { c.columns.iter().for_each(|col| { if col.name().value == "msg" { - assert!(col.extensions.fulltext_options.as_ref().unwrap().is_empty()); + assert!(col + .extensions + .fulltext_index_options + .as_ref() + .unwrap() + .is_empty()); } }); } else { @@ -2041,7 +2046,7 @@ CREATE TABLE log ( let sql2 = r" CREATE TABLE log ( ts TIMESTAMP TIME INDEX, - msg STRING FULLTEXT WITH (analyzer='English', case_sensitive='false') + msg STRING FULLTEXT INDEX WITH (analyzer='English', case_sensitive='false') )"; let result2 = ParserContext::create_with_dialect( sql2, @@ -2053,7 +2058,7 @@ CREATE TABLE log ( if let Statement::CreateTable(c) = &result2[0] { c.columns.iter().for_each(|col| { if col.name().value == "msg" { - let options = col.extensions.fulltext_options.as_ref().unwrap(); + let options = col.extensions.fulltext_index_options.as_ref().unwrap(); assert_eq!(options.len(), 2); assert_eq!(options.get("analyzer").unwrap(), "English"); assert_eq!(options.get("case_sensitive").unwrap(), "false"); @@ -2066,8 +2071,8 @@ CREATE TABLE log ( let sql3 = r" CREATE TABLE log ( ts TIMESTAMP TIME INDEX, - msg1 TINYTEXT FULLTEXT WITH (analyzer='English', case_sensitive='false'), - msg2 CHAR(20) FULLTEXT WITH (analyzer='Chinese', case_sensitive='true') + msg1 TINYTEXT FULLTEXT INDEX WITH (analyzer='English', case_sensitive='false'), + msg2 CHAR(20) FULLTEXT INDEX WITH (analyzer='Chinese', case_sensitive='true') )"; let result3 = ParserContext::create_with_dialect( sql3, @@ -2079,12 +2084,12 @@ CREATE TABLE log ( if let Statement::CreateTable(c) = &result3[0] { c.columns.iter().for_each(|col| { if col.name().value == "msg1" { - let options = col.extensions.fulltext_options.as_ref().unwrap(); + let options = col.extensions.fulltext_index_options.as_ref().unwrap(); assert_eq!(options.len(), 2); assert_eq!(options.get("analyzer").unwrap(), "English"); assert_eq!(options.get("case_sensitive").unwrap(), "false"); } else if col.name().value == "msg2" { - let options = col.extensions.fulltext_options.as_ref().unwrap(); + let options = col.extensions.fulltext_index_options.as_ref().unwrap(); assert_eq!(options.len(), 2); assert_eq!(options.get("analyzer").unwrap(), "Chinese"); assert_eq!(options.get("case_sensitive").unwrap(), "true"); @@ -2100,7 +2105,7 @@ CREATE TABLE log ( let sql = r" CREATE TABLE log ( ts TIMESTAMP TIME INDEX, - msg INT FULLTEXT, + msg INT FULLTEXT INDEX, )"; let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); @@ -2116,7 +2121,7 @@ CREATE TABLE log ( let sql = r" CREATE TABLE log ( ts TIMESTAMP TIME INDEX, - msg STRING FULLTEXT WITH (analyzer='English', analyzer='Chinese') FULLTEXT WITH (case_sensitive='false') + msg STRING FULLTEXT INDEX WITH (analyzer='English', analyzer='Chinese') FULLTEXT INDEX WITH (case_sensitive='false') )"; let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); @@ -2124,7 +2129,7 @@ CREATE TABLE log ( assert!(result .unwrap_err() .to_string() - .contains("duplicated FULLTEXT option")); + .contains("duplicated FULLTEXT INDEX option")); } #[test] @@ -2132,7 +2137,7 @@ CREATE TABLE log ( let sql = r" CREATE TABLE log ( ts TIMESTAMP TIME INDEX, - msg STRING FULLTEXT WITH (analyzer='English', invalid_option='Chinese') + msg STRING FULLTEXT INDEX WITH (analyzer='English', invalid_option='Chinese') )"; let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); @@ -2140,7 +2145,7 @@ CREATE TABLE log ( assert!(result .unwrap_err() .to_string() - .contains("invalid FULLTEXT option")); + .contains("invalid FULLTEXT INDEX option")); } #[test] @@ -2274,4 +2279,200 @@ CREATE TABLE log ( ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); assert!(result.is_err()); } + + #[test] + fn test_parse_column_extensions_vector() { + let sql = "VECTOR(128)"; + let dialect = GenericDialect {}; + let mut tokenizer = Tokenizer::new(&dialect, sql); + let tokens = tokenizer.tokenize().unwrap(); + let mut parser = Parser::new(&dialect).with_tokens(tokens); + let name = Ident::new("vec_col"); + let data_type = DataType::Custom( + ObjectName(vec![Ident::new("VECTOR")]), + vec!["128".to_string()], + ); + let mut extensions = ColumnExtensions::default(); + + let result = + ParserContext::parse_column_extensions(&mut parser, &name, &data_type, &mut extensions); + assert!(result.is_ok()); + assert!(extensions.vector_options.is_some()); + let vector_options = extensions.vector_options.unwrap(); + assert_eq!(vector_options.get(VECTOR_OPT_DIM), Some(&"128".to_string())); + } + + #[test] + fn test_parse_column_extensions_vector_invalid() { + let sql = "VECTOR()"; + let dialect = GenericDialect {}; + let mut tokenizer = Tokenizer::new(&dialect, sql); + let tokens = tokenizer.tokenize().unwrap(); + let mut parser = Parser::new(&dialect).with_tokens(tokens); + let name = Ident::new("vec_col"); + let data_type = DataType::Custom(ObjectName(vec![Ident::new("VECTOR")]), vec![]); + let mut extensions = ColumnExtensions::default(); + + let result = + ParserContext::parse_column_extensions(&mut parser, &name, &data_type, &mut extensions); + assert!(result.is_err()); + } + + #[test] + fn test_parse_column_extensions_indices() { + // Test skipping index + { + let sql = "SKIPPING INDEX"; + let dialect = GenericDialect {}; + let mut tokenizer = Tokenizer::new(&dialect, sql); + let tokens = tokenizer.tokenize().unwrap(); + let mut parser = Parser::new(&dialect).with_tokens(tokens); + let name = Ident::new("col"); + let data_type = DataType::String(None); + let mut extensions = ColumnExtensions::default(); + let result = ParserContext::parse_column_extensions( + &mut parser, + &name, + &data_type, + &mut extensions, + ); + assert!(result.is_ok()); + assert!(extensions.skipping_index_options.is_some()); + } + + // Test fulltext index with options + { + let sql = "FULLTEXT INDEX WITH (analyzer = 'English', case_sensitive = 'true')"; + let dialect = GenericDialect {}; + let mut tokenizer = Tokenizer::new(&dialect, sql); + let tokens = tokenizer.tokenize().unwrap(); + let mut parser = Parser::new(&dialect).with_tokens(tokens); + let name = Ident::new("text_col"); + let data_type = DataType::String(None); + let mut extensions = ColumnExtensions::default(); + let result = ParserContext::parse_column_extensions( + &mut parser, + &name, + &data_type, + &mut extensions, + ); + assert!(result.unwrap()); + assert!(extensions.fulltext_index_options.is_some()); + let fulltext_options = extensions.fulltext_index_options.unwrap(); + assert_eq!( + fulltext_options.get("analyzer"), + Some(&"English".to_string()) + ); + assert_eq!( + fulltext_options.get("case_sensitive"), + Some(&"true".to_string()) + ); + } + + // Test fulltext index with invalid type (should fail) + { + let sql = "FULLTEXT INDEX WITH (analyzer = 'English')"; + let dialect = GenericDialect {}; + let mut tokenizer = Tokenizer::new(&dialect, sql); + let tokens = tokenizer.tokenize().unwrap(); + let mut parser = Parser::new(&dialect).with_tokens(tokens); + let name = Ident::new("num_col"); + let data_type = DataType::Int(None); // Non-string type + let mut extensions = ColumnExtensions::default(); + let result = ParserContext::parse_column_extensions( + &mut parser, + &name, + &data_type, + &mut extensions, + ); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("FULLTEXT index only supports string type")); + } + + // Test fulltext index with invalid option (won't fail, the parser doesn't check the option's content) + { + let sql = "FULLTEXT INDEX WITH (analyzer = 'Invalid', case_sensitive = 'true')"; + let dialect = GenericDialect {}; + let mut tokenizer = Tokenizer::new(&dialect, sql); + let tokens = tokenizer.tokenize().unwrap(); + let mut parser = Parser::new(&dialect).with_tokens(tokens); + let name = Ident::new("text_col"); + let data_type = DataType::String(None); + let mut extensions = ColumnExtensions::default(); + let result = ParserContext::parse_column_extensions( + &mut parser, + &name, + &data_type, + &mut extensions, + ); + assert!(result.unwrap()); + } + + // Test inverted index + { + let sql = "INVERTED INDEX"; + let dialect = GenericDialect {}; + let mut tokenizer = Tokenizer::new(&dialect, sql); + let tokens = tokenizer.tokenize().unwrap(); + let mut parser = Parser::new(&dialect).with_tokens(tokens); + let name = Ident::new("col"); + let data_type = DataType::String(None); + let mut extensions = ColumnExtensions::default(); + let result = ParserContext::parse_column_extensions( + &mut parser, + &name, + &data_type, + &mut extensions, + ); + assert!(result.is_ok()); + assert!(extensions.inverted_index_options.is_some()); + } + + // Test inverted index with options (should fail) + { + let sql = "INVERTED INDEX WITH (analyzer = 'English')"; + let dialect = GenericDialect {}; + let mut tokenizer = Tokenizer::new(&dialect, sql); + let tokens = tokenizer.tokenize().unwrap(); + let mut parser = Parser::new(&dialect).with_tokens(tokens); + let name = Ident::new("col"); + let data_type = DataType::String(None); + let mut extensions = ColumnExtensions::default(); + let result = ParserContext::parse_column_extensions( + &mut parser, + &name, + &data_type, + &mut extensions, + ); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("INVERTED index doesn't support options")); + } + + // Test multiple indices + { + let sql = "SKIPPING INDEX FULLTEXT INDEX"; + let dialect = GenericDialect {}; + let mut tokenizer = Tokenizer::new(&dialect, sql); + let tokens = tokenizer.tokenize().unwrap(); + let mut parser = Parser::new(&dialect).with_tokens(tokens); + let name = Ident::new("col"); + let data_type = DataType::String(None); + let mut extensions = ColumnExtensions::default(); + let result = ParserContext::parse_column_extensions( + &mut parser, + &name, + &data_type, + &mut extensions, + ); + assert!(result.unwrap()); + assert!(extensions.skipping_index_options.is_some()); + assert!(extensions.fulltext_index_options.is_some()); + } + } } diff --git a/src/sql/src/statements.rs b/src/sql/src/statements.rs index 21e9ead437..a9c1d2aa2b 100644 --- a/src/sql/src/statements.rs +++ b/src/sql/src/statements.rs @@ -463,8 +463,6 @@ pub fn has_primary_key_option(column_def: &ColumnDef) -> bool { pub fn column_to_schema( column: &Column, time_index: &str, - invereted_index_cols: &Option>, - primary_keys: &[String], timezone: Option<&Timezone>, ) -> Result { let is_time_index = column.name().value == time_index; @@ -487,20 +485,6 @@ pub fn column_to_schema( column: &column.name().value, })?; - // To keep compatibility, - // 1. if inverted index columns is not set, leave it empty meaning primary key columns will be used - // 2. if inverted index columns is set and non-empty, set selected columns to be inverted indexed - // 3. if inverted index columns is set and empty, set primary key columns to be non-inverted indexed explicitly - if let Some(inverted_index_cols) = invereted_index_cols { - if inverted_index_cols.is_empty() { - if primary_keys.contains(&column.name().value) { - column_schema.insert_inverted_index_placeholder(); - } - } else if inverted_index_cols.contains(&column.name().value) { - column_schema.set_inverted_index(true); - } - } - if let Some(ColumnOption::Comment(c)) = column.options().iter().find_map(|o| { if matches!(o.option, ColumnOption::Comment(_)) { Some(&o.option) @@ -525,6 +509,8 @@ pub fn column_to_schema( .context(SetSkippingIndexOptionSnafu)?; } + column_schema.set_inverted_index(column.extensions.inverted_index_options.is_some()); + Ok(column_schema) } @@ -1382,7 +1368,7 @@ mod tests { extensions: ColumnExtensions::default(), }; - let column_schema = column_to_schema(&column_def, "ts", &None, &[], None).unwrap(); + let column_schema = column_to_schema(&column_def, "ts", None).unwrap(); assert_eq!("col", column_schema.name); assert_eq!( @@ -1392,7 +1378,7 @@ mod tests { assert!(column_schema.is_nullable()); assert!(!column_schema.is_time_index()); - let column_schema = column_to_schema(&column_def, "col", &None, &[], None).unwrap(); + let column_schema = column_to_schema(&column_def, "col", None).unwrap(); assert_eq!("col", column_schema.name); assert_eq!( @@ -1421,7 +1407,7 @@ mod tests { extensions: ColumnExtensions::default(), }; - let column_schema = column_to_schema(&column_def, "ts", &None, &[], None).unwrap(); + let column_schema = column_to_schema(&column_def, "ts", None).unwrap(); assert_eq!("col2", column_schema.name); assert_eq!(ConcreteDataType::string_datatype(), column_schema.data_type); @@ -1456,8 +1442,6 @@ mod tests { let column_schema = column_to_schema( &column, "ts", - &None, - &[], Some(&Timezone::from_tz_string("Asia/Shanghai").unwrap()), ) .unwrap(); @@ -1476,7 +1460,7 @@ mod tests { ); // without timezone - let column_schema = column_to_schema(&column, "ts", &None, &[], None).unwrap(); + let column_schema = column_to_schema(&column, "ts", None).unwrap(); assert_eq!("col", column_schema.name); assert_eq!( @@ -1502,7 +1486,7 @@ mod tests { options: vec![], }, extensions: ColumnExtensions { - fulltext_options: Some( + fulltext_index_options: Some( HashMap::from_iter([ ( COLUMN_FULLTEXT_OPT_KEY_ANALYZER.to_string(), @@ -1517,10 +1501,11 @@ mod tests { ), vector_options: None, skipping_index_options: None, + inverted_index_options: None, }, }; - let column_schema = column_to_schema(&column, "ts", &None, &[], None).unwrap(); + let column_schema = column_to_schema(&column, "ts", None).unwrap(); assert_eq!("col", column_schema.name); assert_eq!(ConcreteDataType::string_datatype(), column_schema.data_type); let fulltext_options = column_schema.fulltext_options().unwrap().unwrap(); diff --git a/src/sql/src/statements/create.rs b/src/sql/src/statements/create.rs index 6fd0333a80..981a51105f 100644 --- a/src/sql/src/statements/create.rs +++ b/src/sql/src/statements/create.rs @@ -65,8 +65,6 @@ pub enum TableConstraint { PrimaryKey { columns: Vec }, /// Time index constraint. TimeIndex { column: Ident }, - /// Inverted index constraint. - InvertedIndex { columns: Vec }, } impl Display for TableConstraint { @@ -78,9 +76,6 @@ impl Display for TableConstraint { TableConstraint::TimeIndex { column } => { write!(f, "TIME INDEX ({})", column) } - TableConstraint::InvertedIndex { columns } => { - write!(f, "INVERTED INDEX ({})", format_list_comma!(columns)) - } } } } @@ -112,12 +107,17 @@ pub struct Column { /// Column extensions for greptimedb dialect. #[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Default, Serialize)] pub struct ColumnExtensions { - /// Fulltext options. - pub fulltext_options: Option, - /// Vector options. + /// Vector type options. pub vector_options: Option, + + /// Fulltext index options. + pub fulltext_index_options: Option, /// Skipping index options. pub skipping_index_options: Option, + /// Inverted index options. + /// + /// Inverted index doesn't have options at present. There won't be any options in that map. + pub inverted_index_options: Option, } impl Column { @@ -152,7 +152,8 @@ impl Display for Column { } write!(f, "{}", self.column_def)?; - if let Some(fulltext_options) = &self.extensions.fulltext_options { + + if let Some(fulltext_options) = &self.extensions.fulltext_index_options { if !fulltext_options.is_empty() { let options = fulltext_options.kv_pairs(); write!(f, " FULLTEXT WITH({})", format_list_comma!(options))?; @@ -169,13 +170,22 @@ impl Display for Column { write!(f, " SKIPPING INDEX")?; } } + + if let Some(inverted_index_options) = &self.extensions.inverted_index_options { + if !inverted_index_options.is_empty() { + let options = inverted_index_options.kv_pairs(); + write!(f, " INVERTED INDEX WITH({})", format_list_comma!(options))?; + } else { + write!(f, " INVERTED INDEX")?; + } + } Ok(()) } } impl ColumnExtensions { pub fn build_fulltext_options(&self) -> Result> { - let Some(options) = self.fulltext_options.as_ref() else { + let Some(options) = self.fulltext_index_options.as_ref() else { return Ok(None); }; diff --git a/src/store-api/src/metadata.rs b/src/store-api/src/metadata.rs index febb0bc28f..b07e1bdb60 100644 --- a/src/store-api/src/metadata.rs +++ b/src/store-api/src/metadata.rs @@ -633,36 +633,12 @@ impl RegionMetadataBuilder { .map(|col| col.column_schema.name.clone()) .collect(); - let pk_as_inverted_index = !self - .column_metadatas - .iter() - .any(|c| c.column_schema.has_inverted_index_key()); - let mut set_inverted_index_for_primary_keys = false; - for add_column in columns { if names.contains(&add_column.column_metadata.column_schema.name) { // Column already exists. continue; } - // Handles using primary key as inverted index. - let has_inverted_index_key = add_column - .column_metadata - .column_schema - .has_inverted_index_key(); - - if pk_as_inverted_index - && has_inverted_index_key - && !set_inverted_index_for_primary_keys - { - self.column_metadatas.iter_mut().for_each(|col| { - if col.semantic_type == SemanticType::Tag { - col.column_schema.set_inverted_index(true); - } - }); - set_inverted_index_for_primary_keys = true; - } - let column_id = add_column.column_metadata.column_id; let semantic_type = add_column.column_metadata.semantic_type; let column_name = add_column.column_metadata.column_schema.name.clone(); @@ -1526,6 +1502,8 @@ mod test { #[test] fn test_add_column_with_inverted_index() { + // only set inverted index to true explicitly will this column be inverted indexed + // a (tag), b (field), c (ts) let metadata = build_test_region_metadata(); let mut builder = RegionMetadataBuilder::from_existing(metadata); @@ -1550,7 +1528,7 @@ mod test { check_columns(&metadata, &["a", "b", "c", "d", "e"]); assert_eq!([1, 4, 5], &metadata.primary_key[..]); let column_metadata = metadata.column_by_name("a").unwrap(); - assert!(column_metadata.column_schema.is_inverted_indexed()); + assert!(!column_metadata.column_schema.is_inverted_indexed()); let column_metadata = metadata.column_by_name("b").unwrap(); assert!(!column_metadata.column_schema.is_inverted_indexed()); let column_metadata = metadata.column_by_name("c").unwrap(); diff --git a/src/table/src/metadata.rs b/src/table/src/metadata.rs index b54f979261..c8a65a8407 100644 --- a/src/table/src/metadata.rs +++ b/src/table/src/metadata.rs @@ -283,34 +283,10 @@ impl TableMeta { let mut columns: Vec = Vec::with_capacity(table_schema.column_schemas().len()); - // When we are setting inverted index for the first time - // (schemas.all(!has_inverted_index_key)). - // We need to make sure the table's primary index's inverted index - // property is set to true. - let pk_as_inverted_index = !self - .schema - .column_schemas() - .iter() - .any(|c| c.has_inverted_index_key()); - - for (i, column_schema) in table_schema.column_schemas().iter().enumerate() { + for column_schema in table_schema.column_schemas().iter() { if column_schema.name == column_name { - // If user explicitly unset an inverted index in primary keys. - // We should invalidate the primary key as inverted index - // on the condition of schemas.all(!has_inverted_index_key). - if !value && self.primary_key_indices.contains(&i) { - let mut new_column_schema = column_schema.clone(); - new_column_schema.insert_inverted_index_placeholder(); - columns.push(new_column_schema); - } else { - let mut new_column_schema = column_schema.clone(); - new_column_schema.set_inverted_index(value); - columns.push(new_column_schema); - } - } else if pk_as_inverted_index && self.primary_key_indices.contains(&i) { - // Need to set inverted_indexed=true for all other columns in primary key. let mut new_column_schema = column_schema.clone(); - new_column_schema.set_inverted_index(true); + new_column_schema.set_inverted_index(value); columns.push(new_column_schema); } else { columns.push(column_schema.clone()); diff --git a/tests/cases/standalone/common/alter/change_col_inverted_index.result b/tests/cases/standalone/common/alter/change_col_inverted_index.result index 0d75504c2e..619c5fc65b 100644 --- a/tests/cases/standalone/common/alter/change_col_inverted_index.result +++ b/tests/cases/standalone/common/alter/change_col_inverted_index.result @@ -34,36 +34,34 @@ SELECT fox FROM fox WHERE MATCHES(fox, '"fox jumps"') ORDER BY ts; SHOW CREATE TABLE fox; -+-------+------------------------------------+ -| Table | Create Table | -+-------+------------------------------------+ -| fox | CREATE TABLE IF NOT EXISTS "fox" ( | -| | "ts" TIMESTAMP(3) NOT NULL, | -| | "fox" STRING NULL, | -| | TIME INDEX ("ts"), | -| | INVERTED INDEX ("fox") | -| | ) | -| | | -| | ENGINE=mito | -| | | -+-------+------------------------------------+ ++-------+-------------------------------------+ +| Table | Create Table | ++-------+-------------------------------------+ +| fox | CREATE TABLE IF NOT EXISTS "fox" ( | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "fox" STRING NULL INVERTED INDEX, | +| | TIME INDEX ("ts") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++-------+-------------------------------------+ -- SQLNESS ARG restart=true SHOW CREATE TABLE fox; -+-------+------------------------------------+ -| Table | Create Table | -+-------+------------------------------------+ -| fox | CREATE TABLE IF NOT EXISTS "fox" ( | -| | "ts" TIMESTAMP(3) NOT NULL, | -| | "fox" STRING NULL, | -| | TIME INDEX ("ts"), | -| | INVERTED INDEX ("fox") | -| | ) | -| | | -| | ENGINE=mito | -| | | -+-------+------------------------------------+ ++-------+-------------------------------------+ +| Table | Create Table | ++-------+-------------------------------------+ +| fox | CREATE TABLE IF NOT EXISTS "fox" ( | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "fox" STRING NULL INVERTED INDEX, | +| | TIME INDEX ("ts") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++-------+-------------------------------------+ SHOW INDEX FROM fox; @@ -127,13 +125,13 @@ Affected Rows: 0 SHOW INDEX FROM test_pk; -+---------+------------+-------------------------+--------------+-------------+-----------+-------------+----------+--------+------+-----------------------------------------------------+---------+---------------+---------+------------+ -| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | -+---------+------------+-------------------------+--------------+-------------+-----------+-------------+----------+--------+------+-----------------------------------------------------+---------+---------------+---------+------------+ -| test_pk | 1 | PRIMARY, INVERTED INDEX | 3 | bar | A | | | | YES | greptime-primary-key-v1, greptime-inverted-index-v1 | | | YES | | -| test_pk | 1 | PRIMARY, INVERTED INDEX | 2 | foo | A | | | | YES | greptime-primary-key-v1, greptime-inverted-index-v1 | | | YES | | -| test_pk | 1 | TIME INDEX | 1 | ts | A | | | | NO | | | | YES | | -+---------+------------+-------------------------+--------------+-------------+-----------+-------------+----------+--------+------+-----------------------------------------------------+---------+---------------+---------+------------+ ++---------+------------+------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------+---------+---------------+---------+------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | ++---------+------------+------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------+---------+---------------+---------+------------+ +| test_pk | 1 | PRIMARY | 3 | bar | A | | | | YES | greptime-primary-key-v1 | | | YES | | +| test_pk | 1 | PRIMARY | 2 | foo | A | | | | YES | greptime-primary-key-v1 | | | YES | | +| test_pk | 1 | TIME INDEX | 1 | ts | A | | | | NO | | | | YES | | ++---------+------------+------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------+---------+---------------+---------+------------+ ALTER TABLE test_pk MODIFY COLUMN foo UNSET INVERTED INDEX; @@ -141,13 +139,13 @@ Affected Rows: 0 SHOW INDEX FROM test_pk; -+---------+------------+-------------------------+--------------+-------------+-----------+-------------+----------+--------+------+-----------------------------------------------------+---------+---------------+---------+------------+ -| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | -+---------+------------+-------------------------+--------------+-------------+-----------+-------------+----------+--------+------+-----------------------------------------------------+---------+---------------+---------+------------+ -| test_pk | 1 | PRIMARY, INVERTED INDEX | 3 | bar | A | | | | YES | greptime-primary-key-v1, greptime-inverted-index-v1 | | | YES | | -| test_pk | 1 | PRIMARY | 2 | foo | A | | | | YES | greptime-primary-key-v1 | | | YES | | -| test_pk | 1 | TIME INDEX | 1 | ts | A | | | | NO | | | | YES | | -+---------+------------+-------------------------+--------------+-------------+-----------+-------------+----------+--------+------+-----------------------------------------------------+---------+---------------+---------+------------+ ++---------+------------+------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------+---------+---------------+---------+------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | ++---------+------------+------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------+---------+---------------+---------+------------+ +| test_pk | 1 | PRIMARY | 3 | bar | A | | | | YES | greptime-primary-key-v1 | | | YES | | +| test_pk | 1 | PRIMARY | 2 | foo | A | | | | YES | greptime-primary-key-v1 | | | YES | | +| test_pk | 1 | TIME INDEX | 1 | ts | A | | | | NO | | | | YES | | ++---------+------------+------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------+---------+---------------+---------+------------+ ALTER TABLE test_pk MODIFY COLUMN bar UNSET INVERTED INDEX; diff --git a/tests/cases/standalone/common/create/create_metric_table.result b/tests/cases/standalone/common/create/create_metric_table.result index a0b6e782dd..41d4dcb351 100644 --- a/tests/cases/standalone/common/create/create_metric_table.result +++ b/tests/cases/standalone/common/create/create_metric_table.result @@ -73,24 +73,23 @@ DESC TABLE phy; SHOW CREATE TABLE phy; -+-------+------------------------------------+ -| Table | Create Table | -+-------+------------------------------------+ -| phy | CREATE TABLE IF NOT EXISTS "phy" ( | -| | "ts" TIMESTAMP(3) NOT NULL, | -| | "val" DOUBLE NULL, | -| | "host" STRING NULL, | -| | "job" STRING NULL, | -| | TIME INDEX ("ts"), | -| | PRIMARY KEY ("host", "job"), | -| | INVERTED INDEX ("host", "job") | -| | ) | -| | | -| | ENGINE=metric | -| | WITH( | -| | physical_metric_table = '' | -| | ) | -+-------+------------------------------------+ ++-------+--------------------------------------+ +| Table | Create Table | ++-------+--------------------------------------+ +| phy | CREATE TABLE IF NOT EXISTS "phy" ( | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "val" DOUBLE NULL, | +| | "host" STRING NULL INVERTED INDEX, | +| | "job" STRING NULL INVERTED INDEX, | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host", "job") | +| | ) | +| | | +| | ENGINE=metric | +| | WITH( | +| | physical_metric_table = '' | +| | ) | ++-------+--------------------------------------+ DESC TABLE t1; diff --git a/tests/cases/standalone/common/create/create_with_fulltext.result b/tests/cases/standalone/common/create/create_with_fulltext.result index fd1ae5777a..e99b15ce6d 100644 --- a/tests/cases/standalone/common/create/create_with_fulltext.result +++ b/tests/cases/standalone/common/create/create_with_fulltext.result @@ -1,6 +1,6 @@ CREATE TABLE log ( ts TIMESTAMP TIME INDEX, - msg STRING FULLTEXT, + msg STRING FULLTEXT INDEX, ); Affected Rows: 0 @@ -26,7 +26,7 @@ Affected Rows: 0 CREATE TABLE log_with_opts ( ts TIMESTAMP TIME INDEX, - msg TEXT FULLTEXT WITH (analyzer='English', case_sensitive='true'), + msg TEXT FULLTEXT INDEX WITH (analyzer='English', case_sensitive='true'), ); Affected Rows: 0 @@ -52,8 +52,8 @@ Affected Rows: 0 CREATE TABLE log_multi_fulltext_cols ( ts TIMESTAMP TIME INDEX, - msg TINYTEXT FULLTEXT, - msg2 VARCHAR FULLTEXT, + msg TINYTEXT FULLTEXT INDEX, + msg2 VARCHAR FULLTEXT INDEX, ); Affected Rows: 0 @@ -83,19 +83,19 @@ CREATE TABLE log_dup_fulltext_opts ( msg TEXT FULLTEXT FULLTEXT, ); -Error: 1004(InvalidArguments), Invalid column option, column name: msg, error: duplicated FULLTEXT option +Error: 1004(InvalidArguments), Invalid column option, column name: msg, error: expect INDEX after FULLTEXT keyword CREATE TABLE log_with_invalid_type ( ts TIMESTAMP TIME INDEX, - msg INT FULLTEXT, + msg INT FULLTEXT INDEX, ); Error: 1004(InvalidArguments), Invalid column option, column name: msg, error: FULLTEXT index only supports string type CREATE TABLE log_with_invalid_option ( ts TIMESTAMP TIME INDEX, - msg TEXT FULLTEXT WITH (analyzer='English', invalid_option='true'), + msg TEXT FULLTEXT INDEX WITH (analyzer='English', invalid_option='true'), ); -Error: 1004(InvalidArguments), Invalid column option, column name: msg, error: invalid FULLTEXT option: invalid_option +Error: 1004(InvalidArguments), Invalid column option, column name: msg, error: invalid FULLTEXT INDEX option: invalid_option diff --git a/tests/cases/standalone/common/create/create_with_fulltext.sql b/tests/cases/standalone/common/create/create_with_fulltext.sql index 27e2328e8d..af4dac7b21 100644 --- a/tests/cases/standalone/common/create/create_with_fulltext.sql +++ b/tests/cases/standalone/common/create/create_with_fulltext.sql @@ -1,6 +1,6 @@ CREATE TABLE log ( ts TIMESTAMP TIME INDEX, - msg STRING FULLTEXT, + msg STRING FULLTEXT INDEX, ); SHOW CREATE TABLE log; @@ -10,7 +10,7 @@ DROP TABLE log; CREATE TABLE log_with_opts ( ts TIMESTAMP TIME INDEX, - msg TEXT FULLTEXT WITH (analyzer='English', case_sensitive='true'), + msg TEXT FULLTEXT INDEX WITH (analyzer='English', case_sensitive='true'), ); SHOW CREATE TABLE log_with_opts; @@ -20,8 +20,8 @@ DROP TABLE log_with_opts; CREATE TABLE log_multi_fulltext_cols ( ts TIMESTAMP TIME INDEX, - msg TINYTEXT FULLTEXT, - msg2 VARCHAR FULLTEXT, + msg TINYTEXT FULLTEXT INDEX, + msg2 VARCHAR FULLTEXT INDEX, ); SHOW CREATE TABLE log_multi_fulltext_cols; @@ -36,10 +36,10 @@ CREATE TABLE log_dup_fulltext_opts ( CREATE TABLE log_with_invalid_type ( ts TIMESTAMP TIME INDEX, - msg INT FULLTEXT, + msg INT FULLTEXT INDEX, ); CREATE TABLE log_with_invalid_option ( ts TIMESTAMP TIME INDEX, - msg TEXT FULLTEXT WITH (analyzer='English', invalid_option='true'), + msg TEXT FULLTEXT INDEX WITH (analyzer='English', invalid_option='true'), ); diff --git a/tests/cases/standalone/common/flow/flow_user_guide.result b/tests/cases/standalone/common/flow/flow_user_guide.result index e0d7575dff..c044fef367 100644 --- a/tests/cases/standalone/common/flow/flow_user_guide.result +++ b/tests/cases/standalone/common/flow/flow_user_guide.result @@ -5,7 +5,7 @@ CREATE TABLE `ngx_access_log` ( `referer` STRING NULL, `method` STRING NULL, `endpoint` STRING NULL, - `trace_id` STRING NULL FULLTEXT, + `trace_id` STRING NULL FULLTEXT INDEX, `protocol` STRING NULL, `status` SMALLINT UNSIGNED NULL, `size` DOUBLE NULL, diff --git a/tests/cases/standalone/common/flow/flow_user_guide.sql b/tests/cases/standalone/common/flow/flow_user_guide.sql index 285d2198dc..d882972393 100644 --- a/tests/cases/standalone/common/flow/flow_user_guide.sql +++ b/tests/cases/standalone/common/flow/flow_user_guide.sql @@ -5,7 +5,7 @@ CREATE TABLE `ngx_access_log` ( `referer` STRING NULL, `method` STRING NULL, `endpoint` STRING NULL, - `trace_id` STRING NULL FULLTEXT, + `trace_id` STRING NULL FULLTEXT INDEX, `protocol` STRING NULL, `status` SMALLINT UNSIGNED NULL, `size` DOUBLE NULL, diff --git a/tests/cases/standalone/common/show/show_create.result b/tests/cases/standalone/common/show/show_create.result index 7a52c8181b..21532b28f3 100644 --- a/tests/cases/standalone/common/show/show_create.result +++ b/tests/cases/standalone/common/show/show_create.result @@ -121,23 +121,22 @@ Affected Rows: 0 show create table phy; -+-------+------------------------------------+ -| Table | Create Table | -+-------+------------------------------------+ -| phy | CREATE TABLE IF NOT EXISTS "phy" ( | -| | "ts" TIMESTAMP(3) NOT NULL, | -| | "val" DOUBLE NULL, | -| | "host" STRING NULL, | -| | TIME INDEX ("ts"), | -| | PRIMARY KEY ("host"), | -| | INVERTED INDEX ("host") | -| | ) | -| | | -| | ENGINE=metric | -| | WITH( | -| | physical_metric_table = '' | -| | ) | -+-------+------------------------------------+ ++-------+--------------------------------------+ +| Table | Create Table | ++-------+--------------------------------------+ +| phy | CREATE TABLE IF NOT EXISTS "phy" ( | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "val" DOUBLE NULL, | +| | "host" STRING NULL INVERTED INDEX, | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host") | +| | ) | +| | | +| | ENGINE=metric | +| | WITH( | +| | physical_metric_table = '' | +| | ) | ++-------+--------------------------------------+ show create table t1; @@ -302,10 +301,9 @@ CREATE TABLE "specify_invereted_index_cols" ( "ts" TIMESTAMP(3) NOT NULL, "val" DOUBLE NULL, "host" STRING NULL, - "job" STRING NULL, + "job" STRING NULL INVERTED INDEX, TIME INDEX ("ts"), PRIMARY KEY ("host", "job"), - INVERTED INDEX ("job") ); Affected Rows: 0 @@ -319,10 +317,9 @@ show create table specify_invereted_index_cols; | | "ts" TIMESTAMP(3) NOT NULL, | | | "val" DOUBLE NULL, | | | "host" STRING NULL, | -| | "job" STRING NULL, | +| | "job" STRING NULL INVERTED INDEX, | | | TIME INDEX ("ts"), | -| | PRIMARY KEY ("host", "job"), | -| | INVERTED INDEX ("job") | +| | PRIMARY KEY ("host", "job") | | | ) | | | | | | ENGINE=mito | @@ -340,7 +337,6 @@ CREATE TABLE "specify_empty_invereted_index_cols" ( "job" STRING NULL, TIME INDEX ("ts"), PRIMARY KEY ("host", "job"), - INVERTED INDEX () ); Affected Rows: 0 @@ -356,8 +352,7 @@ show create table specify_empty_invereted_index_cols; | | "host" STRING NULL, | | | "job" STRING NULL, | | | TIME INDEX ("ts"), | -| | PRIMARY KEY ("host", "job"), | -| | INVERTED INDEX () | +| | PRIMARY KEY ("host", "job") | | | ) | | | | | | ENGINE=mito | @@ -368,3 +363,32 @@ drop table specify_empty_invereted_index_cols; Affected Rows: 0 +CREATE TABLE test_column_constrain_composite_indexes ( + `id` INT SKIPPING INDEX INVERTED INDEX, + host STRING PRIMARY KEY SKIPPING INDEX FULLTEXT INDEX INVERTED INDEX, + ts TIMESTAMP TIME INDEX, +); + +Affected Rows: 0 + +show create table test_column_constrain_composite_indexes; + ++-----------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++-----------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| test_column_constrain_composite_indexes | CREATE TABLE IF NOT EXISTS "test_column_constrain_composite_indexes" ( | +| | "id" INT NULL SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM') INVERTED INDEX, | +| | "host" STRING NULL FULLTEXT WITH(analyzer = 'English', case_sensitive = 'false') SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM') INVERTED INDEX, | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++-----------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +drop table test_column_constrain_composite_indexes; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/show/show_create.sql b/tests/cases/standalone/common/show/show_create.sql index c7a50a2598..a0791caa20 100644 --- a/tests/cases/standalone/common/show/show_create.sql +++ b/tests/cases/standalone/common/show/show_create.sql @@ -122,10 +122,9 @@ CREATE TABLE "specify_invereted_index_cols" ( "ts" TIMESTAMP(3) NOT NULL, "val" DOUBLE NULL, "host" STRING NULL, - "job" STRING NULL, + "job" STRING NULL INVERTED INDEX, TIME INDEX ("ts"), PRIMARY KEY ("host", "job"), - INVERTED INDEX ("job") ); show create table specify_invereted_index_cols; @@ -139,9 +138,18 @@ CREATE TABLE "specify_empty_invereted_index_cols" ( "job" STRING NULL, TIME INDEX ("ts"), PRIMARY KEY ("host", "job"), - INVERTED INDEX () ); show create table specify_empty_invereted_index_cols; drop table specify_empty_invereted_index_cols; + +CREATE TABLE test_column_constrain_composite_indexes ( + `id` INT SKIPPING INDEX INVERTED INDEX, + host STRING PRIMARY KEY SKIPPING INDEX FULLTEXT INDEX INVERTED INDEX, + ts TIMESTAMP TIME INDEX, +); + +show create table test_column_constrain_composite_indexes; + +drop table test_column_constrain_composite_indexes; diff --git a/tests/cases/standalone/common/show/show_index.result b/tests/cases/standalone/common/show/show_index.result index 08093be451..0376443746 100644 --- a/tests/cases/standalone/common/show/show_index.result +++ b/tests/cases/standalone/common/show/show_index.result @@ -1,15 +1,14 @@ CREATE TABLE IF NOT EXISTS system_metrics ( host STRING, - idc STRING FULLTEXT, + idc STRING FULLTEXT INDEX INVERTED INDEX, cpu_util DOUBLE, memory_util DOUBLE, disk_util DOUBLE, desc1 STRING, - desc2 STRING FULLTEXT, - desc3 STRING FULLTEXT, + desc2 STRING FULLTEXT INDEX, + desc3 STRING FULLTEXT INDEX, ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(host, idc), - INVERTED INDEX(idc, desc1, desc2), TIME INDEX(ts) ); @@ -32,25 +31,42 @@ CREATE TABLE IF NOT EXISTS test_no_inverted_index ( c DOUBLE, ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(a, b), - INVERTED INDEX(), TIME INDEX(ts) ); Affected Rows: 0 +show create table test_no_inverted_index; + ++------------------------+-------------------------------------------------------------------------------+ +| Table | Create Table | ++------------------------+-------------------------------------------------------------------------------+ +| test_no_inverted_index | CREATE TABLE IF NOT EXISTS "test_no_inverted_index" ( | +| | "a" STRING NULL, | +| | "b" STRING NULL SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM'), | +| | "c" DOUBLE NULL, | +| | "ts" TIMESTAMP(3) NOT NULL DEFAULT current_timestamp(), | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("a", "b") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++------------------------+-------------------------------------------------------------------------------+ + SHOW INDEX; Error: 2000(InvalidSyntax), Unexpected token while parsing SQL statement, expected: '{FROM | IN} table', found: ; SHOW INDEX FROM test; -+-------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------------------------------------------------------------+---------+---------------+---------+------------+ -| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | -+-------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------------------------------------------------------------+---------+---------------+---------+------------+ -| test | 1 | PRIMARY, INVERTED INDEX | 1 | a | A | | | | YES | greptime-primary-key-v1, greptime-inverted-index-v1 | | | YES | | -| test | 1 | PRIMARY, INVERTED INDEX, SKIPPING INDEX | 2 | b | A | | | | YES | greptime-primary-key-v1, greptime-inverted-index-v1, greptime-bloom-filter-v1 | | | YES | | -| test | 1 | TIME INDEX | 1 | ts | A | | | | NO | | | | YES | | -+-------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------------------------------------------------------------+---------+---------------+---------+------------+ ++-------+------------+-------------------------+--------------+-------------+-----------+-------------+----------+--------+------+---------------------------------------------------+---------+---------------+---------+------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | ++-------+------------+-------------------------+--------------+-------------+-----------+-------------+----------+--------+------+---------------------------------------------------+---------+---------------+---------+------------+ +| test | 1 | PRIMARY | 1 | a | A | | | | YES | greptime-primary-key-v1 | | | YES | | +| test | 1 | PRIMARY, SKIPPING INDEX | 2 | b | A | | | | YES | greptime-primary-key-v1, greptime-bloom-filter-v1 | | | YES | | +| test | 1 | TIME INDEX | 1 | ts | A | | | | NO | | | | YES | | ++-------+------------+-------------------------+--------------+-------------+-----------+-------------+----------+--------+------+---------------------------------------------------+---------+---------------+---------+------------+ SHOW INDEX FROM test_no_inverted_index; @@ -67,8 +83,7 @@ SHOW INDEX FROM system_metrics; +----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+---------------------------------------------------------------------------------+---------+---------------+---------+------------+ | Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | +----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+---------------------------------------------------------------------------------+---------+---------------+---------+------------+ -| system_metrics | 1 | INVERTED INDEX | 6 | desc1 | A | | | | YES | greptime-inverted-index-v1 | | | YES | | -| system_metrics | 1 | INVERTED INDEX, FULLTEXT INDEX | 7 | desc2 | A | | | | YES | greptime-inverted-index-v1, greptime-fulltext-index-v1 | | | YES | | +| system_metrics | 1 | FULLTEXT INDEX | 7 | desc2 | A | | | | YES | greptime-fulltext-index-v1 | | | YES | | | system_metrics | 1 | FULLTEXT INDEX | 8 | desc3 | A | | | | YES | greptime-fulltext-index-v1 | | | YES | | | system_metrics | 1 | PRIMARY | 1 | host | A | | | | YES | greptime-primary-key-v1 | | | YES | | | system_metrics | 1 | PRIMARY, INVERTED INDEX, FULLTEXT INDEX | 2 | idc | A | | | | YES | greptime-primary-key-v1, greptime-inverted-index-v1, greptime-fulltext-index-v1 | | | YES | | @@ -80,8 +95,7 @@ SHOW INDEX FROM system_metrics in public; +----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+---------------------------------------------------------------------------------+---------+---------------+---------+------------+ | Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | +----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+---------------------------------------------------------------------------------+---------+---------------+---------+------------+ -| system_metrics | 1 | INVERTED INDEX | 6 | desc1 | A | | | | YES | greptime-inverted-index-v1 | | | YES | | -| system_metrics | 1 | INVERTED INDEX, FULLTEXT INDEX | 7 | desc2 | A | | | | YES | greptime-inverted-index-v1, greptime-fulltext-index-v1 | | | YES | | +| system_metrics | 1 | FULLTEXT INDEX | 7 | desc2 | A | | | | YES | greptime-fulltext-index-v1 | | | YES | | | system_metrics | 1 | FULLTEXT INDEX | 8 | desc3 | A | | | | YES | greptime-fulltext-index-v1 | | | YES | | | system_metrics | 1 | PRIMARY | 1 | host | A | | | | YES | greptime-primary-key-v1 | | | YES | | | system_metrics | 1 | PRIMARY, INVERTED INDEX, FULLTEXT INDEX | 2 | idc | A | | | | YES | greptime-primary-key-v1, greptime-inverted-index-v1, greptime-fulltext-index-v1 | | | YES | | diff --git a/tests/cases/standalone/common/show/show_index.sql b/tests/cases/standalone/common/show/show_index.sql index ec6a7d1212..ef89ba3ea2 100644 --- a/tests/cases/standalone/common/show/show_index.sql +++ b/tests/cases/standalone/common/show/show_index.sql @@ -1,15 +1,14 @@ CREATE TABLE IF NOT EXISTS system_metrics ( host STRING, - idc STRING FULLTEXT, + idc STRING FULLTEXT INDEX INVERTED INDEX, cpu_util DOUBLE, memory_util DOUBLE, disk_util DOUBLE, desc1 STRING, - desc2 STRING FULLTEXT, - desc3 STRING FULLTEXT, + desc2 STRING FULLTEXT INDEX, + desc3 STRING FULLTEXT INDEX, ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(host, idc), - INVERTED INDEX(idc, desc1, desc2), TIME INDEX(ts) ); @@ -28,10 +27,11 @@ CREATE TABLE IF NOT EXISTS test_no_inverted_index ( c DOUBLE, ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(a, b), - INVERTED INDEX(), TIME INDEX(ts) ); +show create table test_no_inverted_index; + SHOW INDEX; SHOW INDEX FROM test; diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result index fe4e4ada00..4a41154ceb 100644 --- a/tests/cases/standalone/common/system/information_schema.result +++ b/tests/cases/standalone/common/system/information_schema.result @@ -586,24 +586,24 @@ select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME = 'TIME INDEX'; select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME != 'TIME INDEX'; -+--------------------+-------------------+-------------------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ -| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | -+--------------------+-------------------+-------------------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ -| def | public | PRIMARY, INVERTED INDEX | def | greptime | public | numbers | number | 1 | | | | | -+--------------------+-------------------+-------------------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ +| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ +| def | public | PRIMARY | def | greptime | public | numbers | number | 1 | | | | | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME LIKE '%INDEX'; -+--------------------+-------------------+-------------------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ -| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | -+--------------------+-------------------+-------------------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ -| def | public | PRIMARY, INVERTED INDEX | def | greptime | public | numbers | number | 1 | | | | | -+--------------------+-------------------+-------------------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ +++ +++ select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME NOT LIKE '%INDEX'; -++ -++ ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ +| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ +| def | public | PRIMARY | def | greptime | public | numbers | number | 1 | | | | | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME == 'TIME INDEX' AND CONSTRAINT_SCHEMA != 'my_db'; @@ -685,11 +685,11 @@ desc table key_column_usage; select * from key_column_usage; -+--------------------+-------------------+-------------------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ -| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | -+--------------------+-------------------+-------------------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ -| def | public | PRIMARY, INVERTED INDEX | def | greptime | public | numbers | number | 1 | | | | | -+--------------------+-------------------+-------------------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ +| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ +| def | public | PRIMARY | def | greptime | public | numbers | number | 1 | | | | | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ -- tables not implemented DESC TABLE COLUMN_PRIVILEGES;