From 4306cba8660aeec97c88ad326ebe34a7fa870806 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Wed, 19 Jun 2024 21:21:58 -0700 Subject: [PATCH] feat: show database options (#4174) * test: test create table with database ttl * feat: show database options * fix: comment * chore: apply suggestion Co-authored-by: Jeremyhi * chore: fix CR comments and refactor * chore: style Co-authored-by: Weny Xu --------- Co-authored-by: Jeremyhi Co-authored-by: Weny Xu --- src/auth/tests/mod.rs | 5 +- .../src/information_schema/schemata.rs | 46 ++++++++++-- src/catalog/src/information_schema/utils.rs | 15 ++++ src/common/meta/src/key/schema_name.rs | 22 ++++++ src/query/src/sql.rs | 10 ++- src/sql/src/parsers/show_parser.rs | 55 ++++++++++++--- src/sql/src/statements/show.rs | 12 +++- .../common/create/create_database.result | 9 --- .../common/create/create_database.sql | 4 -- .../common/create/create_database_opts.result | 70 +++++++++++++++++++ .../common/create/create_database_opts.sql | 17 +++++ .../common/show/show_databases_tables.result | 18 +++-- .../common/show/show_databases_tables.sql | 10 +-- .../common/system/information_schema.result | 14 ++-- 14 files changed, 263 insertions(+), 44 deletions(-) create mode 100644 tests/cases/standalone/common/create/create_database_opts.result create mode 100644 tests/cases/standalone/common/create/create_database_opts.sql diff --git a/src/auth/tests/mod.rs b/src/auth/tests/mod.rs index 6f7bbb0fe0..310d476ba2 100644 --- a/src/auth/tests/mod.rs +++ b/src/auth/tests/mod.rs @@ -52,7 +52,10 @@ fn test_permission_checker() { let sql_result = checker.check_permission( None, - PermissionReq::SqlStatement(&Statement::ShowDatabases(ShowDatabases::new(ShowKind::All))), + PermissionReq::SqlStatement(&Statement::ShowDatabases(ShowDatabases::new( + ShowKind::All, + false, + ))), ); assert_matches!(sql_result, Ok(PermissionResp::Reject)); diff --git a/src/catalog/src/information_schema/schemata.rs b/src/catalog/src/information_schema/schemata.rs index 0d5196ca1a..d4151e8279 100644 --- a/src/catalog/src/information_schema/schemata.rs +++ b/src/catalog/src/information_schema/schemata.rs @@ -17,6 +17,7 @@ use std::sync::{Arc, Weak}; use arrow_schema::SchemaRef as ArrowSchemaRef; use common_catalog::consts::INFORMATION_SCHEMA_SCHEMATA_TABLE_ID; use common_error::ext::BoxedError; +use common_meta::key::schema_name::SchemaNameKey; use common_recordbatch::adapter::RecordBatchStreamAdapter; use common_recordbatch::{RecordBatch, SendableRecordBatchStream}; use datafusion::execution::TaskContext; @@ -32,15 +33,18 @@ use store_api::storage::{ScanRequest, TableId}; use super::SCHEMATA; use crate::error::{ - CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu, + CreateRecordBatchSnafu, InternalSnafu, Result, SchemaNotFoundSnafu, TableMetadataManagerSnafu, + UpgradeWeakCatalogManagerRefSnafu, }; -use crate::information_schema::{InformationTable, Predicates}; +use crate::information_schema::{utils, InformationTable, Predicates}; use crate::CatalogManager; pub const CATALOG_NAME: &str = "catalog_name"; pub const SCHEMA_NAME: &str = "schema_name"; const DEFAULT_CHARACTER_SET_NAME: &str = "default_character_set_name"; const DEFAULT_COLLATION_NAME: &str = "default_collation_name"; +/// The database options +pub const SCHEMA_OPTS: &str = "options"; const INIT_CAPACITY: usize = 42; /// The `information_schema.schemata` table implementation. @@ -74,6 +78,7 @@ impl InformationSchemaSchemata { false, ), ColumnSchema::new("sql_path", ConcreteDataType::string_datatype(), true), + ColumnSchema::new(SCHEMA_OPTS, ConcreteDataType::string_datatype(), true), ])) } @@ -133,6 +138,7 @@ struct InformationSchemaSchemataBuilder { charset_names: StringVectorBuilder, collation_names: StringVectorBuilder, sql_paths: StringVectorBuilder, + schema_options: StringVectorBuilder, } impl InformationSchemaSchemataBuilder { @@ -150,6 +156,7 @@ impl InformationSchemaSchemataBuilder { charset_names: StringVectorBuilder::with_capacity(INIT_CAPACITY), collation_names: StringVectorBuilder::with_capacity(INIT_CAPACITY), sql_paths: StringVectorBuilder::with_capacity(INIT_CAPACITY), + schema_options: StringVectorBuilder::with_capacity(INIT_CAPACITY), } } @@ -160,21 +167,50 @@ impl InformationSchemaSchemataBuilder { .catalog_manager .upgrade() .context(UpgradeWeakCatalogManagerRefSnafu)?; + let table_metadata_manager = utils::table_meta_manager(&self.catalog_manager)?; let predicates = Predicates::from_scan_request(&request); for schema_name in catalog_manager.schema_names(&catalog_name).await? { - self.add_schema(&predicates, &catalog_name, &schema_name); + let opts = if let Some(table_metadata_manager) = &table_metadata_manager { + let schema_opts = table_metadata_manager + .schema_manager() + .get(SchemaNameKey::new(&catalog_name, &schema_name)) + .await + .context(TableMetadataManagerSnafu)? + .context(SchemaNotFoundSnafu { + catalog: &catalog_name, + schema: &schema_name, + })?; + + Some(format!("{schema_opts}")) + } else { + None + }; + + self.add_schema( + &predicates, + &catalog_name, + &schema_name, + opts.as_deref().unwrap_or(""), + ); } self.finish() } - fn add_schema(&mut self, predicates: &Predicates, catalog_name: &str, schema_name: &str) { + fn add_schema( + &mut self, + predicates: &Predicates, + catalog_name: &str, + schema_name: &str, + schema_options: &str, + ) { let row = [ (CATALOG_NAME, &Value::from(catalog_name)), (SCHEMA_NAME, &Value::from(schema_name)), (DEFAULT_CHARACTER_SET_NAME, &Value::from("utf8")), (DEFAULT_COLLATION_NAME, &Value::from("utf8_bin")), + (SCHEMA_OPTS, &Value::from(schema_options)), ]; if !predicates.eval(&row) { @@ -186,6 +222,7 @@ impl InformationSchemaSchemataBuilder { self.charset_names.push(Some("utf8")); self.collation_names.push(Some("utf8_bin")); self.sql_paths.push(None); + self.schema_options.push(Some(schema_options)); } fn finish(&mut self) -> Result { @@ -195,6 +232,7 @@ impl InformationSchemaSchemataBuilder { Arc::new(self.charset_names.finish()), Arc::new(self.collation_names.finish()), Arc::new(self.sql_paths.finish()), + Arc::new(self.schema_options.finish()), ]; RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu) } diff --git a/src/catalog/src/information_schema/utils.rs b/src/catalog/src/information_schema/utils.rs index e476e1fc91..6a35947182 100644 --- a/src/catalog/src/information_schema/utils.rs +++ b/src/catalog/src/information_schema/utils.rs @@ -15,6 +15,7 @@ use std::sync::{Arc, Weak}; use common_config::Mode; +use common_meta::key::TableMetadataManagerRef; use meta_client::client::MetaClient; use snafu::OptionExt; @@ -51,3 +52,17 @@ pub fn meta_client(catalog_manager: &Weak) -> Result, +) -> Result> { + let catalog_manager = catalog_manager + .upgrade() + .context(UpgradeWeakCatalogManagerRefSnafu)?; + + Ok(catalog_manager + .as_any() + .downcast_ref::() + .map(|manager| manager.table_metadata_manager_ref().clone())) +} diff --git a/src/common/meta/src/key/schema_name.rs b/src/common/meta/src/key/schema_name.rs index 91c4c74bc1..5d1e3b5324 100644 --- a/src/common/meta/src/key/schema_name.rs +++ b/src/common/meta/src/key/schema_name.rs @@ -57,6 +57,17 @@ pub struct SchemaNameValue { pub ttl: Option, } +impl Display for SchemaNameValue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(ttl) = self.ttl { + let ttl = humantime::format_duration(ttl); + write!(f, "ttl='{ttl}'")?; + } + + Ok(()) + } +} + impl TryFrom<&HashMap> for SchemaNameValue { type Error = Error; @@ -233,6 +244,17 @@ mod tests { use super::*; use crate::kv_backend::memory::MemoryKvBackend; + #[test] + fn test_display_schema_value() { + let schema_value = SchemaNameValue { ttl: None }; + assert_eq!("", schema_value.to_string()); + + let schema_value = SchemaNameValue { + ttl: Some(Duration::from_secs(9)), + }; + assert_eq!("ttl='9s'", schema_value.to_string()); + } + #[test] fn test_serialization() { let key = SchemaNameKey::new("my-catalog", "my-schema"); diff --git a/src/query/src/sql.rs b/src/query/src/sql.rs index 038e26572d..9b99637976 100644 --- a/src/query/src/sql.rs +++ b/src/query/src/sql.rs @@ -66,6 +66,7 @@ use crate::planner::DfLogicalPlanner; use crate::QueryEngineRef; const SCHEMAS_COLUMN: &str = "Database"; +const OPTIONS_COLUMN: &str = "Options"; const TABLES_COLUMN: &str = "Tables"; const FIELD_COLUMN: &str = "Field"; const TABLE_TYPE_COLUMN: &str = "Table_type"; @@ -155,7 +156,14 @@ pub async fn show_databases( catalog_manager: &CatalogManagerRef, query_ctx: QueryContextRef, ) -> Result { - let projects = vec![(schemata::SCHEMA_NAME, SCHEMAS_COLUMN)]; + let projects = if stmt.full { + vec![ + (schemata::SCHEMA_NAME, SCHEMAS_COLUMN), + (schemata::SCHEMA_OPTS, OPTIONS_COLUMN), + ] + } else { + vec![(schemata::SCHEMA_NAME, SCHEMAS_COLUMN)] + }; let filters = vec![col(schemata::CATALOG_NAME).eq(lit(query_ctx.current_catalog()))]; let like_field = Some(schemata::SCHEMA_NAME); diff --git a/src/sql/src/parsers/show_parser.rs b/src/sql/src/parsers/show_parser.rs index 5cf8b5fe3c..6e680419d8 100644 --- a/src/sql/src/parsers/show_parser.rs +++ b/src/sql/src/parsers/show_parser.rs @@ -32,7 +32,7 @@ impl<'a> ParserContext<'a> { /// todo(hl) support `show settings`/`show create`/`show users` etc. pub(crate) fn parse_show(&mut self) -> Result { if self.consume_token("DATABASES") || self.consume_token("SCHEMAS") { - self.parse_show_databases() + self.parse_show_databases(false) } else if self.matches_keyword(Keyword::TABLES) { self.parser.next_token(); self.parse_show_tables(false) @@ -75,6 +75,8 @@ impl<'a> ParserContext<'a> { } else if self.consume_token("COLUMNS") || self.consume_token("FIELDS") { // SHOW {COLUMNS | FIELDS} self.parse_show_columns(true) + } else if self.consume_token("DATABASES") || self.consume_token("SCHEMAS") { + self.parse_show_databases(true) } else { self.unsupported(self.peek_token_as_string()) } @@ -341,12 +343,13 @@ impl<'a> ParserContext<'a> { } /// Parses `SHOW DATABASES` statement. - pub fn parse_show_databases(&mut self) -> Result { + pub fn parse_show_databases(&mut self, full: bool) -> Result { let tok = self.parser.next_token().token; match &tok { - Token::EOF | Token::SemiColon => { - Ok(Statement::ShowDatabases(ShowDatabases::new(ShowKind::All))) - } + Token::EOF | Token::SemiColon => Ok(Statement::ShowDatabases(ShowDatabases::new( + ShowKind::All, + full, + ))), Token::Word(w) => match w.keyword { Keyword::LIKE => Ok(Statement::ShowDatabases(ShowDatabases::new( ShowKind::Like(self.parse_identifier().with_context(|_| { @@ -356,6 +359,7 @@ impl<'a> ParserContext<'a> { actual: tok.to_string(), } })?), + full, ))), Keyword::WHERE => Ok(Statement::ShowDatabases(ShowDatabases::new( ShowKind::Where(self.parser.parse_expr().with_context(|_| { @@ -365,6 +369,7 @@ impl<'a> ParserContext<'a> { actual: self.peek_token_as_string(), } })?), + full, ))), _ => self.unsupported(self.peek_token_as_string()), }, @@ -395,7 +400,39 @@ mod tests { assert_matches!( &stmts[0], Statement::ShowDatabases(ShowDatabases { - kind: ShowKind::All + kind: ShowKind::All, + full: false, + }) + ); + } + + #[test] + pub fn test_show_full_databases() { + let sql = "SHOW FULL DATABASES"; + let result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); + let stmts = result.unwrap(); + assert_eq!(1, stmts.len()); + + assert_matches!( + &stmts[0], + Statement::ShowDatabases(ShowDatabases { + kind: ShowKind::All, + full: true, + }) + ); + + let sql = "SHOW FULL DATABASES LIKE 'test%'"; + let result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); + let stmts = result.unwrap(); + assert_eq!(1, stmts.len()); + + assert_matches!( + &stmts[0], + Statement::ShowDatabases(ShowDatabases { + kind: ShowKind::Like(_), + full: true, }) ); } @@ -414,7 +451,8 @@ mod tests { kind: ShowKind::Like(sqlparser::ast::Ident { value: _, quote_style: None, - }) + }), + .. }) ); } @@ -434,7 +472,8 @@ mod tests { left: _, right: _, op: sqlparser::ast::BinaryOperator::Or, - }) + }), + .. }) ); } diff --git a/src/sql/src/statements/show.rs b/src/sql/src/statements/show.rs index 90dad65ead..ff345f255f 100644 --- a/src/sql/src/statements/show.rs +++ b/src/sql/src/statements/show.rs @@ -40,6 +40,7 @@ impl Display for ShowKind { #[derive(Debug, Clone, PartialEq, Eq, Visit, VisitMut)] pub struct ShowDatabases { pub kind: ShowKind, + pub full: bool, } /// The SQL `SHOW COLUMNS` statement @@ -85,15 +86,20 @@ impl Display for ShowIndex { impl ShowDatabases { /// Creates a statement for `SHOW DATABASES` - pub fn new(kind: ShowKind) -> Self { - ShowDatabases { kind } + pub fn new(kind: ShowKind, full: bool) -> Self { + ShowDatabases { kind, full } } } impl Display for ShowDatabases { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let kind = &self.kind; - write!(f, r#"SHOW DATABASES {kind}"#) + + if self.full { + write!(f, r#"SHOW FULL DATABASES {kind}"#) + } else { + write!(f, r#"SHOW DATABASES {kind}"#) + } } } diff --git a/tests/cases/standalone/common/create/create_database.result b/tests/cases/standalone/common/create/create_database.result index 8434c768d6..9612b3115e 100644 --- a/tests/cases/standalone/common/create/create_database.result +++ b/tests/cases/standalone/common/create/create_database.result @@ -10,10 +10,6 @@ create database 'illegal-database'; Affected Rows: 1 -create database mydb with (ttl = '1h'); - -Affected Rows: 1 - show databases; +--------------------+ @@ -22,7 +18,6 @@ show databases; | greptime_private | | illegal-database | | information_schema | -| mydb | | public | +--------------------+ @@ -30,7 +25,3 @@ drop database 'illegal-database'; Affected Rows: 0 -drop database mydb; - -Affected Rows: 0 - diff --git a/tests/cases/standalone/common/create/create_database.sql b/tests/cases/standalone/common/create/create_database.sql index bfbfe6b572..6341ce6f85 100644 --- a/tests/cases/standalone/common/create/create_database.sql +++ b/tests/cases/standalone/common/create/create_database.sql @@ -4,10 +4,6 @@ create database illegal-database; create database 'illegal-database'; -create database mydb with (ttl = '1h'); - show databases; drop database 'illegal-database'; - -drop database mydb; diff --git a/tests/cases/standalone/common/create/create_database_opts.result b/tests/cases/standalone/common/create/create_database_opts.result new file mode 100644 index 0000000000..c824290d71 --- /dev/null +++ b/tests/cases/standalone/common/create/create_database_opts.result @@ -0,0 +1,70 @@ +CREATE DATABASE mydb WITH (ttl = '1h'); + +Affected Rows: 1 + +SHOW DATABASES; + ++--------------------+ +| Database | ++--------------------+ +| greptime_private | +| information_schema | +| mydb | +| public | ++--------------------+ + +SHOW FULL DATABASES; + ++--------------------+----------+ +| Database | Options | ++--------------------+----------+ +| greptime_private | | +| information_schema | | +| mydb | ttl='1h' | +| public | | ++--------------------+----------+ + +USE mydb; + +Affected Rows: 0 + +CREATE TABLE test(host STRING, cpu DOUBLE, ts TIMESTAMP TIME INDEX); + +Affected Rows: 0 + +SHOW CREATE TABLE test; + ++-------+-------------------------------------+ +| Table | Create Table | ++-------+-------------------------------------+ +| test | CREATE TABLE IF NOT EXISTS "test" ( | +| | "host" STRING NULL, | +| | "cpu" DOUBLE NULL, | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | TIME INDEX ("ts") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | ttl = '1h' | +| | ) | ++-------+-------------------------------------+ + +USE public; + +Affected Rows: 0 + +DROP DATABASE mydb; + +Affected Rows: 0 + +SHOW DATABASES; + ++--------------------+ +| Database | ++--------------------+ +| greptime_private | +| information_schema | +| public | ++--------------------+ + diff --git a/tests/cases/standalone/common/create/create_database_opts.sql b/tests/cases/standalone/common/create/create_database_opts.sql new file mode 100644 index 0000000000..fb2e58690e --- /dev/null +++ b/tests/cases/standalone/common/create/create_database_opts.sql @@ -0,0 +1,17 @@ +CREATE DATABASE mydb WITH (ttl = '1h'); + +SHOW DATABASES; + +SHOW FULL DATABASES; + +USE mydb; + +CREATE TABLE test(host STRING, cpu DOUBLE, ts TIMESTAMP TIME INDEX); + +SHOW CREATE TABLE test; + +USE public; + +DROP DATABASE mydb; + +SHOW DATABASES; diff --git a/tests/cases/standalone/common/show/show_databases_tables.result b/tests/cases/standalone/common/show/show_databases_tables.result index e73d5b749d..7b6dfb7a16 100644 --- a/tests/cases/standalone/common/show/show_databases_tables.result +++ b/tests/cases/standalone/common/show/show_databases_tables.result @@ -1,4 +1,4 @@ -show databases; +SHOW DATABASES; +--------------------+ | Database | @@ -8,11 +8,21 @@ show databases; | public | +--------------------+ -use information_schema; +SHOW FULL DATABASES; + ++--------------------+---------+ +| Database | Options | ++--------------------+---------+ +| greptime_private | | +| information_schema | | +| public | | ++--------------------+---------+ + +USE information_schema; Affected Rows: 0 -show tables; +SHOW TABLES; +---------------------------------------+ | Tables | @@ -48,7 +58,7 @@ show tables; | triggers | +---------------------------------------+ -use public; +USE public; Affected Rows: 0 diff --git a/tests/cases/standalone/common/show/show_databases_tables.sql b/tests/cases/standalone/common/show/show_databases_tables.sql index 150f341b84..f5e46a2849 100644 --- a/tests/cases/standalone/common/show/show_databases_tables.sql +++ b/tests/cases/standalone/common/show/show_databases_tables.sql @@ -1,7 +1,9 @@ -show databases; +SHOW DATABASES; -use information_schema; +SHOW FULL DATABASES; -show tables; +USE information_schema; -use public; +SHOW TABLES; + +USE public; diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result index cbe6e2d39f..77077b9a3a 100644 --- a/tests/cases/standalone/common/system/information_schema.result +++ b/tests/cases/standalone/common/system/information_schema.result @@ -322,6 +322,7 @@ select * from information_schema.columns order by table_schema, table_name, colu | greptime | information_schema | schemata | catalog_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | | greptime | information_schema | schemata | default_character_set_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | | greptime | information_schema | schemata | default_collation_name | 4 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | schemata | options | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | | greptime | information_schema | schemata | schema_name | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | | greptime | information_schema | schemata | sql_path | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | | greptime | information_schema | session_status | variable_name | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | @@ -528,16 +529,17 @@ desc table schemata; | default_character_set_name | String | | NO | | FIELD | | default_collation_name | String | | NO | | FIELD | | sql_path | String | | YES | | FIELD | +| options | String | | YES | | FIELD | +----------------------------+--------+-----+------+---------+---------------+ select * from schemata where catalog_name = 'greptime' and schema_name != 'public' order by catalog_name, schema_name; -+--------------+--------------------+----------------------------+------------------------+----------+ -| catalog_name | schema_name | default_character_set_name | default_collation_name | sql_path | -+--------------+--------------------+----------------------------+------------------------+----------+ -| greptime | greptime_private | utf8 | utf8_bin | | -| greptime | information_schema | utf8 | utf8_bin | | -+--------------+--------------------+----------------------------+------------------------+----------+ ++--------------+--------------------+----------------------------+------------------------+----------+---------+ +| catalog_name | schema_name | default_character_set_name | default_collation_name | sql_path | options | ++--------------+--------------------+----------------------------+------------------------+----------+---------+ +| greptime | greptime_private | utf8 | utf8_bin | | | +| greptime | information_schema | utf8 | utf8_bin | | | ++--------------+--------------------+----------------------------+------------------------+----------+---------+ -- test engines select * from engines;