rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-21 23:40:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: unify describe table execution (#1285)

Browse Source
This commit is contained in:
LFC
2023-03-31 09:59:19 +08:00
committed by GitHub
parent 3e51640442
commit fae293310c
14 changed files with 144 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.lock generated
View File

@@ -2429,7 +2429,6 @@ dependencies = [
"tower",
"tower-http",
"url",
"uuid",
]
[[package]]
@@ -2911,6 +2910,7 @@ dependencies = [
"itertools",
"meta-client",
"meta-srv",
"mito",
"moka",
"openmetrics-parser",
"partition",
@@ -2931,6 +2931,7 @@ dependencies = [
"toml",
"tonic",
"tower",
"uuid",
]
[[package]]

1
src/datanode/Cargo.toml
View File

@@ -66,7 +66,6 @@ tonic.workspace = true
tower = { version = "0.4", features = ["full"] }
tower-http = { version = "0.3", features = ["full"] }
url = "2.3.1"
uuid.workspace = true
[dev-dependencies]
axum-test-helper = { git = "https://github.com/sunng87/axum-test-helper.git", branch = "patch-1" }

6
src/datanode/src/instance/sql.rs
View File

@@ -124,11 +124,6 @@ impl Instance {
.execute(SqlRequest::ShowTables(show_tables), query_ctx)
.await
}
QueryStatement::Sql(Statement::DescribeTable(describe_table)) => {
self.sql_handler
.execute(SqlRequest::DescribeTable(describe_table), query_ctx)
.await
}
QueryStatement::Sql(Statement::ShowCreateTable(_show_create_table)) => {
unimplemented!("SHOW CREATE TABLE is unimplemented yet");
}
@@ -185,6 +180,7 @@ impl Instance {
| QueryStatement::Sql(Statement::Use(_))
| QueryStatement::Sql(Statement::Tql(_))
| QueryStatement::Sql(Statement::Delete(_))
| QueryStatement::Sql(Statement::DescribeTable(_))
| QueryStatement::Promql(_) => unreachable!(),
}
}

19
src/datanode/src/sql.rs
View File

@@ -17,17 +17,16 @@ use common_error::prelude::BoxedError;
use common_procedure::ProcedureManagerRef;
use common_query::Output;
use common_telemetry::error;
use query::sql::{describe_table, show_databases, show_tables};
use query::sql::{show_databases, show_tables};
use session::context::QueryContextRef;
use snafu::{OptionExt, ResultExt};
use sql::statements::describe::DescribeTable;
use sql::statements::show::{ShowDatabases, ShowTables};
use table::engine::{EngineContext, TableEngineProcedureRef, TableEngineRef, TableReference};
use table::requests::*;
use table::TableRef;
use crate::error::{
self, CloseTableEngineSnafu, ExecuteSqlSnafu, GetTableSnafu, Result, TableNotFoundSnafu,
CloseTableEngineSnafu, ExecuteSqlSnafu, GetTableSnafu, Result, TableNotFoundSnafu,
};
use crate::instance::sql::table_idents_to_full_name;
@@ -48,7 +47,6 @@ pub enum SqlRequest {
FlushTable(FlushTableRequest),
ShowDatabases(ShowDatabases),
ShowTables(ShowTables),
DescribeTable(DescribeTable),
CopyTable(CopyTableRequest),
}
@@ -97,19 +95,6 @@ impl SqlHandler {
show_tables(req, self.catalog_manager.clone(), query_ctx.clone())
.context(ExecuteSqlSnafu)
}
SqlRequest::DescribeTable(req) => {
let (catalog, schema, table) =
table_idents_to_full_name(req.name(), query_ctx.clone())?;
let table = self
.catalog_manager
.table(&catalog, &schema, &table)
.await
.context(error::CatalogSnafu)?
.with_context(|| TableNotFoundSnafu {
table_name: req.name().to_string(),
})?;
describe_table(table).context(ExecuteSqlSnafu)
}
SqlRequest::FlushTable(req) => self.flush_table(req).await,
};
if let Err(e) = &result {

42
src/datanode/src/sql/create.rs
View File

@@ -313,13 +313,15 @@ mod tests {
use common_base::readable_size::ReadableSize;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::Schema;
use query::parser::QueryLanguageParser;
use session::context::QueryContext;
use sql::dialect::GenericDialect;
use sql::parser::ParserContext;
use sql::statements::statement::Statement;
use super::*;
use crate::error::Error;
use crate::tests::test_util::create_mock_sql_handler;
use crate::tests::test_util::{create_mock_sql_handler, MockInstance};
fn sql_to_statement(sql: &str) -> CreateTable {
let mut res = ParserContext::create_with_dialect(sql, &GenericDialect {}).unwrap();
@@ -522,4 +524,42 @@ mod tests {
schema.column_schema_by_name("memory").unwrap().data_type
);
}
#[tokio::test(flavor = "multi_thread")]
async fn create_table_by_procedure() {
let instance = MockInstance::with_procedure_enabled("create_table_by_procedure").await;
let sql = r#"create table test_table(
host string,
ts timestamp,
cpu double default 0,
memory double,
TIME INDEX (ts),
PRIMARY KEY(host)
) engine=mito with(regions=1);"#;
let stmt = QueryLanguageParser::parse_sql(sql).unwrap();
let output = instance
.inner()
.execute_stmt(stmt, QueryContext::arc())
.await
.unwrap();
assert!(matches!(output, Output::AffectedRows(0)));
// create if not exists
let sql = r#"create table if not exists test_table(
host string,
ts timestamp,
cpu double default 0,
memory double,
TIME INDEX (ts),
PRIMARY KEY(host)
) engine=mito with(regions=1);"#;
let stmt = QueryLanguageParser::parse_sql(sql).unwrap();
let output = instance
.inner()
.execute_stmt(stmt, QueryContext::arc())
.await
.unwrap();
assert!(matches!(output, Output::AffectedRows(0)));
}
}

1
src/datanode/src/tests.rs
View File

@@ -13,6 +13,5 @@
// limitations under the License.
// TODO(LFC): These tests should be moved to frontend crate. They are actually standalone instance tests.
mod instance_test;
mod promql_test;
pub(crate) mod test_util;

10
src/datanode/src/tests/test_util.rs
View File

@@ -207,16 +207,6 @@ pub(crate) async fn setup_test_instance(test_name: &str) -> MockInstance {
MockInstance::new(test_name).await
}
pub async fn check_output_stream(output: Output, expected: String) {
let recordbatches = match output {
Output::Stream(stream) => util::collect_batches(stream).await.unwrap(),
Output::RecordBatches(recordbatches) => recordbatches,
_ => unreachable!(),
};
let pretty_print = recordbatches.pretty_print().unwrap();
assert_eq!(pretty_print, expected, "{}", pretty_print);
}
pub async fn check_unordered_output_stream(output: Output, expected: String) {
let sort_table = |table: String| -> String {
let replaced = table.replace("\\n", "\n");

2
src/frontend/Cargo.toml
View File

@@ -30,6 +30,7 @@ futures = "0.3"
futures-util.workspace = true
itertools = "0.10"
meta-client = { path = "../meta-client" }
mito = { path = "../mito", features = ["test"] }
moka = { version = "0.9", features = ["future"] }
openmetrics-parser = "0.4"
partition = { path = "../partition" }
@@ -56,3 +57,4 @@ meta-srv = { path = "../meta-srv", features = ["mock"] }
strfmt = "0.2"
toml = "0.5"
tower = "0.4"
uuid.workspace = true

33
src/frontend/src/instance.rs
View File

@@ -65,15 +65,17 @@ use snafu::prelude::*;
use sql::dialect::GenericDialect;
use sql::parser::ParserContext;
use sql::statements::copy::CopyTable;
use sql::statements::describe::DescribeTable;
use sql::statements::statement::Statement;
use sql::statements::tql::Tql;
use crate::catalog::FrontendCatalogManager;
use crate::datanode::DatanodeClients;
use crate::error::{
self, Error, ExecLogicalPlanSnafu, ExecutePromqlSnafu, ExecuteStatementSnafu, ExternalSnafu,
InvalidInsertRequestSnafu, MissingMetasrvOptsSnafu, NotSupportedSnafu, ParseQuerySnafu,
ParseSqlSnafu, PlanStatementSnafu, Result, SqlExecInterceptedSnafu,
self, CatalogSnafu, DescribeStatementSnafu, Error, ExecLogicalPlanSnafu, ExecutePromqlSnafu,
ExecuteStatementSnafu, ExternalSnafu, InvalidInsertRequestSnafu, MissingMetasrvOptsSnafu,
NotSupportedSnafu, ParseQuerySnafu, ParseSqlSnafu, PlanStatementSnafu, Result,
SqlExecInterceptedSnafu, TableNotFoundSnafu,
};
use crate::expr_factory::{CreateExprFactoryRef, DefaultCreateExprFactory};
use crate::frontend::FrontendOptions;
@@ -464,6 +466,27 @@ impl Instance {
.context(ExecLogicalPlanSnafu)
}
async fn describe_table(
&self,
stmt: DescribeTable,
query_ctx: QueryContextRef,
) -> Result<Output> {
let (catalog, schema, table) = table_idents_to_full_name(stmt.name(), query_ctx)
.map_err(BoxedError::new)
.context(ExternalSnafu)?;
let table = self
.catalog_manager
.table(&catalog, &schema, &table)
.await
.context(CatalogSnafu)?
.with_context(|| TableNotFoundSnafu {
table_name: stmt.name().to_string(),
})?;
query::sql::describe_table(table).context(DescribeStatementSnafu)
}
async fn query_statement(&self, stmt: Statement, query_ctx: QueryContextRef) -> Result<Output> {
check_permission(self.plugins.clone(), &stmt, &query_ctx)?;
@@ -479,12 +502,14 @@ impl Instance {
}
Statement::Tql(tql) => self.execute_tql(tql, query_ctx).await,
Statement::DescribeTable(stmt) => self.describe_table(stmt, query_ctx).await,
Statement::CreateDatabase(_)
| Statement::CreateExternalTable(_)
| Statement::ShowDatabases(_)
| Statement::CreateTable(_)
| Statement::ShowTables(_)
| Statement::DescribeTable(_)
| Statement::Insert(_)
| Statement::Alter(_)
| Statement::DropTable(_)

16
src/frontend/src/instance/distributed.rs
View File

@@ -46,7 +46,7 @@ use partition::partition::{PartitionBound, PartitionDef};
use query::error::QueryExecutionSnafu;
use query::parser::QueryStatement;
use query::query_engine::StatementHandler;
use query::sql::{describe_table, show_databases, show_tables};
use query::sql::{show_databases, show_tables};
use session::context::QueryContextRef;
use snafu::{ensure, OptionExt, ResultExt};
use sql::ast::Value as SqlValue;
@@ -347,20 +347,6 @@ impl DistInstance {
Statement::ShowTables(stmt) => {
show_tables(stmt, self.catalog_manager.clone(), query_ctx)
}
Statement::DescribeTable(stmt) => {
let (catalog, schema, table) = table_idents_to_full_name(stmt.name(), query_ctx)
.map_err(BoxedError::new)
.context(error::ExternalSnafu)?;
let table = self
.catalog_manager
.table(&catalog, &schema, &table)
.await
.context(CatalogSnafu)?
.with_context(|| TableNotFoundSnafu {
table_name: stmt.name().to_string(),
})?;
describe_table(table)
}
Statement::Insert(insert) => {
let (catalog, schema, table) =
table_idents_to_full_name(insert.table_name(), query_ctx.clone())

3
src/frontend/src/tests.rs
View File

@@ -12,6 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
mod instance_test;
mod test_util;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;

138
src/datanode/src/tests/instance_test.rs → src/frontend/src/tests/instance_test.rs
View File

@@ -19,19 +19,17 @@ use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_query::Output;
use common_recordbatch::util;
use common_telemetry::logging;
use datatypes::data_type::ConcreteDataType;
use datatypes::vectors::{Int64Vector, StringVector, UInt64Vector, VectorRef};
use query::parser::{QueryLanguageParser, QueryStatement};
use session::context::{QueryContext, QueryContextRef};
use snafu::ResultExt;
use sql::statements::statement::Statement;
use servers::query_handler::sql::SqlQueryHandler;
use session::context::QueryContext;
use crate::error::{Error, ExecuteLogicalPlanSnafu, PlanStatementSnafu};
use crate::tests::test_util::{self, check_output_stream, setup_test_instance, MockInstance};
use crate::error::Error;
use crate::tests::test_util::check_output_stream;
use crate::tests::{create_standalone_instance, MockStandaloneInstance};
#[tokio::test(flavor = "multi_thread")]
async fn test_create_database_and_insert_query() {
let instance = MockInstance::new("create_database_and_insert_query").await;
let instance = create_standalone_instance("create_database_and_insert_query").await;
let output = execute_sql(&instance, "create database test").await;
assert!(matches!(output, Output::AffectedRows(1)));
@@ -78,7 +76,8 @@ async fn test_create_database_and_insert_query() {
#[tokio::test(flavor = "multi_thread")]
async fn test_issue477_same_table_name_in_different_databases() {
let instance = MockInstance::new("test_issue477_same_table_name_in_different_databases").await;
let instance =
create_standalone_instance("test_issue477_same_table_name_in_different_databases").await;
// Create database a and b
let output = execute_sql(&instance, "create database a").await;
@@ -145,7 +144,7 @@ async fn test_issue477_same_table_name_in_different_databases() {
.await;
}
async fn assert_query_result(instance: &MockInstance, sql: &str, ts: i64, host: &str) {
async fn assert_query_result(instance: &MockStandaloneInstance, sql: &str, ts: i64, host: &str) {
let query_output = execute_sql(instance, sql).await;
match query_output {
Output::Stream(s) => {
@@ -167,7 +166,7 @@ async fn assert_query_result(instance: &MockInstance, sql: &str, ts: i64, host:
#[tokio::test(flavor = "multi_thread")]
async fn test_execute_insert() {
let instance = setup_test_instance("test_execute_insert").await;
let instance = create_standalone_instance("test_execute_insert").await;
// create table
execute_sql(
@@ -189,7 +188,7 @@ async fn test_execute_insert() {
#[tokio::test(flavor = "multi_thread")]
async fn test_execute_insert_by_select() {
let instance = setup_test_instance("test_execute_insert_by_select").await;
let instance = create_standalone_instance("test_execute_insert_by_select").await;
// create table
execute_sql(
@@ -250,11 +249,12 @@ async fn test_execute_insert_by_select() {
#[tokio::test(flavor = "multi_thread")]
async fn test_execute_insert_query_with_i64_timestamp() {
let instance = MockInstance::new("insert_query_i64_timestamp").await;
let instance = create_standalone_instance("insert_query_i64_timestamp").await;
test_util::create_test_table(instance.inner(), ConcreteDataType::int64_datatype())
.await
.unwrap();
execute_sql(
&instance,
"create table demo(host string, cpu double, memory double, ts bigint time index, primary key (host));",
).await;
let output = execute_sql(
&instance,
@@ -301,7 +301,7 @@ async fn test_execute_insert_query_with_i64_timestamp() {
#[tokio::test(flavor = "multi_thread")]
async fn test_execute_query() {
let instance = MockInstance::new("execute_query").await;
let instance = create_standalone_instance("execute_query").await;
let output = execute_sql(&instance, "select sum(number) from numbers limit 20").await;
match output {
@@ -321,7 +321,7 @@ async fn test_execute_query() {
#[tokio::test(flavor = "multi_thread")]
async fn test_execute_show_databases_tables() {
let instance = MockInstance::new("execute_show_databases_tables").await;
let instance = create_standalone_instance("execute_show_databases_tables").await;
let output = execute_sql(&instance, "show databases").await;
match output {
@@ -363,13 +363,10 @@ async fn test_execute_show_databases_tables() {
_ => unreachable!(),
}
// creat a table
test_util::create_test_table(
instance.inner(),
ConcreteDataType::timestamp_millisecond_datatype(),
)
.await
.unwrap();
execute_sql(
&instance,
"create table demo(host string, cpu double, memory double, ts timestamp time index, primary key (host));",
).await;
let output = execute_sql(&instance, "show tables").await;
match output {
@@ -400,7 +397,7 @@ async fn test_execute_show_databases_tables() {
#[tokio::test(flavor = "multi_thread")]
pub async fn test_execute_create() {
let instance = MockInstance::new("execute_create").await;
let instance = create_standalone_instance("execute_create").await;
let output = execute_sql(
&instance,
@@ -419,7 +416,7 @@ pub async fn test_execute_create() {
#[tokio::test]
async fn test_rename_table() {
let instance = MockInstance::new("test_rename_table_local").await;
let instance = create_standalone_instance("test_rename_table_local").await;
let output = execute_sql(&instance, "create database db").await;
assert!(matches!(output, Output::AffectedRows(1)));
@@ -475,7 +472,7 @@ async fn test_rename_table() {
#[tokio::test]
async fn test_create_table_after_rename_table() {
let instance = MockInstance::new("test_rename_table_local").await;
let instance = create_standalone_instance("test_rename_table_local").await;
let output = execute_sql(&instance, "create database db").await;
assert!(matches!(output, Output::AffectedRows(1)));
@@ -525,7 +522,7 @@ async fn test_create_table_after_rename_table() {
#[tokio::test(flavor = "multi_thread")]
async fn test_alter_table() {
let instance = setup_test_instance("test_alter_table").await;
let instance = create_standalone_instance("test_alter_table").await;
// create table
execute_sql(
@@ -612,7 +609,7 @@ async fn test_alter_table() {
}
async fn test_insert_with_default_value_for_type(type_name: &str) {
let instance = MockInstance::new("execute_create").await;
let instance = create_standalone_instance("execute_create").await;
let create_sql = format!(
r#"create table test_table(
@@ -663,7 +660,7 @@ async fn test_insert_with_default_value() {
#[tokio::test(flavor = "multi_thread")]
async fn test_use_database() {
let instance = MockInstance::new("test_use_database").await;
let instance = create_standalone_instance("test_use_database").await;
let output = execute_sql(&instance, "create database db1").await;
assert!(matches!(output, Output::AffectedRows(1)));
@@ -722,7 +719,7 @@ async fn test_use_database() {
#[tokio::test(flavor = "multi_thread")]
async fn test_delete() {
let instance = MockInstance::new("test_delete").await;
let instance = create_standalone_instance("test_delete").await;
let output = execute_sql(
&instance,
@@ -774,7 +771,7 @@ async fn test_execute_copy_to_s3() {
logging::init_default_ut_logging();
if let Ok(bucket) = env::var("GT_S3_BUCKET") {
if !bucket.is_empty() {
let instance = setup_test_instance("test_execute_copy_to_s3").await;
let instance = create_standalone_instance("test_execute_copy_to_s3").await;
// setups
execute_sql(
@@ -813,7 +810,7 @@ async fn test_execute_copy_from_s3() {
logging::init_default_ut_logging();
if let Ok(bucket) = env::var("GT_S3_BUCKET") {
if !bucket.is_empty() {
let instance = setup_test_instance("test_execute_copy_from_s3").await;
let instance = create_standalone_instance("test_execute_copy_from_s3").await;
// setups
execute_sql(
@@ -908,89 +905,26 @@ async fn test_execute_copy_from_s3() {
}
}
#[tokio::test(flavor = "multi_thread")]
async fn test_create_by_procedure() {
common_telemetry::init_default_ut_logging();
let instance = MockInstance::with_procedure_enabled("create_by_procedure").await;
let output = execute_sql(
&instance,
r#"create table test_table(
host string,
ts timestamp,
cpu double default 0,
memory double,
TIME INDEX (ts),
PRIMARY KEY(host)
) engine=mito with(regions=1);"#,
)
.await;
assert!(matches!(output, Output::AffectedRows(0)));
// Create if not exists
let output = execute_sql(
&instance,
r#"create table if not exists test_table(
host string,
ts timestamp,
cpu double default 0,
memory double,
TIME INDEX (ts),
PRIMARY KEY(host)
) engine=mito with(regions=1);"#,
)
.await;
assert!(matches!(output, Output::AffectedRows(0)));
}
async fn execute_sql(instance: &MockInstance, sql: &str) -> Output {
async fn execute_sql(instance: &MockStandaloneInstance, sql: &str) -> Output {
execute_sql_in_db(instance, sql, DEFAULT_SCHEMA_NAME).await
}
async fn try_execute_sql(
instance: &MockInstance,
instance: &MockStandaloneInstance,
sql: &str,
) -> Result<Output, crate::error::Error> {
try_execute_sql_in_db(instance, sql, DEFAULT_SCHEMA_NAME).await
}
async fn try_execute_sql_in_db(
instance: &MockInstance,
instance: &MockStandaloneInstance,
sql: &str,
db: &str,
) -> Result<Output, crate::error::Error> {
let query_ctx = Arc::new(QueryContext::with(DEFAULT_CATALOG_NAME, db));
async fn plan_exec(
instance: &MockInstance,
stmt: QueryStatement,
query_ctx: QueryContextRef,
) -> Result<Output, Error> {
let engine = instance.inner().query_engine();
let plan = engine
.planner()
.plan(stmt, query_ctx.clone())
.await
.context(PlanStatementSnafu)?;
engine
.execute(plan, query_ctx)
.await
.context(ExecuteLogicalPlanSnafu)
}
let stmt = QueryLanguageParser::parse_sql(sql).unwrap();
match stmt {
QueryStatement::Sql(Statement::Query(_)) | QueryStatement::Sql(Statement::Delete(_)) => {
plan_exec(instance, stmt, query_ctx).await
}
QueryStatement::Sql(Statement::Insert(ref insert)) if insert.is_insert_select() => {
plan_exec(instance, stmt, query_ctx).await
}
_ => instance.inner().execute_stmt(stmt, query_ctx).await,
}
instance.instance.do_query(sql, query_ctx).await.remove(0)
}
async fn execute_sql_in_db(instance: &MockInstance, sql: &str, db: &str) -> Output {
async fn execute_sql_in_db(instance: &MockStandaloneInstance, sql: &str, db: &str) -> Output {
try_execute_sql_in_db(instance, sql, db).await.unwrap()
}

26
src/frontend/src/tests/test_util.rs Normal file
View File

@@ -0,0 +1,26 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use common_query::Output;
use common_recordbatch::util;
pub(crate) async fn check_output_stream(output: Output, expected: String) {
let recordbatches = match output {
Output::Stream(stream) => util::collect_batches(stream).await.unwrap(),
Output::RecordBatches(recordbatches) => recordbatches,
_ => unreachable!(),
};
let pretty_print = recordbatches.pretty_print().unwrap();
assert_eq!(pretty_print, expected, "{}", pretty_print);
}

2
tests/cases/standalone/alter/rename_table.result
View File

@@ -31,7 +31,7 @@ Affected Rows: 0
DESC TABLE t;
Error: 4001(TableNotFound), Table not found: t
Error: 4001(TableNotFound), Table `t` not exist
SELECT * FROM t;