rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-25 17:30:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: table and database conflicts (#491)

Browse Source 
* fix: table conflicts in different database, #483

* feat: support db query param in prometheus remoting read/write

* feat: support db query param in influxdb line protocol

* fix: make schema_name work in gRPC

* fix: table data path

* fix: table manifest dir

* feat: adds opendal logging layer to object store

* Update src/frontend/src/instance.rs

Co-authored-by: LFC <bayinamine@gmail.com>

* Update src/frontend/src/instance.rs

Co-authored-by: LFC <bayinamine@gmail.com>

* Update src/servers/src/line_writer.rs

Co-authored-by: Lei, Huang <6406592+v0y4g3r@users.noreply.github.com>

* Update src/servers/src/line_writer.rs

Co-authored-by: Lei, Huang <6406592+v0y4g3r@users.noreply.github.com>

* fix: compile error

* ci: use larger runner for running coverage

* fix: address already in use in test

Co-authored-by: LFC <bayinamine@gmail.com>
Co-authored-by: Lei, Huang <6406592+v0y4g3r@users.noreply.github.com>
This commit is contained in:
dennis zhuang
2022-11-14 23:16:52 +08:00
committed by GitHub
parent 76732d6506
commit 448e8f139e
26 changed files with 514 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/datanode/src/instance.rs
View File

@@ -9,6 +9,7 @@ use frontend::frontend::Mode;
use log_store::fs::{config::LogConfig, log::LocalFileLogStore};
use meta_client::client::{MetaClient, MetaClientBuilder};
use meta_client::MetaClientOpts;
use object_store::layers::LoggingLayer;
use object_store::{services::fs::Builder, util, ObjectStore};
use query::query_engine::{QueryEngineFactory, QueryEngineRef};
use snafu::prelude::*;
@@ -156,7 +157,9 @@ pub(crate) async fn new_object_store(store_config: &ObjectStoreConfig) -> Result
.build()
.context(error::InitBackendSnafu { dir: &data_dir })?;
Ok(ObjectStore::new(accessor))
let object_store = ObjectStore::new(accessor).layer(LoggingLayer); // Add logging
Ok(object_store)
}
/// Create metasrv client instance and spawn heartbeat loop.

5
src/datanode/src/instance/grpc.rs
View File

@@ -4,7 +4,7 @@ use api::v1::{
ObjectExpr, ObjectResult, SelectExpr,
};
use async_trait::async_trait;
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_catalog::consts::DEFAULT_CATALOG_NAME;
use common_error::ext::ErrorExt;
use common_error::status_code::StatusCode;
use common_insert::insertion_expr_to_request;
@@ -152,9 +152,8 @@ impl GrpcQueryHandler for Instance {
async fn do_query(&self, query: ObjectExpr) -> servers::error::Result<ObjectResult> {
let object_resp = match query.expr {
Some(object_expr::Expr::Insert(insert_expr)) => {
// TODO(dennis): retrieve schema name from DatabaseRequest
let catalog_name = DEFAULT_CATALOG_NAME;
let schema_name = DEFAULT_SCHEMA_NAME;
let schema_name = &insert_expr.schema_name;
let table_name = &insert_expr.table_name;
let expr = insert_expr
.expr

14
src/datanode/src/sql.rs
View File

@@ -5,7 +5,7 @@ use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_query::Output;
use snafu::{OptionExt, ResultExt};
use sql::statements::show::{ShowDatabases, ShowTables};
use table::engine::{EngineContext, TableEngineRef};
use table::engine::{EngineContext, TableEngineRef, TableReference};
use table::requests::*;
use table::TableRef;
@@ -54,11 +54,15 @@ impl SqlHandler {
}
}
pub(crate) fn get_table(&self, table_name: &str) -> Result<TableRef> {
pub(crate) fn get_table<'a>(&self, table_ref: &'a TableReference) -> Result<TableRef> {
self.table_engine
.get_table(&EngineContext::default(), table_name)
.context(GetTableSnafu { table_name })?
.context(TableNotFoundSnafu { table_name })
.get_table(&EngineContext::default(), table_ref)
.with_context(|_| GetTableSnafu {
table_name: table_ref.to_string(),
})?
.with_context(|| TableNotFoundSnafu {
table_name: table_ref.to_string(),
})
}
pub(crate) fn get_default_catalog(&self) -> Result<CatalogProviderRef> {

25
src/datanode/src/sql/alter.rs
View File

@@ -1,8 +1,9 @@
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_query::Output;
use snafu::prelude::*;
use sql::statements::alter::{AlterTable, AlterTableOperation};
use sql::statements::{column_def_to_schema, table_idents_to_full_name};
use table::engine::EngineContext;
use table::engine::{EngineContext, TableReference};
use table::requests::{AddColumnRequest, AlterKind, AlterTableRequest};
use crate::error::{self, Result};
@@ -11,15 +12,29 @@ use crate::sql::SqlHandler;
impl SqlHandler {
pub(crate) async fn alter(&self, req: AlterTableRequest) -> Result<Output> {
let ctx = EngineContext {};
let table_name = &req.table_name.clone();
let catalog_name = req.catalog_name.as_deref().unwrap_or(DEFAULT_CATALOG_NAME);
let schema_name = req.schema_name.as_deref().unwrap_or(DEFAULT_SCHEMA_NAME);
let table_name = &req.table_name.to_string();
let table_ref = TableReference {
catalog: catalog_name,
schema: schema_name,
table: table_name,
};
let full_table_name = table_ref.to_string();
ensure!(
self.table_engine.table_exists(&ctx, table_name),
error::TableNotFoundSnafu { table_name }
self.table_engine.table_exists(&ctx, &table_ref),
error::TableNotFoundSnafu {
table_name: &full_table_name,
}
);
self.table_engine
.alter_table(&ctx, req)
.await
.context(error::AlterTableSnafu { table_name })?;
.context(error::AlterTableSnafu {
table_name: full_table_name,
})?;
// Tried in MySQL, it really prints "Affected Rows: 0".
Ok(Output::AffectedRows(0))
}

19
src/datanode/src/sql/insert.rs
View File

@@ -7,6 +7,7 @@ use snafu::OptionExt;
use snafu::ResultExt;
use sql::ast::Value as SqlValue;
use sql::statements::{self, insert::Insert};
use table::engine::TableReference;
use table::requests::*;
use crate::error::{
@@ -17,13 +18,19 @@ use crate::sql::{SqlHandler, SqlRequest};
impl SqlHandler {
pub(crate) async fn insert(&self, req: InsertRequest) -> Result<Output> {
let table_name = &req.table_name.to_string();
let table = self.get_table(table_name)?;
// FIXME(dennis): table_ref is used in InsertSnafu and the req is consumed
// in `insert`, so we have to clone catalog_name etc.
let table_ref = TableReference {
catalog: &req.catalog_name.to_string(),
schema: &req.schema_name.to_string(),
table: &req.table_name.to_string(),
};
let affected_rows = table
.insert(req)
.await
.context(InsertSnafu { table_name })?;
let table = self.get_table(&table_ref)?;
let affected_rows = table.insert(req).await.with_context(|_| InsertSnafu {
table_name: table_ref.to_string(),
})?;
Ok(Output::AffectedRows(affected_rows))
}

11
src/datanode/src/tests/grpc_test.rs
View File

@@ -24,12 +24,13 @@ use crate::tests::test_util::{self, TestGuard};
async fn setup_grpc_server(
name: &str,
port: usize,
datanode_port: usize,
frontend_port: usize,
) -> (String, TestGuard, Arc<GrpcServer>, Arc<GrpcServer>) {
common_telemetry::init_default_ut_logging();
let (mut opts, guard) = test_util::create_tmp_dir_and_datanode_opts(name);
let datanode_grpc_addr = format!("127.0.0.1:{}", port);
let datanode_grpc_addr = format!("127.0.0.1:{}", datanode_port);
opts.rpc_addr = datanode_grpc_addr.clone();
let instance = Arc::new(Instance::with_mock_meta_client(&opts).await.unwrap());
instance.start().await.unwrap();
@@ -43,7 +44,7 @@ async fn setup_grpc_server(
.unwrap(),
);
let fe_grpc_addr = format!("127.0.0.1:{}", port + 1);
let fe_grpc_addr = format!("127.0.0.1:{}", frontend_port);
let fe_opts = FrontendOptions {
mode: Standalone,
datanode_rpc_addr: datanode_grpc_addr.clone(),
@@ -95,7 +96,7 @@ async fn setup_grpc_server(
#[tokio::test(flavor = "multi_thread")]
async fn test_auto_create_table() {
let (addr, _guard, fe_grpc_server, dn_grpc_server) =
setup_grpc_server("auto_create_table", 3991).await;
setup_grpc_server("auto_create_table", 3992, 3993).await;
let grpc_client = Client::with_urls(vec![addr]);
let db = Database::new("greptime", grpc_client);
@@ -162,7 +163,7 @@ fn expect_data() -> (Column, Column, Column, Column) {
async fn test_insert_and_select() {
common_telemetry::init_default_ut_logging();
let (addr, _guard, fe_grpc_server, dn_grpc_server) =
setup_grpc_server("insert_and_select", 3990).await;
setup_grpc_server("insert_and_select", 3990, 3991).await;
let grpc_client = Client::with_urls(vec![addr]);

102
src/datanode/src/tests/instance_test.rs
View File

@@ -1,4 +1,4 @@
use arrow::array::{Int64Array, UInt64Array};
use arrow::array::{Int64Array, UInt64Array, Utf8Array};
use common_query::Output;
use common_recordbatch::util;
use datafusion::arrow_print;
@@ -64,6 +64,106 @@ async fn test_create_database_and_insert_query() {
_ => unreachable!(),
}
}
#[tokio::test(flavor = "multi_thread")]
async fn test_issue477_same_table_name_in_different_databases() {
common_telemetry::init_default_ut_logging();
let (opts, _guard) =
test_util::create_tmp_dir_and_datanode_opts("create_database_and_insert_query");
let instance = Instance::with_mock_meta_client(&opts).await.unwrap();
instance.start().await.unwrap();
// Create database a and b
let output = instance.execute_sql("create database a").await.unwrap();
assert!(matches!(output, Output::AffectedRows(1)));
let output = instance.execute_sql("create database b").await.unwrap();
assert!(matches!(output, Output::AffectedRows(1)));
// Create table a.demo and b.demo
let output = instance
.execute_sql(
r#"create table a.demo(
host STRING,
ts bigint,
TIME INDEX(ts)
)"#,
)
.await
.unwrap();
assert!(matches!(output, Output::AffectedRows(1)));
let output = instance
.execute_sql(
r#"create table b.demo(
host STRING,
ts bigint,
TIME INDEX(ts)
)"#,
)
.await
.unwrap();
assert!(matches!(output, Output::AffectedRows(1)));
// Insert different data into a.demo and b.demo
let output = instance
.execute_sql(
r#"insert into a.demo(host, ts) values
('host1', 1655276557000)
"#,
)
.await
.unwrap();
assert!(matches!(output, Output::AffectedRows(1)));
let output = instance
.execute_sql(
r#"insert into b.demo(host, ts) values
('host2',1655276558000)
"#,
)
.await
.unwrap();
assert!(matches!(output, Output::AffectedRows(1)));
// Query data and assert
assert_query_result(
&instance,
"select host,ts from a.demo order by ts",
1655276557000,
"host1",
)
.await;
assert_query_result(
&instance,
"select host,ts from b.demo order by ts",
1655276558000,
"host2",
)
.await;
}
async fn assert_query_result(instance: &Instance, sql: &str, ts: i64, host: &str) {
let query_output = instance.execute_sql(sql).await.unwrap();
match query_output {
Output::Stream(s) => {
let batches = util::collect(s).await.unwrap();
let columns = batches[0].df_recordbatch.columns();
assert_eq!(2, columns.len());
assert_eq!(
&Utf8Array::<i32>::from_slice(&[host]),
columns[0]
.as_any()
.downcast_ref::<Utf8Array<i32>>()
.unwrap()
);
assert_eq!(
&Int64Array::from_slice(&[ts]),
columns[1].as_any().downcast_ref::<Int64Array>().unwrap()
);
}
_ => unreachable!(),
}
}
#[tokio::test(flavor = "multi_thread")]
async fn test_execute_insert() {