rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-29 19:30:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: support to create external table (#1372)

Browse Source 
* feat: support to create external table

* chore: apply suggestions from CR

* test: add create external table without ts type

* chore: apply suggestions from CR

* fix: fix import typo

* refactor: move consts to table crate

* chore: apply suggestions from CR

* refactor: rename create_table_schema
This commit is contained in:
Weny Xu
2023-04-24 15:43:12 +09:00
committed by GitHub
parent 17daf4cdff
commit f2167663b2
26 changed files with 527 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/datanode/Cargo.toml
View File

@@ -30,6 +30,7 @@ datafusion.workspace = true
datafusion-common.workspace = true
datafusion-expr.workspace = true
datatypes = { path = "../datatypes" }
file-table-engine = { path = "../file-table-engine" }
futures = "0.3"
futures-util.workspace = true
hyper = { version = "0.14", features = ["full"] }

27
src/datanode/src/error.rs
View File

@@ -16,6 +16,7 @@ use std::any::Any;
use common_error::prelude::*;
use common_procedure::ProcedureId;
use serde_json::error::Error as JsonError;
use snafu::Location;
use storage::error::Error as StorageError;
use table::error::Error as TableError;
@@ -305,6 +306,18 @@ pub enum Error {
source: common_time::error::Error,
},
#[snafu(display("Failed to infer schema: {}", source))]
InferSchema {
#[snafu(backtrace)]
source: query::error::Error,
},
#[snafu(display("Failed to prepare immutable table: {}", source))]
PrepareImmutableTable {
#[snafu(backtrace)]
source: query::error::Error,
},
#[snafu(display("Failed to access catalog, source: {}", source))]
Catalog {
#[snafu(backtrace)]
@@ -314,6 +327,7 @@ pub enum Error {
#[snafu(display("Failed to find table {} from catalog, source: {}", table_name, source))]
FindTable {
table_name: String,
#[snafu(backtrace)]
source: catalog::error::Error,
},
@@ -424,6 +438,12 @@ pub enum Error {
#[snafu(backtrace)]
source: BoxedError,
},
#[snafu(display("Failed to encode object into json, source: {}", source))]
EncodeJson {
location: Location,
source: JsonError,
},
}
pub type Result<T> = std::result::Result<T, Error>;
@@ -464,6 +484,8 @@ impl ErrorExt for Error {
ConvertSchema { source, .. } | VectorComputation { source } => source.status_code(),
InferSchema { source, .. } => source.status_code(),
ColumnValuesNumberMismatch { .. }
| InvalidSql { .. }
| NotSupportSql { .. }
@@ -479,7 +501,10 @@ impl ErrorExt for Error {
| DatabaseNotFound { .. }
| MissingNodeId { .. }
| MissingMetasrvOpts { .. }
| ColumnNoneDefaultValue { .. } => StatusCode::InvalidArguments,
| ColumnNoneDefaultValue { .. }
| PrepareImmutableTable { .. } => StatusCode::InvalidArguments,
EncodeJson { .. } => StatusCode::Unexpected,
// TODO(yingwen): Further categorize http error.
StartServer { .. }

35
src/datanode/src/instance.rs
View File

@@ -27,6 +27,7 @@ use common_procedure::local::{LocalManager, ManagerConfig};
use common_procedure::store::state_store::ObjectStateStore;
use common_procedure::ProcedureManagerRef;
use common_telemetry::logging::info;
use file_table_engine::engine::immutable::ImmutableFileTableEngine;
use log_store::raft_engine::log_store::RaftEngineLogStore;
use log_store::LogConfig;
use meta_client::client::{MetaClient, MetaClientBuilder};
@@ -109,7 +110,7 @@ impl Instance {
let object_store = new_object_store(&opts.storage.store).await?;
let log_store = Arc::new(create_log_store(&opts.wal).await?);
let table_engine = Arc::new(DefaultEngine::new(
let mito_engine = Arc::new(DefaultEngine::new(
TableEngineConfig::default(),
EngineImpl::new(
StorageEngineConfig::from(opts),
@@ -117,19 +118,30 @@ impl Instance {
object_store.clone(),
compaction_scheduler,
),
object_store,
object_store.clone(),
));
let mut engine_procedures = HashMap::with_capacity(2);
engine_procedures.insert(
table_engine.name().to_string(),
table_engine.clone() as TableEngineProcedureRef,
mito_engine.name().to_string(),
mito_engine.clone() as TableEngineProcedureRef,
);
// TODO(yingwen): Insert the file table engine into `engine_procedures`
// once #1372 is ready.
let immutable_file_engine = Arc::new(ImmutableFileTableEngine::new(
file_table_engine::config::EngineConfig::default(),
object_store.clone(),
));
engine_procedures.insert(
immutable_file_engine.name().to_string(),
immutable_file_engine.clone() as TableEngineProcedureRef,
);
let engine_manager = Arc::new(
MemoryTableEngineManager::new(table_engine.clone())
.with_engine_procedures(engine_procedures),
MemoryTableEngineManager::with(vec![
mito_engine.clone(),
immutable_file_engine.clone(),
])
.with_engine_procedures(engine_procedures),
);
// create remote catalog manager
@@ -198,12 +210,13 @@ impl Instance {
// Register all procedures.
if let Some(procedure_manager) = &procedure_manager {
// Register procedures of the mito engine.
table_engine.register_procedure_loaders(&**procedure_manager);
mito_engine.register_procedure_loaders(&**procedure_manager);
immutable_file_engine.register_procedure_loaders(&**procedure_manager);
// Register procedures in table-procedure crate.
table_procedure::register_procedure_loaders(
catalog_manager.clone(),
table_engine.clone(),
table_engine.clone(),
mito_engine.clone(),
mito_engine.clone(),
&**procedure_manager,
);
// TODO(yingwen): Register procedures of the file table engine once #1372

21
src/datanode/src/instance/sql.rs
View File

@@ -83,6 +83,27 @@ impl Instance {
.execute(SqlRequest::CreateTable(request), query_ctx)
.await
}
Statement::CreateExternalTable(create_external_table) => {
let table_id = self
.table_id_provider
.as_ref()
.context(TableIdProviderNotFoundSnafu)?
.next_table_id()
.await
.context(BumpTableIdSnafu)?;
let name = create_external_table.name.clone();
let (catalog, schema, table) = table_idents_to_full_name(&name, query_ctx.clone())?;
let table_ref = TableReference::full(&catalog, &schema, &table);
let request = self
.sql_handler
.create_external_to_request(table_id, create_external_table, &table_ref)
.await?;
let table_id = request.id;
info!("Creating external table: {table_ref}, table id = {table_id}",);
self.sql_handler
.execute(SqlRequest::CreateTable(request), query_ctx)
.await
}
Statement::Alter(alter_table) => {
let name = alter_table.table_name().clone();
let (catalog, schema, table) = table_idents_to_full_name(&name, query_ctx.clone())?;

13
src/datanode/src/server/grpc.rs
View File

@@ -13,6 +13,7 @@
// limitations under the License.
use api::v1::{AlterExpr, CreateTableExpr, DropTableExpr, FlushTableExpr};
use common_catalog::consts::IMMUTABLE_FILE_ENGINE;
use common_grpc_expr::{alter_expr_to_request, create_expr_to_request};
use common_query::Output;
use common_telemetry::info;
@@ -58,7 +59,9 @@ impl Instance {
table_id
};
let request = create_expr_to_request(table_id, expr).context(CreateExprToRequestSnafu)?;
let require_time_index = expr.engine != IMMUTABLE_FILE_ENGINE;
let request = create_expr_to_request(table_id, expr, require_time_index)
.context(CreateExprToRequestSnafu)?;
self.sql_handler()
.execute(SqlRequest::CreateTable(request), QueryContext::arc())
@@ -118,7 +121,7 @@ mod tests {
async fn test_create_expr_to_request() {
common_telemetry::init_default_ut_logging();
let expr = testing_create_expr();
let request = create_expr_to_request(1024, expr).unwrap();
let request = create_expr_to_request(1024, expr, true).unwrap();
assert_eq!(request.id, MIN_USER_TABLE_ID);
assert_eq!(request.catalog_name, "greptime".to_string());
assert_eq!(request.schema_name, "public".to_string());
@@ -130,7 +133,7 @@ mod tests {
let mut expr = testing_create_expr();
expr.primary_keys = vec!["host".to_string(), "not-exist-column".to_string()];
let result = create_expr_to_request(1025, expr);
let result = create_expr_to_request(1025, expr, true);
let err_msg = result.unwrap_err().to_string();
assert!(
err_msg.contains("Column `not-exist-column` not found in table `my-metrics`"),
@@ -142,11 +145,11 @@ mod tests {
#[test]
fn test_create_table_schema() {
let mut expr = testing_create_expr();
let schema = create_table_schema(&expr).unwrap();
let schema = create_table_schema(&expr, true).unwrap();
assert_eq!(schema, expected_table_schema());
expr.time_index = "not-exist-column".to_string();
let result = create_table_schema(&expr);
let result = create_table_schema(&expr, true);
let err_msg = result.unwrap_err().to_string();
assert!(
err_msg.contains("Missing timestamp column"),

1
src/datanode/src/sql.rs
View File

@@ -32,6 +32,7 @@ use crate::instance::sql::table_idents_to_full_name;
mod alter;
mod create;
mod create_external;
mod drop_table;
mod flush_table;
pub(crate) mod insert;

61
src/datanode/src/sql/create_external.rs Normal file
View File

@@ -0,0 +1,61 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use file_table_engine::table::immutable::ImmutableFileTableOptions;
use query::sql::prepare_immutable_file_table_files_and_schema;
use snafu::ResultExt;
use sql::statements::create::CreateExternalTable;
use table::engine::TableReference;
use table::metadata::TableId;
use table::requests::{CreateTableRequest, TableOptions, IMMUTABLE_TABLE_META_KEY};
use crate::error::{self, Result};
use crate::sql::SqlHandler;
impl SqlHandler {
pub(crate) async fn create_external_to_request(
&self,
table_id: TableId,
stmt: CreateExternalTable,
table_ref: &TableReference<'_>,
) -> Result<CreateTableRequest> {
let mut options = stmt.options;
let (files, schema) =
prepare_immutable_file_table_files_and_schema(&options, &stmt.columns)
.await
.context(error::PrepareImmutableTableSnafu)?;
let meta = ImmutableFileTableOptions { files };
options.insert(
IMMUTABLE_TABLE_META_KEY.to_string(),
serde_json::to_string(&meta).context(error::EncodeJsonSnafu)?,
);
Ok(CreateTableRequest {
id: table_id,
catalog_name: table_ref.catalog.to_string(),
schema_name: table_ref.schema.to_string(),
table_name: table_ref.table.to_string(),
desc: None,
schema,
region_numbers: vec![0],
primary_key_indices: vec![0],
create_if_not_exists: stmt.if_not_exists,
table_options: TableOptions::try_from(&options)
.context(error::UnrecognizedTableOptionSnafu)?,
engine: stmt.engine,
})
}
}