TableEngine and SqlHandler impl (#45)

* Impl TableEngine, bridge to storage

* Impl sql handler to process insert sql

* fix: minor changes and typo

* test: add datanode test

* test: add table-engine test

* fix: code style

* refactor: split out insert mod from sql and minor changes by CR

* refactor: replace with_context with context

src/table-engine/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "table-engine"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+async-trait = "0.1"
+chrono = { version = "0.4", features = ["serde"] }
+common-error = {path = "../common/error" }
+common-query = {path = "../common/query" }
+common-recordbatch = {path = "../common/recordbatch" }
+common-telemetry = {path = "../common/telemetry" }
+snafu = { version = "0.7", features = ["backtraces"] }
+storage ={ path = "../storage" }
+store-api ={ path = "../store-api" }
+table = { path = "../table" }
+
+[dev-dependencies]
+datatypes = { path = "../datatypes" }
+tokio = { version = "1.18", features = ["full"] }

src/table-engine/src/engine.rs

@@ -0,0 +1,251 @@
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+use std::sync::RwLock;
+
+use async_trait::async_trait;
+use snafu::ResultExt;
+use store_api::storage::ConcreteDataType;
+use store_api::storage::{
+    self as store, ColumnDescriptorBuilder, ColumnFamilyDescriptorBuilder,
+    EngineContext as StorageContext, Region, RegionDescriptor, RegionId, RegionMeta,
+    RowKeyDescriptorBuilder, StorageEngine,
+};
+use table::engine::{EngineContext, TableEngine};
+use table::requests::{AlterTableRequest, CreateTableRequest, DropTableRequest};
+use table::{
+    metadata::{TableId, TableInfoBuilder, TableMetaBuilder, TableType},
+    table::TableRef,
+};
+
+use crate::error::{CreateTableSnafu, Error, Result};
+use crate::table::MitoTable;
+
+pub const DEFAULT_ENGINE: &str = "mito";
+
+/// [TableEngine] implementation.
+///
+/// About mito <https://en.wikipedia.org/wiki/Alfa_Romeo_MiTo>.
+/// "you can't be a true petrolhead until you've owned an Alfa Romeo" -- by Jeremy Clarkson
+#[derive(Clone)]
+pub struct MitoEngine<Store: StorageEngine> {
+    inner: Arc<MitoEngineInner<Store>>,
+}
+
+impl<Store: StorageEngine> MitoEngine<Store> {
+    pub fn new(storage_engine: Store) -> Self {
+        Self {
+            inner: Arc::new(MitoEngineInner::new(storage_engine)),
+        }
+    }
+}
+
+#[async_trait]
+impl<Store: StorageEngine> TableEngine for MitoEngine<Store> {
+    type Error = Error;
+
+    async fn create_table(
+        &self,
+        ctx: &EngineContext,
+        request: CreateTableRequest,
+    ) -> Result<TableRef> {
+        self.inner.create_table(ctx, request).await
+    }
+
+    async fn alter_table(
+        &self,
+        _ctx: &EngineContext,
+        _request: AlterTableRequest,
+    ) -> Result<TableRef> {
+        unimplemented!();
+    }
+
+    fn get_table(&self, ctx: &EngineContext, name: &str) -> Result<Option<TableRef>> {
+        self.inner.get_table(ctx, name)
+    }
+
+    fn table_exists(&self, _ctx: &EngineContext, _name: &str) -> bool {
+        unimplemented!();
+    }
+
+    async fn drop_table(&self, _ctx: &EngineContext, _request: DropTableRequest) -> Result<()> {
+        unimplemented!();
+    }
+}
+
+/// FIXME(boyan) impl system catalog to keep table metadata.
+struct MitoEngineInner<Store: StorageEngine> {
+    tables: RwLock<HashMap<String, TableRef>>,
+    storage_engine: Store,
+    next_table_id: AtomicU64,
+}
+
+impl<Store: StorageEngine> MitoEngineInner<Store> {
+    fn new(storage_engine: Store) -> Self {
+        Self {
+            tables: RwLock::new(HashMap::default()),
+            storage_engine,
+            next_table_id: AtomicU64::new(0),
+        }
+    }
+
+    fn next_table_id(&self) -> TableId {
+        self.next_table_id.fetch_add(1, Ordering::Relaxed)
+    }
+}
+
+impl<Store: StorageEngine> MitoEngineInner<Store> {
+    async fn create_table(
+        &self,
+        _ctx: &EngineContext,
+        request: CreateTableRequest,
+    ) -> Result<TableRef> {
+        //FIXME(boyan): we only supports creating a demo table right now
+        //The create table sql is like:
+        //    create table demo(host string,
+        //                                  ts int64,
+        //                                  cpu float64,
+        //                                  memory float64,
+        //                                  PRIMARY KEY(ts, host)) with regions=1;
+
+        //TODO(boyan): supports multi regions
+        let region_id: RegionId = 0;
+        let name = store::gen_region_name(region_id);
+
+        let host_column =
+            ColumnDescriptorBuilder::new(0, "host", ConcreteDataType::string_datatype())
+                .is_nullable(false)
+                .build();
+        let cpu_column =
+            ColumnDescriptorBuilder::new(1, "cpu", ConcreteDataType::float64_datatype())
+                .is_nullable(true)
+                .build();
+        let memory_column =
+            ColumnDescriptorBuilder::new(2, "memory", ConcreteDataType::float64_datatype())
+                .is_nullable(true)
+                .build();
+        let ts_column =
+            ColumnDescriptorBuilder::new(0, "ts", ConcreteDataType::int64_datatype()).build();
+
+        let row_key = RowKeyDescriptorBuilder::new(ts_column)
+            .push_column(host_column)
+            .enable_version_column(false)
+            .build();
+
+        let default_cf = ColumnFamilyDescriptorBuilder::default()
+            .push_column(cpu_column)
+            .push_column(memory_column)
+            .build();
+
+        let region = self
+            .storage_engine
+            .create_region(
+                &StorageContext::default(),
+                RegionDescriptor {
+                    id: region_id,
+                    name,
+                    row_key,
+                    default_cf,
+                    extra_cfs: Vec::default(),
+                },
+            )
+            .await
+            .map_err(|e| Box::new(e) as _)
+            .context(CreateTableSnafu)?;
+
+        // Use region meta schema instead of request schema
+        let table_meta = TableMetaBuilder::new(region.in_memory_metadata().schema().clone())
+            .engine(DEFAULT_ENGINE)
+            .build();
+
+        let table_name = request.name;
+        let table_info = TableInfoBuilder::new(table_name.clone(), table_meta)
+            .table_id(self.next_table_id())
+            .table_version(0u64)
+            .table_type(TableType::Base)
+            .desc(request.desc)
+            .build();
+
+        let table = Arc::new(MitoTable::new(table_info, region));
+
+        self.tables
+            .write()
+            .unwrap()
+            .insert(table_name, table.clone());
+
+        Ok(table)
+    }
+
+    fn get_table(&self, _ctx: &EngineContext, name: &str) -> Result<Option<TableRef>> {
+        Ok(self.tables.read().unwrap().get(name).cloned())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use datatypes::schema::{ColumnSchema, Schema};
+    use datatypes::vectors::*;
+    use storage::EngineImpl;
+    use table::requests::InsertRequest;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_creat_table_insert() {
+        let column_schemas = vec![
+            ColumnSchema::new("host", ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new("ts", ConcreteDataType::int64_datatype(), true),
+            ColumnSchema::new("cpu", ConcreteDataType::float64_datatype(), true),
+            ColumnSchema::new("memory", ConcreteDataType::float64_datatype(), true),
+        ];
+
+        let table_engine = MitoEngine::<EngineImpl>::new(EngineImpl::new());
+
+        let table_name = "demo";
+        let schema = Arc::new(Schema::new(column_schemas));
+        let table = table_engine
+            .create_table(
+                &EngineContext::default(),
+                CreateTableRequest {
+                    name: table_name.to_string(),
+                    desc: Some(" a test table".to_string()),
+                    schema: schema.clone(),
+                },
+            )
+            .await
+            .unwrap();
+
+        assert_eq!(TableType::Base, table.table_type());
+        assert_eq!(schema, table.schema());
+
+        let insert_req = InsertRequest {
+            table_name: table_name.to_string(),
+            columns_values: HashMap::default(),
+        };
+        assert_eq!(0, table.insert(insert_req).await.unwrap());
+
+        let mut columns_values: HashMap<String, VectorRef> = HashMap::with_capacity(4);
+        columns_values.insert(
+            "host".to_string(),
+            Arc::new(StringVector::from(vec!["host1", "host2"])),
+        );
+        columns_values.insert(
+            "cpu".to_string(),
+            Arc::new(Float64Vector::from_vec(vec![55.5, 66.6])),
+        );
+        columns_values.insert(
+            "memory".to_string(),
+            Arc::new(Float64Vector::from_vec(vec![1024f64, 4096f64])),
+        );
+        columns_values.insert(
+            "ts".to_string(),
+            Arc::new(Int64Vector::from_vec(vec![1, 2])),
+        );
+
+        let insert_req = InsertRequest {
+            table_name: table_name.to_string(),
+            columns_values,
+        };
+        assert_eq!(2, table.insert(insert_req).await.unwrap());
+    }
+}

src/table-engine/src/error.rs

@@ -0,0 +1,35 @@
+use std::any::Any;
+
+use common_error::prelude::*;
+
+// TODO(boyan): use ErrorExt instead.
+pub type BoxedError = Box<dyn std::error::Error + Send + Sync>;
+
+#[derive(Debug, Snafu)]
+#[snafu(visibility(pub))]
+pub enum Error {
+    #[snafu(display("Fail to create table, source: {}", source))]
+    CreateTable {
+        source: BoxedError,
+        backtrace: Backtrace,
+    },
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl ErrorExt for Error {
+    fn status_code(&self) -> StatusCode {
+        match self {
+            //TODO: should return the source's status code after use ErrorExt in BoxedError.
+            Error::CreateTable { .. } => StatusCode::InvalidArguments,
+        }
+    }
+
+    fn backtrace_opt(&self) -> Option<&Backtrace> {
+        ErrorCompat::backtrace(self)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/table-engine/src/lib.rs

@@ -0,0 +1,3 @@
+pub mod engine;
+pub mod error;
+pub mod table;

src/table-engine/src/table.rs

@@ -0,0 +1,94 @@
+use std::any::Any;
+
+use async_trait::async_trait;
+use common_query::logical_plan::Expr;
+use common_recordbatch::SendableRecordBatchStream;
+use snafu::OptionExt;
+use store_api::storage::SchemaRef;
+use store_api::storage::{PutOperation, Region, WriteContext, WriteRequest};
+use table::error::{Error as TableError, MissingColumnSnafu, Result as TableResult};
+use table::requests::InsertRequest;
+use table::{
+    metadata::{TableInfo, TableType},
+    table::Table,
+};
+
+/// [Table] implementation.
+pub struct MitoTable<R: Region> {
+    table_info: TableInfo,
+    //TODO(dennis): a table contains multi regions
+    region: R,
+}
+
+#[async_trait]
+impl<R: Region> Table for MitoTable<R> {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.table_info.meta.schema.clone()
+    }
+
+    async fn insert(&self, request: InsertRequest) -> TableResult<usize> {
+        if request.columns_values.is_empty() {
+            return Ok(0);
+        }
+
+        let mut write_request = R::WriteRequest::new(self.schema());
+
+        //FIXME(boyan): we can only insert to demo table right now
+        let mut put_op = <<R as Region>::WriteRequest as WriteRequest>::PutOp::new();
+        let mut columns_values = request.columns_values;
+        let key_columns = vec!["ts", "host"];
+        let value_columns = vec!["cpu", "memory"];
+        //Add row key and columns
+        for name in key_columns {
+            put_op
+                .add_key_column(
+                    name,
+                    columns_values
+                        .get(name)
+                        .context(MissingColumnSnafu { name })?
+                        .clone(),
+                )
+                .map_err(TableError::new)?;
+        }
+        // Add vaue columns
+        let mut rows_num = 0;
+        for name in value_columns {
+            if let Some(v) = columns_values.remove(name) {
+                rows_num = v.len();
+                put_op.add_value_column(name, v).map_err(TableError::new)?;
+            }
+        }
+        write_request.put(put_op).map_err(TableError::new)?;
+
+        let _resp = self
+            .region
+            .write(&WriteContext::default(), write_request)
+            .await
+            .map_err(TableError::new)?;
+
+        Ok(rows_num)
+    }
+
+    fn table_type(&self) -> TableType {
+        self.table_info.table_type
+    }
+
+    async fn scan(
+        &self,
+        _projection: &Option<Vec<usize>>,
+        _filters: &[Expr],
+        _limit: Option<usize>,
+    ) -> TableResult<SendableRecordBatchStream> {
+        unimplemented!();
+    }
+}
+
+impl<R: Region> MitoTable<R> {
+    pub fn new(table_info: TableInfo, region: R) -> Self {
+        Self { table_info, region }
+    }
+}