TableEngine and SqlHandler impl (#45)

* Impl TableEngine, bridge to storage

* Impl sql handler to process insert sql

* fix: minor changes and typo

* test: add datanode test

* test: add table-engine test

* fix: code style

* refactor: split out insert mod from sql and minor changes by CR

* refactor: replace with_context with context

src/table/src/engine.rs

@@ -1,3 +1,45 @@
+use common_error::ext::ErrorExt;
+
+use crate::requests::{AlterTableRequest, CreateTableRequest, DropTableRequest};
+use crate::TableRef;
+
 /// Table engine abstraction.
 #[async_trait::async_trait]
-pub trait Engine {}
+pub trait TableEngine: Send + Sync + Clone {
+    type Error: ErrorExt + Send + Sync + 'static;
+
+    /// Create a table by given request.
+    ///
+    /// Return the created table.
+    async fn create_table(
+        &self,
+        ctx: &EngineContext,
+        request: CreateTableRequest,
+    ) -> Result<TableRef, Self::Error>;
+
+    /// Alter table schema, options etc. by given request,
+    ///
+    /// Returns the table after altered.
+    async fn alter_table(
+        &self,
+        ctx: &EngineContext,
+        request: AlterTableRequest,
+    ) -> Result<TableRef, Self::Error>;
+
+    /// Returns the table by it's name.
+    fn get_table(&self, ctx: &EngineContext, name: &str) -> Result<Option<TableRef>, Self::Error>;
+
+    /// Returns true when the given table is exists.
+    fn table_exists(&self, ctx: &EngineContext, name: &str) -> bool;
+
+    /// Drops the given table.
+    async fn drop_table(
+        &self,
+        ctx: &EngineContext,
+        request: DropTableRequest,
+    ) -> Result<(), Self::Error>;
+}
+
+/// Storage engine context.
+#[derive(Debug, Clone, Default)]
+pub struct EngineContext {}

src/table/src/error.rs

@@ -23,6 +23,9 @@ pub enum InnerError {
         backtrace: Backtrace,
     },
 
+    #[snafu(display("Missing column when insert, column : {}", name))]
+    MissingColumn { name: String, backtrace: Backtrace },
+
     #[snafu(display("Not expected to run ExecutionPlan more than once"))]
     ExecuteRepeatedly { backtrace: Backtrace },
 }

src/table/src/lib.rs

@@ -1,5 +1,7 @@
-mod engine;
+pub mod engine;
 pub mod error;
+pub mod metadata;
+pub mod requests;
 pub mod table;
 
 pub use crate::table::{Table, TableRef};

src/table/src/metadata.rs

@@ -0,0 +1,168 @@
+use std::collections::HashMap;
+
+use chrono::{DateTime, NaiveDateTime, TimeZone, Utc};
+use datatypes::schema::SchemaRef;
+
+pub type TableId = u64;
+pub type TableVersion = u64;
+
+/// Indicates whether and how a filter expression can be handled by a
+/// Table for table scans.
+#[derive(Debug, Clone, PartialEq)]
+pub enum FilterPushDownType {
+    /// The expression cannot be used by the provider.
+    Unsupported,
+    /// The expression can be used to help minimise the data retrieved,
+    /// but the provider cannot guarantee that all returned tuples
+    /// satisfy the filter. The Filter plan node containing this expression
+    /// will be preserved.
+    Inexact,
+    /// The provider guarantees that all returned data satisfies this
+    /// filter expression. The Filter plan node containing this expression
+    /// will be removed.
+    Exact,
+}
+
+/// Indicates the type of this table for metadata/catalog purposes.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum TableType {
+    /// An ordinary physical table.
+    Base,
+    /// A non-materialised table that itself uses a query internally to provide data.
+    View,
+    /// A transient table.
+    Temporary,
+}
+
+#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Eq, PartialEq, Default)]
+pub struct TableIdent {
+    pub table_id: TableId,
+    pub version: TableVersion,
+}
+
+#[derive(Clone, Debug)]
+pub struct TableMeta {
+    pub schema: SchemaRef,
+    pub engine: String,
+    pub engine_options: HashMap<String, String>,
+    pub options: HashMap<String, String>,
+    pub created_on: DateTime<Utc>,
+}
+
+#[derive(Clone, Debug)]
+pub struct TableInfo {
+    pub ident: TableIdent,
+    pub name: String,
+    pub desc: Option<String>,
+    pub meta: TableMeta,
+    pub table_type: TableType,
+}
+
+impl TableIdent {
+    pub fn new(table_id: TableId) -> Self {
+        Self {
+            table_id,
+            version: 0,
+        }
+    }
+}
+
+pub struct TableMetaBuilder {
+    schema: SchemaRef,
+    engine: String,
+    engine_options: HashMap<String, String>,
+    options: HashMap<String, String>,
+}
+
+impl TableMetaBuilder {
+    pub fn new(schema: SchemaRef) -> Self {
+        Self {
+            schema,
+            engine: String::default(),
+            engine_options: HashMap::default(),
+            options: HashMap::default(),
+        }
+    }
+
+    pub fn engine(mut self, engine: impl Into<String>) -> Self {
+        self.engine = engine.into();
+        self
+    }
+
+    pub fn table_option(mut self, name: &str, val: &str) -> Self {
+        self.options.insert(name.to_string(), val.to_string());
+        self
+    }
+
+    pub fn engine_option(mut self, name: &str, val: &str) -> Self {
+        self.engine_options
+            .insert(name.to_string(), val.to_string());
+        self
+    }
+
+    pub fn build(self) -> TableMeta {
+        TableMeta {
+            schema: self.schema,
+            engine: self.engine,
+            engine_options: self.engine_options,
+            options: self.options,
+            // TODO(dennis): use time utilities helper function
+            created_on: Utc.from_utc_datetime(&NaiveDateTime::from_timestamp(0, 0)),
+        }
+    }
+}
+
+pub struct TableInfoBuilder {
+    ident: TableIdent,
+    name: String,
+    desc: Option<String>,
+    meta: TableMeta,
+    table_type: TableType,
+}
+
+impl TableInfoBuilder {
+    pub fn new(name: impl Into<String>, meta: TableMeta) -> Self {
+        Self {
+            ident: TableIdent::new(0),
+            name: name.into(),
+            desc: None,
+            meta,
+            table_type: TableType::Base,
+        }
+    }
+
+    pub fn table_id(mut self, id: impl Into<TableId>) -> Self {
+        self.ident.table_id = id.into();
+        self
+    }
+
+    pub fn table_version(mut self, version: impl Into<TableVersion>) -> Self {
+        self.ident.version = version.into();
+        self
+    }
+
+    pub fn table_type(mut self, table_type: TableType) -> Self {
+        self.table_type = table_type;
+        self
+    }
+
+    pub fn metadata(mut self, meta: TableMeta) -> Self {
+        self.meta = meta;
+        self
+    }
+
+    pub fn desc(mut self, desc: Option<String>) -> Self {
+        self.desc = desc;
+        self
+    }
+
+    pub fn build(self) -> TableInfo {
+        TableInfo {
+            ident: self.ident,
+            name: self.name,
+            desc: self.desc,
+            meta: self.meta,
+            table_type: self.table_type,
+        }
+    }
+}

src/table/src/requests.rs

@@ -0,0 +1,24 @@
+//! Table and TableEngine requests
+use std::collections::HashMap;
+
+use datatypes::prelude::VectorRef;
+use datatypes::schema::SchemaRef;
+
+/// Insert request
+pub struct InsertRequest {
+    pub table_name: String,
+    pub columns_values: HashMap<String, VectorRef>,
+}
+
+/// Create table request
+pub struct CreateTableRequest {
+    pub name: String,
+    pub desc: Option<String>,
+    pub schema: SchemaRef,
+}
+
+/// Alter table request
+pub struct AlterTableRequest {}
+
+/// Drop table request
+pub struct DropTableRequest {}

src/table/src/table.rs

@@ -2,70 +2,15 @@ pub mod adapter;
 pub mod numbers;
 
 use std::any::Any;
-use std::collections::HashMap;
 use std::sync::Arc;
 
-use chrono::DateTime;
-use chrono::Utc;
 use common_query::logical_plan::Expr;
 use common_recordbatch::SendableRecordBatchStream;
-use datatypes::schema::{Schema, SchemaRef};
+use datatypes::schema::SchemaRef;
 
 use crate::error::Result;
-
-pub type TableId = u64;
-pub type TableVersion = u64;
-
-/// Indicates whether and how a filter expression can be handled by a
-/// Table for table scans.
-#[derive(Debug, Clone, PartialEq)]
-pub enum TableProviderFilterPushDown {
-    /// The expression cannot be used by the provider.
-    Unsupported,
-    /// The expression can be used to help minimise the data retrieved,
-    /// but the provider cannot guarantee that all returned tuples
-    /// satisfy the filter. The Filter plan node containing this expression
-    /// will be preserved.
-    Inexact,
-    /// The provider guarantees that all returned data satisfies this
-    /// filter expression. The Filter plan node containing this expression
-    /// will be removed.
-    Exact,
-}
-
-/// Indicates the type of this table for metadata/catalog purposes.
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum TableType {
-    /// An ordinary physical table.
-    Base,
-    /// A non-materialised table that itself uses a query internally to provide data.
-    View,
-    /// A transient table.
-    Temporary,
-}
-
-#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Eq, PartialEq, Default)]
-pub struct TableIdent {
-    pub table_id: TableId,
-    pub version: TableVersion,
-}
-
-#[derive(Debug)]
-pub struct TableInfo {
-    pub ident: TableIdent,
-    pub name: String,
-    pub desc: Option<String>,
-    pub meta: TableMeta,
-}
-
-#[derive(Clone, Debug)]
-pub struct TableMeta {
-    pub schema: Arc<Schema>,
-    pub engine: String,
-    pub engine_options: HashMap<String, String>,
-    pub options: HashMap<String, String>,
-    pub created_on: DateTime<Utc>,
-}
+use crate::metadata::{FilterPushDownType, TableType};
+use crate::requests::InsertRequest;
 
 /// Table abstraction.
 #[async_trait::async_trait]
@@ -82,6 +27,11 @@ pub trait Table: Send + Sync {
         TableType::Base
     }
 
+    /// Insert values into table.
+    async fn insert(&self, _request: InsertRequest) -> Result<usize> {
+        unimplemented!();
+    }
+
     /// Scan the table and returns a SendableRecordBatchStream.
     async fn scan(
         &self,
@@ -96,8 +46,8 @@ pub trait Table: Send + Sync {
 
     /// Tests whether the table provider can make use of a filter expression
     /// to optimise data retrieval.
-    fn supports_filter_pushdown(&self, _filter: &Expr) -> Result<TableProviderFilterPushDown> {
-        Ok(TableProviderFilterPushDown::Unsupported)
+    fn supports_filter_pushdown(&self, _filter: &Expr) -> Result<FilterPushDownType> {
+        Ok(FilterPushDownType::Unsupported)
     }
 }
 

src/table/src/table/adapter.rs

@@ -31,7 +31,7 @@ use futures::Stream;
 use snafu::prelude::*;
 
 use crate::error::{self, Result};
-use crate::table::{Table, TableProviderFilterPushDown, TableRef, TableType};
+use crate::table::{FilterPushDownType, Table, TableRef, TableType};
 
 /// Greptime SendableRecordBatchStream -> datafusion ExecutionPlan.
 struct ExecutionPlanAdapter {
@@ -139,7 +139,7 @@ impl TableProvider for DfTableProviderAdapter {
         filters: &[DfExpr],
         limit: Option<usize>,
     ) -> DfResult<Arc<dyn ExecutionPlan>> {
-        let filters: Vec<Expr> = filters.iter().map(Clone::clone).map(Expr::new).collect();
+        let filters: Vec<Expr> = filters.iter().map(Clone::clone).map(Into::into).collect();
 
         let stream = self.table.scan(projection, &filters, limit).await?;
         Ok(Arc::new(ExecutionPlanAdapter {
@@ -151,13 +151,11 @@ impl TableProvider for DfTableProviderAdapter {
     fn supports_filter_pushdown(&self, filter: &DfExpr) -> DfResult<DfTableProviderFilterPushDown> {
         let p = self
             .table
-            .supports_filter_pushdown(&Expr::new(filter.clone()))?;
+            .supports_filter_pushdown(&filter.clone().into())?;
         match p {
-            TableProviderFilterPushDown::Unsupported => {
-                Ok(DfTableProviderFilterPushDown::Unsupported)
-            }
-            TableProviderFilterPushDown::Inexact => Ok(DfTableProviderFilterPushDown::Inexact),
-            TableProviderFilterPushDown::Exact => Ok(DfTableProviderFilterPushDown::Exact),
+            FilterPushDownType::Unsupported => Ok(DfTableProviderFilterPushDown::Unsupported),
+            FilterPushDownType::Inexact => Ok(DfTableProviderFilterPushDown::Inexact),
+            FilterPushDownType::Exact => Ok(DfTableProviderFilterPushDown::Exact),
         }
     }
 }
@@ -223,17 +221,15 @@ impl Table for TableAdapter {
         )))
     }
 
-    fn supports_filter_pushdown(&self, filter: &Expr) -> Result<TableProviderFilterPushDown> {
+    fn supports_filter_pushdown(&self, filter: &Expr) -> Result<FilterPushDownType> {
         match self
             .table_provider
             .supports_filter_pushdown(filter.df_expr())
             .context(error::DatafusionSnafu)?
         {
-            DfTableProviderFilterPushDown::Unsupported => {
-                Ok(TableProviderFilterPushDown::Unsupported)
-            }
-            DfTableProviderFilterPushDown::Inexact => Ok(TableProviderFilterPushDown::Inexact),
-            DfTableProviderFilterPushDown::Exact => Ok(TableProviderFilterPushDown::Exact),
+            DfTableProviderFilterPushDown::Unsupported => Ok(FilterPushDownType::Unsupported),
+            DfTableProviderFilterPushDown::Inexact => Ok(FilterPushDownType::Inexact),
+            DfTableProviderFilterPushDown::Exact => Ok(FilterPushDownType::Exact),
         }
     }
 }