feat: frontend instance (#238)

* feat: frontend instance

* no need to carry column length in `Column` proto

* add more tests

* rebase develop

* create a new variant with already provisioned RecordBatches in Output

* resolve code review comments

* new frontend instance does not connect datanode grpc

* add more tests

* add more tests

* rebase develop

Co-authored-by: luofucong <luofucong@greptime.com>

src/api/Cargo.toml

@@ -6,7 +6,9 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+datatypes = { path = "../datatypes" }
 prost = "0.11"
+snafu = { version = "0.7", features = ["backtraces"] }
 tonic = "0.8"
 
 [build-dependencies]

src/api/greptime/v1/column.proto

@@ -29,6 +29,10 @@ message Column {
     repeated bool bool_values = 11;
     repeated bytes binary_values = 12;
     repeated string string_values = 13;
+
+    repeated int32 date_values = 14;
+    repeated int64 datetime_values = 15;
+    repeated int64 ts_millis_values = 16;
   }
   // The array of non-null values in this column.
   //
@@ -43,6 +47,9 @@ message Column {
   // Mask maps the positions of null values.
   // If a bit in null_mask is 1, it indicates that the column value at that position is null.
   bytes null_mask = 4;
+
+  // Helpful in creating vector from column.
+  ColumnDataType datatype = 5;
 }
 
 message ColumnDef {

src/api/greptime/v1/database.proto

@@ -38,7 +38,23 @@ message PhysicalPlan {
 
 message InsertExpr {
   string table_name = 1;
-  repeated bytes values = 2;
+
+  message Values {
+    repeated bytes values = 1;
+  }
+
+  oneof expr {
+    Values values = 2;
+
+    // TODO(LFC): Remove field "sql" in InsertExpr.
+    // When Frontend instance received an insertion SQL (`insert into ...`), it's anticipated to parse the SQL and
+    // assemble the values to insert to feed Datanode. In other words, inserting data through Datanode instance's GRPC
+    // interface shouldn't use SQL directly.
+    // Then why the "sql" field exists here? It's because the Frontend needs table schema to create the values to insert,
+    // which is currently not able to find anywhere. (Maybe the table schema is suppose to be fetched from Meta?)
+    // The "sql" field is meant to be removed in the future.
+    string sql = 3;
+  }
 }
 
 // TODO(jiachun)

src/api/src/error.rs

@@ -0,0 +1,18 @@
+use datatypes::prelude::ConcreteDataType;
+use snafu::prelude::*;
+use snafu::Backtrace;
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+#[derive(Debug, Snafu)]
+#[snafu(visibility(pub))]
+pub enum Error {
+    #[snafu(display("Unknown proto column datatype: {}", datatype))]
+    UnknownColumnDataType { datatype: i32, backtrace: Backtrace },
+
+    #[snafu(display("Failed to create column datatype from {:?}", from))]
+    IntoColumnDataType {
+        from: ConcreteDataType,
+        backtrace: Backtrace,
+    },
+}

src/api/src/helper.rs

@@ -0,0 +1,230 @@
+use datatypes::prelude::ConcreteDataType;
+use snafu::prelude::*;
+
+use crate::error::{self, Result};
+use crate::v1::ColumnDataType;
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct ColumnDataTypeWrapper(ColumnDataType);
+
+impl ColumnDataTypeWrapper {
+    pub fn try_new(datatype: i32) -> Result<Self> {
+        let datatype = ColumnDataType::from_i32(datatype)
+            .context(error::UnknownColumnDataTypeSnafu { datatype })?;
+        Ok(Self(datatype))
+    }
+
+    pub fn datatype(&self) -> ColumnDataType {
+        self.0
+    }
+}
+
+impl From<ColumnDataTypeWrapper> for ConcreteDataType {
+    fn from(datatype: ColumnDataTypeWrapper) -> Self {
+        match datatype.0 {
+            ColumnDataType::Boolean => ConcreteDataType::boolean_datatype(),
+            ColumnDataType::Int8 => ConcreteDataType::int8_datatype(),
+            ColumnDataType::Int16 => ConcreteDataType::int16_datatype(),
+            ColumnDataType::Int32 => ConcreteDataType::int32_datatype(),
+            ColumnDataType::Int64 => ConcreteDataType::int64_datatype(),
+            ColumnDataType::Uint8 => ConcreteDataType::uint8_datatype(),
+            ColumnDataType::Uint16 => ConcreteDataType::uint16_datatype(),
+            ColumnDataType::Uint32 => ConcreteDataType::uint32_datatype(),
+            ColumnDataType::Uint64 => ConcreteDataType::uint64_datatype(),
+            ColumnDataType::Float32 => ConcreteDataType::float32_datatype(),
+            ColumnDataType::Float64 => ConcreteDataType::float64_datatype(),
+            ColumnDataType::Binary => ConcreteDataType::binary_datatype(),
+            ColumnDataType::String => ConcreteDataType::string_datatype(),
+            ColumnDataType::Date => ConcreteDataType::date_datatype(),
+            ColumnDataType::Datetime => ConcreteDataType::datetime_datatype(),
+            ColumnDataType::Timestamp => ConcreteDataType::timestamp_millis_datatype(),
+        }
+    }
+}
+
+impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
+    type Error = error::Error;
+
+    fn try_from(datatype: ConcreteDataType) -> Result<Self> {
+        let datatype = ColumnDataTypeWrapper(match datatype {
+            ConcreteDataType::Boolean(_) => ColumnDataType::Boolean,
+            ConcreteDataType::Int8(_) => ColumnDataType::Int8,
+            ConcreteDataType::Int16(_) => ColumnDataType::Int16,
+            ConcreteDataType::Int32(_) => ColumnDataType::Int32,
+            ConcreteDataType::Int64(_) => ColumnDataType::Int64,
+            ConcreteDataType::UInt8(_) => ColumnDataType::Uint8,
+            ConcreteDataType::UInt16(_) => ColumnDataType::Uint16,
+            ConcreteDataType::UInt32(_) => ColumnDataType::Uint32,
+            ConcreteDataType::UInt64(_) => ColumnDataType::Uint64,
+            ConcreteDataType::Float32(_) => ColumnDataType::Float32,
+            ConcreteDataType::Float64(_) => ColumnDataType::Float64,
+            ConcreteDataType::Binary(_) => ColumnDataType::Binary,
+            ConcreteDataType::String(_) => ColumnDataType::String,
+            ConcreteDataType::Date(_) => ColumnDataType::Date,
+            ConcreteDataType::DateTime(_) => ColumnDataType::Datetime,
+            ConcreteDataType::Timestamp(_) => ColumnDataType::Timestamp,
+            ConcreteDataType::Null(_) | ConcreteDataType::List(_) => {
+                return error::IntoColumnDataTypeSnafu { from: datatype }.fail()
+            }
+        });
+        Ok(datatype)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_concrete_datatype_from_column_datatype() {
+        assert_eq!(
+            ConcreteDataType::boolean_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Boolean).into()
+        );
+        assert_eq!(
+            ConcreteDataType::int8_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Int8).into()
+        );
+        assert_eq!(
+            ConcreteDataType::int16_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Int16).into()
+        );
+        assert_eq!(
+            ConcreteDataType::int32_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Int32).into()
+        );
+        assert_eq!(
+            ConcreteDataType::int64_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Int64).into()
+        );
+        assert_eq!(
+            ConcreteDataType::uint8_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Uint8).into()
+        );
+        assert_eq!(
+            ConcreteDataType::uint16_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Uint16).into()
+        );
+        assert_eq!(
+            ConcreteDataType::uint32_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Uint32).into()
+        );
+        assert_eq!(
+            ConcreteDataType::uint64_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Uint64).into()
+        );
+        assert_eq!(
+            ConcreteDataType::float32_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Float32).into()
+        );
+        assert_eq!(
+            ConcreteDataType::float64_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Float64).into()
+        );
+        assert_eq!(
+            ConcreteDataType::binary_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Binary).into()
+        );
+        assert_eq!(
+            ConcreteDataType::string_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::String).into()
+        );
+        assert_eq!(
+            ConcreteDataType::date_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Date).into()
+        );
+        assert_eq!(
+            ConcreteDataType::datetime_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Datetime).into()
+        );
+        assert_eq!(
+            ConcreteDataType::timestamp_millis_datatype(),
+            ColumnDataTypeWrapper(ColumnDataType::Timestamp).into()
+        );
+    }
+
+    #[test]
+    fn test_column_datatype_from_concrete_datatype() {
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Boolean),
+            ConcreteDataType::boolean_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Int8),
+            ConcreteDataType::int8_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Int16),
+            ConcreteDataType::int16_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Int32),
+            ConcreteDataType::int32_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Int64),
+            ConcreteDataType::int64_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Uint8),
+            ConcreteDataType::uint8_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Uint16),
+            ConcreteDataType::uint16_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Uint32),
+            ConcreteDataType::uint32_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Uint64),
+            ConcreteDataType::uint64_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Float32),
+            ConcreteDataType::float32_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Float64),
+            ConcreteDataType::float64_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Binary),
+            ConcreteDataType::binary_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::String),
+            ConcreteDataType::string_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Date),
+            ConcreteDataType::date_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Datetime),
+            ConcreteDataType::datetime_datatype().try_into().unwrap()
+        );
+        assert_eq!(
+            ColumnDataTypeWrapper(ColumnDataType::Timestamp),
+            ConcreteDataType::timestamp_millis_datatype()
+                .try_into()
+                .unwrap()
+        );
+
+        let result: Result<ColumnDataTypeWrapper> = ConcreteDataType::null_datatype().try_into();
+        assert!(result.is_err());
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Failed to create column datatype from Null(NullType)"
+        );
+
+        let result: Result<ColumnDataTypeWrapper> =
+            ConcreteDataType::list_datatype(ConcreteDataType::boolean_datatype()).try_into();
+        assert!(result.is_err());
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            "Failed to create column datatype from List(ListType { inner: Boolean(BooleanType) })"
+        );
+    }
+}

src/api/src/lib.rs

@@ -1,3 +1,5 @@
+pub mod error;
+pub mod helper;
 pub mod serde;
 pub mod v1;
 

src/api/src/serde.rs

@@ -138,6 +138,7 @@ mod tests {
             semantic_type: SEMANTIC_TAG,
             values: Some(values),
             null_mask,
+            ..Default::default()
         };
         InsertBatch {
             columns: vec![column],
@@ -156,6 +157,7 @@ mod tests {
             semantic_type: SEMANTIC_TAG,
             values: Some(values),
             null_mask,
+            ..Default::default()
         };
         SelectResult {
             columns: vec![column],