fix: opentsdb/influxdb tags are not put to primary key indices (#506)

src/client/Cargo.toml

@@ -12,6 +12,7 @@ common-error = { path = "../common/error" }
 common-grpc = { path = "../common/grpc" }
 common-query = { path = "../common/query" }
 common-recordbatch = { path = "../common/recordbatch" }
+common-insert = { path = "../common/insert" }
 common-time = { path = "../common/time" }
 datafusion = { git = "https://github.com/apache/arrow-datafusion.git", branch = "arrow2", features = [
     "simd",

src/client/src/database.rs

@@ -1,27 +1,24 @@
 use std::sync::Arc;
 
-use api::helper::ColumnDataTypeWrapper;
 use api::v1::codec::SelectResult as GrpcSelectResult;
 use api::v1::{
-    column::Values, object_expr, object_result, select_expr, Column, ColumnDataType,
-    DatabaseRequest, ExprHeader, InsertExpr, MutateResult as GrpcMutateResult, ObjectExpr,
-    ObjectResult as GrpcObjectResult, PhysicalPlan, SelectExpr,
+    object_expr, object_result, select_expr, DatabaseRequest, ExprHeader, InsertExpr,
+    MutateResult as GrpcMutateResult, ObjectExpr, ObjectResult as GrpcObjectResult, PhysicalPlan,
+    SelectExpr,
 };
-use common_base::BitVec;
 use common_error::status_code::StatusCode;
 use common_grpc::AsExcutionPlan;
 use common_grpc::DefaultAsPlanImpl;
+use common_insert::column_to_vector;
 use common_query::Output;
 use common_recordbatch::{RecordBatch, RecordBatches};
-use common_time::date::Date;
-use common_time::datetime::DateTime;
-use common_time::timestamp::Timestamp;
 use datafusion::physical_plan::ExecutionPlan;
 use datatypes::prelude::*;
 use datatypes::schema::{ColumnSchema, Schema};
 use snafu::{ensure, OptionExt, ResultExt};
 
 use crate::error;
+use crate::error::ColumnToVectorSnafu;
 use crate::{
     error::{ConvertSchemaSnafu, DatanodeSnafu, DecodeSelectSnafu, EncodePhysicalSnafu},
     Client, Result,
@@ -199,7 +196,9 @@ impl TryFrom<ObjectResult> for Output {
                 let vectors = select
                     .columns
                     .iter()
-                    .map(|column| column_to_vector(column, select.row_count))
+                    .map(|column| {
+                        column_to_vector(column, select.row_count).context(ColumnToVectorSnafu)
+                    })
                     .collect::<Result<Vec<VectorRef>>>()?;
 
                 let column_schemas = select
@@ -237,99 +236,10 @@ impl TryFrom<ObjectResult> for Output {
     }
 }
 
-fn column_to_vector(column: &Column, rows: u32) -> Result<VectorRef> {
-    let wrapper =
-        ColumnDataTypeWrapper::try_new(column.datatype).context(error::ColumnDataTypeSnafu)?;
-    let column_datatype = wrapper.datatype();
-
-    let rows = rows as usize;
-    let mut vector = VectorBuilder::with_capacity(wrapper.into(), rows);
-
-    if let Some(values) = &column.values {
-        let values = collect_column_values(column_datatype, values);
-        let mut values_iter = values.into_iter();
-
-        let null_mask = BitVec::from_slice(&column.null_mask);
-        let mut nulls_iter = null_mask.iter().by_vals().fuse();
-
-        for i in 0..rows {
-            if let Some(true) = nulls_iter.next() {
-                vector.push_null();
-            } else {
-                let value_ref = values_iter.next().context(error::InvalidColumnProtoSnafu {
-                    err_msg: format!(
-                        "value not found at position {} of column {}",
-                        i, &column.column_name
-                    ),
-                })?;
-                vector
-                    .try_push_ref(value_ref)
-                    .context(error::CreateVectorSnafu)?;
-            }
-        }
-    } else {
-        (0..rows).for_each(|_| vector.push_null());
-    }
-    Ok(vector.finish())
-}
-
-fn collect_column_values(column_datatype: ColumnDataType, values: &Values) -> Vec<ValueRef> {
-    macro_rules! collect_values {
-        ($value: expr, $mapper: expr) => {
-            $value.iter().map($mapper).collect::<Vec<ValueRef>>()
-        };
-    }
-
-    match column_datatype {
-        ColumnDataType::Boolean => collect_values!(values.bool_values, |v| ValueRef::from(*v)),
-        ColumnDataType::Int8 => collect_values!(values.i8_values, |v| ValueRef::from(*v as i8)),
-        ColumnDataType::Int16 => {
-            collect_values!(values.i16_values, |v| ValueRef::from(*v as i16))
-        }
-        ColumnDataType::Int32 => {
-            collect_values!(values.i32_values, |v| ValueRef::from(*v))
-        }
-        ColumnDataType::Int64 => {
-            collect_values!(values.i64_values, |v| ValueRef::from(*v as i64))
-        }
-        ColumnDataType::Uint8 => {
-            collect_values!(values.u8_values, |v| ValueRef::from(*v as u8))
-        }
-        ColumnDataType::Uint16 => {
-            collect_values!(values.u16_values, |v| ValueRef::from(*v as u16))
-        }
-        ColumnDataType::Uint32 => {
-            collect_values!(values.u32_values, |v| ValueRef::from(*v))
-        }
-        ColumnDataType::Uint64 => {
-            collect_values!(values.u64_values, |v| ValueRef::from(*v as u64))
-        }
-        ColumnDataType::Float32 => collect_values!(values.f32_values, |v| ValueRef::from(*v)),
-        ColumnDataType::Float64 => collect_values!(values.f64_values, |v| ValueRef::from(*v)),
-        ColumnDataType::Binary => {
-            collect_values!(values.binary_values, |v| ValueRef::from(v.as_slice()))
-        }
-        ColumnDataType::String => {
-            collect_values!(values.string_values, |v| ValueRef::from(v.as_str()))
-        }
-        ColumnDataType::Date => {
-            collect_values!(values.date_values, |v| ValueRef::Date(Date::new(*v)))
-        }
-        ColumnDataType::Datetime => {
-            collect_values!(values.datetime_values, |v| ValueRef::DateTime(
-                DateTime::new(*v)
-            ))
-        }
-        ColumnDataType::Timestamp => {
-            collect_values!(values.ts_millis_values, |v| ValueRef::Timestamp(
-                Timestamp::from_millis(*v)
-            ))
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
+    use api::helper::ColumnDataTypeWrapper;
+    use api::v1::Column;
     use datanode::server::grpc::select::{null_mask, values};
     use datatypes::vectors::{
         BinaryVector, BooleanVector, DateTimeVector, DateVector, Float32Vector, Float64Vector,

src/client/src/error.rs

@@ -48,24 +48,12 @@ pub enum Error {
     #[snafu(display("Mutate result has failure {}", failure))]
     MutateFailure { failure: u32, backtrace: Backtrace },
 
-    #[snafu(display("Invalid column proto: {}", err_msg))]
-    InvalidColumnProto {
-        err_msg: String,
-        backtrace: Backtrace,
-    },
-
     #[snafu(display("Column datatype error, source: {}", source))]
     ColumnDataType {
         #[snafu(backtrace)]
         source: api::error::Error,
     },
 
-    #[snafu(display("Failed to create vector, source: {}", source))]
-    CreateVector {
-        #[snafu(backtrace)]
-        source: datatypes::error::Error,
-    },
-
     #[snafu(display("Failed to create RecordBatches, source: {}", source))]
     CreateRecordBatches {
         #[snafu(backtrace)]
@@ -97,6 +85,12 @@ pub enum Error {
         #[snafu(backtrace)]
         source: common_grpc::error::Error,
     },
+
+    #[snafu(display("Failed to convert column to vector, source: {}", source))]
+    ColumnToVector {
+        #[snafu(backtrace)]
+        source: common_insert::error::Error,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -112,15 +106,13 @@ impl ErrorExt for Error {
             | Error::Datanode { .. }
             | Error::EncodePhysical { .. }
             | Error::MutateFailure { .. }
-            | Error::InvalidColumnProto { .. }
             | Error::ColumnDataType { .. }
             | Error::MissingField { .. } => StatusCode::Internal,
-            Error::ConvertSchema { source } | Error::CreateVector { source } => {
-                source.status_code()
-            }
+            Error::ConvertSchema { source } => source.status_code(),
             Error::CreateRecordBatches { source } => source.status_code(),
             Error::CreateChannel { source, .. } => source.status_code(),
             Error::IllegalGrpcClientState { .. } => StatusCode::Unexpected,
+            Error::ColumnToVector { source, .. } => source.status_code(),
         }
     }