mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-05-24 00:40:40 +00:00
feat: struct value and vector (#7033)
* feat: struct value Signed-off-by: Ning Sun <sunning@greptime.com> * feat: update for proto module * feat: wip struct type * feat: implement more vector operations * feat: make datatype and api * feat: reoslve some compilation issues * feat: resolve all compilation issues * chore: format update * test: resolve tests * test: test and refactor value-to-pb * feat: add more tests and fix for value types * chore: remove dbg * feat: test and fix iterator * fix: resolve struct_type issue * refactor: use vec for struct items * chore: update proto to main branch * refactor: address some of review issues * refactor: update for further review * Add validation on new methods * feat: update struct/list json serialization * refactor: reimplement get in struct_vector * refactor: struct vector functions * refactor: fix lint issue * refactor: address review comments --------- Signed-off-by: Ning Sun <sunning@greptime.com>
This commit is contained in:
Ning Sun
GitHub
@@ -15,7 +15,6 @@
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
use common_base::BitVec;
|
||||
use common_decimal::Decimal128;
|
||||
use common_decimal::decimal128::{DECIMAL128_DEFAULT_SCALE, DECIMAL128_MAX_PRECISION};
|
||||
use common_time::time::Time;
|
||||
@@ -24,9 +23,12 @@ use common_time::{Date, IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth
|
||||
use datatypes::prelude::{ConcreteDataType, ValueRef};
|
||||
use datatypes::scalars::ScalarVector;
|
||||
use datatypes::types::{
|
||||
Int8Type, Int16Type, IntervalType, TimeType, TimestampType, UInt8Type, UInt16Type,
|
||||
Int8Type, Int16Type, IntervalType, StructField, StructType, TimeType, TimestampType, UInt8Type,
|
||||
UInt16Type,
|
||||
};
|
||||
use datatypes::value::{
|
||||
ListValue, ListValueRef, OrderedF32, OrderedF64, StructValue, StructValueRef, Value,
|
||||
};
|
||||
use datatypes::value::{OrderedF32, OrderedF64, Value};
|
||||
use datatypes::vectors::{
|
||||
BinaryVector, BooleanVector, DateVector, Decimal128Vector, Float32Vector, Float64Vector,
|
||||
Int32Vector, Int64Vector, IntervalDayTimeVector, IntervalMonthDayNanoVector,
|
||||
@@ -42,14 +44,14 @@ use greptime_proto::v1::query_request::Query;
|
||||
use greptime_proto::v1::value::ValueData;
|
||||
use greptime_proto::v1::{
|
||||
self, ColumnDataTypeExtension, DdlRequest, DecimalTypeExtension, JsonTypeExtension,
|
||||
QueryRequest, Row, SemanticType, VectorTypeExtension,
|
||||
ListTypeExtension, QueryRequest, Row, SemanticType, StructTypeExtension, VectorTypeExtension,
|
||||
};
|
||||
use paste::paste;
|
||||
use snafu::prelude::*;
|
||||
|
||||
use crate::error::{self, InconsistentTimeUnitSnafu, InvalidTimeUnitSnafu, Result};
|
||||
use crate::v1::column::Values;
|
||||
use crate::v1::{Column, ColumnDataType, Value as GrpcValue};
|
||||
use crate::v1::{ColumnDataType, Value as GrpcValue};
|
||||
|
||||
/// ColumnDataTypeWrapper is a wrapper of ColumnDataType and ColumnDataTypeExtension.
|
||||
/// It could be used to convert with ConcreteDataType.
|
||||
@@ -85,7 +87,7 @@ impl ColumnDataTypeWrapper {
|
||||
|
||||
/// Get a tuple of ColumnDataType and ColumnDataTypeExtension.
|
||||
pub fn to_parts(&self) -> (ColumnDataType, Option<ColumnDataTypeExtension>) {
|
||||
(self.datatype, self.datatype_ext)
|
||||
(self.datatype, self.datatype_ext.clone())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,6 +161,45 @@ impl From<ColumnDataTypeWrapper> for ConcreteDataType {
|
||||
ConcreteDataType::vector_default_datatype()
|
||||
}
|
||||
}
|
||||
ColumnDataType::List => {
|
||||
if let Some(TypeExt::ListType(d)) = datatype_wrapper
|
||||
.datatype_ext
|
||||
.as_ref()
|
||||
.and_then(|datatype_ext| datatype_ext.type_ext.as_ref())
|
||||
{
|
||||
let item_type = ColumnDataTypeWrapper {
|
||||
datatype: d.datatype(),
|
||||
datatype_ext: d.datatype_extension.clone().map(|d| *d),
|
||||
};
|
||||
ConcreteDataType::list_datatype(item_type.into())
|
||||
} else {
|
||||
// invalid state: type extension not found
|
||||
ConcreteDataType::null_datatype()
|
||||
}
|
||||
}
|
||||
ColumnDataType::Struct => {
|
||||
if let Some(TypeExt::StructType(d)) = datatype_wrapper
|
||||
.datatype_ext
|
||||
.as_ref()
|
||||
.and_then(|datatype_ext| datatype_ext.type_ext.as_ref())
|
||||
{
|
||||
let fields = d
|
||||
.fields
|
||||
.iter()
|
||||
.map(|f| {
|
||||
let field_type = ColumnDataTypeWrapper {
|
||||
datatype: f.datatype(),
|
||||
datatype_ext: f.datatype_extension.clone(),
|
||||
};
|
||||
StructField::new(f.name.to_string(), field_type.into(), true)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
ConcreteDataType::struct_datatype(StructType::from(fields))
|
||||
} else {
|
||||
// invalid state: type extension not found
|
||||
ConcreteDataType::null_datatype()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -249,6 +290,39 @@ impl ColumnDataTypeWrapper {
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a list datatype with the given item type.
|
||||
pub fn list_datatype(item_type: ColumnDataTypeWrapper) -> Self {
|
||||
ColumnDataTypeWrapper {
|
||||
datatype: ColumnDataType::List,
|
||||
datatype_ext: Some(ColumnDataTypeExtension {
|
||||
type_ext: Some(TypeExt::ListType(Box::new(ListTypeExtension {
|
||||
datatype: item_type.datatype() as i32,
|
||||
datatype_extension: item_type.datatype_ext.map(Box::new),
|
||||
}))),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a struct datatype with the given field tuples (name, datatype).
|
||||
pub fn struct_datatype(fields: Vec<(String, ColumnDataTypeWrapper)>) -> Self {
|
||||
let struct_fields = fields
|
||||
.into_iter()
|
||||
.map(|(name, datatype)| greptime_proto::v1::StructField {
|
||||
name,
|
||||
datatype: datatype.datatype() as i32,
|
||||
datatype_extension: datatype.datatype_ext,
|
||||
})
|
||||
.collect();
|
||||
ColumnDataTypeWrapper {
|
||||
datatype: ColumnDataType::Struct,
|
||||
datatype_ext: Some(ColumnDataTypeExtension {
|
||||
type_ext: Some(TypeExt::StructType(StructTypeExtension {
|
||||
fields: struct_fields,
|
||||
})),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
|
||||
@@ -290,9 +364,9 @@ impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
|
||||
ConcreteDataType::Decimal128(_) => ColumnDataType::Decimal128,
|
||||
ConcreteDataType::Json(_) => ColumnDataType::Json,
|
||||
ConcreteDataType::Vector(_) => ColumnDataType::Vector,
|
||||
ConcreteDataType::List(_) => ColumnDataType::List,
|
||||
ConcreteDataType::Struct(_) => ColumnDataType::Struct,
|
||||
ConcreteDataType::Null(_)
|
||||
| ConcreteDataType::List(_)
|
||||
| ConcreteDataType::Struct(_)
|
||||
| ConcreteDataType::Dictionary(_)
|
||||
| ConcreteDataType::Duration(_) => {
|
||||
return error::IntoColumnDataTypeSnafu { from: datatype }.fail();
|
||||
@@ -321,6 +395,40 @@ impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
|
||||
})),
|
||||
})
|
||||
}
|
||||
ColumnDataType::List => {
|
||||
if let Some(list_type) = datatype.as_list() {
|
||||
let list_item_type =
|
||||
ColumnDataTypeWrapper::try_from(list_type.item_type().clone())?;
|
||||
Some(ColumnDataTypeExtension {
|
||||
type_ext: Some(TypeExt::ListType(Box::new(ListTypeExtension {
|
||||
datatype: list_item_type.datatype.into(),
|
||||
datatype_extension: list_item_type.datatype_ext.map(Box::new),
|
||||
}))),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
ColumnDataType::Struct => {
|
||||
if let Some(struct_type) = datatype.as_struct() {
|
||||
let mut fields = Vec::with_capacity(struct_type.fields().len());
|
||||
for field in struct_type.fields() {
|
||||
let field_type =
|
||||
ColumnDataTypeWrapper::try_from(field.data_type().clone())?;
|
||||
let proto_field = crate::v1::StructField {
|
||||
name: field.name().to_string(),
|
||||
datatype: field_type.datatype.into(),
|
||||
datatype_extension: field_type.datatype_ext,
|
||||
};
|
||||
fields.push(proto_field);
|
||||
}
|
||||
Some(ColumnDataTypeExtension {
|
||||
type_ext: Some(TypeExt::StructType(StructTypeExtension { fields })),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
Ok(Self {
|
||||
@@ -448,56 +556,17 @@ pub fn values_with_capacity(datatype: ColumnDataType, capacity: usize) -> Values
|
||||
binary_values: Vec::with_capacity(capacity),
|
||||
..Default::default()
|
||||
},
|
||||
ColumnDataType::List => Values {
|
||||
list_values: Vec::with_capacity(capacity),
|
||||
..Default::default()
|
||||
},
|
||||
ColumnDataType::Struct => Values {
|
||||
struct_values: Vec::with_capacity(capacity),
|
||||
..Default::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// The type of vals must be same.
|
||||
pub fn push_vals(column: &mut Column, origin_count: usize, vector: VectorRef) {
|
||||
let values = column.values.get_or_insert_with(Values::default);
|
||||
let mut null_mask = BitVec::from_slice(&column.null_mask);
|
||||
let len = vector.len();
|
||||
null_mask.reserve_exact(origin_count + len);
|
||||
null_mask.extend(BitVec::repeat(false, len));
|
||||
|
||||
(0..len).for_each(|idx| match vector.get(idx) {
|
||||
Value::Null => null_mask.set(idx + origin_count, true),
|
||||
Value::Boolean(val) => values.bool_values.push(val),
|
||||
Value::UInt8(val) => values.u8_values.push(val.into()),
|
||||
Value::UInt16(val) => values.u16_values.push(val.into()),
|
||||
Value::UInt32(val) => values.u32_values.push(val),
|
||||
Value::UInt64(val) => values.u64_values.push(val),
|
||||
Value::Int8(val) => values.i8_values.push(val.into()),
|
||||
Value::Int16(val) => values.i16_values.push(val.into()),
|
||||
Value::Int32(val) => values.i32_values.push(val),
|
||||
Value::Int64(val) => values.i64_values.push(val),
|
||||
Value::Float32(val) => values.f32_values.push(*val),
|
||||
Value::Float64(val) => values.f64_values.push(*val),
|
||||
Value::String(val) => values.string_values.push(val.as_utf8().to_string()),
|
||||
Value::Binary(val) => values.binary_values.push(val.to_vec()),
|
||||
Value::Date(val) => values.date_values.push(val.val()),
|
||||
Value::Timestamp(val) => match val.unit() {
|
||||
TimeUnit::Second => values.timestamp_second_values.push(val.value()),
|
||||
TimeUnit::Millisecond => values.timestamp_millisecond_values.push(val.value()),
|
||||
TimeUnit::Microsecond => values.timestamp_microsecond_values.push(val.value()),
|
||||
TimeUnit::Nanosecond => values.timestamp_nanosecond_values.push(val.value()),
|
||||
},
|
||||
Value::Time(val) => match val.unit() {
|
||||
TimeUnit::Second => values.time_second_values.push(val.value()),
|
||||
TimeUnit::Millisecond => values.time_millisecond_values.push(val.value()),
|
||||
TimeUnit::Microsecond => values.time_microsecond_values.push(val.value()),
|
||||
TimeUnit::Nanosecond => values.time_nanosecond_values.push(val.value()),
|
||||
},
|
||||
Value::IntervalYearMonth(val) => values.interval_year_month_values.push(val.to_i32()),
|
||||
Value::IntervalDayTime(val) => values.interval_day_time_values.push(val.to_i64()),
|
||||
Value::IntervalMonthDayNano(val) => values
|
||||
.interval_month_day_nano_values
|
||||
.push(convert_month_day_nano_to_pb(val)),
|
||||
Value::Decimal128(val) => values.decimal128_values.push(convert_to_pb_decimal128(val)),
|
||||
Value::List(_) | Value::Duration(_) => unreachable!(),
|
||||
});
|
||||
column.null_mask = null_mask.into_vec();
|
||||
}
|
||||
|
||||
/// Returns the type name of the [Request].
|
||||
pub fn request_type(request: &Request) -> &'static str {
|
||||
match request {
|
||||
@@ -555,7 +624,7 @@ pub fn convert_to_pb_decimal128(v: Decimal128) -> v1::Decimal128 {
|
||||
|
||||
pub fn pb_value_to_value_ref<'a>(
|
||||
value: &'a v1::Value,
|
||||
datatype_ext: &'a Option<ColumnDataTypeExtension>,
|
||||
datatype_ext: Option<&'a ColumnDataTypeExtension>,
|
||||
) -> ValueRef<'a> {
|
||||
let Some(value) = &value.value_data else {
|
||||
return ValueRef::Null;
|
||||
@@ -622,6 +691,77 @@ pub fn pb_value_to_value_ref<'a>(
|
||||
))
|
||||
}
|
||||
}
|
||||
ValueData::ListValue(list) => {
|
||||
let list_datatype_ext = datatype_ext
|
||||
.as_ref()
|
||||
.and_then(|ext| {
|
||||
if let Some(TypeExt::ListType(l)) = &ext.type_ext {
|
||||
Some(l)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.expect("list must contain datatype ext");
|
||||
let item_type = ConcreteDataType::from(ColumnDataTypeWrapper::new(
|
||||
list_datatype_ext.datatype(),
|
||||
list_datatype_ext
|
||||
.datatype_extension
|
||||
.as_ref()
|
||||
.map(|ext| *ext.clone()),
|
||||
));
|
||||
let items = list
|
||||
.items
|
||||
.iter()
|
||||
.map(|item| {
|
||||
pb_value_to_value_ref(item, list_datatype_ext.datatype_extension.as_deref())
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let list_value = ListValueRef::RefList {
|
||||
val: items,
|
||||
item_datatype: item_type.clone(),
|
||||
};
|
||||
ValueRef::List(list_value)
|
||||
}
|
||||
|
||||
ValueData::StructValue(struct_value) => {
|
||||
let struct_datatype_ext = datatype_ext
|
||||
.as_ref()
|
||||
.and_then(|ext| {
|
||||
if let Some(TypeExt::StructType(s)) = &ext.type_ext {
|
||||
Some(s)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.expect("struct must contain datatype ext");
|
||||
|
||||
let struct_fields = struct_datatype_ext
|
||||
.fields
|
||||
.iter()
|
||||
.map(|field| {
|
||||
let field_type = ConcreteDataType::from(ColumnDataTypeWrapper::new(
|
||||
field.datatype(),
|
||||
field.datatype_extension.clone(),
|
||||
));
|
||||
let field_name = field.name.to_string();
|
||||
StructField::new(field_name, field_type, true)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let items = struct_value
|
||||
.items
|
||||
.iter()
|
||||
.zip(struct_datatype_ext.fields.iter())
|
||||
.map(|(item, field)| pb_value_to_value_ref(item, field.datatype_extension.as_ref()))
|
||||
.collect::<Vec<ValueRef>>();
|
||||
|
||||
let struct_value_ref = StructValueRef::RefList {
|
||||
val: items,
|
||||
fields: StructType::new(struct_fields),
|
||||
};
|
||||
ValueRef::Struct(struct_value_ref)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -896,8 +1036,8 @@ pub fn is_column_type_value_eq(
|
||||
}
|
||||
|
||||
/// Convert value into proto's value.
|
||||
pub fn to_proto_value(value: Value) -> Option<v1::Value> {
|
||||
let proto_value = match value {
|
||||
pub fn to_proto_value(value: Value) -> v1::Value {
|
||||
match value {
|
||||
Value::Null => v1::Value { value_data: None },
|
||||
Value::Boolean(v) => v1::Value {
|
||||
value_data: Some(ValueData::BoolValue(v)),
|
||||
@@ -983,10 +1123,34 @@ pub fn to_proto_value(value: Value) -> Option<v1::Value> {
|
||||
Value::Decimal128(v) => v1::Value {
|
||||
value_data: Some(ValueData::Decimal128Value(convert_to_pb_decimal128(v))),
|
||||
},
|
||||
Value::List(_) | Value::Duration(_) => return None,
|
||||
};
|
||||
Value::List(list_value) => v1::Value {
|
||||
value_data: Some(ValueData::ListValue(v1::ListValue {
|
||||
items: convert_list_to_pb_values(list_value),
|
||||
})),
|
||||
},
|
||||
Value::Struct(struct_value) => v1::Value {
|
||||
value_data: Some(ValueData::StructValue(v1::StructValue {
|
||||
items: convert_struct_to_pb_values(struct_value),
|
||||
})),
|
||||
},
|
||||
Value::Duration(_) => v1::Value { value_data: None },
|
||||
}
|
||||
}
|
||||
|
||||
Some(proto_value)
|
||||
fn convert_list_to_pb_values(list_value: ListValue) -> Vec<v1::Value> {
|
||||
list_value
|
||||
.take_items()
|
||||
.into_iter()
|
||||
.map(to_proto_value)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn convert_struct_to_pb_values(struct_value: StructValue) -> Vec<v1::Value> {
|
||||
struct_value
|
||||
.take_items()
|
||||
.into_iter()
|
||||
.map(to_proto_value)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Returns the [ColumnDataTypeWrapper] of the value.
|
||||
@@ -1021,6 +1185,8 @@ pub fn proto_value_type(value: &v1::Value) -> Option<ColumnDataType> {
|
||||
ValueData::IntervalDayTimeValue(_) => ColumnDataType::IntervalDayTime,
|
||||
ValueData::IntervalMonthDayNanoValue(_) => ColumnDataType::IntervalMonthDayNano,
|
||||
ValueData::Decimal128Value(_) => ColumnDataType::Decimal128,
|
||||
ValueData::ListValue(_) => ColumnDataType::List,
|
||||
ValueData::StructValue(_) => ColumnDataType::Struct,
|
||||
};
|
||||
Some(value_type)
|
||||
}
|
||||
@@ -1075,7 +1241,23 @@ pub fn value_to_grpc_value(value: Value) -> GrpcValue {
|
||||
convert_month_day_nano_to_pb(v),
|
||||
)),
|
||||
Value::Decimal128(v) => Some(ValueData::Decimal128Value(convert_to_pb_decimal128(v))),
|
||||
Value::List(_) | Value::Duration(_) => unreachable!(),
|
||||
Value::List(list_value) => {
|
||||
let items = list_value
|
||||
.take_items()
|
||||
.into_iter()
|
||||
.map(value_to_grpc_value)
|
||||
.collect();
|
||||
Some(ValueData::ListValue(v1::ListValue { items }))
|
||||
}
|
||||
Value::Struct(struct_value) => {
|
||||
let items = struct_value
|
||||
.take_items()
|
||||
.into_iter()
|
||||
.map(value_to_grpc_value)
|
||||
.collect();
|
||||
Some(ValueData::StructValue(v1::StructValue { items }))
|
||||
}
|
||||
Value::Duration(_) => unreachable!(),
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1173,15 +1355,11 @@ mod tests {
|
||||
TimeMillisecondType, TimeSecondType, TimestampMillisecondType, TimestampSecondType,
|
||||
UInt32Type,
|
||||
};
|
||||
use datatypes::vectors::{
|
||||
BooleanVector, IntervalDayTimeVector, IntervalMonthDayNanoVector, IntervalYearMonthVector,
|
||||
TimeMicrosecondVector, TimeMillisecondVector, TimeNanosecondVector, TimeSecondVector,
|
||||
TimestampMicrosecondVector, TimestampMillisecondVector, TimestampNanosecondVector,
|
||||
TimestampSecondVector, Vector,
|
||||
};
|
||||
use datatypes::vectors::BooleanVector;
|
||||
use paste::paste;
|
||||
|
||||
use super::*;
|
||||
use crate::v1::Column;
|
||||
|
||||
#[test]
|
||||
fn test_values_with_capacity() {
|
||||
@@ -1260,6 +1438,14 @@ mod tests {
|
||||
let values = values_with_capacity(ColumnDataType::Vector, 2);
|
||||
let values = values.binary_values;
|
||||
assert_eq!(2, values.capacity());
|
||||
|
||||
let values = values_with_capacity(ColumnDataType::List, 2);
|
||||
let values = values.list_values;
|
||||
assert_eq!(2, values.capacity());
|
||||
|
||||
let values = values_with_capacity(ColumnDataType::Struct, 2);
|
||||
let values = values.struct_values;
|
||||
assert_eq!(2, values.capacity());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1352,6 +1538,37 @@ mod tests {
|
||||
ConcreteDataType::vector_datatype(3),
|
||||
ColumnDataTypeWrapper::vector_datatype(3).into()
|
||||
);
|
||||
assert_eq!(
|
||||
ConcreteDataType::list_datatype(ConcreteDataType::string_datatype()),
|
||||
ColumnDataTypeWrapper::list_datatype(ColumnDataTypeWrapper::string_datatype()).into()
|
||||
);
|
||||
let struct_type = StructType::new(vec![
|
||||
StructField::new("id".to_string(), ConcreteDataType::int64_datatype(), true),
|
||||
StructField::new(
|
||||
"name".to_string(),
|
||||
ConcreteDataType::string_datatype(),
|
||||
true,
|
||||
),
|
||||
StructField::new("age".to_string(), ConcreteDataType::int32_datatype(), true),
|
||||
StructField::new(
|
||||
"address".to_string(),
|
||||
ConcreteDataType::string_datatype(),
|
||||
true,
|
||||
),
|
||||
]);
|
||||
assert_eq!(
|
||||
ConcreteDataType::struct_datatype(struct_type.clone()),
|
||||
ColumnDataTypeWrapper::struct_datatype(vec![
|
||||
("id".to_string(), ColumnDataTypeWrapper::int64_datatype()),
|
||||
("name".to_string(), ColumnDataTypeWrapper::string_datatype()),
|
||||
("age".to_string(), ColumnDataTypeWrapper::int32_datatype()),
|
||||
(
|
||||
"address".to_string(),
|
||||
ColumnDataTypeWrapper::string_datatype()
|
||||
)
|
||||
])
|
||||
.into()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1455,176 +1672,29 @@ mod tests {
|
||||
"Failed to create column datatype from Null(NullType)"
|
||||
);
|
||||
|
||||
let result: Result<ColumnDataTypeWrapper> =
|
||||
ConcreteDataType::list_datatype(ConcreteDataType::boolean_datatype()).try_into();
|
||||
assert!(result.is_err());
|
||||
assert_eq!(
|
||||
result.unwrap_err().to_string(),
|
||||
"Failed to create column datatype from List(ListType { item_type: Boolean(BooleanType) })"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_column_put_timestamp_values() {
|
||||
let mut column = Column {
|
||||
column_name: "test".to_string(),
|
||||
semantic_type: 0,
|
||||
values: Some(Values {
|
||||
..Default::default()
|
||||
}),
|
||||
null_mask: vec![],
|
||||
datatype: 0,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let vector = Arc::new(TimestampNanosecondVector::from_vec(vec![1, 2, 3]));
|
||||
push_vals(&mut column, 3, vector);
|
||||
assert_eq!(
|
||||
vec![1, 2, 3],
|
||||
column.values.as_ref().unwrap().timestamp_nanosecond_values
|
||||
ColumnDataTypeWrapper::list_datatype(ColumnDataTypeWrapper::int16_datatype()),
|
||||
ConcreteDataType::list_datatype(ConcreteDataType::int16_datatype())
|
||||
.try_into()
|
||||
.expect("Failed to create column datatype from List(ListType { item_type: Int16(Int16Type) })")
|
||||
);
|
||||
|
||||
let vector = Arc::new(TimestampMillisecondVector::from_vec(vec![4, 5, 6]));
|
||||
push_vals(&mut column, 3, vector);
|
||||
assert_eq!(
|
||||
vec![4, 5, 6],
|
||||
column.values.as_ref().unwrap().timestamp_millisecond_values
|
||||
);
|
||||
|
||||
let vector = Arc::new(TimestampMicrosecondVector::from_vec(vec![7, 8, 9]));
|
||||
push_vals(&mut column, 3, vector);
|
||||
assert_eq!(
|
||||
vec![7, 8, 9],
|
||||
column.values.as_ref().unwrap().timestamp_microsecond_values
|
||||
);
|
||||
|
||||
let vector = Arc::new(TimestampSecondVector::from_vec(vec![10, 11, 12]));
|
||||
push_vals(&mut column, 3, vector);
|
||||
assert_eq!(
|
||||
vec![10, 11, 12],
|
||||
column.values.as_ref().unwrap().timestamp_second_values
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_column_put_time_values() {
|
||||
let mut column = Column {
|
||||
column_name: "test".to_string(),
|
||||
semantic_type: 0,
|
||||
values: Some(Values {
|
||||
..Default::default()
|
||||
}),
|
||||
null_mask: vec![],
|
||||
datatype: 0,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let vector = Arc::new(TimeNanosecondVector::from_vec(vec![1, 2, 3]));
|
||||
push_vals(&mut column, 3, vector);
|
||||
assert_eq!(
|
||||
vec![1, 2, 3],
|
||||
column.values.as_ref().unwrap().time_nanosecond_values
|
||||
);
|
||||
|
||||
let vector = Arc::new(TimeMillisecondVector::from_vec(vec![4, 5, 6]));
|
||||
push_vals(&mut column, 3, vector);
|
||||
assert_eq!(
|
||||
vec![4, 5, 6],
|
||||
column.values.as_ref().unwrap().time_millisecond_values
|
||||
);
|
||||
|
||||
let vector = Arc::new(TimeMicrosecondVector::from_vec(vec![7, 8, 9]));
|
||||
push_vals(&mut column, 3, vector);
|
||||
assert_eq!(
|
||||
vec![7, 8, 9],
|
||||
column.values.as_ref().unwrap().time_microsecond_values
|
||||
);
|
||||
|
||||
let vector = Arc::new(TimeSecondVector::from_vec(vec![10, 11, 12]));
|
||||
push_vals(&mut column, 3, vector);
|
||||
assert_eq!(
|
||||
vec![10, 11, 12],
|
||||
column.values.as_ref().unwrap().time_second_values
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_column_put_interval_values() {
|
||||
let mut column = Column {
|
||||
column_name: "test".to_string(),
|
||||
semantic_type: 0,
|
||||
values: Some(Values {
|
||||
..Default::default()
|
||||
}),
|
||||
null_mask: vec![],
|
||||
datatype: 0,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let vector = Arc::new(IntervalYearMonthVector::from_vec(vec![1, 2, 3]));
|
||||
push_vals(&mut column, 3, vector);
|
||||
assert_eq!(
|
||||
vec![1, 2, 3],
|
||||
column.values.as_ref().unwrap().interval_year_month_values
|
||||
);
|
||||
|
||||
let vector = Arc::new(IntervalDayTimeVector::from_vec(vec![
|
||||
IntervalDayTime::new(0, 4).into(),
|
||||
IntervalDayTime::new(0, 5).into(),
|
||||
IntervalDayTime::new(0, 6).into(),
|
||||
]));
|
||||
push_vals(&mut column, 3, vector);
|
||||
assert_eq!(
|
||||
vec![4, 5, 6],
|
||||
column.values.as_ref().unwrap().interval_day_time_values
|
||||
);
|
||||
|
||||
let vector = Arc::new(IntervalMonthDayNanoVector::from_vec(vec![
|
||||
IntervalMonthDayNano::new(0, 0, 7).into(),
|
||||
IntervalMonthDayNano::new(0, 0, 8).into(),
|
||||
IntervalMonthDayNano::new(0, 0, 9).into(),
|
||||
]));
|
||||
let len = vector.len();
|
||||
push_vals(&mut column, 3, vector);
|
||||
(0..len).for_each(|i| {
|
||||
assert_eq!(
|
||||
7 + i as i64,
|
||||
column
|
||||
.values
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.interval_month_day_nano_values
|
||||
.get(i)
|
||||
.unwrap()
|
||||
.nanoseconds
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_column_put_vector() {
|
||||
use crate::v1::SemanticType;
|
||||
// Some(false), None, Some(true), Some(true)
|
||||
let mut column = Column {
|
||||
column_name: "test".to_string(),
|
||||
semantic_type: SemanticType::Field as i32,
|
||||
values: Some(Values {
|
||||
bool_values: vec![false, true, true],
|
||||
..Default::default()
|
||||
}),
|
||||
null_mask: vec![2],
|
||||
datatype: ColumnDataType::Boolean as i32,
|
||||
..Default::default()
|
||||
};
|
||||
let row_count = 4;
|
||||
|
||||
let vector = Arc::new(BooleanVector::from(vec![Some(true), None, Some(false)]));
|
||||
push_vals(&mut column, row_count, vector);
|
||||
// Some(false), None, Some(true), Some(true), Some(true), None, Some(false)
|
||||
let bool_values = column.values.unwrap().bool_values;
|
||||
assert_eq!(vec![false, true, true, true, false], bool_values);
|
||||
let null_mask = column.null_mask;
|
||||
assert_eq!(34, null_mask[0]);
|
||||
ColumnDataTypeWrapper::struct_datatype(vec![
|
||||
("a".to_string(), ColumnDataTypeWrapper::int64_datatype()),
|
||||
(
|
||||
"a.a".to_string(),
|
||||
ColumnDataTypeWrapper::list_datatype(ColumnDataTypeWrapper::string_datatype())
|
||||
)
|
||||
]),
|
||||
ConcreteDataType::struct_datatype(StructType::new(vec![
|
||||
StructField::new("a".to_string(), ConcreteDataType::int64_datatype(), true),
|
||||
StructField::new(
|
||||
"a.a".to_string(),
|
||||
ConcreteDataType::list_datatype(ConcreteDataType::string_datatype()), true
|
||||
)
|
||||
])).try_into().expect("Failed to create column datatype from Struct(StructType { fields: [StructField { name: \"a\", data_type: Int64(Int64Type) }, StructField { name: \"a.a\", data_type: List(ListType { item_type: String(StringType) }) }] })")
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2000,4 +2070,50 @@ mod tests {
|
||||
assert_eq!(pb_decimal.lo, 123);
|
||||
assert_eq!(pb_decimal.hi, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_list_to_pb_value() {
|
||||
let value = Value::List(ListValue::new(
|
||||
vec![Value::Boolean(true)],
|
||||
ConcreteDataType::boolean_datatype(),
|
||||
));
|
||||
|
||||
let pb_value = to_proto_value(value);
|
||||
|
||||
match pb_value.value_data.unwrap() {
|
||||
ValueData::ListValue(pb_list_value) => {
|
||||
assert_eq!(pb_list_value.items.len(), 1);
|
||||
}
|
||||
_ => panic!("Unexpected value type"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_struct_to_pb_value() {
|
||||
let items = vec![Value::Boolean(true), Value::String("tom".into())];
|
||||
|
||||
let value = Value::Struct(
|
||||
StructValue::try_new(
|
||||
items,
|
||||
StructType::new(vec![
|
||||
StructField::new(
|
||||
"a.a".to_string(),
|
||||
ConcreteDataType::boolean_datatype(),
|
||||
true,
|
||||
),
|
||||
StructField::new("a.b".to_string(), ConcreteDataType::string_datatype(), true),
|
||||
]),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
|
||||
let pb_value = to_proto_value(value);
|
||||
|
||||
match pb_value.value_data.unwrap() {
|
||||
ValueData::StructValue(pb_struct_value) => {
|
||||
assert_eq!(pb_struct_value.items.len(), 2);
|
||||
}
|
||||
_ => panic!("Unexpected value type"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,8 +37,10 @@ const SKIPPING_INDEX_GRPC_KEY: &str = "skipping_index";
|
||||
|
||||
/// Tries to construct a `ColumnSchema` from the given `ColumnDef`.
|
||||
pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
|
||||
let data_type =
|
||||
ColumnDataTypeWrapper::try_new(column_def.data_type, column_def.datatype_extension)?;
|
||||
let data_type = ColumnDataTypeWrapper::try_new(
|
||||
column_def.data_type,
|
||||
column_def.datatype_extension.clone(),
|
||||
)?;
|
||||
|
||||
let constraint = if column_def.default_constraint.is_empty() {
|
||||
None
|
||||
|
||||
Reference in New Issue
Block a user