rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-06-01 04:40:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: impl timestamp type, value and vectors (#226)

Browse Source 
* wip: impl timestamp data type

* add timestamp vectors

* adapt to recent changes to vector module

* fix all unit test

* rebase develop

* fix slice

* change default time unit to millisecond

* add more tests

* fix some CR comments

* fix some CR comments

* fix clippy

* fix some cr comments

* fix some CR comments

* fix some CR comments

* remove time unit in LogicalTypeId::Timestamp
This commit is contained in:
Lei, Huang
2022-09-09 11:43:30 +08:00
committed by GitHub
parent 82dfe78321
commit 9366e77407
46 changed files with 1003 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/datanode/src/server/grpc/create.rs
View File

@@ -111,6 +111,7 @@ fn create_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
ColumnDataType::String => ConcreteDataType::string_datatype(),
ColumnDataType::Date => ConcreteDataType::date_datatype(),
ColumnDataType::Datetime => ConcreteDataType::datetime_datatype(),
ColumnDataType::Timestamp => ConcreteDataType::timestamp_millis_datatype(),
};
Ok(ColumnSchema {
name: column_def.name.clone(),

19
src/datanode/src/server/grpc/insert.rs
View File

@@ -6,6 +6,7 @@ use std::{
use api::v1::{codec::InsertBatch, column::Values, Column, InsertExpr};
use common_base::BitVec;
use common_time::timestamp::Timestamp;
use datatypes::{data_type::ConcreteDataType, value::Value, vectors::VectorBuilder};
use snafu::{ensure, OptionExt, ResultExt};
use table::{requests::InsertRequest, Table};
@@ -170,7 +171,23 @@ fn convert_values(data_type: &ConcreteDataType, values: Values) -> Vec<Value> {
.into_iter()
.map(|val| val.into())
.collect(),
_ => unimplemented!(),
ConcreteDataType::DateTime(_) => values
.i64_values
.into_iter()
.map(|v| Value::DateTime(v.into()))
.collect(),
ConcreteDataType::Date(_) => values
.i32_values
.into_iter()
.map(|v| Value::Date(v.into()))
.collect(),
ConcreteDataType::Timestamp(_) => values
.i64_values
.into_iter()
.map(|v| Value::Timestamp(Timestamp::from_millis(v)))
.collect(),
ConcreteDataType::Null(_) => unreachable!(),
ConcreteDataType::List(_) => unreachable!(),
}
}

41
src/datanode/src/server/grpc/select.rs
View File

@@ -100,10 +100,10 @@ fn null_mask(arrays: &Vec<Arc<dyn Array>>, row_count: usize) -> Vec<u8> {
}
macro_rules! convert_arrow_array_to_grpc_vals {
($data_type: expr, $arrays: ident, $(($Type: ident, $CastType: ty, $field: ident, $MapFunction: expr)), +) => {
($data_type: expr, $arrays: ident, $(($Type: pat, $CastType: ty, $field: ident, $MapFunction: expr)), +) => {
match $data_type {
$(
arrow::datatypes::DataType::$Type => {
$Type => {
let mut vals = Values::default();
for array in $arrays {
let array = array.as_any().downcast_ref::<$CastType>().with_context(|| ConversionSnafu {
@@ -119,8 +119,8 @@ macro_rules! convert_arrow_array_to_grpc_vals {
)+
_ => unimplemented!(),
}
};
}
fn values(arrays: &[Arc<dyn Array>]) -> Result<Values> {
@@ -129,29 +129,34 @@ fn values(arrays: &[Arc<dyn Array>]) -> Result<Values> {
}
let data_type = arrays[0].data_type();
use arrow::datatypes::DataType;
convert_arrow_array_to_grpc_vals!(
data_type, arrays,
(Boolean, BooleanArray, bool_values, |x| {x}),
(DataType::Boolean, BooleanArray, bool_values, |x| {x}),
(Int8, PrimitiveArray<i8>, i8_values, |x| {*x as i32}),
(Int16, PrimitiveArray<i16>, i16_values, |x| {*x as i32}),
(Int32, PrimitiveArray<i32>, i32_values, |x| {*x}),
(Int64, PrimitiveArray<i64>, i64_values, |x| {*x}),
(DataType::Int8, PrimitiveArray<i8>, i8_values, |x| {*x as i32}),
(DataType::Int16, PrimitiveArray<i16>, i16_values, |x| {*x as i32}),
(DataType::Int32, PrimitiveArray<i32>, i32_values, |x| {*x}),
(DataType::Int64, PrimitiveArray<i64>, i64_values, |x| {*x}),
(UInt8, PrimitiveArray<u8>, u8_values, |x| {*x as u32}),
(UInt16, PrimitiveArray<u16>, u16_values, |x| {*x as u32}),
(UInt32, PrimitiveArray<u32>, u32_values, |x| {*x}),
(UInt64, PrimitiveArray<u64>, u64_values, |x| {*x}),
(DataType::UInt8, PrimitiveArray<u8>, u8_values, |x| {*x as u32}),
(DataType::UInt16, PrimitiveArray<u16>, u16_values, |x| {*x as u32}),
(DataType::UInt32, PrimitiveArray<u32>, u32_values, |x| {*x}),
(DataType::UInt64, PrimitiveArray<u64>, u64_values, |x| {*x}),
(Float32, PrimitiveArray<f32>, f32_values, |x| {*x}),
(Float64, PrimitiveArray<f64>, f64_values, |x| {*x}),
(DataType::Float32, PrimitiveArray<f32>, f32_values, |x| {*x}),
(DataType::Float64, PrimitiveArray<f64>, f64_values, |x| {*x}),
(Binary, BinaryArray, binary_values, |x| {x.into()}),
(LargeBinary, BinaryArray, binary_values, |x| {x.into()}),
(DataType::Binary, BinaryArray, binary_values, |x| {x.into()}),
(DataType::LargeBinary, BinaryArray, binary_values, |x| {x.into()}),
(Utf8, StringArray, string_values, |x| {x.into()}),
(LargeUtf8, StringArray, string_values, |x| {x.into()})
(DataType::Utf8, StringArray, string_values, |x| {x.into()}),
(DataType::LargeUtf8, StringArray, string_values, |x| {x.into()}),
(DataType::Date32, PrimitiveArray<i32>, i32_values, |x| {*x as i32}),
(DataType::Date64, PrimitiveArray<i64>, i64_values, |x| {*x as i64}),
(DataType::Timestamp(arrow::datatypes::TimeUnit::Millisecond, _), PrimitiveArray<i64>, i64_values, |x| {*x} )
)
}

1
src/datanode/src/sql.rs
View File

@@ -121,6 +121,7 @@ fn sql_data_type_to_concrete_data_type(data_type: &SqlDataType) -> Result<Concre
}
.fail(),
},
SqlDataType::Timestamp => Ok(ConcreteDataType::timestamp_millis_datatype()),
_ => error::SqlTypeNotSupportedSnafu {
t: data_type.clone(),
}

4
src/datanode/src/sql/create.rs
View File

@@ -329,5 +329,9 @@ mod tests {
SqlDataType::Custom(ObjectName(vec![Ident::new("datetime")])),
ConcreteDataType::datetime_datatype(),
);
check_type(
SqlDataType::Timestamp,
ConcreteDataType::timestamp_millis_datatype(),
);
}
}

5
src/datanode/src/sql/insert.rs
View File

@@ -156,7 +156,6 @@ fn parse_sql_value(
parse_string_to_value(s.to_owned(), data_type)?
}
_ => todo!("Other sql value"),
})
}
@@ -220,8 +219,10 @@ fn sql_number_to_value(data_type: &ConcreteDataType, n: &str) -> Result<Value> {
(Int32, i32),
(Int64, i64),
(Float64, f64),
(Float32, f32)
(Float32, f32),
(Timestamp, i64)
)
// TODO(hl): also Date/DateTime
}
fn parse_sql_number<R: FromStr + std::fmt::Debug>(n: &str) -> Result<R>

2
src/datanode/src/tests/grpc_test.rs
View File

@@ -147,7 +147,7 @@ fn testing_create_expr() -> CreateExpr {
},
ColumnDef {
name: "ts".to_string(),
data_type: 4, // int64
data_type: 15, // timestamp
is_nullable: true,
},
];

4
src/datanode/src/tests/http_test.rs
View File

@@ -64,7 +64,7 @@ async fn test_sql_api() {
let body = res.text().await;
assert_eq!(
body,
r#"{"success":true,"output":{"Rows":[{"schema":{"fields":[{"name":"host","data_type":"Utf8","is_nullable":false,"metadata":{}},{"name":"cpu","data_type":"Float64","is_nullable":true,"metadata":{}},{"name":"memory","data_type":"Float64","is_nullable":true,"metadata":{}},{"name":"ts","data_type":"Int64","is_nullable":true,"metadata":{}}],"metadata":{"greptime:timestamp_column":"ts","greptime:version":"0"}},"columns":[["host"],[66.6],[1024.0],[0]]}]}}"#
r#"{"success":true,"output":{"Rows":[{"schema":{"fields":[{"name":"host","data_type":"Utf8","is_nullable":false,"metadata":{}},{"name":"cpu","data_type":"Float64","is_nullable":true,"metadata":{}},{"name":"memory","data_type":"Float64","is_nullable":true,"metadata":{}},{"name":"ts","data_type":{"Timestamp":["Millisecond",null]},"is_nullable":true,"metadata":{}}],"metadata":{"greptime:timestamp_column":"ts","greptime:version":"0"}},"columns":[["host"],[66.6],[1024.0],[0]]}]}}"#
);
// select with projections
@@ -77,7 +77,7 @@ async fn test_sql_api() {
let body = res.text().await;
assert_eq!(
body,
r#"{"success":true,"output":{"Rows":[{"schema":{"fields":[{"name":"cpu","data_type":"Float64","is_nullable":true,"metadata":{}},{"name":"ts","data_type":"Int64","is_nullable":true,"metadata":{}}],"metadata":{"greptime:timestamp_column":"ts","greptime:version":"0"}},"columns":[[66.6],[0]]}]}}"#
r#"{"success":true,"output":{"Rows":[{"schema":{"fields":[{"name":"cpu","data_type":"Float64","is_nullable":true,"metadata":{}},{"name":"ts","data_type":{"Timestamp":["Millisecond",null]},"is_nullable":true,"metadata":{}}],"metadata":{"greptime:timestamp_column":"ts","greptime:version":"0"}},"columns":[[66.6],[0]]}]}}"#
);
}

14
src/datanode/src/tests/instance_test.rs
View File

@@ -125,13 +125,13 @@ async fn test_alter_table() {
let pretty_print = arrow_print::write(&recordbatch);
let pretty_print = pretty_print.lines().collect::<Vec<&str>>();
let expected = vec![
"+-------+------+-----+--------+--------+",
"| host | ts | cpu | memory | my_tag |",
"+-------+------+-----+--------+--------+",
"| host1 | 1000 | 1.1 | 100 | |",
"| host2 | 2000 | 2.2 | 200 | hello |",
"| host3 | 3000 | 3.3 | 300 | |",
"+-------+------+-----+--------+--------+",
"+-------+---------------------+-----+--------+--------+",
"| host | ts | cpu | memory | my_tag |",
"+-------+---------------------+-----+--------+--------+",
"| host1 | 1970-01-01 00:00:01 | 1.1 | 100 | |",
"| host2 | 1970-01-01 00:00:02 | 2.2 | 200 | hello |",
"| host3 | 1970-01-01 00:00:03 | 3.3 | 300 | |",
"+-------+---------------------+-----+--------+--------+",
];
assert_eq!(pretty_print, expected);
}

2
src/datanode/src/tests/test_util.rs
View File

@@ -48,7 +48,7 @@ pub async fn create_test_table(instance: &Instance) -> Result<()> {
ColumnSchema::new("host", ConcreteDataType::string_datatype(), false),
ColumnSchema::new("cpu", ConcreteDataType::float64_datatype(), true),
ColumnSchema::new("memory", ConcreteDataType::float64_datatype(), true),
ColumnSchema::new("ts", ConcreteDataType::int64_datatype(), true),
ColumnSchema::new("ts", ConcreteDataType::timestamp_millis_datatype(), true),
];
let table_name = "demo";