From 51a4d660b78586be068ea14d78e780b17c841376 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Mon, 12 Jun 2023 10:21:49 +0100 Subject: [PATCH] feat(to_unixtime): add timestamp types as arguments (#1632) * feat(to_unixtime): add timestamp types as arguments * feat(to_unixtime): change the return type * feat(to_unixtime): address code review issues * feat(to_unixtime): fix fmt issue --- .../src/scalars/timestamp/to_unixtime.rs | 197 ++++++++++++++++-- .../standalone/common/select/dummy.result | 61 ++++++ .../cases/standalone/common/select/dummy.sql | 18 ++ 3 files changed, 260 insertions(+), 16 deletions(-) diff --git a/src/common/function/src/scalars/timestamp/to_unixtime.rs b/src/common/function/src/scalars/timestamp/to_unixtime.rs index 9922525164..e07adc793d 100644 --- a/src/common/function/src/scalars/timestamp/to_unixtime.rs +++ b/src/common/function/src/scalars/timestamp/to_unixtime.rs @@ -16,13 +16,16 @@ use std::fmt; use std::str::FromStr; use std::sync::Arc; -use common_query::error::{self, Result, UnsupportedInputDataTypeSnafu}; +use common_query::error::{InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu}; use common_query::prelude::{Signature, Volatility}; use common_time::timestamp::TimeUnit; use common_time::Timestamp; use datatypes::prelude::ConcreteDataType; -use datatypes::types::StringType; -use datatypes::vectors::{Int64Vector, StringVector, Vector, VectorRef}; +use datatypes::types::TimestampType; +use datatypes::vectors::{ + Int64Vector, StringVector, TimestampMicrosecondVector, TimestampMillisecondVector, + TimestampNanosecondVector, TimestampSecondVector, Vector, VectorRef, +}; use snafu::ensure; use crate::scalars::function::{Function, FunctionContext}; @@ -42,18 +45,33 @@ fn convert_to_seconds(arg: &str) -> Option { } } +fn process_vector(vector: &dyn Vector) -> Vec> { + (0..vector.len()) + .map(|i| paste::expr!((vector.get(i)).as_timestamp().map(|ts| ts.value()))) + .collect::>>() +} + impl Function for ToUnixtimeFunction { fn name(&self) -> &str { NAME } fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result { - Ok(ConcreteDataType::timestamp_second_datatype()) + Ok(ConcreteDataType::int64_datatype()) } fn signature(&self) -> Signature { - Signature::exact( - vec![ConcreteDataType::String(StringType)], + Signature::uniform( + 1, + vec![ + ConcreteDataType::string_datatype(), + ConcreteDataType::int32_datatype(), + ConcreteDataType::int64_datatype(), + ConcreteDataType::timestamp_second_datatype(), + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_microsecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype(), + ], Volatility::Immutable, ) } @@ -61,7 +79,7 @@ impl Function for ToUnixtimeFunction { fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result { ensure!( columns.len() == 1, - error::InvalidFuncArgsSnafu { + InvalidFuncArgsSnafu { err_msg: format!( "The length of the args is not correct, expect exactly one, have: {}", columns.len() @@ -79,6 +97,42 @@ impl Function for ToUnixtimeFunction { .collect::>(), ))) } + ConcreteDataType::Int64(_) | ConcreteDataType::Int32(_) => { + let array = columns[0].to_arrow_array(); + Ok(Arc::new(Int64Vector::try_from_arrow_array(&array).unwrap())) + } + ConcreteDataType::Timestamp(ts) => { + let array = columns[0].to_arrow_array(); + let value = match ts { + TimestampType::Second(_) => { + let vector = paste::expr!(TimestampSecondVector::try_from_arrow_array( + array + ) + .unwrap()); + process_vector(&vector) + } + TimestampType::Millisecond(_) => { + let vector = paste::expr!( + TimestampMillisecondVector::try_from_arrow_array(array).unwrap() + ); + process_vector(&vector) + } + TimestampType::Microsecond(_) => { + let vector = paste::expr!( + TimestampMicrosecondVector::try_from_arrow_array(array).unwrap() + ); + process_vector(&vector) + } + TimestampType::Nanosecond(_) => { + let vector = paste::expr!(TimestampNanosecondVector::try_from_arrow_array( + array + ) + .unwrap()); + process_vector(&vector) + } + }; + Ok(Arc::new(Int64Vector::from(value))) + } _ => UnsupportedInputDataTypeSnafu { function: NAME, datatypes: columns.iter().map(|c| c.data_type()).collect::>(), @@ -97,28 +151,37 @@ impl fmt::Display for ToUnixtimeFunction { #[cfg(test)] mod tests { use common_query::prelude::TypeSignature; - use datatypes::prelude::ConcreteDataType; - use datatypes::types::StringType; + use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder}; + use datatypes::scalars::ScalarVector; + use datatypes::timestamp::TimestampSecond; use datatypes::value::Value; - use datatypes::vectors::StringVector; + use datatypes::vectors::{StringVector, TimestampSecondVector}; use super::{ToUnixtimeFunction, *}; use crate::scalars::Function; #[test] - fn test_to_unixtime() { + fn test_string_to_unixtime() { let f = ToUnixtimeFunction::default(); assert_eq!("to_unixtime", f.name()); assert_eq!( - ConcreteDataType::timestamp_second_datatype(), + ConcreteDataType::int64_datatype(), f.return_type(&[]).unwrap() ); assert!(matches!(f.signature(), - Signature { - type_signature: TypeSignature::Exact(valid_types), - volatility: Volatility::Immutable - } if valid_types == vec![ConcreteDataType::String(StringType)] + Signature { + type_signature: TypeSignature::Uniform(1, valid_types), + volatility: Volatility::Immutable + } if valid_types == vec![ + ConcreteDataType::string_datatype(), + ConcreteDataType::int32_datatype(), + ConcreteDataType::int64_datatype(), + ConcreteDataType::timestamp_second_datatype(), + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_microsecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype(), + ] )); let times = vec![ @@ -145,4 +208,106 @@ mod tests { } } } + + #[test] + fn test_int_to_unixtime() { + let f = ToUnixtimeFunction::default(); + assert_eq!("to_unixtime", f.name()); + assert_eq!( + ConcreteDataType::int64_datatype(), + f.return_type(&[]).unwrap() + ); + + assert!(matches!(f.signature(), + Signature { + type_signature: TypeSignature::Uniform(1, valid_types), + volatility: Volatility::Immutable + } if valid_types == vec![ + ConcreteDataType::string_datatype(), + ConcreteDataType::int32_datatype(), + ConcreteDataType::int64_datatype(), + ConcreteDataType::timestamp_second_datatype(), + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_microsecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype(), + ] + )); + + let times = vec![Some(3_i64), None, Some(5_i64), None]; + let results = vec![Some(3), None, Some(5), None]; + let args: Vec = vec![Arc::new(Int64Vector::from(times.clone()))]; + let vector = f.eval(FunctionContext::default(), &args).unwrap(); + assert_eq!(4, vector.len()); + for (i, _t) in times.iter().enumerate() { + let v = vector.get(i); + if i == 1 || i == 3 { + assert_eq!(Value::Null, v); + continue; + } + match v { + Value::Int64(ts) => { + assert_eq!(ts, (*results.get(i).unwrap()).unwrap()); + } + _ => unreachable!(), + } + } + } + + #[test] + fn test_timestamp_to_unixtime() { + let f = ToUnixtimeFunction::default(); + assert_eq!("to_unixtime", f.name()); + assert_eq!( + ConcreteDataType::int64_datatype(), + f.return_type(&[]).unwrap() + ); + + assert!(matches!(f.signature(), + Signature { + type_signature: TypeSignature::Uniform(1, valid_types), + volatility: Volatility::Immutable + } if valid_types == vec![ + ConcreteDataType::string_datatype(), + ConcreteDataType::int32_datatype(), + ConcreteDataType::int64_datatype(), + ConcreteDataType::timestamp_second_datatype(), + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_microsecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype(), + ] + )); + + let times: Vec> = vec![ + Some(TimestampSecond::new(123)), + None, + Some(TimestampSecond::new(42)), + None, + ]; + let results = vec![Some(123), None, Some(42), None]; + let ts_vector: TimestampSecondVector = build_vector_from_slice(×); + let args: Vec = vec![Arc::new(ts_vector)]; + let vector = f.eval(FunctionContext::default(), &args).unwrap(); + assert_eq!(4, vector.len()); + for (i, _t) in times.iter().enumerate() { + let v = vector.get(i); + if i == 1 || i == 3 { + assert_eq!(Value::Null, v); + continue; + } + match v { + Value::Int64(ts) => { + assert_eq!(ts, (*results.get(i).unwrap()).unwrap()); + } + _ => unreachable!(), + } + } + } + + fn build_vector_from_slice(items: &[Option>]) -> T { + let mut builder = T::Builder::with_capacity(items.len()); + for item in items { + builder.push(*item); + } + builder.finish() + } } diff --git a/tests/cases/standalone/common/select/dummy.result b/tests/cases/standalone/common/select/dummy.result index 759f7b57c9..0887fdd7f8 100644 --- a/tests/cases/standalone/common/select/dummy.result +++ b/tests/cases/standalone/common/select/dummy.result @@ -42,3 +42,64 @@ select TO_UNIXTIME('2023-03-01T06:35:02Z'); | 1677652502 | +-------------------------------------------+ +select TO_UNIXTIME(2); + ++-----------------------+ +| to_unixtime(Int64(2)) | ++-----------------------+ +| 2 | ++-----------------------+ + +create table test_unixtime(a int, b timestamp time index); + +Affected Rows: 0 + +DESC TABLE test_unixtime; + ++-------+----------------------+------+---------+---------------+ +| Field | Type | Null | Default | Semantic Type | ++-------+----------------------+------+---------+---------------+ +| a | Int32 | YES | | FIELD | +| b | TimestampMillisecond | NO | | TIME INDEX | ++-------+----------------------+------+---------+---------------+ + +insert into test_unixtime values(27, 27); + +Affected Rows: 1 + +select * from test_unixtime; + ++----+-------------------------+ +| a | b | ++----+-------------------------+ +| 27 | 1970-01-01T00:00:00.027 | ++----+-------------------------+ + +select a from test_unixtime; + ++----+ +| a | ++----+ +| 27 | ++----+ + +select b from test_unixtime; + ++-------------------------+ +| b | ++-------------------------+ +| 1970-01-01T00:00:00.027 | ++-------------------------+ + +select TO_UNIXTIME(b) from test_unixtime; + ++------------------------------+ +| to_unixtime(test_unixtime.b) | ++------------------------------+ +| 27 | ++------------------------------+ + +DROP TABLE test_unixtime; + +Affected Rows: 1 + diff --git a/tests/cases/standalone/common/select/dummy.sql b/tests/cases/standalone/common/select/dummy.sql index 9c2a3944cf..5bf16da864 100644 --- a/tests/cases/standalone/common/select/dummy.sql +++ b/tests/cases/standalone/common/select/dummy.sql @@ -11,3 +11,21 @@ select "A"; select * where "a" = "A"; select TO_UNIXTIME('2023-03-01T06:35:02Z'); + +select TO_UNIXTIME(2); + +create table test_unixtime(a int, b timestamp time index); + +DESC TABLE test_unixtime; + +insert into test_unixtime values(27, 27); + +select * from test_unixtime; + +select a from test_unixtime; + +select b from test_unixtime; + +select TO_UNIXTIME(b) from test_unixtime; + +DROP TABLE test_unixtime;