From d20191572e1f34074b50d6304198a3e104ad2f22 Mon Sep 17 00:00:00 2001 From: evenyag Date: Fri, 22 Apr 2022 19:19:48 +0800 Subject: [PATCH 1/9] feat: Implement PrimitiveType and PrimitiveVector --- Cargo.lock | 98 +++++++++++++++++++++ src/common/src/lib.rs | 6 ++ src/datatypes/Cargo.toml | 3 + src/datatypes/src/data_type.rs | 18 ++++ src/datatypes/src/lib.rs | 5 ++ src/datatypes/src/type_id.rs | 30 +++++++ src/datatypes/src/types.rs | 2 + src/datatypes/src/types/primitive_traits.rs | 28 ++++++ src/datatypes/src/types/primitive_type.rs | 73 +++++++++++++++ src/datatypes/src/value.rs | 59 +++++++++++++ src/datatypes/src/vectors.rs | 26 ++++++ src/datatypes/src/vectors/primitive.rs | 33 +++++++ 12 files changed, 381 insertions(+) create mode 100644 src/datatypes/src/data_type.rs create mode 100644 src/datatypes/src/type_id.rs create mode 100644 src/datatypes/src/types.rs create mode 100644 src/datatypes/src/types/primitive_traits.rs create mode 100644 src/datatypes/src/types/primitive_type.rs create mode 100644 src/datatypes/src/value.rs create mode 100644 src/datatypes/src/vectors.rs create mode 100644 src/datatypes/src/vectors/primitive.rs diff --git a/Cargo.lock b/Cargo.lock index 9cade898fe..4a4d27f706 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,20 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "arrow2" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e387b20dd573a96f36b173d9027483898f944d696521afd74e2caa3c813d86e" +dependencies = [ + "bytemuck", + "chrono", + "either", + "hash_hasher", + "num-traits", + "simdutf8", +] + [[package]] name = "async-trait" version = "0.1.53" @@ -13,12 +27,48 @@ dependencies = [ "syn", ] +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bytemuck" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdead85bdec19c194affaeeb670c0e41fe23de31459efd1c174d049269cf02cc" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562e382481975bc61d11275ac5e62a19abd00b0547d99516a415336f183dcd0e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "common" version = "0.1.0" @@ -33,6 +83,11 @@ dependencies = [ [[package]] name = "datatypes" version = "0.1.0" +dependencies = [ + "arrow2", + "common", + "paste", +] [[package]] name = "doc-comment" @@ -40,6 +95,18 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "hash_hasher" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c" + [[package]] name = "heck" version = "0.3.3" @@ -66,10 +133,35 @@ version = "0.1.0" name = "logical-plans" version = "0.1.0" +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + [[package]] name = "object-store" version = "0.1.0" +[[package]] +name = "paste" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" + [[package]] name = "proc-macro2" version = "1.0.37" @@ -92,6 +184,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "simdutf8" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" + [[package]] name = "snafu" version = "0.7.0" diff --git a/src/common/src/lib.rs b/src/common/src/lib.rs index 8b13789179..ae09dd0f2c 100644 --- a/src/common/src/lib.rs +++ b/src/common/src/lib.rs @@ -1 +1,7 @@ +/// Bytes buffer. +#[derive(Debug, Default, Clone)] +pub struct Bytes(Vec); +/// String buffer with arbitrary encoding. +#[derive(Debug, Default, Clone)] +pub struct StringBytes(Vec); diff --git a/src/datatypes/Cargo.toml b/src/datatypes/Cargo.toml index f32b26efb9..00fe122272 100644 --- a/src/datatypes/Cargo.toml +++ b/src/datatypes/Cargo.toml @@ -6,3 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +common = { path = "../common" } +arrow2 = "0.10" +paste = "1.0" diff --git a/src/datatypes/src/data_type.rs b/src/datatypes/src/data_type.rs new file mode 100644 index 0000000000..122a2c173a --- /dev/null +++ b/src/datatypes/src/data_type.rs @@ -0,0 +1,18 @@ +use std::sync::Arc; + +use crate::type_id::LogicalTypeId; +use crate::value::Value; + +/// Data type abstraction. +pub trait DataType: std::fmt::Debug { + /// Name of this data type. + fn name(&self) -> &str; + + /// Returns id of the Logical data type. + fn logical_type_id(&self) -> LogicalTypeId; + + /// Returns the default value of this type. + fn default_value(&self) -> Value; +} + +pub type DataTypeRef = Arc; diff --git a/src/datatypes/src/lib.rs b/src/datatypes/src/lib.rs index 6bde67a2b1..3b22def878 100644 --- a/src/datatypes/src/lib.rs +++ b/src/datatypes/src/lib.rs @@ -1 +1,6 @@ +mod data_type; mod schema; +pub mod type_id; +mod types; +pub mod value; +pub mod vectors; diff --git a/src/datatypes/src/type_id.rs b/src/datatypes/src/type_id.rs new file mode 100644 index 0000000000..e299594caa --- /dev/null +++ b/src/datatypes/src/type_id.rs @@ -0,0 +1,30 @@ +/// Unique identifier for logical data type. +#[derive(Debug)] +pub enum LogicalTypeId { + Null, + + // Numeric types: + Boolean, + Int8, + Int16, + Int32, + Int64, + UInt8, + UInt16, + UInt32, + UInt64, + Float32, + Float64, + + // String types: + String, + Binary, + + // Date & Time types: + /// Date representing the elapsed time since UNIX epoch (1970-01-01) + /// in days (32 bits). + Date, + /// Datetime representing the elapsed time since UNIX epoch (1970-01-01) in + /// seconds/milliseconds/microseconds/nanoseconds, determined by precision. + DateTime, +} diff --git a/src/datatypes/src/types.rs b/src/datatypes/src/types.rs new file mode 100644 index 0000000000..fd30e143a9 --- /dev/null +++ b/src/datatypes/src/types.rs @@ -0,0 +1,2 @@ +pub mod primitive_traits; +pub mod primitive_type; diff --git a/src/datatypes/src/types/primitive_traits.rs b/src/datatypes/src/types/primitive_traits.rs new file mode 100644 index 0000000000..6402c7a727 --- /dev/null +++ b/src/datatypes/src/types/primitive_traits.rs @@ -0,0 +1,28 @@ +use arrow2::types::NativeType; + +use crate::value::Value; + +/// Primitive type. +pub trait Primitive: PartialOrd + Default + Clone + Copy + Into + NativeType { + /// Largest numeric type this primitive type can be cast to. + type LargestType: Primitive; +} + +macro_rules! impl_primitive { + ($Type:ident, $LargestType: ident) => { + impl Primitive for $Type { + type LargestType = $LargestType; + } + }; +} + +impl_primitive!(u8, u64); +impl_primitive!(u16, u64); +impl_primitive!(u32, u64); +impl_primitive!(u64, u64); +impl_primitive!(i8, i64); +impl_primitive!(i16, i64); +impl_primitive!(i32, i64); +impl_primitive!(i64, i64); +impl_primitive!(f32, f64); +impl_primitive!(f64, f64); diff --git a/src/datatypes/src/types/primitive_type.rs b/src/datatypes/src/types/primitive_type.rs new file mode 100644 index 0000000000..ade799acfe --- /dev/null +++ b/src/datatypes/src/types/primitive_type.rs @@ -0,0 +1,73 @@ +use std::marker::PhantomData; +use std::sync::Arc; + +use crate::data_type::{DataType, DataTypeRef}; +use crate::type_id::LogicalTypeId; +use crate::types::primitive_traits::Primitive; +use crate::value::Value; + +pub struct PrimitiveType { + _phantom: PhantomData, +} + +impl PrimitiveType { + pub fn new() -> Self { + Self { + _phantom: PhantomData, + } + } +} + +/// Create a new [DataTypeRef] from a primitive type. +pub trait CreateDataType { + fn create_data_type() -> DataTypeRef; +} + +macro_rules! impl_create_data_type { + ($Type:ident) => { + paste::paste! { + impl CreateDataType for $Type { + fn create_data_type() -> DataTypeRef { + Arc::new(PrimitiveType::<$Type>::new()) + } + } + } + }; +} + +macro_rules! impl_numeric { + ($Type:ident, $TypeId:ident) => { + impl DataType for PrimitiveType<$Type> { + fn name(&self) -> &str { + stringify!($TypeId) + } + + fn logical_type_id(&self) -> LogicalTypeId { + LogicalTypeId::$TypeId + } + + fn default_value(&self) -> Value { + $Type::default().into() + } + } + + impl std::fmt::Debug for PrimitiveType<$Type> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", self.name()) + } + } + + impl_create_data_type!($Type); + }; +} + +impl_numeric!(u8, UInt8); +impl_numeric!(u16, UInt16); +impl_numeric!(u32, UInt32); +impl_numeric!(u64, UInt64); +impl_numeric!(i8, Int8); +impl_numeric!(i16, Int16); +impl_numeric!(i32, Int32); +impl_numeric!(i64, Int64); +impl_numeric!(f32, Float32); +impl_numeric!(f64, Float64); diff --git a/src/datatypes/src/value.rs b/src/datatypes/src/value.rs new file mode 100644 index 0000000000..063ec321f5 --- /dev/null +++ b/src/datatypes/src/value.rs @@ -0,0 +1,59 @@ +use common::{Bytes, StringBytes}; + +/// Value holds a single arbitrary value of any [DataType](crate::data_type::DataType). +#[derive(Debug)] +pub enum Value { + Null, + + // Numeric types: + Boolean(bool), + UInt8(u8), + UInt16(u16), + UInt32(u32), + UInt64(u64), + Int8(i8), + Int16(i16), + Int32(i32), + Int64(i64), + Float32(f32), + Float64(f64), + + // String types: + String(StringBytes), + Binary(Bytes), + + // Date & Time types: + Date(i32), + DateTime(i64), +} + +macro_rules! impl_from { + ($Variant:ident, $Type:ident) => { + impl From<$Type> for Value { + fn from(value: $Type) -> Self { + Value::$Variant(value) + } + } + + impl From> for Value { + fn from(value: Option<$Type>) -> Self { + match value { + Some(v) => Value::$Variant(v), + None => Value::Null, + } + } + } + }; +} + +impl_from!(Boolean, bool); +impl_from!(UInt8, u8); +impl_from!(UInt16, u16); +impl_from!(UInt32, u32); +impl_from!(UInt64, u64); +impl_from!(Int8, i8); +impl_from!(Int16, i16); +impl_from!(Int32, i32); +impl_from!(Int64, i64); +impl_from!(Float32, f32); +impl_from!(Float64, f64); diff --git a/src/datatypes/src/vectors.rs b/src/datatypes/src/vectors.rs new file mode 100644 index 0000000000..cbb95f5c33 --- /dev/null +++ b/src/datatypes/src/vectors.rs @@ -0,0 +1,26 @@ +pub mod primitive; + +use std::any::Any; +use std::sync::Arc; + +use crate::data_type::DataTypeRef; + +/// Vector of data values. +pub trait Vector: Send + Sync { + /// Returns the data type of the vector. + fn data_type(&self) -> DataTypeRef; + + /// Returns the vector as [Any](std::any::Any) so that it can be + /// downcast to a specific implementation. + fn as_any(&self) -> &dyn Any; + + /// Returns number of elements in the vector. + fn len(&self) -> usize; + + /// Returns whether the vector is empty. + fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +pub type VectorRef = Arc; diff --git a/src/datatypes/src/vectors/primitive.rs b/src/datatypes/src/vectors/primitive.rs new file mode 100644 index 0000000000..b58fe5e1c6 --- /dev/null +++ b/src/datatypes/src/vectors/primitive.rs @@ -0,0 +1,33 @@ +use std::any::Any; + +use arrow2::array::PrimitiveArray; + +use crate::data_type::DataTypeRef; +use crate::types::primitive_traits::Primitive; +use crate::types::primitive_type::CreateDataType; +use crate::vectors::Vector; + +/// Vector for primitive data types. +pub struct PrimitiveVector { + array: PrimitiveArray, +} + +impl PrimitiveVector { + pub fn new(array: PrimitiveArray) -> Self { + Self { array } + } +} + +impl Vector for PrimitiveVector { + fn data_type(&self) -> DataTypeRef { + T::create_data_type() + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn len(&self) -> usize { + self.array.len() + } +} From c9c5e69adf6e0c2dd8c87fdba82a573a8a8ddc97 Mon Sep 17 00:00:00 2001 From: evenyag Date: Sun, 24 Apr 2022 13:45:12 +0800 Subject: [PATCH 2/9] feat: Implement BinaryType and BinaryVector --- src/common/src/bytes.rs | 7 ++++++ src/common/src/lib.rs | 8 +------ src/datatypes/src/data_type.rs | 2 +- src/datatypes/src/lib.rs | 5 +++++ src/datatypes/src/prelude.rs | 3 +++ src/datatypes/src/types.rs | 1 + src/datatypes/src/types/binary_type.rs | 30 ++++++++++++++++++++++++++ src/datatypes/src/value.rs | 4 +++- src/datatypes/src/vectors.rs | 3 +++ src/datatypes/src/vectors/binary.rs | 26 ++++++++++++++++++++++ 10 files changed, 80 insertions(+), 9 deletions(-) create mode 100644 src/common/src/bytes.rs create mode 100644 src/datatypes/src/prelude.rs create mode 100644 src/datatypes/src/types/binary_type.rs create mode 100644 src/datatypes/src/vectors/binary.rs diff --git a/src/common/src/bytes.rs b/src/common/src/bytes.rs new file mode 100644 index 0000000000..ae09dd0f2c --- /dev/null +++ b/src/common/src/bytes.rs @@ -0,0 +1,7 @@ +/// Bytes buffer. +#[derive(Debug, Default, Clone)] +pub struct Bytes(Vec); + +/// String buffer with arbitrary encoding. +#[derive(Debug, Default, Clone)] +pub struct StringBytes(Vec); diff --git a/src/common/src/lib.rs b/src/common/src/lib.rs index ae09dd0f2c..ad0049c4c2 100644 --- a/src/common/src/lib.rs +++ b/src/common/src/lib.rs @@ -1,7 +1 @@ -/// Bytes buffer. -#[derive(Debug, Default, Clone)] -pub struct Bytes(Vec); - -/// String buffer with arbitrary encoding. -#[derive(Debug, Default, Clone)] -pub struct StringBytes(Vec); +pub mod bytes; diff --git a/src/datatypes/src/data_type.rs b/src/datatypes/src/data_type.rs index 122a2c173a..4cd9f6fd15 100644 --- a/src/datatypes/src/data_type.rs +++ b/src/datatypes/src/data_type.rs @@ -4,7 +4,7 @@ use crate::type_id::LogicalTypeId; use crate::value::Value; /// Data type abstraction. -pub trait DataType: std::fmt::Debug { +pub trait DataType: std::fmt::Debug + Send + Sync { /// Name of this data type. fn name(&self) -> &str; diff --git a/src/datatypes/src/lib.rs b/src/datatypes/src/lib.rs index 3b22def878..5c7b8b66d9 100644 --- a/src/datatypes/src/lib.rs +++ b/src/datatypes/src/lib.rs @@ -1,6 +1,11 @@ mod data_type; +pub mod prelude; mod schema; pub mod type_id; mod types; pub mod value; pub mod vectors; + +use arrow2::array::BinaryArray; + +pub type LargeBinaryArray = BinaryArray; diff --git a/src/datatypes/src/prelude.rs b/src/datatypes/src/prelude.rs new file mode 100644 index 0000000000..1aa4775354 --- /dev/null +++ b/src/datatypes/src/prelude.rs @@ -0,0 +1,3 @@ +pub use crate::data_type::{DataType, DataTypeRef}; +pub use crate::type_id::LogicalTypeId; +pub use crate::value::Value; diff --git a/src/datatypes/src/types.rs b/src/datatypes/src/types.rs index fd30e143a9..02fa6bc718 100644 --- a/src/datatypes/src/types.rs +++ b/src/datatypes/src/types.rs @@ -1,2 +1,3 @@ +pub mod binary_type; pub mod primitive_traits; pub mod primitive_type; diff --git a/src/datatypes/src/types/binary_type.rs b/src/datatypes/src/types/binary_type.rs new file mode 100644 index 0000000000..c5d5a3ed18 --- /dev/null +++ b/src/datatypes/src/types/binary_type.rs @@ -0,0 +1,30 @@ +use std::sync::Arc; + +use common::bytes::StringBytes; + +use crate::data_type::{DataType, DataTypeRef}; +use crate::type_id::LogicalTypeId; +use crate::value::Value; + +#[derive(Debug, Default)] +pub struct BinaryType; + +impl BinaryType { + pub fn arc() -> DataTypeRef { + Arc::new(Self) + } +} + +impl DataType for BinaryType { + fn name(&self) -> &str { + "Binary" + } + + fn logical_type_id(&self) -> LogicalTypeId { + LogicalTypeId::String + } + + fn default_value(&self) -> Value { + StringBytes::default().into() + } +} diff --git a/src/datatypes/src/value.rs b/src/datatypes/src/value.rs index 063ec321f5..3a7f597eaf 100644 --- a/src/datatypes/src/value.rs +++ b/src/datatypes/src/value.rs @@ -1,4 +1,4 @@ -use common::{Bytes, StringBytes}; +use common::bytes::{Bytes, StringBytes}; /// Value holds a single arbitrary value of any [DataType](crate::data_type::DataType). #[derive(Debug)] @@ -57,3 +57,5 @@ impl_from!(Int32, i32); impl_from!(Int64, i64); impl_from!(Float32, f32); impl_from!(Float64, f64); +impl_from!(String, StringBytes); +impl_from!(Binary, Bytes); diff --git a/src/datatypes/src/vectors.rs b/src/datatypes/src/vectors.rs index cbb95f5c33..9adac163b6 100644 --- a/src/datatypes/src/vectors.rs +++ b/src/datatypes/src/vectors.rs @@ -1,3 +1,4 @@ +pub mod binary; pub mod primitive; use std::any::Any; @@ -8,6 +9,8 @@ use crate::data_type::DataTypeRef; /// Vector of data values. pub trait Vector: Send + Sync { /// Returns the data type of the vector. + /// + /// This may require heap allocation. fn data_type(&self) -> DataTypeRef; /// Returns the vector as [Any](std::any::Any) so that it can be diff --git a/src/datatypes/src/vectors/binary.rs b/src/datatypes/src/vectors/binary.rs new file mode 100644 index 0000000000..230a0ce611 --- /dev/null +++ b/src/datatypes/src/vectors/binary.rs @@ -0,0 +1,26 @@ +use std::any::Any; + +use crate::data_type::DataTypeRef; +use crate::types::binary_type::BinaryType; +use crate::vectors::Vector; +use crate::LargeBinaryArray; + +/// Vector of binary strings. +#[derive(Debug)] +pub struct BinaryVector { + array: LargeBinaryArray, +} + +impl Vector for BinaryVector { + fn data_type(&self) -> DataTypeRef { + BinaryType::arc() + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn len(&self) -> usize { + self.array.len() + } +} From 445fd7571212889169fc69c0ef5ebe302f6f56a2 Mon Sep 17 00:00:00 2001 From: evenyag Date: Sun, 24 Apr 2022 17:22:56 +0800 Subject: [PATCH 3/9] feat: Implement ScalarVector for primitive/binary vector --- src/datatypes/src/lib.rs | 6 +- src/datatypes/src/scalar.rs | 86 ++++++++++++++++++++++++++ src/datatypes/src/vectors/binary.rs | 48 +++++++++++++- src/datatypes/src/vectors/primitive.rs | 74 +++++++++++++++++++++- 4 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 src/datatypes/src/scalar.rs diff --git a/src/datatypes/src/lib.rs b/src/datatypes/src/lib.rs index 5c7b8b66d9..513520cf5e 100644 --- a/src/datatypes/src/lib.rs +++ b/src/datatypes/src/lib.rs @@ -1,11 +1,15 @@ +#![feature(generic_associated_types)] + mod data_type; pub mod prelude; +mod scalar; mod schema; pub mod type_id; mod types; pub mod value; pub mod vectors; -use arrow2::array::BinaryArray; +use arrow2::array::{BinaryArray, MutableBinaryArray}; pub type LargeBinaryArray = BinaryArray; +pub type MutableLargeBinaryArray = MutableBinaryArray; diff --git a/src/datatypes/src/scalar.rs b/src/datatypes/src/scalar.rs new file mode 100644 index 0000000000..96cc69fc09 --- /dev/null +++ b/src/datatypes/src/scalar.rs @@ -0,0 +1,86 @@ +use crate::vectors::Vector; + +/// A sub trait of Vector to add scalar operation support. +// This implementation refers to Datebend's [ScalarColumn](https://github.com/datafuselabs/databend/blob/main/common/datavalues/src/scalars/type_.rs) +// and skyzh's [type-exercise-in-rust](https://github.com/skyzh/type-exercise-in-rust). +pub trait ScalarVector: Vector { + /// The reference item of this vector. + type RefItem<'a>: Copy + where + Self: 'a; + + /// Iterator type of this vector. + type Iter<'a>: Iterator>> + where + Self: 'a; + + /// Builder type to build this vector. + type Builder: ScalarVectorBuilder; + + /// Returns the reference to an element at given position. + /// + /// Note: `get()` has bad performance, avoid call this function inside loop. + fn get_data(&self, idx: usize) -> Option>; + + /// Returns iterator of current vector. + fn iter_data(&self) -> Self::Iter<'_>; +} + +/// A trait over all vector builders. +pub trait ScalarVectorBuilder { + type VectorType: ScalarVector; + + /// Create a new builder with initial `capacity`. + fn with_capacity(capacity: usize) -> Self; + + /// Push a value into the builder. + fn push(&mut self, value: Option<::RefItem<'_>>); + + /// Finish build and return a new vector. + fn finish(self) -> Self::VectorType; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::vectors::binary::BinaryVector; + use crate::vectors::primitive::Int32Vector; + + fn build_vector_from_slice(items: &[Option>]) -> T { + let mut builder = T::Builder::with_capacity(items.len()); + for item in items { + builder.push(*item); + } + builder.finish() + } + + fn assert_vector_eq<'a, T: ScalarVector>(expect: &[Option>], vector: &'a T) + where + T::RefItem<'a>: PartialEq + std::fmt::Debug, + { + for (a, b) in expect.iter().zip(vector.iter_data()) { + assert_eq!(*a, b); + } + } + + #[test] + fn test_build_i32_vector() { + let expect = vec![Some(1), Some(2), Some(3), None, Some(5)]; + let vector: Int32Vector = build_vector_from_slice(&expect); + assert_vector_eq(&expect, &vector); + } + + #[test] + fn test_build_binary_vector() { + let expect: Vec> = vec![ + Some(b"a"), + Some(b"b"), + Some(b"c"), + None, + Some(b"e"), + Some(b""), + ]; + let vector: BinaryVector = build_vector_from_slice(&expect); + assert_vector_eq(&expect, &vector); + } +} diff --git a/src/datatypes/src/vectors/binary.rs b/src/datatypes/src/vectors/binary.rs index 230a0ce611..27eb29ae6a 100644 --- a/src/datatypes/src/vectors/binary.rs +++ b/src/datatypes/src/vectors/binary.rs @@ -1,9 +1,13 @@ use std::any::Any; +use arrow2::array::BinaryValueIter; +use arrow2::bitmap::utils::ZipValidity; + use crate::data_type::DataTypeRef; +use crate::scalar::{ScalarVector, ScalarVectorBuilder}; use crate::types::binary_type::BinaryType; use crate::vectors::Vector; -use crate::LargeBinaryArray; +use crate::{LargeBinaryArray, MutableLargeBinaryArray}; /// Vector of binary strings. #[derive(Debug)] @@ -24,3 +28,45 @@ impl Vector for BinaryVector { self.array.len() } } + +impl ScalarVector for BinaryVector { + type RefItem<'a> = &'a [u8]; + type Iter<'a> = ZipValidity<'a, &'a [u8], BinaryValueIter<'a, i64>>; + type Builder = BinaryVectorBuilder; + + fn get_data(&self, idx: usize) -> Option> { + if idx < self.len() { + Some(self.array.value(idx)) + } else { + None + } + } + + fn iter_data(&self) -> Self::Iter<'_> { + self.array.iter() + } +} + +pub struct BinaryVectorBuilder { + mutable_array: MutableLargeBinaryArray, +} + +impl ScalarVectorBuilder for BinaryVectorBuilder { + type VectorType = BinaryVector; + + fn with_capacity(capacity: usize) -> Self { + Self { + mutable_array: MutableLargeBinaryArray::with_capacity(capacity), + } + } + + fn push(&mut self, value: Option<::RefItem<'_>>) { + self.mutable_array.push(value); + } + + fn finish(self) -> Self::VectorType { + BinaryVector { + array: self.mutable_array.into(), + } + } +} diff --git a/src/datatypes/src/vectors/primitive.rs b/src/datatypes/src/vectors/primitive.rs index b58fe5e1c6..73a1c14991 100644 --- a/src/datatypes/src/vectors/primitive.rs +++ b/src/datatypes/src/vectors/primitive.rs @@ -1,8 +1,11 @@ use std::any::Any; +use std::slice::Iter; -use arrow2::array::PrimitiveArray; +use arrow2::array::{MutablePrimitiveArray, PrimitiveArray}; +use arrow2::bitmap::utils::ZipValidity; use crate::data_type::DataTypeRef; +use crate::scalar::{ScalarVector, ScalarVectorBuilder}; use crate::types::primitive_traits::Primitive; use crate::types::primitive_type::CreateDataType; use crate::vectors::Vector; @@ -31,3 +34,72 @@ impl Vector for PrimitiveVector { self.array.len() } } + +impl ScalarVector for PrimitiveVector { + type RefItem<'a> = T; + type Iter<'a> = PrimitiveIter<'a, T>; + type Builder = PrimitiveVectorBuilder; + + fn get_data(&self, idx: usize) -> Option> { + if idx < self.len() { + Some(self.array.value(idx)) + } else { + None + } + } + + fn iter_data(&self) -> Self::Iter<'_> { + PrimitiveIter { + iter: self.array.iter(), + } + } +} + +pub type UInt8Vector = PrimitiveVector; +pub type UInt16Vector = PrimitiveVector; +pub type UInt32Vector = PrimitiveVector; +pub type UInt64Vector = PrimitiveVector; + +pub type Int8Vector = PrimitiveVector; +pub type Int16Vector = PrimitiveVector; +pub type Int32Vector = PrimitiveVector; +pub type Int64Vector = PrimitiveVector; + +pub type Float32Vector = PrimitiveVector; +pub type Float64Vector = PrimitiveVector; + +pub struct PrimitiveIter<'a, T> { + iter: ZipValidity<'a, &'a T, Iter<'a, T>>, +} + +impl<'a, T: Copy> Iterator for PrimitiveIter<'a, T> { + type Item = Option; + + fn next(&mut self) -> Option> { + self.iter.next().map(|v| v.copied()) + } +} + +pub struct PrimitiveVectorBuilder { + mutable_array: MutablePrimitiveArray, +} + +impl ScalarVectorBuilder for PrimitiveVectorBuilder { + type VectorType = PrimitiveVector; + + fn with_capacity(capacity: usize) -> Self { + Self { + mutable_array: MutablePrimitiveArray::with_capacity(capacity), + } + } + + fn push(&mut self, value: Option<::RefItem<'_>>) { + self.mutable_array.push(value); + } + + fn finish(self) -> Self::VectorType { + PrimitiveVector { + array: self.mutable_array.into(), + } + } +} From 7a015d4f2a5a30afa66b2059e349ce7d148a2861 Mon Sep 17 00:00:00 2001 From: evenyag Date: Mon, 25 Apr 2022 10:42:16 +0800 Subject: [PATCH 4/9] feat: Allow DataType/Vector converting into arrow's type --- src/datatypes/src/data_type.rs | 5 +++++ src/datatypes/src/types/binary_type.rs | 5 +++++ src/datatypes/src/types/primitive_type.rs | 6 ++++++ src/datatypes/src/vectors.rs | 5 +++++ src/datatypes/src/vectors/binary.rs | 6 ++++++ src/datatypes/src/vectors/primitive.rs | 6 ++++++ 6 files changed, 33 insertions(+) diff --git a/src/datatypes/src/data_type.rs b/src/datatypes/src/data_type.rs index 4cd9f6fd15..77174a43a3 100644 --- a/src/datatypes/src/data_type.rs +++ b/src/datatypes/src/data_type.rs @@ -1,5 +1,7 @@ use std::sync::Arc; +use arrow2::datatypes::DataType as ArrowDataType; + use crate::type_id::LogicalTypeId; use crate::value::Value; @@ -13,6 +15,9 @@ pub trait DataType: std::fmt::Debug + Send + Sync { /// Returns the default value of this type. fn default_value(&self) -> Value; + + /// Convert this type as [arrow2::datatypes::DataType]. + fn as_arrow_type(&self) -> ArrowDataType; } pub type DataTypeRef = Arc; diff --git a/src/datatypes/src/types/binary_type.rs b/src/datatypes/src/types/binary_type.rs index c5d5a3ed18..18c2a0fa1b 100644 --- a/src/datatypes/src/types/binary_type.rs +++ b/src/datatypes/src/types/binary_type.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use arrow2::datatypes::DataType as ArrowDataType; use common::bytes::StringBytes; use crate::data_type::{DataType, DataTypeRef}; @@ -27,4 +28,8 @@ impl DataType for BinaryType { fn default_value(&self) -> Value { StringBytes::default().into() } + + fn as_arrow_type(&self) -> ArrowDataType { + ArrowDataType::LargeBinary + } } diff --git a/src/datatypes/src/types/primitive_type.rs b/src/datatypes/src/types/primitive_type.rs index ade799acfe..3bef260359 100644 --- a/src/datatypes/src/types/primitive_type.rs +++ b/src/datatypes/src/types/primitive_type.rs @@ -1,6 +1,8 @@ use std::marker::PhantomData; use std::sync::Arc; +use arrow2::datatypes::DataType as ArrowDataType; + use crate::data_type::{DataType, DataTypeRef}; use crate::type_id::LogicalTypeId; use crate::types::primitive_traits::Primitive; @@ -49,6 +51,10 @@ macro_rules! impl_numeric { fn default_value(&self) -> Value { $Type::default().into() } + + fn as_arrow_type(&self) -> ArrowDataType { + ArrowDataType::$TypeId + } } impl std::fmt::Debug for PrimitiveType<$Type> { diff --git a/src/datatypes/src/vectors.rs b/src/datatypes/src/vectors.rs index 9adac163b6..c90106c2ea 100644 --- a/src/datatypes/src/vectors.rs +++ b/src/datatypes/src/vectors.rs @@ -4,6 +4,8 @@ pub mod primitive; use std::any::Any; use std::sync::Arc; +use arrow2::array::ArrayRef; + use crate::data_type::DataTypeRef; /// Vector of data values. @@ -24,6 +26,9 @@ pub trait Vector: Send + Sync { fn is_empty(&self) -> bool { self.len() == 0 } + + /// Convert this vector to a new arrow [ArrayRef]. + fn to_arrow_array(&self) -> ArrayRef; } pub type VectorRef = Arc; diff --git a/src/datatypes/src/vectors/binary.rs b/src/datatypes/src/vectors/binary.rs index 27eb29ae6a..6b27c5a855 100644 --- a/src/datatypes/src/vectors/binary.rs +++ b/src/datatypes/src/vectors/binary.rs @@ -1,5 +1,7 @@ use std::any::Any; +use std::sync::Arc; +use arrow2::array::ArrayRef; use arrow2::array::BinaryValueIter; use arrow2::bitmap::utils::ZipValidity; @@ -27,6 +29,10 @@ impl Vector for BinaryVector { fn len(&self) -> usize { self.array.len() } + + fn to_arrow_array(&self) -> ArrayRef { + Arc::new(self.array.clone()) + } } impl ScalarVector for BinaryVector { diff --git a/src/datatypes/src/vectors/primitive.rs b/src/datatypes/src/vectors/primitive.rs index 73a1c14991..0caa466884 100644 --- a/src/datatypes/src/vectors/primitive.rs +++ b/src/datatypes/src/vectors/primitive.rs @@ -1,6 +1,8 @@ use std::any::Any; use std::slice::Iter; +use std::sync::Arc; +use arrow2::array::ArrayRef; use arrow2::array::{MutablePrimitiveArray, PrimitiveArray}; use arrow2::bitmap::utils::ZipValidity; @@ -33,6 +35,10 @@ impl Vector for PrimitiveVector { fn len(&self) -> usize { self.array.len() } + + fn to_arrow_array(&self) -> ArrayRef { + Arc::new(self.array.clone()) + } } impl ScalarVector for PrimitiveVector { From 6535f5c4cda6b8c4f71bb5744e1c6bdcf0b17dc8 Mon Sep 17 00:00:00 2001 From: evenyag Date: Mon, 25 Apr 2022 10:45:04 +0800 Subject: [PATCH 5/9] refactor: Rename CreateDataType to DataTypeBuilder --- src/datatypes/src/types/primitive_type.rs | 12 ++++++------ src/datatypes/src/vectors/primitive.rs | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/datatypes/src/types/primitive_type.rs b/src/datatypes/src/types/primitive_type.rs index 3bef260359..4bba31e857 100644 --- a/src/datatypes/src/types/primitive_type.rs +++ b/src/datatypes/src/types/primitive_type.rs @@ -21,15 +21,15 @@ impl PrimitiveType { } /// Create a new [DataTypeRef] from a primitive type. -pub trait CreateDataType { - fn create_data_type() -> DataTypeRef; +pub trait DataTypeBuilder { + fn build_data_type() -> DataTypeRef; } -macro_rules! impl_create_data_type { +macro_rules! impl_build_data_type { ($Type:ident) => { paste::paste! { - impl CreateDataType for $Type { - fn create_data_type() -> DataTypeRef { + impl DataTypeBuilder for $Type { + fn build_data_type() -> DataTypeRef { Arc::new(PrimitiveType::<$Type>::new()) } } @@ -63,7 +63,7 @@ macro_rules! impl_numeric { } } - impl_create_data_type!($Type); + impl_build_data_type!($Type); }; } diff --git a/src/datatypes/src/vectors/primitive.rs b/src/datatypes/src/vectors/primitive.rs index 0caa466884..e7fa024169 100644 --- a/src/datatypes/src/vectors/primitive.rs +++ b/src/datatypes/src/vectors/primitive.rs @@ -9,7 +9,7 @@ use arrow2::bitmap::utils::ZipValidity; use crate::data_type::DataTypeRef; use crate::scalar::{ScalarVector, ScalarVectorBuilder}; use crate::types::primitive_traits::Primitive; -use crate::types::primitive_type::CreateDataType; +use crate::types::primitive_type::DataTypeBuilder; use crate::vectors::Vector; /// Vector for primitive data types. @@ -23,9 +23,9 @@ impl PrimitiveVector { } } -impl Vector for PrimitiveVector { +impl Vector for PrimitiveVector { fn data_type(&self) -> DataTypeRef { - T::create_data_type() + T::build_data_type() } fn as_any(&self) -> &dyn Any { @@ -41,7 +41,7 @@ impl Vector for PrimitiveVector { } } -impl ScalarVector for PrimitiveVector { +impl ScalarVector for PrimitiveVector { type RefItem<'a> = T; type Iter<'a> = PrimitiveIter<'a, T>; type Builder = PrimitiveVectorBuilder; @@ -90,7 +90,7 @@ pub struct PrimitiveVectorBuilder { mutable_array: MutablePrimitiveArray, } -impl ScalarVectorBuilder for PrimitiveVectorBuilder { +impl ScalarVectorBuilder for PrimitiveVectorBuilder { type VectorType = PrimitiveVector; fn with_capacity(capacity: usize) -> Self { From 7f075d2c0fec887c696fda60e9ac2b9cf7a6ce5c Mon Sep 17 00:00:00 2001 From: evenyag Date: Mon, 25 Apr 2022 11:04:28 +0800 Subject: [PATCH 6/9] refactor: Rename scalar mod to scalars --- src/datatypes/src/lib.rs | 2 +- src/datatypes/src/{scalar.rs => scalars.rs} | 0 src/datatypes/src/vectors/binary.rs | 2 +- src/datatypes/src/vectors/primitive.rs | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename src/datatypes/src/{scalar.rs => scalars.rs} (100%) diff --git a/src/datatypes/src/lib.rs b/src/datatypes/src/lib.rs index 513520cf5e..7f3e4ebb8e 100644 --- a/src/datatypes/src/lib.rs +++ b/src/datatypes/src/lib.rs @@ -2,7 +2,7 @@ mod data_type; pub mod prelude; -mod scalar; +mod scalars; mod schema; pub mod type_id; mod types; diff --git a/src/datatypes/src/scalar.rs b/src/datatypes/src/scalars.rs similarity index 100% rename from src/datatypes/src/scalar.rs rename to src/datatypes/src/scalars.rs diff --git a/src/datatypes/src/vectors/binary.rs b/src/datatypes/src/vectors/binary.rs index 6b27c5a855..bb6d5dc775 100644 --- a/src/datatypes/src/vectors/binary.rs +++ b/src/datatypes/src/vectors/binary.rs @@ -6,7 +6,7 @@ use arrow2::array::BinaryValueIter; use arrow2::bitmap::utils::ZipValidity; use crate::data_type::DataTypeRef; -use crate::scalar::{ScalarVector, ScalarVectorBuilder}; +use crate::scalars::{ScalarVector, ScalarVectorBuilder}; use crate::types::binary_type::BinaryType; use crate::vectors::Vector; use crate::{LargeBinaryArray, MutableLargeBinaryArray}; diff --git a/src/datatypes/src/vectors/primitive.rs b/src/datatypes/src/vectors/primitive.rs index e7fa024169..ac1e99447d 100644 --- a/src/datatypes/src/vectors/primitive.rs +++ b/src/datatypes/src/vectors/primitive.rs @@ -7,7 +7,7 @@ use arrow2::array::{MutablePrimitiveArray, PrimitiveArray}; use arrow2::bitmap::utils::ZipValidity; use crate::data_type::DataTypeRef; -use crate::scalar::{ScalarVector, ScalarVectorBuilder}; +use crate::scalars::{ScalarVector, ScalarVectorBuilder}; use crate::types::primitive_traits::Primitive; use crate::types::primitive_type::DataTypeBuilder; use crate::vectors::Vector; From 49081d818c7b0fc1c9580f274eb4501aa42ae81f Mon Sep 17 00:00:00 2001 From: evenyag Date: Mon, 25 Apr 2022 11:50:04 +0800 Subject: [PATCH 7/9] refactor: Divide common crate into small crates --- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- src/common/{ => base}/Cargo.toml | 2 +- src/common/{ => base}/src/bytes.rs | 0 src/common/{ => base}/src/lib.rs | 0 src/datatypes/Cargo.toml | 2 +- src/datatypes/src/types/binary_type.rs | 2 +- src/datatypes/src/value.rs | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) rename src/common/{ => base}/Cargo.toml (90%) rename src/common/{ => base}/src/bytes.rs (100%) rename src/common/{ => base}/src/lib.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 4a4d27f706..0d0aa96d36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,6 +33,10 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base" +version = "0.1.0" + [[package]] name = "bytemuck" version = "1.9.1" @@ -69,10 +73,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "common" -version = "0.1.0" - [[package]] name = "datanode" version = "0.1.0" @@ -85,7 +85,7 @@ name = "datatypes" version = "0.1.0" dependencies = [ "arrow2", - "common", + "base", "paste", ] diff --git a/Cargo.toml b/Cargo.toml index b63440d6e3..deff6444b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] members = [ - "src/common", + "src/common/base", "src/datanode", "src/datatypes", "src/log-store", diff --git a/src/common/Cargo.toml b/src/common/base/Cargo.toml similarity index 90% rename from src/common/Cargo.toml rename to src/common/base/Cargo.toml index b7723d9c7f..6916ddc11b 100644 --- a/src/common/Cargo.toml +++ b/src/common/base/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "common" +name = "base" version = "0.1.0" edition = "2021" diff --git a/src/common/src/bytes.rs b/src/common/base/src/bytes.rs similarity index 100% rename from src/common/src/bytes.rs rename to src/common/base/src/bytes.rs diff --git a/src/common/src/lib.rs b/src/common/base/src/lib.rs similarity index 100% rename from src/common/src/lib.rs rename to src/common/base/src/lib.rs diff --git a/src/datatypes/Cargo.toml b/src/datatypes/Cargo.toml index 00fe122272..5b17af9837 100644 --- a/src/datatypes/Cargo.toml +++ b/src/datatypes/Cargo.toml @@ -6,6 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -common = { path = "../common" } +base = { path = "../common/base" } arrow2 = "0.10" paste = "1.0" diff --git a/src/datatypes/src/types/binary_type.rs b/src/datatypes/src/types/binary_type.rs index 18c2a0fa1b..34f729e617 100644 --- a/src/datatypes/src/types/binary_type.rs +++ b/src/datatypes/src/types/binary_type.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use arrow2::datatypes::DataType as ArrowDataType; -use common::bytes::StringBytes; +use base::bytes::StringBytes; use crate::data_type::{DataType, DataTypeRef}; use crate::type_id::LogicalTypeId; diff --git a/src/datatypes/src/value.rs b/src/datatypes/src/value.rs index 3a7f597eaf..41b43f54ce 100644 --- a/src/datatypes/src/value.rs +++ b/src/datatypes/src/value.rs @@ -1,4 +1,4 @@ -use common::bytes::{Bytes, StringBytes}; +use base::bytes::{Bytes, StringBytes}; /// Value holds a single arbitrary value of any [DataType](crate::data_type::DataType). #[derive(Debug)] From 58f19b2a33ca16e537af9688c5601c210e4f142f Mon Sep 17 00:00:00 2001 From: evenyag Date: Mon, 25 Apr 2022 14:24:25 +0800 Subject: [PATCH 8/9] refactor(datatypes): pub use some types --- src/datatypes/src/prelude.rs | 2 ++ src/datatypes/src/types.rs | 10 +++++++--- src/datatypes/src/vectors.rs | 2 ++ src/datatypes/src/vectors/binary.rs | 2 +- src/datatypes/src/vectors/primitive.rs | 6 ++---- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/datatypes/src/prelude.rs b/src/datatypes/src/prelude.rs index 1aa4775354..de9b056a39 100644 --- a/src/datatypes/src/prelude.rs +++ b/src/datatypes/src/prelude.rs @@ -1,3 +1,5 @@ pub use crate::data_type::{DataType, DataTypeRef}; +pub use crate::scalars::{ScalarVector, ScalarVectorBuilder}; pub use crate::type_id::LogicalTypeId; pub use crate::value::Value; +pub use crate::vectors::{Vector, VectorRef}; diff --git a/src/datatypes/src/types.rs b/src/datatypes/src/types.rs index 02fa6bc718..61982e99f9 100644 --- a/src/datatypes/src/types.rs +++ b/src/datatypes/src/types.rs @@ -1,3 +1,7 @@ -pub mod binary_type; -pub mod primitive_traits; -pub mod primitive_type; +mod binary_type; +mod primitive_traits; +mod primitive_type; + +pub use binary_type::BinaryType; +pub use primitive_traits::Primitive; +pub use primitive_type::{DataTypeBuilder, PrimitiveType}; diff --git a/src/datatypes/src/vectors.rs b/src/datatypes/src/vectors.rs index c90106c2ea..1ea36c32a2 100644 --- a/src/datatypes/src/vectors.rs +++ b/src/datatypes/src/vectors.rs @@ -5,6 +5,8 @@ use std::any::Any; use std::sync::Arc; use arrow2::array::ArrayRef; +pub use binary::*; +pub use primitive::*; use crate::data_type::DataTypeRef; diff --git a/src/datatypes/src/vectors/binary.rs b/src/datatypes/src/vectors/binary.rs index bb6d5dc775..60b49dd827 100644 --- a/src/datatypes/src/vectors/binary.rs +++ b/src/datatypes/src/vectors/binary.rs @@ -7,7 +7,7 @@ use arrow2::bitmap::utils::ZipValidity; use crate::data_type::DataTypeRef; use crate::scalars::{ScalarVector, ScalarVectorBuilder}; -use crate::types::binary_type::BinaryType; +use crate::types::BinaryType; use crate::vectors::Vector; use crate::{LargeBinaryArray, MutableLargeBinaryArray}; diff --git a/src/datatypes/src/vectors/primitive.rs b/src/datatypes/src/vectors/primitive.rs index ac1e99447d..95d1266843 100644 --- a/src/datatypes/src/vectors/primitive.rs +++ b/src/datatypes/src/vectors/primitive.rs @@ -2,14 +2,12 @@ use std::any::Any; use std::slice::Iter; use std::sync::Arc; -use arrow2::array::ArrayRef; -use arrow2::array::{MutablePrimitiveArray, PrimitiveArray}; +use arrow2::array::{ArrayRef, MutablePrimitiveArray, PrimitiveArray}; use arrow2::bitmap::utils::ZipValidity; use crate::data_type::DataTypeRef; use crate::scalars::{ScalarVector, ScalarVectorBuilder}; -use crate::types::primitive_traits::Primitive; -use crate::types::primitive_type::DataTypeBuilder; +use crate::types::{DataTypeBuilder, Primitive}; use crate::vectors::Vector; /// Vector for primitive data types. From b505c87e8d9256f38c818206e1cb5e7acdd3edec Mon Sep 17 00:00:00 2001 From: evenyag Date: Tue, 26 Apr 2022 11:48:22 +0800 Subject: [PATCH 9/9] refactor: Rename base to common-base --- Cargo.lock | 10 +++++----- src/common/base/Cargo.toml | 2 +- src/datatypes/Cargo.toml | 2 +- src/datatypes/src/types/binary_type.rs | 2 +- src/datatypes/src/value.rs | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d0aa96d36..cfea96319f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,10 +33,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "base" -version = "0.1.0" - [[package]] name = "bytemuck" version = "1.9.1" @@ -73,6 +69,10 @@ dependencies = [ "num-traits", ] +[[package]] +name = "common-base" +version = "0.1.0" + [[package]] name = "datanode" version = "0.1.0" @@ -85,7 +85,7 @@ name = "datatypes" version = "0.1.0" dependencies = [ "arrow2", - "base", + "common-base", "paste", ] diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml index 6916ddc11b..afcd0e6983 100644 --- a/src/common/base/Cargo.toml +++ b/src/common/base/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "base" +name = "common-base" version = "0.1.0" edition = "2021" diff --git a/src/datatypes/Cargo.toml b/src/datatypes/Cargo.toml index 5b17af9837..7365737cc3 100644 --- a/src/datatypes/Cargo.toml +++ b/src/datatypes/Cargo.toml @@ -6,6 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -base = { path = "../common/base" } arrow2 = "0.10" +common-base = { path = "../common/base" } paste = "1.0" diff --git a/src/datatypes/src/types/binary_type.rs b/src/datatypes/src/types/binary_type.rs index 34f729e617..18f5c86813 100644 --- a/src/datatypes/src/types/binary_type.rs +++ b/src/datatypes/src/types/binary_type.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use arrow2::datatypes::DataType as ArrowDataType; -use base::bytes::StringBytes; +use common_base::bytes::StringBytes; use crate::data_type::{DataType, DataTypeRef}; use crate::type_id::LogicalTypeId; diff --git a/src/datatypes/src/value.rs b/src/datatypes/src/value.rs index 41b43f54ce..8f6dd3b0de 100644 --- a/src/datatypes/src/value.rs +++ b/src/datatypes/src/value.rs @@ -1,4 +1,4 @@ -use base::bytes::{Bytes, StringBytes}; +use common_base::bytes::{Bytes, StringBytes}; /// Value holds a single arbitrary value of any [DataType](crate::data_type::DataType). #[derive(Debug)]