diff --git a/src/datatypes/src/arrow_array.rs b/src/datatypes/src/arrow_array.rs index ca2cb6cc48..3444598ede 100644 --- a/src/datatypes/src/arrow_array.rs +++ b/src/datatypes/src/arrow_array.rs @@ -12,11 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use arrow::array::{ - self, Array, BinaryArray as ArrowBinaryArray, ListArray, - MutableBinaryArray as ArrowMutableBinaryArray, MutableUtf8Array, PrimitiveArray, Utf8Array, -}; -use arrow::datatypes::DataType as ArrowDataType; +use arrow::array::{self, Array, ListArray, PrimitiveArray}; +use arrow::datatypes::DataType; use common_time::timestamp::Timestamp; use snafu::OptionExt; @@ -24,10 +21,10 @@ use crate::error::{ConversionSnafu, Result}; use crate::prelude::ConcreteDataType; use crate::value::{ListValue, Value}; -pub type BinaryArray = ArrowBinaryArray; -pub type MutableBinaryArray = ArrowMutableBinaryArray; -pub type MutableStringArray = MutableUtf8Array; -pub type StringArray = Utf8Array; +pub type BinaryArray = arrow::array::LargeBinaryArray; +pub type MutableBinaryArray = arrow::array::LargeBinaryBuilder; +pub type StringArray = arrow::array::StringArray; +pub type MutableStringArray = arrow::array::StringBuilder; macro_rules! cast_array { ($arr: ident, $CastType: ty) => { @@ -45,37 +42,29 @@ pub fn arrow_array_get(array: &dyn Array, idx: usize) -> Result { } let result = match array.data_type() { - ArrowDataType::Null => Value::Null, - ArrowDataType::Boolean => { - Value::Boolean(cast_array!(array, array::BooleanArray).value(idx)) - } - ArrowDataType::Binary | ArrowDataType::LargeBinary => { + DataType::Null => Value::Null, + DataType::Boolean => Value::Boolean(cast_array!(array, array::BooleanArray).value(idx)), + DataType::Binary | DataType::LargeBinary => { Value::Binary(cast_array!(array, BinaryArray).value(idx).into()) } - ArrowDataType::Int8 => Value::Int8(cast_array!(array, PrimitiveArray::).value(idx)), - ArrowDataType::Int16 => Value::Int16(cast_array!(array, PrimitiveArray::).value(idx)), - ArrowDataType::Int32 => Value::Int32(cast_array!(array, PrimitiveArray::).value(idx)), - ArrowDataType::Int64 => Value::Int64(cast_array!(array, PrimitiveArray::).value(idx)), - ArrowDataType::UInt8 => Value::UInt8(cast_array!(array, PrimitiveArray::).value(idx)), - ArrowDataType::UInt16 => { - Value::UInt16(cast_array!(array, PrimitiveArray::).value(idx)) - } - ArrowDataType::UInt32 => { - Value::UInt32(cast_array!(array, PrimitiveArray::).value(idx)) - } - ArrowDataType::UInt64 => { - Value::UInt64(cast_array!(array, PrimitiveArray::).value(idx)) - } - ArrowDataType::Float32 => { + DataType::Int8 => Value::Int8(cast_array!(array, PrimitiveArray::).value(idx)), + DataType::Int16 => Value::Int16(cast_array!(array, PrimitiveArray::).value(idx)), + DataType::Int32 => Value::Int32(cast_array!(array, PrimitiveArray::).value(idx)), + DataType::Int64 => Value::Int64(cast_array!(array, PrimitiveArray::).value(idx)), + DataType::UInt8 => Value::UInt8(cast_array!(array, PrimitiveArray::).value(idx)), + DataType::UInt16 => Value::UInt16(cast_array!(array, PrimitiveArray::).value(idx)), + DataType::UInt32 => Value::UInt32(cast_array!(array, PrimitiveArray::).value(idx)), + DataType::UInt64 => Value::UInt64(cast_array!(array, PrimitiveArray::).value(idx)), + DataType::Float32 => { Value::Float32(cast_array!(array, PrimitiveArray::).value(idx).into()) } - ArrowDataType::Float64 => { + DataType::Float64 => { Value::Float64(cast_array!(array, PrimitiveArray::).value(idx).into()) } - ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => { + DataType::Utf8 | DataType::LargeUtf8 => { Value::String(cast_array!(array, StringArray).value(idx).into()) } - ArrowDataType::Timestamp(t, _) => { + DataType::Timestamp(t, _) => { let value = cast_array!(array, PrimitiveArray::).value(idx); let unit = match ConcreteDataType::from_arrow_time_unit(t) { ConcreteDataType::Timestamp(t) => t.unit, @@ -83,7 +72,7 @@ pub fn arrow_array_get(array: &dyn Array, idx: usize) -> Result { }; Value::Timestamp(Timestamp::new(value, unit)) } - ArrowDataType::List(_) => { + DataType::List(_) => { let array = cast_array!(array, ListArray::).value(idx); let inner_datatype = ConcreteDataType::try_from(array.data_type())?; let values = (0..array.len()) diff --git a/src/datatypes/src/types/primitive_traits.rs b/src/datatypes/src/types/primitive_traits.rs index e900ba217e..14c98b42e0 100644 --- a/src/datatypes/src/types/primitive_traits.rs +++ b/src/datatypes/src/types/primitive_traits.rs @@ -14,8 +14,7 @@ use std::cmp::Ordering; -use arrow::compute::arithmetics::basic::NativeArithmetics; -use arrow::types::NativeType; +use arrow::datatypes::ArrowPrimitiveType; use num::NumCast; use crate::prelude::Scalar; @@ -29,9 +28,8 @@ pub trait Primitive: + Copy + Into + IntoValueRef<'static> - + NativeType + + ArrowPrimitiveType + serde::Serialize - + NativeArithmetics + NumCast + Scalar { diff --git a/src/datatypes/src/vectors/binary.rs b/src/datatypes/src/vectors/binary.rs index 817b29bca0..7be3dc6a8e 100644 --- a/src/datatypes/src/vectors/binary.rs +++ b/src/datatypes/src/vectors/binary.rs @@ -15,8 +15,8 @@ use std::any::Any; use std::sync::Arc; -use arrow::array::{Array, ArrayRef, BinaryValueIter, MutableArray}; -use arrow::bitmap::utils::ZipValidity; +use arrow::array::{Array, ArrayRef}; +use arrow::array::{ArrayIter, GenericByteArray}; use snafu::{OptionExt, ResultExt}; use crate::arrow_array::{BinaryArray, MutableBinaryArray}; @@ -106,7 +106,7 @@ impl Vector for BinaryVector { impl ScalarVector for BinaryVector { type OwnedItem = Vec; type RefItem<'a> = &'a [u8]; - type Iter<'a> = ZipValidity<'a, &'a [u8], BinaryValueIter<'a, i64>>; + type Iter<'a> = ArrayIter<&'a BinaryArray>; type Builder = BinaryVectorBuilder; fn get_data(&self, idx: usize) -> Option> {